From 4ce31d389da26d0aac6fb110f372039f90673b65 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 16:14:47 +0000 Subject: [PATCH 01/27] make BusHandle concrete --- crates/aggregator/src/committee_finalizer.rs | 6 +- crates/aggregator/src/ext.rs | 14 +-- crates/aggregator/src/plaintext_aggregator.rs | 4 +- crates/aggregator/src/publickey_aggregator.rs | 4 +- .../src/threshold_plaintext_aggregator.rs | 4 +- crates/ciphernode-builder/src/ciphernode.rs | 4 +- .../src/ciphernode_builder.rs | 2 +- crates/data/src/sled_store.rs | 5 +- crates/entrypoint/src/helpers/datastore.rs | 4 +- crates/entrypoint/src/helpers/shutdown.rs | 2 +- .../entrypoint/src/start/aggregator_start.rs | 2 +- crates/entrypoint/src/start/start.rs | 2 +- crates/events/src/bus_handle.rs | 67 +++++----- crates/events/src/enclave_event/mod.rs | 115 +++++++++++++----- crates/events/src/eventbus.rs | 6 +- crates/events/src/eventbus_factory.rs | 3 +- crates/events/src/lib.rs | 1 + crates/events/src/sequencer.rs | 29 +++++ crates/events/src/traits.rs | 4 +- crates/evm/src/bonding_registry_sol.rs | 4 +- crates/evm/src/ciphernode_registry_sol.rs | 12 +- crates/evm/src/enclave_sol.rs | 2 +- crates/evm/src/enclave_sol_reader.rs | 2 +- crates/evm/src/enclave_sol_writer.rs | 6 +- crates/evm/src/event_reader.rs | 8 +- .../evm/src/historical_event_coordinator.rs | 6 +- crates/fhe/src/ext.rs | 4 +- crates/keyshare/src/ext.rs | 8 +- crates/keyshare/src/keyshare.rs | 4 +- crates/keyshare/src/threshold_keyshare.rs | 4 +- crates/logger/src/logger.rs | 4 +- crates/net/src/document_publisher.rs | 23 ++-- crates/net/src/net_event_translator.rs | 8 +- crates/request/src/router.rs | 8 +- crates/sortition/src/ciphernode_selector.rs | 6 +- crates/sortition/src/sortition.rs | 6 +- crates/test-helpers/src/lib.rs | 6 +- crates/test-helpers/src/plaintext_writer.rs | 2 +- crates/test-helpers/src/public_key_writer.rs | 2 +- crates/tests/tests/integration.rs | 9 +- crates/tests/tests/integration_legacy.rs | 20 ++- 41 files changed, 257 insertions(+), 175 deletions(-) create mode 100644 crates/events/src/sequencer.rs diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index aef3847ea2..97efc533ad 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -16,19 +16,19 @@ use tracing::{error, info}; /// CommitteeFinalizer is an actor that listens to CommitteeRequested events and dispatches /// CommitteeFinalizeRequested events after the submission deadline has passed. pub struct CommitteeFinalizer { - bus: BusHandle, + bus: BusHandle, pending_committees: HashMap, } impl CommitteeFinalizer { - pub fn new(bus: &BusHandle) -> Self { + pub fn new(bus: &BusHandle) -> Self { Self { bus: bus.clone(), pending_committees: HashMap::new(), } } - pub fn attach(bus: &BusHandle) -> Addr { + pub fn attach(bus: &BusHandle) -> Addr { let addr = CommitteeFinalizer::new(bus).start(); bus.subscribe_all( diff --git a/crates/aggregator/src/ext.rs b/crates/aggregator/src/ext.rs index abdbbdf489..72c9dbe020 100644 --- a/crates/aggregator/src/ext.rs +++ b/crates/aggregator/src/ext.rs @@ -28,12 +28,12 @@ use e3_sortition::Sortition; #[deprecated = "In favour of ThresholdPlaintextAggregatorExtension"] pub struct PlaintextAggregatorExtension { - bus: BusHandle, + bus: BusHandle, sortition: Addr, } impl PlaintextAggregatorExtension { - pub fn create(bus: &BusHandle, sortition: &Addr) -> Box { + pub fn create(bus: &BusHandle, sortition: &Addr) -> Box { Box::new(Self { bus: bus.clone(), sortition: sortition.clone(), @@ -148,11 +148,11 @@ impl E3Extension for PlaintextAggregatorExtension { } pub struct PublicKeyAggregatorExtension { - bus: BusHandle, + bus: BusHandle, } impl PublicKeyAggregatorExtension { - pub fn create(bus: &BusHandle) -> Box { + pub fn create(bus: &BusHandle) -> Box { Box::new(Self { bus: bus.clone() }) } } @@ -233,7 +233,7 @@ impl E3Extension for PublicKeyAggregatorExtension { fn create_publickey_aggregator( fhe: Arc, - bus: BusHandle, + bus: BusHandle, e3_id: E3id, sync_state: Persistable, ) -> Recipient { @@ -247,14 +247,14 @@ fn create_publickey_aggregator( } pub struct ThresholdPlaintextAggregatorExtension { - bus: BusHandle, + bus: BusHandle, sortition: Addr, multithread: Addr, } impl ThresholdPlaintextAggregatorExtension { pub fn create( - bus: &BusHandle, + bus: &BusHandle, sortition: &Addr, multithread: &Addr, ) -> Box { diff --git a/crates/aggregator/src/plaintext_aggregator.rs b/crates/aggregator/src/plaintext_aggregator.rs index afc416255b..583e1c4f84 100644 --- a/crates/aggregator/src/plaintext_aggregator.rs +++ b/crates/aggregator/src/plaintext_aggregator.rs @@ -72,7 +72,7 @@ struct ComputeAggregate { #[deprecated = "To be replaced by ThresholdPlaintextAggregator"] pub struct PlaintextAggregator { fhe: Arc, - bus: BusHandle, + bus: BusHandle, sortition: Addr, e3_id: E3id, state: Persistable, @@ -80,7 +80,7 @@ pub struct PlaintextAggregator { pub struct PlaintextAggregatorParams { pub fhe: Arc, - pub bus: BusHandle, + pub bus: BusHandle, pub sortition: Addr, pub e3_id: E3id, } diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index 4c66dbd129..e100504669 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -56,14 +56,14 @@ struct ComputeAggregate { pub struct PublicKeyAggregator { fhe: Arc, - bus: BusHandle, + bus: BusHandle, e3_id: E3id, state: Persistable, } pub struct PublicKeyAggregatorParams { pub fhe: Arc, - pub bus: BusHandle, + pub bus: BusHandle, pub e3_id: E3id, } diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 4471338185..81e84f9338 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -122,7 +122,7 @@ pub struct ComputeAggregate { pub struct ThresholdPlaintextAggregator { multithread: Addr, - bus: BusHandle, + bus: BusHandle, sortition: Addr, e3_id: E3id, state: Persistable, @@ -130,7 +130,7 @@ pub struct ThresholdPlaintextAggregator { pub struct ThresholdPlaintextAggregatorParams { pub multithread: Addr, - pub bus: BusHandle, + pub bus: BusHandle, pub sortition: Addr, pub e3_id: E3id, } diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 6bf9775d47..3c02ee1d29 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -12,7 +12,7 @@ use e3_events::{BusHandle, EnclaveEvent, EventBus, HistoryCollector}; pub struct CiphernodeHandle { pub address: String, pub store: DataStore, - pub bus: BusHandle, + pub bus: BusHandle, pub history: Option>>, pub errors: Option>>, } @@ -21,7 +21,7 @@ impl CiphernodeHandle { pub fn new( address: String, store: DataStore, - bus: BusHandle, + bus: BusHandle, history: Option>>, errors: Option>>, ) -> Self { diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 861b567a31..d404fb4ed3 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -16,7 +16,7 @@ use e3_aggregator::ext::{ use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{DataStore, InMemStore, Repositories, RepositoriesFactory}; -use e3_events::{EnclaveEvent, EventBus, EventBusConfig}; +use e3_events::{EnclaveEvent, EventBus, EventBusConfig, Stored, Unstored}; use e3_evm::{ helpers::{ load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 87f3d87c79..b86e29cda5 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -21,7 +21,7 @@ use tracing::{error, info}; pub struct SledStore { db: Option, - bus: BusHandle, + bus: BusHandle, // Only used for Shutdown } impl Actor for SledStore { @@ -29,7 +29,7 @@ impl Actor for SledStore { } impl SledStore { - pub fn new(bus: &BusHandle, path: &PathBuf) -> Result> { + pub fn new(bus: &BusHandle, path: &PathBuf) -> Result> { info!("Starting SledStore with {:?}", path); let db = SledDb::new(PathBuf::from(path))?; @@ -108,7 +108,6 @@ impl Handler for SledStore { } } } - impl Handler for SledStore { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { diff --git a/crates/entrypoint/src/helpers/datastore.rs b/crates/entrypoint/src/helpers/datastore.rs index 39fc5129d0..3f9677c86e 100644 --- a/crates/entrypoint/src/helpers/datastore.rs +++ b/crates/entrypoint/src/helpers/datastore.rs @@ -13,7 +13,7 @@ use e3_data::{DataStore, InMemStore, SledDb, SledStore}; use e3_data::{Repositories, RepositoriesFactory}; use e3_events::{get_enclave_bus_handle, BusHandle, EnclaveEvent}; -pub fn get_sled_store(bus: &BusHandle, db_file: &PathBuf) -> Result { +pub fn get_sled_store(bus: &BusHandle, db_file: &PathBuf) -> Result { Ok((&SledStore::new(bus, db_file)?).into()) } @@ -21,7 +21,7 @@ pub fn get_in_mem_store() -> DataStore { (&InMemStore::new(true).start()).into() } -pub fn setup_datastore(config: &AppConfig, bus: &BusHandle) -> Result { +pub fn setup_datastore(config: &AppConfig, bus: &BusHandle) -> Result { let store: DataStore = if !config.use_in_mem_store() { get_sled_store(&bus, &config.db_file())? } else { diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index ae4e6123b2..2f2d66d700 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -14,7 +14,7 @@ use tokio::{ }; use tracing::{error, info}; -pub async fn listen_for_shutdown(bus: BusHandle, mut handle: JoinHandle>) { +pub async fn listen_for_shutdown(bus: BusHandle, mut handle: JoinHandle>) { let mut sigterm = signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal stream"); select! { diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index 3789ba2f1b..1bf1a79c07 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -27,7 +27,7 @@ pub async fn execute( pubkey_write_path: Option, plaintext_write_path: Option, experimental_trbfv: bool, -) -> Result<(BusHandle, JoinHandle>, String)> { +) -> Result<(BusHandle, JoinHandle>, String)> { let bus = get_enclave_bus_handle(); let rng = Arc::new(Mutex::new(ChaCha20Rng::from_rng(OsRng)?)); let store = setup_datastore(config, &bus)?; diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 9ba3e3ab35..211e439ddb 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -26,7 +26,7 @@ pub async fn execute( config: &AppConfig, address: Address, experimental_trbfv: bool, -) -> Result<(BusHandle, JoinHandle>, String)> { +) -> Result<(BusHandle, JoinHandle>, String)> { let rng = Arc::new(Mutex::new(rand_chacha::ChaCha20Rng::from_rng(OsRng)?)); let bus = get_enclave_bus_handle(); diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index b0e2e3135c..400a4275d6 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -4,24 +4,30 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Addr, Recipient}; +use std::marker::PhantomData; + +use actix::{Actor, Addr, Recipient}; use crate::{ + sequencer::Sequencer, traits::{ - CompositeEvent, ErrorDispatcher, ErrorFactory, Event, EventConstructorWithTimestamp, - EventFactory, EventPublisher, EventSubscriber, + ErrorDispatcher, ErrorFactory, Event, EventConstructorWithTimestamp, EventFactory, + EventPublisher, EventSubscriber, }, - ErrorEvent, EventBus, Subscribe, + EnclaveErrorType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, Stored, Subscribe, + Unstored, }; #[derive(Clone, Debug)] -pub struct BusHandle { - bus: Addr>, +pub struct BusHandle { + bus: Addr>>, + seq: Addr, } -impl BusHandle { - pub fn new(bus: Addr>) -> Self { - Self { bus } +impl BusHandle { + pub fn new(bus: Addr>>) -> Self { + let seq = Sequencer::new(&bus).start(); + Self { bus, seq } } pub fn bus(&self) -> Addr> { @@ -29,56 +35,53 @@ impl BusHandle { } } -impl EventPublisher for BusHandle { - fn publish(&self, data: impl Into) { +impl EventPublisher> for BusHandle { + fn publish(&self, data: impl Into) { let evt = self.event_from(data); - self.bus.do_send(evt); + self.seq.do_send(evt); } - fn publish_from_remote(&self, data: impl Into, ts: u128) { + fn publish_from_remote(&self, data: impl Into, ts: u128) { let evt = self.event_from_remote_source(data, ts); - self.bus.do_send(evt) + self.seq.do_send(evt) } - fn naked_dispatch(&self, event: E) { - self.bus.do_send(event); + fn naked_dispatch(&self, event: EnclaveEvent) { + self.seq.do_send(event); } } -impl ErrorDispatcher for BusHandle -where - E: CompositeEvent, -{ - fn err(&self, err_type: E::ErrType, error: impl Into) { +impl ErrorDispatcher> for BusHandle { + fn err(&self, err_type: EnclaveErrorType, error: impl Into) { let evt = self.event_from_error(err_type, error); - self.bus.do_send(evt); + self.seq.do_send(evt); } } -impl EventFactory for BusHandle { - fn event_from(&self, data: impl Into) -> E { +impl EventFactory> for BusHandle { + fn event_from(&self, data: impl Into) -> EnclaveEvent { // TODO: add self.hcl.tick() - E::new_with_timestamp(data.into(), 0) + EnclaveEvent::::new_with_timestamp(data.into(), 0) } fn event_from_remote_source(&self, data: impl Into, ts: u128) -> E { // TODO: add self.hcl.receive(ts) - E::new_with_timestamp(data.into(), ts) + EnclaveEvent::::new_with_timestamp(data.into(), ts) } } -impl ErrorFactory for BusHandle { - fn event_from_error(&self, err_type: E::ErrType, error: impl Into) -> E { - E::from_error(err_type, error) +impl ErrorFactory> for BusHandle { + fn event_from_error(&self, err_type: EnclaveErrorType, error: impl Into) -> E { + EnclaveEvent::::from_error(err_type, error) } } -impl EventSubscriber for BusHandle { - fn subscribe(&self, event_type: &str, recipient: Recipient) { +impl EventSubscriber> for BusHandle { + fn subscribe(&self, event_type: &str, recipient: Recipient>) { self.bus.do_send(Subscribe::new(event_type, recipient)) } - fn subscribe_all(&self, event_types: &[&str], recipient: Recipient) { + fn subscribe_all(&self, event_types: &[&str], recipient: Recipient>) { for event_type in event_types.into_iter() { self.bus .do_send(Subscribe::new(*event_type, recipient.clone())); diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 0048b85890..515cc903b6 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -66,7 +66,7 @@ use crate::{ E3id, EventId, }; use actix::Message; -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::{ fmt::{self}, hash::Hash, @@ -123,14 +123,45 @@ impl EnclaveEventData { } } +pub trait SeqState: Clone + std::fmt::Debug + 'static { + type Seq: Unpin + + Sync + + Send + + Serialize + + DeserializeOwned + + Clone + + PartialEq + + Eq + + Hash + + std::fmt::Debug; +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Unstored; + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Stored; + +impl SeqState for Unstored { + type Seq = (); +} + +impl SeqState for Stored { + type Seq = u64; +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EnclaveEvent { +pub struct EnclaveEvent { id: EventId, payload: EnclaveEventData, + seq: S::Seq, } -impl EnclaveEvent { +impl EnclaveEvent +where + S: SeqState, +{ pub fn to_bytes(&self) -> Result, bincode::Error> { bincode::serialize(self) } @@ -140,15 +171,23 @@ impl EnclaveEvent { } pub fn get_id(&self) -> EventId { - self.clone().into() + self.into() + } +} + +impl EnclaveEvent { + pub fn into_stored(self, seq: u64) -> EnclaveEvent { + EnclaveEvent:: { + id: self.id, + payload: self.payload, + seq, + } } } -impl Event for EnclaveEvent { +impl Event for EnclaveEvent { type Id = EventId; type Data = EnclaveEventData; - // type FromError = anyhow::Error; - // type ErrType = EnclaveErrorType; fn event_type(&self) -> String { self.payload.event_type() @@ -166,7 +205,7 @@ impl Event for EnclaveEvent { } } -impl ErrorEvent for EnclaveEvent { +impl ErrorEvent for EnclaveEvent { type ErrType = EnclaveErrorType; type FromError = anyhow::Error; @@ -176,17 +215,24 @@ impl ErrorEvent for EnclaveEvent { EnclaveEvent { payload: payload.into(), id, + seq: (), } } } -impl From for EventId { - fn from(value: EnclaveEvent) -> Self { +impl From> for EventId { + fn from(value: EnclaveEvent) -> Self { value.id } } -impl EnclaveEvent { +impl From<&EnclaveEvent> for EventId { + fn from(value: &EnclaveEvent) -> Self { + value.id.clone() + } +} + +impl EnclaveEvent { pub fn get_e3_id(&self) -> Option { match self.payload { EnclaveEventData::KeyshareCreated(ref data) => Some(data.e3_id.clone()), @@ -238,35 +284,42 @@ impl_into_event_data!( ThresholdShareCreated ); -impl TryFrom<&EnclaveEvent> for EnclaveError { - type Error = anyhow::Error; - fn try_from(value: &EnclaveEvent) -> Result { - value.clone().try_into() - } -} - -impl TryFrom for EnclaveError { - type Error = anyhow::Error; - fn try_from(value: EnclaveEvent) -> Result { - if let EnclaveEventData::EnclaveError(data) = value.payload.clone() { - Ok(data) - } else { - return Err(anyhow::anyhow!("Not an enclave error {:?}", value)); - } - } -} +// impl TryFrom<&EnclaveEvent> for EnclaveError { +// type Error = anyhow::Error; +// fn try_from(value: &EnclaveEvent) -> Result { +// value.clone().try_into() +// } +// } +// +// impl TryFrom for EnclaveError { +// type Error = anyhow::Error; +// fn try_from(value: EnclaveEvent) -> Result { +// if let EnclaveEventData::EnclaveError(data) = value.payload.clone() { +// Ok(data) +// } else { +// return Err(anyhow::anyhow!("Not an enclave error {:?}", value)); +// } +// } +// } -impl fmt::Display for EnclaveEvent { +impl fmt::Display for EnclaveEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&format!("{:?}", self)) } } -impl EventConstructorWithTimestamp for EnclaveEvent { +impl EventConstructorWithTimestamp for EnclaveEvent { fn new_with_timestamp(data: Self::Data, _ts: u128) -> Self { let payload = data.into(); let id = EventId::hash(&payload); // hcl.receive(remote_ts)?; - EnclaveEvent { id, payload } + EnclaveEvent { + id, + payload, + seq: (), + } } } + +#[cfg(test)] +mod tests {} diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 883663d9fb..23c22c0b81 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::traits::{ErrorEvent, Event}; -use crate::{prelude::*, BusHandle, CompositeEvent}; +use crate::{prelude::*, BusHandle, CompositeEvent, EnclaveEvent, Stored}; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; use std::collections::{HashMap, VecDeque}; @@ -133,8 +133,8 @@ impl Handler for EventBus { } } -impl From>> for BusHandle { - fn from(value: Addr>) -> Self { +impl From>>> for BusHandle { + fn from(value: Addr>>) -> Self { BusHandle::new(value) } } diff --git a/crates/events/src/eventbus_factory.rs b/crates/events/src/eventbus_factory.rs index f7312d6f71..24dd404a96 100644 --- a/crates/events/src/eventbus_factory.rs +++ b/crates/events/src/eventbus_factory.rs @@ -18,6 +18,7 @@ use crate::EnclaveEvent; use crate::EventBus; use crate::HistoryCollector; use crate::Subscribe; +use crate::Unstored; // The singleton factory using once_cell pub struct EventBusFactory { @@ -92,6 +93,6 @@ pub fn get_error_collector() -> Addr> { EventBusFactory::instance().get_error_collector() } -pub fn get_enclave_bus_handle() -> BusHandle { +pub fn get_enclave_bus_handle() -> BusHandle { get_enclave_event_bus().into() } diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 1f356b3a47..eac423f6e7 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -15,6 +15,7 @@ pub mod hlc; mod ordered_set; pub mod prelude; mod seed; +mod sequencer; mod traits; pub use bus_handle::*; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs new file mode 100644 index 0000000000..a1e73e136d --- /dev/null +++ b/crates/events/src/sequencer.rs @@ -0,0 +1,29 @@ +use actix::{Actor, Addr, Handler}; + +use crate::{EnclaveEvent, EventBus, Stored, Unstored}; + +pub struct Sequencer { + bus: Addr>>, + seq: u64, +} + +impl Sequencer { + pub fn new(bus: &Addr>>) -> Self { + Self { + bus: bus.clone(), + seq: 0, + } + } +} + +impl Actor for Sequencer { + type Context = actix::Context; +} + +impl Handler> for Sequencer { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + self.seq += 1; + self.bus.do_send(msg.into_stored(self.seq)) + } +} diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index ba98fda9da..de29f13c67 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -87,6 +87,6 @@ pub trait EventConstructorWithTimestamp: Event + Sized { fn new_with_timestamp(data: Self::Data, ts: u128) -> Self; } -pub trait CompositeEvent: ErrorEvent + EventConstructorWithTimestamp {} +pub trait CompositeEvent: EventConstructorWithTimestamp {} -impl CompositeEvent for E where E: Sized + Event + ErrorEvent + EventConstructorWithTimestamp {} +impl CompositeEvent for E where E: Sized + Event + EventConstructorWithTimestamp {} diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index af199634ae..4ec776957a 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -145,7 +145,7 @@ pub struct BondingRegistrySolReader; impl BondingRegistrySolReader { pub async fn attach

( processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, repository: &Repository, @@ -179,7 +179,7 @@ pub struct BondingRegistrySol; impl BondingRegistrySol { pub async fn attach

( processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, repository: &Repository, diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index a6cf0918f4..150d334834 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -218,7 +218,7 @@ pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { pub async fn attach

( processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, repository: &Repository, @@ -250,12 +250,12 @@ impl CiphernodeRegistrySolReader { pub struct CiphernodeRegistrySolWriter

{ provider: EthProvider

, contract_address: Address, - bus: BusHandle, + bus: BusHandle, } impl CiphernodeRegistrySolWriter

{ pub async fn new( - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: Address, ) -> Result { @@ -267,7 +267,7 @@ impl CiphernodeRegistrySolWriter } pub async fn attach( - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, is_aggregator: bool, @@ -511,7 +511,7 @@ pub struct CiphernodeRegistrySol; impl CiphernodeRegistrySol { pub async fn attach

( processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, repository: &Repository, @@ -535,7 +535,7 @@ impl CiphernodeRegistrySol { } pub async fn attach_writer

( - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, is_aggregator: bool, diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 387116f92c..6f93c9265d 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -19,7 +19,7 @@ pub struct EnclaveSol; impl EnclaveSol { pub async fn attach( processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, read_provider: EthProvider, write_provider: EthProvider, contract_address: &str, diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index 3a2a938005..e37e4d0ec1 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -107,7 +107,7 @@ pub struct EnclaveSolReader; impl EnclaveSolReader { pub async fn attach

( processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, repository: &Repository, diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 35aade7e7f..6b64ed11b8 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -35,12 +35,12 @@ sol!( pub struct EnclaveSolWriter

{ provider: EthProvider

, contract_address: Address, - bus: BusHandle, + bus: BusHandle, } impl EnclaveSolWriter

{ pub fn new( - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: Address, ) -> Result { @@ -52,7 +52,7 @@ impl EnclaveSolWriter

{ } pub async fn attach( - bus: &BusHandle, + bus: &BusHandle, provider: EthProvider

, contract_address: &str, ) -> Result>> { diff --git a/crates/evm/src/event_reader.rs b/crates/evm/src/event_reader.rs index eec8f1e463..30b3b679f0 100644 --- a/crates/evm/src/event_reader.rs +++ b/crates/evm/src/event_reader.rs @@ -54,7 +54,7 @@ pub struct EvmEventReaderParams

{ contract_address: Address, start_block: Option, processor: Recipient, - bus: BusHandle, + bus: BusHandle, state: Persistable, rpc_url: String, } @@ -83,7 +83,7 @@ pub struct EvmEventReader

{ /// Processor to forward events an actor processor: Recipient, /// Event bus for error propagation only - bus: BusHandle, + bus: BusHandle, /// The auto persistable state of the event reader state: Persistable, /// The RPC URL for the provider @@ -113,7 +113,7 @@ impl EvmEventReader

{ contract_address: &str, start_block: Option, processor: &Recipient, - bus: &BusHandle, + bus: &BusHandle, repository: &Repository, rpc_url: String, ) -> Result> { @@ -191,7 +191,7 @@ async fn stream_from_evm( extractor: fn(&LogData, Option<&B256>, u64) -> Option, mut shutdown: oneshot::Receiver<()>, start_block: Option, - bus: &BusHandle, + bus: &BusHandle, rpc_url: String, ) { let chain_id = provider.chain_id(); diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs index 12f1b9e466..042a58f04b 100644 --- a/crates/evm/src/historical_event_coordinator.rs +++ b/crates/evm/src/historical_event_coordinator.rs @@ -30,13 +30,13 @@ pub struct HistoricalEventCoordinator { /// Buffered events during historical sync buffered_events: Vec, /// Target to forward events to (typically EventBus) - target: BusHandle, + target: BusHandle, /// Whether we've started forwarding (after Start message) started: bool, } impl HistoricalEventCoordinator { - pub fn new(target: BusHandle) -> Self { + pub fn new(target: BusHandle) -> Self { Self { registered_count: 0, completed_count: 0, @@ -46,7 +46,7 @@ impl HistoricalEventCoordinator { } } - pub fn setup(target: BusHandle) -> Addr { + pub fn setup(target: BusHandle) -> Addr { Self::new(target).start() } diff --git a/crates/fhe/src/ext.rs b/crates/fhe/src/ext.rs index 8c568f34f8..20de58d4b2 100644 --- a/crates/fhe/src/ext.rs +++ b/crates/fhe/src/ext.rs @@ -20,11 +20,11 @@ pub const FHE_KEY: TypedKey> = TypedKey::new("fhe"); /// TODO: move these to each package with access on MyStruct::launcher() pub struct FheExtension { rng: SharedRng, - bus: BusHandle, + bus: BusHandle, } impl FheExtension { - pub fn create(bus: &BusHandle, rng: &SharedRng) -> Box { + pub fn create(bus: &BusHandle, rng: &SharedRng) -> Box { Box::new(Self { rng: rng.clone(), bus: bus.clone(), diff --git a/crates/keyshare/src/ext.rs b/crates/keyshare/src/ext.rs index c7bb8c95f4..a7ed81ca3a 100644 --- a/crates/keyshare/src/ext.rs +++ b/crates/keyshare/src/ext.rs @@ -20,13 +20,13 @@ use e3_request::{E3Context, E3ContextSnapshot, E3Extension, META_KEY}; use std::sync::Arc; pub struct KeyshareExtension { - bus: BusHandle, + bus: BusHandle, address: String, cipher: Arc, } impl KeyshareExtension { - pub fn create(bus: &BusHandle, address: &str, cipher: &Arc) -> Box { + pub fn create(bus: &BusHandle, address: &str, cipher: &Arc) -> Box { Box::new(Self { bus: bus.clone(), address: address.to_owned(), @@ -117,7 +117,7 @@ impl E3Extension for KeyshareExtension { } pub struct ThresholdKeyshareExtension { - bus: BusHandle, + bus: BusHandle, cipher: Arc, address: String, multithread: Addr, @@ -125,7 +125,7 @@ pub struct ThresholdKeyshareExtension { impl ThresholdKeyshareExtension { pub fn create( - bus: &BusHandle, + bus: &BusHandle, cipher: &Arc, multithread: &Addr, address: &str, diff --git a/crates/keyshare/src/keyshare.rs b/crates/keyshare/src/keyshare.rs index 065009e39e..3db9dff093 100644 --- a/crates/keyshare/src/keyshare.rs +++ b/crates/keyshare/src/keyshare.rs @@ -19,7 +19,7 @@ use tracing::warn; pub struct Keyshare { fhe: Arc, - bus: BusHandle, + bus: BusHandle, secret: Persistable>, address: String, cipher: Arc, @@ -30,7 +30,7 @@ impl Actor for Keyshare { } pub struct KeyshareParams { - pub bus: BusHandle, + pub bus: BusHandle, pub secret: Persistable>, pub fhe: Arc, pub address: String, diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index ccea059247..470cebb2e2 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -269,14 +269,14 @@ impl TryInto for ThresholdKeyshareState { } pub struct ThresholdKeyshareParams { - pub bus: BusHandle, + pub bus: BusHandle, pub cipher: Arc, pub multithread: Addr, pub state: Persistable, } pub struct ThresholdKeyshare { - bus: BusHandle, + bus: BusHandle, cipher: Arc, decryption_key_collector: Option>, multithread: Addr, diff --git a/crates/logger/src/logger.rs b/crates/logger/src/logger.rs index ea222bbe4f..9367736d9a 100644 --- a/crates/logger/src/logger.rs +++ b/crates/logger/src/logger.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Addr, Context, Handler}; -use e3_events::{prelude::Event, EnclaveEvent, EnclaveEventData, EventBus, Subscribe}; +use e3_events::{prelude::Event, EnclaveEvent, EnclaveEventData, EventBus, SeqState, Subscribe}; use std::marker::PhantomData; use tracing::{error, info}; @@ -46,7 +46,7 @@ impl Handler for SimpleLogger { } } -impl EventLogging for EnclaveEvent { +impl EventLogging for EnclaveEvent { fn log(&self, logger_name: &str) { match self.get_data() { EnclaveEventData::EnclaveError(_) => error!(event=%self, "ERROR!"), diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index d5be42ec57..2a96462170 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -40,7 +40,7 @@ const KADEMLIA_BROADCAST_TIMEOUT: Duration = Duration::from_secs(30); /// bus pub struct DocumentPublisher { /// Enclave EventBus - bus: BusHandle, + bus: BusHandle, /// NetCommand sender to forward commands to the NetInterface tx: mpsc::Sender, /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so @@ -55,7 +55,7 @@ pub struct DocumentPublisher { impl DocumentPublisher { /// Create a new NetEventTranslator actor pub fn new( - bus: &BusHandle, + bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, topic: impl Into, @@ -81,7 +81,7 @@ impl DocumentPublisher { /// Setup the DocumentPublisher and start listening for GossipEvents pub fn setup( - bus: &BusHandle, + bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, topic: impl Into, @@ -253,7 +253,7 @@ pub async fn handle_publish_document_requested( pub async fn handle_document_published_notification( net_cmds: mpsc::Sender, net_events: Arc>, - bus: BusHandle, + bus: BusHandle, ids: HashMap, event: DocumentPublishedNotification, ) -> Result<()> { @@ -374,7 +374,7 @@ async fn broadcast_document_published_notification( /// Convert between ThresholdShareCreated and DocumentPublished events pub struct EventConverter { - bus: BusHandle, + bus: BusHandle, } #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -399,10 +399,10 @@ impl ReceivableDocument { } impl EventConverter { - pub fn new(bus: &BusHandle) -> Self { + pub fn new(bus: &BusHandle) -> Self { Self { bus: bus.clone() } } - pub fn setup(bus: &BusHandle) -> Addr { + pub fn setup(bus: &BusHandle) -> Addr { let addr = Self::new(bus).start(); bus.subscribe("ThresholdShareCreated", addr.clone().into()); bus.subscribe("DocumentReceived", addr.clone().into()); @@ -499,7 +499,7 @@ mod tests { fn setup_test() -> ( DefaultGuard, - BusHandle, + BusHandle, mpsc::Sender, mpsc::Receiver, broadcast::Sender, @@ -517,10 +517,9 @@ mod tests { let guard = tracing::subscriber::set_default(subscriber); - let bus: BusHandle = - EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) + .start() + .into(); let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); let (net_evt_tx, net_evt_rx) = broadcast::channel(100); let net_evt_rx = Arc::new(net_evt_rx); diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index f23905184f..e6797c892e 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -34,7 +34,7 @@ use tracing::{error, info, instrument, trace}; /// NetEventTranslator Actor converts between EventBus events and Libp2p events forwarding them to a /// NetInterface for propagation over the p2p network pub struct NetEventTranslator { - bus: BusHandle, + bus: BusHandle, tx: mpsc::Sender, sent_events: HashSet, topic: String, @@ -51,7 +51,7 @@ struct LibP2pEvent(pub Vec); impl NetEventTranslator { /// Create a new NetEventTranslator actor - pub fn new(bus: &BusHandle, tx: &mpsc::Sender, topic: &str) -> Self { + pub fn new(bus: &BusHandle, tx: &mpsc::Sender, topic: &str) -> Self { Self { bus: bus.clone(), tx: tx.clone(), @@ -61,7 +61,7 @@ impl NetEventTranslator { } pub fn setup( - bus: &BusHandle, + bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, topic: &str, @@ -108,7 +108,7 @@ impl NetEventTranslator { /// Spawn a Libp2p interface and hook it up to this actor #[instrument(name = "libp2p", skip_all)] pub async fn setup_with_interface( - bus: BusHandle, + bus: BusHandle, peers: Vec, cipher: &Arc, quic_port: u16, diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index a3edf20f48..e8df316ebf 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -105,19 +105,19 @@ pub struct E3Router { /// A buffer for events to send to the buffer: EventBuffer, /// The EventBus - bus: BusHandle, + bus: BusHandle, /// A repository for storing snapshots store: Repository, } pub struct E3RouterParams { extensions: Arc>>, - bus: BusHandle, + bus: BusHandle, store: Repository, } impl E3Router { - pub fn builder(bus: &BusHandle, store: DataStore) -> E3RouterBuilder { + pub fn builder(bus: &BusHandle, store: DataStore) -> E3RouterBuilder { let repositories = store.repositories(); let builder = E3RouterBuilder { bus: bus.clone(), @@ -282,7 +282,7 @@ impl FromSnapshotWithParams for E3Router { /// Builder for E3Router pub struct E3RouterBuilder { - pub bus: BusHandle, + pub bus: BusHandle, pub extensions: Vec>, pub store: Repository, } diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 0d2d278a57..1962c066de 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -18,7 +18,7 @@ use e3_request::MetaRepositoryFactory; use tracing::info; pub struct CiphernodeSelector { - bus: BusHandle, + bus: BusHandle, sortition: Addr, address: String, data_store: DataStore, @@ -30,7 +30,7 @@ impl Actor for CiphernodeSelector { impl CiphernodeSelector { pub fn new( - bus: &BusHandle, + bus: &BusHandle, sortition: &Addr, address: &str, data_store: &DataStore, @@ -44,7 +44,7 @@ impl CiphernodeSelector { } pub fn attach( - bus: &BusHandle, + bus: &BusHandle, sortition: &Addr, address: &str, data_store: &DataStore, diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 09ecc5beb9..3e5a8a375b 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -139,7 +139,7 @@ pub struct Sortition { /// Persistent map of `chain_id -> NodeStateStore`. node_state: Persistable>, /// Event bus for error reporting and enclave event subscription. - bus: BusHandle, + bus: BusHandle, /// Persistent map of finalized committees per E3 finalized_committees: Persistable>>, } @@ -148,7 +148,7 @@ pub struct Sortition { #[derive(Debug)] pub struct SortitionParams { /// Event bus address. - pub bus: BusHandle, + pub bus: BusHandle, /// Persisted per-chain backend map. pub backends: Persistable>, /// Node state store per chain @@ -169,7 +169,7 @@ impl Sortition { #[instrument(name = "sortition_attach", skip_all)] pub async fn attach( - bus: &BusHandle, + bus: &BusHandle, backends_store: Repository>, node_state_store: Repository>, committees_store: Repository>>, diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 2057b52416..7fd8980ca8 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -72,7 +72,7 @@ pub fn create_crp_bytes_params( pub fn get_common_setup( param_set: Option, ) -> Result<( - BusHandle, + BusHandle, SharedRng, Seed, Arc, @@ -159,13 +159,13 @@ pub fn create_random_eth_addrs(how_many: u32) -> Vec { /// Test helper to add addresses to the committee by creating events on the event bus #[derive(Clone, Debug)] pub struct AddToCommittee { - bus: BusHandle, + bus: BusHandle, count: usize, chain_id: u64, } impl AddToCommittee { - pub fn new(bus: &BusHandle, chain_id: u64) -> Self { + pub fn new(bus: &BusHandle, chain_id: u64) -> Self { Self { bus: bus.clone(), chain_id, diff --git a/crates/test-helpers/src/plaintext_writer.rs b/crates/test-helpers/src/plaintext_writer.rs index 5e77a12a4e..c74a111d3c 100644 --- a/crates/test-helpers/src/plaintext_writer.rs +++ b/crates/test-helpers/src/plaintext_writer.rs @@ -17,7 +17,7 @@ pub struct PlaintextWriter { } impl PlaintextWriter { - pub fn attach(path: &PathBuf, bus: BusHandle) -> Addr { + pub fn attach(path: &PathBuf, bus: BusHandle) -> Addr { let addr = Self { path: path.to_owned(), } diff --git a/crates/test-helpers/src/public_key_writer.rs b/crates/test-helpers/src/public_key_writer.rs index bdd2523725..e2c7bec70f 100644 --- a/crates/test-helpers/src/public_key_writer.rs +++ b/crates/test-helpers/src/public_key_writer.rs @@ -16,7 +16,7 @@ pub struct PublicKeyWriter { } impl PublicKeyWriter { - pub fn attach(path: &PathBuf, bus: BusHandle) -> Addr { + pub fn attach(path: &PathBuf, bus: BusHandle) -> Addr { let addr = Self { path: path.to_owned(), } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 4a5f1ede59..8e931bc833 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -33,7 +33,7 @@ pub fn save_snapshot(file_name: &str, bytes: &[u8]) { } async fn setup_score_sortition_environment( - bus: &BusHandle, + bus: &BusHandle, eth_addrs: &Vec, chain_id: u64, ) -> Result<()> { @@ -118,10 +118,9 @@ async fn test_trbfv_actor() -> Result<()> { let rng = create_shared_rng_from_u64(42); // Create "trigger" bus - let bus: BusHandle = - EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) + .start() + .into(); // Parameters (128bits of security) let (degree, plaintext_modulus, moduli) = ( diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 0c0ba861c3..96a788ef08 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -47,7 +47,7 @@ use tokio::sync::{broadcast, Mutex}; use tokio::time::sleep; async fn setup_local_ciphernode( - bus: &BusHandle, + bus: &BusHandle, rng: &SharedRng, logging: bool, addr: &str, @@ -102,7 +102,7 @@ fn generate_pk_shares( } async fn create_local_ciphernodes( - bus: &BusHandle, + bus: &BusHandle, rng: &SharedRng, count: u32, cipher: &Arc, @@ -120,7 +120,7 @@ async fn create_local_ciphernodes( } async fn setup_score_sortition_environment( - bus: &BusHandle, + bus: &BusHandle, eth_addrs: &Vec, chain_id: u64, ) -> Result<()> { @@ -443,10 +443,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { // Setup elements in test let (cmd_tx, mut cmd_rx) = mpsc::channel(100); // Transmit byte events to the network let (event_tx, _) = broadcast::channel(100); // Receive byte events from the network - let bus: BusHandle = - EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) + .start() + .into(); let history_collector = HistoryCollector::::new().start(); bus.subscribe("*", history_collector.clone().recipient()); let event_rx = Arc::new(event_tx.subscribe()); @@ -620,10 +619,9 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // Setup elements in test let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network let (event_tx, event_rx) = broadcast::channel(100); // Receive byte events from the network - let bus: BusHandle = - EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) + .start() + .into(); let history_collector = HistoryCollector::::new().start(); bus.subscribe("*", history_collector.clone().recipient()); From 8c18fcbdb32d6c712e93e6cd6048c00bb59cf4f5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 17:14:03 +0000 Subject: [PATCH 02/27] separation of stored and unstored enclave events --- .../src/ciphernode_builder.rs | 2 +- crates/events/src/bus_handle.rs | 32 ++++++++++++----- crates/events/src/enclave_event/mod.rs | 34 +++++++++---------- crates/events/src/eventbus.rs | 10 +++--- crates/evm/tests/integration.rs | 6 ++-- crates/request/src/context.rs | 4 +-- crates/request/src/router.rs | 22 ++++++------ 7 files changed, 62 insertions(+), 48 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index d404fb4ed3..861b567a31 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -16,7 +16,7 @@ use e3_aggregator::ext::{ use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{DataStore, InMemStore, Repositories, RepositoriesFactory}; -use e3_events::{EnclaveEvent, EventBus, EventBusConfig, Stored, Unstored}; +use e3_events::{EnclaveEvent, EventBus, EventBusConfig}; use e3_evm::{ helpers::{ load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 400a4275d6..4c10c333d9 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -4,15 +4,13 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::marker::PhantomData; - use actix::{Actor, Addr, Recipient}; use crate::{ sequencer::Sequencer, traits::{ - ErrorDispatcher, ErrorFactory, Event, EventConstructorWithTimestamp, EventFactory, - EventPublisher, EventSubscriber, + ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, + EventSubscriber, }, EnclaveErrorType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, Stored, Subscribe, Unstored, @@ -30,11 +28,21 @@ impl BusHandle { Self { bus, seq } } - pub fn bus(&self) -> Addr> { + pub fn bus(&self) -> Addr>> { self.bus.clone() } } +#[cfg(test)] +impl BusHandle { + pub fn test_stored_event_from( + &self, + data: impl Into, + ) -> EnclaveEvent { + EnclaveEvent::::new_with_timestamp(data.into(), 0).into_stored(42) + } +} + impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) { let evt = self.event_from(data); @@ -64,19 +72,27 @@ impl EventFactory> for BusHandle { EnclaveEvent::::new_with_timestamp(data.into(), 0) } - fn event_from_remote_source(&self, data: impl Into, ts: u128) -> E { + fn event_from_remote_source( + &self, + data: impl Into, + ts: u128, + ) -> EnclaveEvent { // TODO: add self.hcl.receive(ts) EnclaveEvent::::new_with_timestamp(data.into(), ts) } } impl ErrorFactory> for BusHandle { - fn event_from_error(&self, err_type: EnclaveErrorType, error: impl Into) -> E { + fn event_from_error( + &self, + err_type: EnclaveErrorType, + error: impl Into, + ) -> EnclaveEvent { EnclaveEvent::::from_error(err_type, error) } } -impl EventSubscriber> for BusHandle { +impl EventSubscriber> for BusHandle { fn subscribe(&self, event_type: &str, recipient: Recipient>) { self.bus.do_send(Subscribe::new(event_type, recipient)) } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 515cc903b6..0525833268 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -284,23 +284,23 @@ impl_into_event_data!( ThresholdShareCreated ); -// impl TryFrom<&EnclaveEvent> for EnclaveError { -// type Error = anyhow::Error; -// fn try_from(value: &EnclaveEvent) -> Result { -// value.clone().try_into() -// } -// } -// -// impl TryFrom for EnclaveError { -// type Error = anyhow::Error; -// fn try_from(value: EnclaveEvent) -> Result { -// if let EnclaveEventData::EnclaveError(data) = value.payload.clone() { -// Ok(data) -// } else { -// return Err(anyhow::anyhow!("Not an enclave error {:?}", value)); -// } -// } -// } +impl TryFrom<&EnclaveEvent> for EnclaveError { + type Error = anyhow::Error; + fn try_from(value: &EnclaveEvent) -> Result { + value.clone().try_into() + } +} + +impl TryFrom> for EnclaveError { + type Error = anyhow::Error; + fn try_from(value: EnclaveEvent) -> Result { + if let EnclaveEventData::EnclaveError(data) = value.payload.clone() { + Ok(data) + } else { + return Err(anyhow::anyhow!("Not an enclave error {:?}", value)); + } + } +} impl fmt::Display for EnclaveEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 23c22c0b81..a25d13296b 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::traits::{ErrorEvent, Event}; -use crate::{prelude::*, BusHandle, CompositeEvent, EnclaveEvent, Stored}; +use crate::{prelude::*, BusHandle, EnclaveEvent, Stored}; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; use std::collections::{HashMap, VecDeque}; @@ -70,7 +70,7 @@ impl EventBus { addr } - pub fn error(source: &Addr>) -> Addr> { + pub fn error(source: &Addr>) -> Addr> { let addr = HistoryCollector::::new().start(); source.do_send(Subscribe::new("EnclaveError", addr.clone().recipient())); addr @@ -417,10 +417,8 @@ impl Handler for HistoryCollector { ////////////////////////////////////////////////////////////////////////////// /// Function to help with testing when we want to maintain a vec of events -pub fn new_event_bus_with_history() -> (BusHandle, Addr>) -{ - let bus: BusHandle = EventBus::::default().start().into(); - +pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) { + let bus: BusHandle = EventBus::>::default().start().into(); let history = HistoryCollector::new().start(); bus.subscribe("*", history.clone().recipient()); (bus, history) diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index a315276c2a..cfe40fcab4 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -18,7 +18,7 @@ use e3_data::Repository; use e3_entrypoint::helpers::datastore::get_in_mem_store; use e3_events::{ new_event_bus_with_history, prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, - HistoryCollector, Shutdown, TestEvent, + HistoryCollector, Shutdown, TestEvent, Unstored, }; use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmEventReader, HistoricalEventCoordinator}; use std::time::Duration; @@ -269,7 +269,9 @@ async fn ensure_resume_after_shutdown() -> Result<()> { // Ensure shutdown doesn't cause event to be lost. sleep(Duration::from_millis(10)).await; - addr1.send(bus.event_from(Shutdown)).await?; + addr1 + .send(EnclaveEvent::::new_with_timestamp(Shutdown.into(), 4321).into_stored(42)) + .await?; for msg in ["these", "are", "not", "lost"] { contract diff --git a/crates/request/src/context.rs b/crates/request/src/context.rs index d45d3b44a0..7d25ae756e 100644 --- a/crates/request/src/context.rs +++ b/crates/request/src/context.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use e3_data::{ Checkpoint, FromSnapshotWithParams, Repositories, RepositoriesFactory, Repository, Snapshot, }; -use e3_events::{E3id, EnclaveEvent}; +use e3_events::{E3id, EnclaveEvent, Stored}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; @@ -91,7 +91,7 @@ impl E3Context { }); } - pub fn forward_message_now(&self, msg: &EnclaveEvent) { + pub fn forward_message_now(&self, msg: &EnclaveEvent) { self.recipients().into_iter().for_each(|(_, recipient)| { if let Some(act) = recipient { act.do_send(msg.clone()); diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index e8df316ebf..e9e1436949 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -11,6 +11,7 @@ use crate::E3ContextSnapshot; use crate::E3MetaExtension; use crate::RouterRepositoryFactory; use actix::AsyncContext; +use actix::Message; use actix::{Actor, Addr, Context, Handler}; use anyhow::*; use async_trait::async_trait; @@ -151,8 +152,11 @@ impl Handler for E3Router { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { // If we are shutting down then bail on anything else - if let EnclaveEventData::Shutdown(data) = msg.get_data() { - ctx.notify(data.clone()); + if let EnclaveEventData::Shutdown(_) = msg.get_data() { + for (_, ctx) in self.contexts.iter() { + ctx.forward_message_now(&msg) + } + return; } @@ -207,16 +211,6 @@ impl Handler for E3Router { } } -impl Handler for E3Router { - type Result = (); - fn handle(&mut self, msg: Shutdown, _ctx: &mut Self::Context) -> Self::Result { - let shutdown_evt = self.bus.event_from(msg); - for (_, ctx) in self.contexts.iter() { - ctx.forward_message_now(&shutdown_evt) - } - } -} - #[derive(Serialize, Deserialize)] pub struct E3RouterSnapshot { contexts: Vec, @@ -311,3 +305,7 @@ impl E3RouterBuilder { Ok(addr) } } + +#[derive(Message, Clone, Debug)] +#[rtype("()")] +struct EventWrapper(T); From 3bfbbb20381f289a17ffee006a0f85bfcf00edc4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 17:16:37 +0000 Subject: [PATCH 03/27] header --- crates/events/src/sequencer.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index a1e73e136d..61dd3ea718 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Actor, Addr, Handler}; use crate::{EnclaveEvent, EventBus, Stored, Unstored}; From a11e76160f1d3315f936bed5a828e6c36adcd5a4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 17:40:16 +0000 Subject: [PATCH 04/27] add simple test --- crates/events/src/bus_handle.rs | 8 +++-- crates/events/src/enclave_event/test_event.rs | 9 ++++++ crates/events/src/sequencer.rs | 29 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 4c10c333d9..7394959086 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -12,8 +12,8 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EnclaveErrorType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, Stored, Subscribe, - Unstored, + EnclaveErrorType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, + Stored, Subscribe, Unstored, }; #[derive(Clone, Debug)] @@ -31,6 +31,10 @@ impl BusHandle { pub fn bus(&self) -> Addr>> { self.bus.clone() } + + pub fn history(&self) -> Addr>> { + EventBus::>::history(&self.bus) + } } #[cfg(test)] diff --git a/crates/events/src/enclave_event/test_event.rs b/crates/events/src/enclave_event/test_event.rs index a4cbe09a93..ab26068958 100644 --- a/crates/events/src/enclave_event/test_event.rs +++ b/crates/events/src/enclave_event/test_event.rs @@ -14,6 +14,15 @@ pub struct TestEvent { pub entropy: u64, } +impl TestEvent { + pub fn new(msg: &str, entropy: u64) -> Self { + Self { + msg: msg.to_owned(), + entropy, + } + } +} + #[cfg(test)] use std::fmt::{self, Display}; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 61dd3ea718..e11b2d6a0a 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -33,3 +33,32 @@ impl Handler> for Sequencer { self.bus.do_send(msg.into_stored(self.seq)) } } + +#[cfg(test)] +mod tests { + + use crate::{prelude::*, BusHandle, EnclaveEvent, EventBus, TakeEvents, TestEvent}; + use actix::Actor; + + #[actix::test] + async fn it_adds_numbers_to_events() -> anyhow::Result<()> { + let bus: BusHandle = EventBus::::default().start().into(); + let history = bus.history(); + + let event_data = vec![ + TestEvent::new("one", 1), + TestEvent::new("two", 2), + TestEvent::new("three", 3), + ]; + + event_data.iter().cloned().for_each(|d| bus.publish(d)); + let expected = event_data + .into_iter() + .map(|d| EnclaveEvent::new_with_timestamp(d.clone().into(), 0).into_stored(d.entropy)) + .collect::>(); + let events = history.send(TakeEvents::new(3)).await?; + + assert_eq!(events, expected); + Ok(()) + } +} From 6ab1e1e1ec9cdff2b97d92dcc62f660d269a898b Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 17:51:41 +0000 Subject: [PATCH 05/27] make test_only function --- crates/events/src/bus_handle.rs | 10 ---------- crates/events/src/enclave_event/mod.rs | 11 +++++++---- crates/events/src/sequencer.rs | 4 ++-- crates/evm/tests/integration.rs | 6 +++++- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 7394959086..21bdcaae90 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -37,16 +37,6 @@ impl BusHandle { } } -#[cfg(test)] -impl BusHandle { - pub fn test_stored_event_from( - &self, - data: impl Into, - ) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into(), 0).into_stored(42) - } -} - impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) { let evt = self.event_from(data); diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 0525833268..4a2c57ad2c 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -185,6 +185,12 @@ impl EnclaveEvent { } } +impl EnclaveEvent { + pub fn test_only_create_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { + EnclaveEvent::::new_with_timestamp(data, time).into_stored(seq) + } +} + impl Event for EnclaveEvent { type Id = EventId; type Data = EnclaveEventData; @@ -318,8 +324,5 @@ impl EventConstructorWithTimestamp for EnclaveEvent { payload, seq: (), } - } -} + } -#[cfg(test)] -mod tests {} diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index e11b2d6a0a..1924fcfaea 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -41,7 +41,7 @@ mod tests { use actix::Actor; #[actix::test] - async fn it_adds_numbers_to_events() -> anyhow::Result<()> { + async fn it_adds_seqence_numbers_to_events() -> anyhow::Result<()> { let bus: BusHandle = EventBus::::default().start().into(); let history = bus.history(); @@ -54,7 +54,7 @@ mod tests { event_data.iter().cloned().for_each(|d| bus.publish(d)); let expected = event_data .into_iter() - .map(|d| EnclaveEvent::new_with_timestamp(d.clone().into(), 0).into_stored(d.entropy)) + .map(|d| EnclaveEvent::test_only_create_stored_event(d.clone().into(), 0, d.entropy)) .collect::>(); let events = history.send(TakeEvents::new(3)).await?; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index cfe40fcab4..298f0d07c6 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -270,7 +270,11 @@ async fn ensure_resume_after_shutdown() -> Result<()> { // Ensure shutdown doesn't cause event to be lost. sleep(Duration::from_millis(10)).await; addr1 - .send(EnclaveEvent::::new_with_timestamp(Shutdown.into(), 4321).into_stored(42)) + .send(EnclaveEvent::test_only_create_stored_event( + Shutdown.into(), + 4321, + 42, + )) .await?; for msg in ["these", "are", "not", "lost"] { From 40d882f8137636cc1fc97b4f3848ecd4c133b761 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 17:57:20 +0000 Subject: [PATCH 06/27] fix test --- crates/events/src/enclave_event/mod.rs | 4 ++-- crates/tests/tests/integration_legacy.rs | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 4a2c57ad2c..69c2ae34d0 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -324,5 +324,5 @@ impl EventConstructorWithTimestamp for EnclaveEvent { payload, seq: (), } - } - + } +} diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 96a788ef08..2867027a87 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -506,8 +506,14 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { assert_eq!( *msgs.lock().await, vec![ - GossipData::GossipBytes(bus.event_from(evt_1.clone()).to_bytes()?), - GossipData::GossipBytes(bus.event_from(evt_2.clone()).to_bytes()?) + GossipData::GossipBytes( + EnclaveEvent::test_only_create_stored_event(evt_1.clone().into(), 0, 1) + .to_bytes()? + ), + GossipData::GossipBytes( + EnclaveEvent::test_only_create_stored_event(evt_2.clone().into(), 0, 2) + .to_bytes()? + ) ], // notice no local events "NetEventTranslator did not transmit correct events to the network" ); From a1b31010c5674251ff3a46dca3cfdfe038250143 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 18:09:34 +0000 Subject: [PATCH 07/27] only allow new stored events to be created in tests --- Cargo.lock | 1 + crates/events/Cargo.toml | 5 +++++ crates/events/src/enclave_event/mod.rs | 3 ++- crates/events/src/sequencer.rs | 2 +- crates/evm/tests/integration.rs | 6 +----- crates/tests/Cargo.toml | 3 +++ crates/tests/tests/integration_legacy.rs | 6 ++---- 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46eab9324b..007ebfa2dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,6 +2925,7 @@ dependencies = [ "chrono", "derivative", "e3-crypto", + "e3-events", "e3-trbfv", "e3-utils", "futures-util", diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index bbdb604ae7..8acf3f958f 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -29,5 +29,10 @@ e3-crypto = { workspace = true } e3-trbfv = { workspace = true } e3-utils = { workspace = true } +[features] +test-helpers = [] # ensure test-helpers is available for integration tests + [dev-dependencies] proptest = { workspace = true } +e3-events = { workspace = true, features = ["test-helpers"] } + diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 69c2ae34d0..dd37818456 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -185,8 +185,9 @@ impl EnclaveEvent { } } +#[cfg(feature = "test-helpers")] impl EnclaveEvent { - pub fn test_only_create_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { + pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { EnclaveEvent::::new_with_timestamp(data, time).into_stored(seq) } } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 1924fcfaea..e4fd1b2a39 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -54,7 +54,7 @@ mod tests { event_data.iter().cloned().for_each(|d| bus.publish(d)); let expected = event_data .into_iter() - .map(|d| EnclaveEvent::test_only_create_stored_event(d.clone().into(), 0, d.entropy)) + .map(|d| EnclaveEvent::new_stored_event(d.clone().into(), 0, d.entropy)) .collect::>(); let events = history.send(TakeEvents::new(3)).await?; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 298f0d07c6..85054837a9 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -270,11 +270,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { // Ensure shutdown doesn't cause event to be lost. sleep(Duration::from_millis(10)).await; addr1 - .send(EnclaveEvent::test_only_create_stored_event( - Shutdown.into(), - 4321, - 42, - )) + .send(EnclaveEvent::new_stored_event(Shutdown.into(), 4321, 42)) .await?; for msg in ["these", "are", "not", "lost"] { diff --git a/crates/tests/Cargo.toml b/crates/tests/Cargo.toml index 022576f966..fe94458915 100644 --- a/crates/tests/Cargo.toml +++ b/crates/tests/Cargo.toml @@ -43,3 +43,6 @@ tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } zeroize = { workspace = true } + +[dev-dependencies] +e3-events = { workspace = true, features = ["test-helpers"] } diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 2867027a87..36fcd3d15c 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -507,12 +507,10 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { *msgs.lock().await, vec![ GossipData::GossipBytes( - EnclaveEvent::test_only_create_stored_event(evt_1.clone().into(), 0, 1) - .to_bytes()? + EnclaveEvent::new_stored_event(evt_1.clone().into(), 0, 1).to_bytes()? ), GossipData::GossipBytes( - EnclaveEvent::test_only_create_stored_event(evt_2.clone().into(), 0, 2) - .to_bytes()? + EnclaveEvent::new_stored_event(evt_2.clone().into(), 0, 2).to_bytes()? ) ], // notice no local events "NetEventTranslator did not transmit correct events to the network" From ba82f13f18a3d365a932d8b2f3ae51b3a01b7602 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 18:33:57 +0000 Subject: [PATCH 08/27] use dereferencing to get bus --- crates/ciphernode-builder/src/ciphernode.rs | 2 +- crates/entrypoint/src/start/aggregator_start.rs | 2 +- crates/entrypoint/src/start/start.rs | 2 +- crates/events/src/bus_handle.rs | 13 +++++++++---- crates/events/src/eventbus_factory.rs | 1 - crates/evm/tests/integration.rs | 2 +- crates/tests/tests/integration.rs | 4 ++-- crates/tests/tests/integration_legacy.rs | 2 +- 8 files changed, 16 insertions(+), 12 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 3c02ee1d29..5c7670aa63 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -35,7 +35,7 @@ impl CiphernodeHandle { } pub fn bus(&self) -> Addr> { - self.bus.bus() + (*self.bus).clone() } pub fn history(&self) -> Option>> { diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index 1bf1a79c07..4a3a2a574a 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -35,7 +35,7 @@ pub async fn execute( let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) - .with_source_bus(&bus.bus()) + .with_source_bus(&*bus) .with_datastore(store) .with_chains(&config.chains()) .with_sortition_score() diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 211e439ddb..d390d89fe1 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -36,7 +36,7 @@ pub async fn execute( let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) .with_address(&address.to_string()) - .with_source_bus(&bus.bus()) + .with_source_bus(&*bus) .with_datastore(store) .with_sortition_score() .with_chains(&config.chains()) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 21bdcaae90..3fc7ca4aa4 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::ops::Deref; + use actix::{Actor, Addr, Recipient}; use crate::{ @@ -28,15 +30,18 @@ impl BusHandle { Self { bus, seq } } - pub fn bus(&self) -> Addr>> { - self.bus.clone() - } - pub fn history(&self) -> Addr>> { EventBus::>::history(&self.bus) } } +impl Deref for BusHandle { + type Target = Addr>>; + fn deref(&self) -> &Self::Target { + &self.bus + } +} + impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) { let evt = self.event_from(data); diff --git a/crates/events/src/eventbus_factory.rs b/crates/events/src/eventbus_factory.rs index 24dd404a96..098912906c 100644 --- a/crates/events/src/eventbus_factory.rs +++ b/crates/events/src/eventbus_factory.rs @@ -18,7 +18,6 @@ use crate::EnclaveEvent; use crate::EventBus; use crate::HistoryCollector; use crate::Subscribe; -use crate::Unstored; // The singleton factory using once_cell pub struct EventBusFactory { diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 85054837a9..55bd774c09 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -18,7 +18,7 @@ use e3_data::Repository; use e3_entrypoint::helpers::datastore::get_in_mem_store; use e3_events::{ new_event_bus_with_history, prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, - HistoryCollector, Shutdown, TestEvent, Unstored, + HistoryCollector, Shutdown, TestEvent, }; use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmEventReader, HistoricalEventCoordinator}; use std::time::Duration; diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 8e931bc833..5ed9c86f43 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -180,7 +180,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_pubkey_aggregation() .with_sortition_score() .with_threshold_plaintext_aggregation() - .testmode_with_forked_bus(&bus.bus()) + .testmode_with_forked_bus(&*bus) .with_logging() .build() .await @@ -193,7 +193,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_injected_multithread(multithread.clone()) .with_trbfv() .with_sortition_score() - .testmode_with_forked_bus(&bus.bus()) + .testmode_with_forked_bus(&*bus) .with_logging() .build() .await diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 36fcd3d15c..2218e6a1f6 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -57,7 +57,7 @@ async fn setup_local_ciphernode( let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) .with_keyshare() .with_address(addr) - .testmode_with_forked_bus(&bus.bus()) + .testmode_with_forked_bus(&*bus) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() From a29046773f1657076414e4546ca149c192e74b38 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 18:43:38 +0000 Subject: [PATCH 09/27] use producer and consumer to reflect the reality of how we use the seq/bus --- crates/ciphernode-builder/src/ciphernode.rs | 4 +- .../entrypoint/src/start/aggregator_start.rs | 4 +- crates/entrypoint/src/start/start.rs | 4 +- crates/events/src/bus_handle.rs | 37 +++++++++---------- crates/test-helpers/src/lib.rs | 4 +- crates/tests/tests/integration.rs | 4 +- crates/tests/tests/integration_legacy.rs | 2 +- 7 files changed, 29 insertions(+), 30 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 5c7670aa63..6b16382ce4 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -34,8 +34,8 @@ impl CiphernodeHandle { } } - pub fn bus(&self) -> Addr> { - (*self.bus).clone() + pub fn bus(&self) -> &BusHandle { + &self.bus } pub fn history(&self) -> Option>> { diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index 4a3a2a574a..dfbf9beeb1 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -9,7 +9,7 @@ use e3_ciphernode_builder::CiphernodeBuilder; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; -use e3_events::{get_enclave_bus_handle, BusHandle, EnclaveEvent}; +use e3_events::{get_enclave_bus_handle, BusHandle}; use e3_net::{NetEventTranslator, NetRepositoryFactory}; use e3_test_helpers::{PlaintextWriter, PublicKeyWriter}; use rand::SeedableRng; @@ -35,7 +35,7 @@ pub async fn execute( let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) - .with_source_bus(&*bus) + .with_source_bus(bus.consumer()) .with_datastore(store) .with_chains(&config.chains()) .with_sortition_score() diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index d390d89fe1..87bd527969 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -10,8 +10,8 @@ use e3_ciphernode_builder::CiphernodeBuilder; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; +use e3_events::get_enclave_bus_handle; use e3_events::BusHandle; -use e3_events::{get_enclave_bus_handle, EnclaveEvent}; use e3_net::{NetEventTranslator, NetRepositoryFactory}; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; @@ -36,7 +36,7 @@ pub async fn execute( let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) .with_address(&address.to_string()) - .with_source_bus(&*bus) + .with_source_bus(bus.consumer()) .with_datastore(store) .with_sortition_score() .with_chains(&config.chains()) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 3fc7ca4aa4..66dd5e0365 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -4,8 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::ops::Deref; - use actix::{Actor, Addr, Recipient}; use crate::{ @@ -20,48 +18,49 @@ use crate::{ #[derive(Clone, Debug)] pub struct BusHandle { - bus: Addr>>, - seq: Addr, + consumer: Addr>>, + producer: Addr, } impl BusHandle { - pub fn new(bus: Addr>>) -> Self { - let seq = Sequencer::new(&bus).start(); - Self { bus, seq } + pub fn new(consumer: Addr>>) -> Self { + let producer = Sequencer::new(&consumer).start(); + Self { consumer, producer } } pub fn history(&self) -> Addr>> { - EventBus::>::history(&self.bus) + EventBus::>::history(&self.consumer) + } + + pub fn producer(&self) -> &Addr { + &self.producer } -} -impl Deref for BusHandle { - type Target = Addr>>; - fn deref(&self) -> &Self::Target { - &self.bus + pub fn consumer(&self) -> &Addr>> { + &self.consumer } } impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) { let evt = self.event_from(data); - self.seq.do_send(evt); + self.producer.do_send(evt); } fn publish_from_remote(&self, data: impl Into, ts: u128) { let evt = self.event_from_remote_source(data, ts); - self.seq.do_send(evt) + self.producer.do_send(evt) } fn naked_dispatch(&self, event: EnclaveEvent) { - self.seq.do_send(event); + self.producer.do_send(event); } } impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EnclaveErrorType, error: impl Into) { let evt = self.event_from_error(err_type, error); - self.seq.do_send(evt); + self.producer.do_send(evt); } } @@ -93,12 +92,12 @@ impl ErrorFactory> for BusHandle { impl EventSubscriber> for BusHandle { fn subscribe(&self, event_type: &str, recipient: Recipient>) { - self.bus.do_send(Subscribe::new(event_type, recipient)) + self.consumer.do_send(Subscribe::new(event_type, recipient)) } fn subscribe_all(&self, event_types: &[&str], recipient: Recipient>) { for event_type in event_types.into_iter() { - self.bus + self.consumer .do_send(Subscribe::new(*event_type, recipient.clone())); } } diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 7fd8980ca8..5e40e73798 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -126,9 +126,9 @@ pub fn get_common_setup( /// ``` pub fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { for node in nodes.iter() { - let source = &node.bus(); + let source = node.bus().consumer(); for (_, node) in nodes.iter().enumerate() { - let dest = &node.bus(); + let dest = node.bus().consumer(); if source != dest { EventBus::pipe_filter( source, diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 5ed9c86f43..5599bd3d8e 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -180,7 +180,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_pubkey_aggregation() .with_sortition_score() .with_threshold_plaintext_aggregation() - .testmode_with_forked_bus(&*bus) + .testmode_with_forked_bus(bus.consumer()) .with_logging() .build() .await @@ -193,7 +193,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_injected_multithread(multithread.clone()) .with_trbfv() .with_sortition_score() - .testmode_with_forked_bus(&*bus) + .testmode_with_forked_bus(bus.consumer()) .with_logging() .build() .await diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 2218e6a1f6..07b15ca227 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -57,7 +57,7 @@ async fn setup_local_ciphernode( let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) .with_keyshare() .with_address(addr) - .testmode_with_forked_bus(&*bus) + .testmode_with_forked_bus(bus.consumer()) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() From a94f112974e74146b9ad093780aae968f0c9d16e Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 22:35:24 +0000 Subject: [PATCH 10/27] get default hlc being constructed --- Cargo.lock | 1 + .../src/ciphernode_builder.rs | 4 +-- crates/data/src/sled_store.rs | 7 ----- crates/events/Cargo.toml | 1 + crates/events/src/bus_handle.rs | 28 +++++++++++++++++-- crates/events/src/eventbus.rs | 10 ++----- crates/events/src/eventbus_factory.rs | 5 +++- crates/events/src/hlc.rs | 16 +++++++++++ crates/events/src/sequencer.rs | 7 ++++- crates/net/src/document_publisher.rs | 7 ++--- crates/test-helpers/src/ciphernode_system.rs | 2 +- 11 files changed, 63 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 007ebfa2dd..f4f3cdb0cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2931,6 +2931,7 @@ dependencies = [ "futures-util", "once_cell", "proptest", + "rand 0.8.5", "serde", "sha2", "strum", diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 861b567a31..61741fc5ca 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -16,7 +16,7 @@ use e3_aggregator::ext::{ use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{DataStore, InMemStore, Repositories, RepositoriesFactory}; -use e3_events::{EnclaveEvent, EventBus, EventBusConfig}; +use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig}; use e3_evm::{ helpers::{ load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, @@ -299,7 +299,7 @@ impl CiphernodeBuilder { }; // Get a handle from the event bus - let bus = local_bus.into(); + let bus = BusHandle::new(local_bus); let addr = if let Some(addr) = self.address.clone() { info!("Using eth address = {}", addr); diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index b86e29cda5..0bea7217b1 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -43,13 +43,6 @@ impl SledStore { Ok(store) } - - pub fn from_db(db: SledDb) -> Result { - Ok(Self { - db: Some(db), - bus: get_enclave_bus_handle(), - }) - } } impl Handler for SledStore { diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index 8acf3f958f..aa58f52e17 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -19,6 +19,7 @@ chrono = { workspace = true } derivative = { workspace = true } futures-util = { workspace = true } once_cell = { workspace = true } +rand = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } strum = { workspace = true } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 66dd5e0365..983a6e2b69 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -4,9 +4,13 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::sync::Arc; + use actix::{Actor, Addr, Recipient}; +use derivative::Derivative; use crate::{ + hlc::Hlc, sequencer::Sequencer, traits::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, @@ -16,16 +20,24 @@ use crate::{ Stored, Subscribe, Unstored, }; -#[derive(Clone, Debug)] +#[derive(Clone, Derivative)] +#[derivative(Debug)] pub struct BusHandle { consumer: Addr>>, producer: Addr, + #[derivative(Debug = "ignore")] + hlc: Arc, } impl BusHandle { pub fn new(consumer: Addr>>) -> Self { let producer = Sequencer::new(&consumer).start(); - Self { consumer, producer } + let hlc = Hlc::default(); + Self { + consumer, + producer, + hlc: Arc::new(hlc), + } } pub fn history(&self) -> Addr>> { @@ -102,3 +114,15 @@ impl EventSubscriber> for BusHandle { } } } + +impl Into for Addr> { + fn into(self) -> BusHandle { + BusHandle::new(self) + } +} + +impl Into for &Addr> { + fn into(self) -> BusHandle { + BusHandle::new(self.clone()) + } +} diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index a25d13296b..1e2912279a 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -4,6 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::hlc::Hlc; use crate::traits::{ErrorEvent, Event}; use crate::{prelude::*, BusHandle, EnclaveEvent, Stored}; use actix::prelude::*; @@ -133,12 +134,6 @@ impl Handler for EventBus { } } -impl From>>> for BusHandle { - fn from(value: Addr>>) -> Self { - BusHandle::new(value) - } -} - ////////////////////////////////////////////////////////////////////////////// // Subscribe Message ////////////////////////////////////////////////////////////////////////////// @@ -418,7 +413,8 @@ impl Handler for HistoryCollector { /// Function to help with testing when we want to maintain a vec of events pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) { - let bus: BusHandle = EventBus::>::default().start().into(); + let consumer = EventBus::>::default().start(); + let bus: BusHandle = BusHandle::new(consumer); let history = HistoryCollector::new().start(); bus.subscribe("*", history.clone().recipient()); (bus, history) diff --git a/crates/events/src/eventbus_factory.rs b/crates/events/src/eventbus_factory.rs index 098912906c..5db50b732f 100644 --- a/crates/events/src/eventbus_factory.rs +++ b/crates/events/src/eventbus_factory.rs @@ -12,6 +12,7 @@ use std::any::TypeId; use std::collections::HashMap; use std::sync::Mutex; +use crate::hlc::Hlc; use crate::traits::Event; use crate::BusHandle; use crate::EnclaveEvent; @@ -60,6 +61,7 @@ impl EventBusFactory { event_bus } + pub fn get_error_collector(&self) -> Addr> { let type_id = TypeId::of::(); let mut error_collector_cache = self.error_collector_cache.lock().unwrap(); @@ -93,5 +95,6 @@ pub fn get_error_collector() -> Addr> { } pub fn get_enclave_bus_handle() -> BusHandle { - get_enclave_event_bus().into() + let bus = get_enclave_event_bus(); + BusHandle::new(bus) } diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index 4cc36f3239..7a8cb3c11a 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use rand::Rng; +use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; use thiserror::Error; @@ -173,6 +175,13 @@ struct HlcInner { counter: u32, } +impl Default for Hlc { + fn default() -> Self { + let random_id: u32 = rand::thread_rng().gen(); + Self::new(random_id) + } +} + impl Hlc { const DEFAULT_MAX_DRIFT: u64 = 60_000_000; // 60 sec @@ -185,6 +194,13 @@ impl Hlc { } } + pub fn from_str(node: &str) -> Self { + let mut h = DefaultHasher::new(); + node.hash(&mut h); + let id: u64 = h.finish(); + Self::new(id as u32) + } + pub fn with_state(ts: u64, counter: u32, node: u32) -> Self { Self { inner: Mutex::new(HlcInner { ts, counter }), diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index e4fd1b2a39..edea6be5a9 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -42,7 +42,7 @@ mod tests { #[actix::test] async fn it_adds_seqence_numbers_to_events() -> anyhow::Result<()> { - let bus: BusHandle = EventBus::::default().start().into(); + let bus = BusHandle::new(EventBus::::default().start()); let history = bus.history(); let event_data = vec![ @@ -61,4 +61,9 @@ mod tests { assert_eq!(events, expected); Ok(()) } + + #[actix::test] + async fn test_hlc_events() -> anyhow::Result<()> { + Ok(()) + } } diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 2a96462170..21693d70cf 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -486,7 +486,7 @@ mod tests { use actix::Addr; use anyhow::{bail, Result}; use e3_events::{ - BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EnclaveError, + hlc::Hlc, BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EnclaveError, EnclaveEvent, EventBus, EventBusConfig, GetEvents, HistoryCollector, PublishDocumentRequested, TakeEvents, }; @@ -517,9 +517,8 @@ mod tests { let guard = tracing::subscriber::set_default(subscriber); - let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + let consumer = EventBus::::new(EventBusConfig { deduplicate: true }).start(); + let bus = BusHandle::new(consumer); let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); let (net_evt_tx, net_evt_rx) = broadcast::channel(100); let net_evt_rx = Arc::new(net_evt_rx); diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index dd62af9ca5..632f120957 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -197,7 +197,7 @@ mod tests { use super::*; use actix::prelude::*; use e3_data::InMemStore; - use e3_events::{BusHandle, EventBus, EventBusConfig}; + use e3_events::{hlc::Hlc, BusHandle, EventBus, EventBusConfig}; async fn mock_setup_node(address: String) -> Result { // Create mock actors for the test From 520c928f20944b821e375e55df16d6a47dd7a601 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 2 Dec 2025 22:42:21 +0000 Subject: [PATCH 11/27] add timestamps --- crates/events/src/bus_handle.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 983a6e2b69..0bfa70245a 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -78,8 +78,8 @@ impl ErrorDispatcher> for BusHandle { impl EventFactory> for BusHandle { fn event_from(&self, data: impl Into) -> EnclaveEvent { - // TODO: add self.hcl.tick() - EnclaveEvent::::new_with_timestamp(data.into(), 0) + let ts = self.hlc.tick().unwrap(); // XXX: + EnclaveEvent::::new_with_timestamp(data.into(), ts.into()) } fn event_from_remote_source( @@ -87,8 +87,8 @@ impl EventFactory> for BusHandle { data: impl Into, ts: u128, ) -> EnclaveEvent { - // TODO: add self.hcl.receive(ts) - EnclaveEvent::::new_with_timestamp(data.into(), ts) + let ts = self.hlc.receive(&ts.into()).unwrap(); // XXX: + EnclaveEvent::::new_with_timestamp(data.into(), ts.into()) } } From 1a0c4fb8baa805d2ffb9883d76088650992a6a18 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 4 Dec 2025 07:53:14 +0000 Subject: [PATCH 12/27] refactor to use result everywhere --- crates/aggregator/src/committee_finalizer.rs | 11 +- crates/aggregator/src/ext.rs | 18 +- crates/aggregator/src/plaintext_aggregator.rs | 2 +- crates/aggregator/src/publickey_aggregator.rs | 5 +- crates/aggregator/src/test_req_res.rs | 168 ++++++++++++++++++ .../src/threshold_plaintext_aggregator.rs | 2 +- crates/ciphernode-builder/src/ciphernode.rs | 2 +- crates/data/src/sled_store.rs | 8 +- crates/entrypoint/src/helpers/shutdown.rs | 7 +- crates/events/src/bus_handle.rs | 39 ++-- .../events/src/enclave_event/enclave_error.rs | 25 ++- crates/events/src/enclave_event/mod.rs | 2 +- crates/events/src/sequencer.rs | 5 +- crates/events/src/traits.rs | 9 +- crates/evm/src/bonding_registry_sol.rs | 2 +- crates/evm/src/ciphernode_registry_sol.rs | 8 +- crates/evm/src/enclave_sol_reader.rs | 2 +- crates/evm/src/enclave_sol_writer.rs | 9 +- crates/evm/src/event_reader.rs | 12 +- .../evm/src/historical_event_coordinator.rs | 39 ++-- crates/fhe/src/ext.rs | 16 +- crates/keyshare/src/ext.rs | 20 +-- crates/keyshare/src/keyshare.rs | 114 ++++++------ crates/keyshare/src/threshold_keyshare.rs | 6 +- crates/net/src/document_publisher.rs | 25 +-- crates/request/src/router.rs | 103 +++++------ crates/sortition/src/ciphernode_selector.rs | 40 +++-- crates/sortition/src/sortition.rs | 26 +-- crates/test-helpers/src/ciphernode_system.rs | 2 +- crates/test-helpers/src/lib.rs | 2 +- crates/tests/tests/integration.rs | 12 +- crates/tests/tests/integration_legacy.rs | 38 ++-- 32 files changed, 484 insertions(+), 295 deletions(-) create mode 100644 crates/aggregator/src/test_req_res.rs diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index 97efc533ad..069e8a3886 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -6,8 +6,8 @@ use actix::prelude::*; use e3_events::{ - prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EnclaveEvent, - EnclaveEventData, Shutdown, + prelude::*, trap, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EType, + EnclaveEvent, EnclaveEventData, Shutdown, }; use std::collections::HashMap; use std::time::Duration; @@ -112,8 +112,11 @@ impl Handler for CommitteeFinalizer { move |act, _ctx| { info!(e3_id = %e3_id_clone, "Dispatching CommitteeFinalizeRequested event"); - bus.publish(CommitteeFinalizeRequested { - e3_id: e3_id_clone.clone(), + trap(EType::Sortition, &act.bus.clone(), || { + bus.publish(CommitteeFinalizeRequested { + e3_id: e3_id_clone.clone(), + })?; + Ok(()) }); act.pending_committees.remove(&e3_id_clone.to_string()); diff --git a/crates/aggregator/src/ext.rs b/crates/aggregator/src/ext.rs index 72c9dbe020..1e97ab4ac5 100644 --- a/crates/aggregator/src/ext.rs +++ b/crates/aggregator/src/ext.rs @@ -19,7 +19,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use e3_data::{AutoPersist, Persistable, RepositoriesFactory}; use e3_events::{prelude::*, E3id}; -use e3_events::{BusHandle, EnclaveErrorType, EnclaveEvent, EnclaveEventData}; +use e3_events::{BusHandle, EType, EnclaveEvent, EnclaveEventData}; use e3_fhe::ext::FHE_KEY; use e3_fhe::Fhe; use e3_multithread::Multithread; @@ -54,7 +54,7 @@ impl E3Extension for PlaintextAggregatorExtension { let Some(fhe) = ctx.get_dependency(FHE_KEY) else { self.bus.err( - EnclaveErrorType::PlaintextAggregation, + EType::PlaintextAggregation, anyhow!(ERROR_PLAINTEXT_FHE_MISSING), ); return; @@ -62,7 +62,7 @@ impl E3Extension for PlaintextAggregatorExtension { let Some(ref meta) = ctx.get_dependency(META_KEY) else { self.bus.err( - EnclaveErrorType::PlaintextAggregation, + EType::PlaintextAggregation, anyhow!(ERROR_PLAINTEXT_META_MISSING), ); return; @@ -74,7 +74,7 @@ impl E3Extension for PlaintextAggregatorExtension { // This is a single ciphertext for the legacy PlaintextAggregator let Some(single_ciphertext) = data.ciphertext_output.first() else { self.bus.err( - EnclaveErrorType::PlaintextAggregation, + EType::PlaintextAggregation, anyhow!("Could not extract ciphertext from array"), ); return; @@ -122,7 +122,7 @@ impl E3Extension for PlaintextAggregatorExtension { // Get deps let Some(fhe) = ctx.get_dependency(FHE_KEY) else { self.bus.err( - EnclaveErrorType::PlaintextAggregation, + EType::PlaintextAggregation, anyhow!(ERROR_PLAINTEXT_FHE_MISSING), ); return Ok(()); @@ -170,14 +170,14 @@ impl E3Extension for PublicKeyAggregatorExtension { let Some(fhe) = ctx.get_dependency(FHE_KEY) else { self.bus.err( - EnclaveErrorType::PublickeyAggregation, + EType::PublickeyAggregation, anyhow!(ERROR_PUBKEY_FHE_MISSING), ); return; }; let Some(ref meta) = ctx.get_dependency(META_KEY) else { self.bus.err( - EnclaveErrorType::PublickeyAggregation, + EType::PublickeyAggregation, anyhow!(ERROR_PUBKEY_META_MISSING), ); return; @@ -211,7 +211,7 @@ impl E3Extension for PublicKeyAggregatorExtension { // Get deps let Some(fhe) = ctx.get_dependency(FHE_KEY) else { self.bus.err( - EnclaveErrorType::PublickeyAggregation, + EType::PublickeyAggregation, anyhow!(ERROR_PUBKEY_FHE_MISSING), ); @@ -278,7 +278,7 @@ impl E3Extension for ThresholdPlaintextAggregatorExtension { let Some(ref meta) = ctx.get_dependency(META_KEY) else { self.bus.err( - EnclaveErrorType::PlaintextAggregation, + EType::PlaintextAggregation, anyhow!(ERROR_TRBFV_PLAINTEXT_META_MISSING), ); return; diff --git a/crates/aggregator/src/plaintext_aggregator.rs b/crates/aggregator/src/plaintext_aggregator.rs index 583e1c4f84..f78b85508d 100644 --- a/crates/aggregator/src/plaintext_aggregator.rs +++ b/crates/aggregator/src/plaintext_aggregator.rs @@ -239,7 +239,7 @@ impl Handler for PlaintextAggregator { e3_id: self.e3_id.clone(), }; - self.bus.publish(event); + self.bus.publish(event)?; Ok(()) } diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index e100504669..1d022b1fec 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -12,7 +12,6 @@ use e3_events::{ PublicKeyAggregated, Seed, }; use e3_fhe::{Fhe, GetAggregatePublicKey}; -use e3_sortition::Sortition; use e3_utils::ArcBytes; use std::sync::Arc; use tracing::{error, info}; @@ -176,7 +175,7 @@ impl Handler for PublicKeyAggregator { impl Handler for PublicKeyAggregator { type Result = Result<()>; - fn handle(&mut self, msg: ComputeAggregate, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: ComputeAggregate, _: &mut Self::Context) -> Self::Result { info!("Computing Aggregate PublicKey..."); let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { keyshares: msg.keyshares, @@ -198,7 +197,7 @@ impl Handler for PublicKeyAggregator { e3_id: msg.e3_id, nodes, }; - self.bus.publish(event); + self.bus.publish(event)?; } Ok(()) } diff --git a/crates/aggregator/src/test_req_res.rs b/crates/aggregator/src/test_req_res.rs new file mode 100644 index 0000000000..5a537206b3 --- /dev/null +++ b/crates/aggregator/src/test_req_res.rs @@ -0,0 +1,168 @@ +/** + * This is an example of how to best use actix to avoid sending stuff over async boundaries + **/ +use std::{ops::Deref, rc::Rc}; + +use actix::{Actor, Addr, AsyncContext, Context, Handler, Message, Recipient}; + +struct Alice { + bob: Addr, + result: Option, +} + +impl Actor for Alice { + type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + self.bob + .do_send(Responder::new(Greet::new("Alice"), ctx.address())) + } +} + +impl Alice { + pub fn new(bob: &Addr) -> Self { + Self { + bob: bob.clone(), + result: None, + } + } +} + +impl Handler for Alice { + type Result = (); + fn handle(&mut self, msg: GreetResponse, _: &mut Self::Context) -> Self::Result { + self.result = Some(msg.value); + } +} + +impl Handler for Alice { + type Result = Option; + fn handle(&mut self, msg: GetInner, _: &mut Self::Context) -> Self::Result { + self.result.take() + } +} + +struct Bob { + value: Rc, +} + +impl Actor for Bob { + type Context = Context; +} + +impl Bob { + pub fn new(value: &str) -> Self { + Self { + value: Rc::new(value.into()), + } + } +} + +impl Handler> for Bob { + type Result = (); + fn handle( + &mut self, + msg: Responder, + _: &mut Self::Context, + ) -> Self::Result { + let reply = &format!("Hello {}! Regards, {}", msg.name, self.value); + + // do some sync work maybe... + // could store the responder locally + + msg.reply(GreetResponse::new(reply)); + } +} + +#[derive(Message)] +#[rtype("()")] +struct Greet { + name: String, +} + +impl Greet { + pub fn new(name: &str) -> Self { + Self { + name: name.to_owned(), + } + } +} + +#[derive(Message)] +#[rtype("()")] +struct GreetResponse { + value: String, +} + +impl GreetResponse { + pub fn new(value: &str) -> Self { + Self { + value: value.to_owned(), + } + } +} + +/////////////////////////////////////////////////////////////////////// + +#[derive(Message)] +#[rtype("Option")] +struct GetInner; + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{Alice, Bob, GetInner}; + use actix::prelude::*; + use tokio::time::sleep; + + #[actix::test] + async fn test_things() -> anyhow::Result<()> { + let bob = Bob::new("Blurg").start(); + let alice = Alice::new(&bob).start(); + sleep(Duration::from_millis(1)).await; + alice.send(GetInner).await?.unwrap(); + + Ok(()) + } +} + +/////////////////////////////////////////////////////////////////////// + +#[derive(Message)] +#[rtype("()")] +struct Responder +where + U: Message + Send, + U::Result: Send, +{ + value: T, + sender: Recipient, +} + +impl Responder +where + U: Message + Send, + U::Result: Send, +{ + pub fn new(value: T, sender: impl Into>) -> Self { + Self { + value, + sender: sender.into(), + } + } + pub fn reply(&self, msg: U) { + let sender = &self.sender; + sender.do_send(msg); + } +} + +impl Deref for Responder +where + U: Message + Send, + U::Result: Send, +{ + type Target = T; + fn deref(&self) -> &Self::Target { + &self.value + } +} diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 81e84f9338..1c5ee9e8b7 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -342,7 +342,7 @@ impl Handler for ThresholdPlaintextAggregator { }; info!("Dispatching plaintext event {:?}", event); - act.bus.publish(event); + act.bus.publish(event)?; Ok(()) }), ) diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 6b16382ce4..2e67d36187 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -6,7 +6,7 @@ use actix::Addr; use e3_data::{DataStore, InMemStore, StoreAddr}; -use e3_events::{BusHandle, EnclaveEvent, EventBus, HistoryCollector}; +use e3_events::{BusHandle, EnclaveEvent, HistoryCollector}; #[derive(Clone, Debug)] pub struct CiphernodeHandle { diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 0bea7217b1..4facb8aeca 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -8,7 +8,7 @@ use crate::{Get, Insert, InsertSync, Remove}; use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::{Context, Result}; use e3_events::{ - get_enclave_bus_handle, prelude::*, BusHandle, EnclaveErrorType, EnclaveEvent, EnclaveEventData, + get_enclave_bus_handle, prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData, }; use once_cell::sync::Lazy; use sled::Db; @@ -51,7 +51,7 @@ impl Handler for SledStore { fn handle(&mut self, event: Insert, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { match db.insert(event) { - Err(err) => self.bus.err(EnclaveErrorType::Data, err), + Err(err) => self.bus.err(EType::Data, err), _ => (), } } @@ -76,7 +76,7 @@ impl Handler for SledStore { fn handle(&mut self, event: Remove, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { match db.remove(event) { - Err(err) => self.bus.err(EnclaveErrorType::Data, err), + Err(err) => self.bus.err(EType::Data, err), _ => (), } } @@ -91,7 +91,7 @@ impl Handler for SledStore { return match db.get(event) { Ok(v) => v, Err(err) => { - self.bus.err(EnclaveErrorType::Data, err); + self.bus.err(EType::Data, err); None } }; diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 2f2d66d700..1ff0d6748d 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::Result; -use e3_events::{prelude::*, BusHandle, EnclaveEvent, Shutdown}; +use e3_events::{prelude::*, BusHandle, Shutdown}; use std::time::Duration; use tokio::{ select, @@ -22,7 +22,10 @@ pub async fn listen_for_shutdown(bus: BusHandle, mut handle: JoinHandle (), + Err(e) => error!("Shutdown failed to publish! {e}") + } // Wait for all events to propagate tokio::time::sleep(Duration::from_secs(2)).await; diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 0bfa70245a..9d7dca5ff2 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use actix::{Actor, Addr, Recipient}; +use anyhow::Result; use derivative::Derivative; use crate::{ @@ -16,8 +17,8 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EnclaveErrorType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, - Stored, Subscribe, Unstored, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, Stored, + Subscribe, Unstored, }; #[derive(Clone, Derivative)] @@ -54,14 +55,16 @@ impl BusHandle { } impl EventPublisher> for BusHandle { - fn publish(&self, data: impl Into) { - let evt = self.event_from(data); + fn publish(&self, data: impl Into) -> Result<()> { + let evt = self.event_from(data)?; self.producer.do_send(evt); + Ok(()) } - fn publish_from_remote(&self, data: impl Into, ts: u128) { - let evt = self.event_from_remote_source(data, ts); - self.producer.do_send(evt) + fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { + let evt = self.event_from_remote_source(data, ts)?; + self.producer.do_send(evt); + Ok(()) } fn naked_dispatch(&self, event: EnclaveEvent) { @@ -70,32 +73,38 @@ impl EventPublisher> for BusHandle { } impl ErrorDispatcher> for BusHandle { - fn err(&self, err_type: EnclaveErrorType, error: impl Into) { + fn err(&self, err_type: EType, error: impl Into) { let evt = self.event_from_error(err_type, error); self.producer.do_send(evt); } } impl EventFactory> for BusHandle { - fn event_from(&self, data: impl Into) -> EnclaveEvent { - let ts = self.hlc.tick().unwrap(); // XXX: - EnclaveEvent::::new_with_timestamp(data.into(), ts.into()) + fn event_from(&self, data: impl Into) -> Result> { + let ts = self.hlc.tick()?; + Ok(EnclaveEvent::::new_with_timestamp( + data.into(), + ts.into(), + )) } fn event_from_remote_source( &self, data: impl Into, ts: u128, - ) -> EnclaveEvent { - let ts = self.hlc.receive(&ts.into()).unwrap(); // XXX: - EnclaveEvent::::new_with_timestamp(data.into(), ts.into()) + ) -> Result> { + let ts = self.hlc.receive(&ts.into())?; + Ok(EnclaveEvent::::new_with_timestamp( + data.into(), + ts.into(), + )) } } impl ErrorFactory> for BusHandle { fn event_from_error( &self, - err_type: EnclaveErrorType, + err_type: EType, error: impl Into, ) -> EnclaveEvent { EnclaveEvent::::from_error(err_type, error) diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 1487cc7836..a69b9d5707 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -8,14 +8,16 @@ use actix::Message; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; +use crate::{BusHandle, ErrorDispatcher}; + pub trait FromError { - fn from_error(err_type: EnclaveErrorType, error: impl Into) -> Self; + fn from_error(err_type: EType, error: impl Into) -> Self; } #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct EnclaveError { - pub err_type: EnclaveErrorType, + pub err_type: EType, pub message: String, } @@ -26,7 +28,7 @@ impl Display for EnclaveError { } #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum EnclaveErrorType { +pub enum EType { Evm, KeyGeneration, PublickeyAggregation, @@ -35,10 +37,11 @@ pub enum EnclaveErrorType { Decryption, Sortition, Data, + Event, } impl EnclaveError { - pub fn new(err_type: EnclaveErrorType, message: impl Into) -> Self { + pub fn new(err_type: EType, message: impl Into) -> Self { Self { err_type, message: message.into().to_string(), @@ -47,10 +50,22 @@ impl EnclaveError { } impl FromError for EnclaveError { - fn from_error(err_type: EnclaveErrorType, error: impl Into) -> Self { + fn from_error(err_type: EType, error: impl Into) -> Self { Self { err_type, message: error.into(), } } } + +/// Function to run a closure that returns a result. If result is an Err variant it is trapped and +/// sent to the bus as an ErrorEvent +pub fn trap(err_type: EType, bus: &BusHandle, runner: F) +where + F: FnOnce() -> anyhow::Result<()>, +{ + match runner() { + Ok(_) => (), + Err(e) => bus.err(err_type, e), + } +} diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index dd37818456..984158d056 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -213,7 +213,7 @@ impl Event for EnclaveEvent { } impl ErrorEvent for EnclaveEvent { - type ErrType = EnclaveErrorType; + type ErrType = EType; type FromError = anyhow::Error; fn from_error(err_type: Self::ErrType, msg: impl Into) -> Self { diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index edea6be5a9..ecf60f64df 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -51,7 +51,10 @@ mod tests { TestEvent::new("three", 3), ]; - event_data.iter().cloned().for_each(|d| bus.publish(d)); + for d in event_data.clone() { + bus.publish(d)?; + } + let expected = event_data .into_iter() .map(|d| EnclaveEvent::new_stored_event(d.clone().into(), 0, d.entropy)) diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index de29f13c67..12386e85e5 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Message, Recipient}; +use anyhow::Result; use std::fmt::Display; use std::hash::Hash; @@ -37,12 +38,12 @@ pub trait EventFactory { /// Create a new event from the given event data, apply a local HLC timestamp. /// /// This method should be used for events that have originated locally. - fn event_from(&self, data: impl Into) -> E; + fn event_from(&self, data: impl Into) -> Result; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering. /// /// This method should be used for events that originated from remote sources. - fn event_from_remote_source(&self, data: impl Into, ts: u128) -> E; + fn event_from_remote_source(&self, data: impl Into, ts: u128) -> Result; } /// An ErrorFactory creates errors. @@ -57,12 +58,12 @@ pub trait EventPublisher { /// to the event bus. /// /// This method should be used for events that have originated locally. - fn publish(&self, data: impl Into); + fn publish(&self, data: impl Into) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. /// /// This method should be used for events that originated from remote sources. - fn publish_from_remote(&self, data: impl Into, ts: u128); + fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()>; /// Dispatch the given event without applying any HLC transformation. fn naked_dispatch(&self, event: E); } diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index 4ec776957a..20652f3554 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -16,7 +16,7 @@ use alloy::{ }; use anyhow::Result; use e3_data::Repository; -use e3_events::{BusHandle, EnclaveEvent, EnclaveEventData}; +use e3_events::{BusHandle, EnclaveEventData}; use tracing::{error, info, trace}; sol!( diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 150d334834..d02192b0e2 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -18,7 +18,7 @@ use alloy::{ use anyhow::Result; use e3_data::Repository; use e3_events::{ - prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EnclaveErrorType, + prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, EnclaveEvent, EnclaveEventData, EventSubscriber, OrderedSet, PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; @@ -361,7 +361,7 @@ impl Handler } Err(err) => { error!("Failed to submit ticket: {:?}", err); - bus.err(EnclaveErrorType::Evm, err); + bus.err(EType::Evm, err); } } }) @@ -391,7 +391,7 @@ impl Handler { error!("Failed to finalize committee: {:?}", err); - bus.err(EnclaveErrorType::Evm, err); + bus.err(EType::Evm, err); } } }) @@ -419,7 +419,7 @@ impl Handler { info!(tx=%receipt.transaction_hash, "Committee published to registry"); } - Err(err) => bus.err(EnclaveErrorType::Evm, err), + Err(err) => bus.err(EType::Evm, err), } }) } diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index e37e4d0ec1..fa1f6f8f67 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -13,7 +13,7 @@ use alloy::providers::Provider; use alloy::{sol, sol_types::SolEvent}; use anyhow::Result; use e3_data::Repository; -use e3_events::{BusHandle, E3id, EnclaveEvent, EnclaveEventData}; +use e3_events::{BusHandle, E3id, EnclaveEventData}; use e3_utils::utility_types::ArcBytes; use num_bigint::BigUint; use tracing::{error, info, trace}; diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 6b64ed11b8..e6bc7e59da 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -22,7 +22,7 @@ use e3_events::BusHandle; use e3_events::EnclaveEvent; use e3_events::EnclaveEventData; use e3_events::Shutdown; -use e3_events::{E3id, EnclaveErrorType, PlaintextAggregated}; +use e3_events::{E3id, EType, PlaintextAggregated}; use tracing::info; sol!( @@ -99,10 +99,7 @@ impl Handler Handler { bus.err( - EnclaveErrorType::Evm, + EType::Evm, anyhow::anyhow!("Error publishing plaintext output: {:?}", err), ); } diff --git a/crates/evm/src/event_reader.rs b/crates/evm/src/event_reader.rs index 30b3b679f0..39d2035253 100644 --- a/crates/evm/src/event_reader.rs +++ b/crates/evm/src/event_reader.rs @@ -14,7 +14,7 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use anyhow::{anyhow, Result}; use e3_data::{AutoPersist, Persistable, Repository}; -use e3_events::{prelude::*, EnclaveErrorType, EnclaveEvent, EnclaveEventData, EventId}; +use e3_events::{prelude::*, EType, EnclaveEvent, EnclaveEventData, EventId}; use e3_events::{BusHandle, Event}; use futures_util::stream::StreamExt; use std::collections::HashSet; @@ -156,7 +156,7 @@ impl Actor for EvmEventReader

{ let extractor = self.extractor; let Some(shutdown) = self.shutdown_rx.take() else { - bus.err(EnclaveErrorType::Evm, anyhow!("shutdown already called")); + bus.err(EType::Evm, anyhow!("shutdown already called")); return; }; @@ -203,7 +203,7 @@ async fn stream_from_evm( rpc_url ); bus.err( - EnclaveErrorType::Evm, + EType::Evm, anyhow!( "Misconfiguration: Attempted to query historical events from genesis on a non-local node. \ Please specify a `start_block` for contract address {contract_address} on chain {chain_id} using rpc {rpc_url}" @@ -235,7 +235,7 @@ async fn stream_from_evm( } Err(e) => { error!("Failed to fetch historical events: {}", e); - bus.err(EnclaveErrorType::Evm, anyhow!(e)); + bus.err(EType::Evm, anyhow!(e)); return; } } @@ -277,7 +277,7 @@ async fn stream_from_evm( } } Err(e) => { - bus.err(EnclaveErrorType::Evm, anyhow!("{}", e)); + bus.err(EType::Evm, anyhow!("{}", e)); } } @@ -344,7 +344,7 @@ impl Handler for EvmEventReader< Ok(state) }) { Ok(_) => (), - Err(err) => self.bus.err(EnclaveErrorType::Evm, err), + Err(err) => self.bus.err(EType::Evm, err), } } } diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs index 042a58f04b..5b02e33b89 100644 --- a/crates/evm/src/historical_event_coordinator.rs +++ b/crates/evm/src/historical_event_coordinator.rs @@ -6,7 +6,7 @@ use crate::EnclaveEvmEvent; use actix::prelude::*; -use e3_events::{prelude::*, BusHandle, EnclaveEvent, EnclaveEventData}; +use e3_events::{prelude::*, trap, BusHandle, EType, EnclaveEventData}; use tracing::info; #[derive(Clone)] @@ -54,19 +54,20 @@ impl HistoricalEventCoordinator { self.registered_count > 0 && self.registered_count == self.completed_count } - fn flush_buffered_events(&mut self) { + fn flush_buffered_events(&mut self) -> anyhow::Result<()> { // Ordering by block number. But we should also consider the tx_index and log_index. self.buffered_events.sort_by_key(|e| e.block); let count = self.buffered_events.len(); for BufferedEvent { event, .. } in self.buffered_events.drain(..) { - self.target.publish(event); + self.target.publish(event)?; } info!( "HistoricalEventCoordinator: replay complete, published {} ordered events", count ); + Ok(()) } } @@ -82,13 +83,14 @@ impl Handler for HistoricalEventCoordinator { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { - match msg { + trap(EType::Evm, &self.target.clone(), || match msg { EnclaveEvmEvent::RegisterReader => { self.registered_count += 1; info!( total_registered = self.registered_count, "Reader registered with coordinator" ); + Ok(()) } EnclaveEvmEvent::HistoricalSyncComplete => { @@ -101,8 +103,9 @@ impl Handler for HistoricalEventCoordinator { if self.started && self.all_readers_complete() { info!("All readers completed historical sync, flushing buffered events"); - self.flush_buffered_events(); + self.flush_buffered_events()?; } + Ok(()) } EnclaveEvmEvent::Event { event, block } => { @@ -111,10 +114,11 @@ impl Handler for HistoricalEventCoordinator { self.buffered_events.push(BufferedEvent { block, event }); } } else { - self.target.publish(event); + self.target.publish(event)?; } + Ok(()) } - } + }) } } @@ -122,14 +126,17 @@ impl Handler for HistoricalEventCoordinator { type Result = (); fn handle(&mut self, _msg: CoordinatorStart, _ctx: &mut Self::Context) -> Self::Result { - info!( - registered_readers = self.registered_count, - "Starting HistoricalEventCoordinator" - ); - self.started = true; - - if self.all_readers_complete() { - self.flush_buffered_events(); - } + trap(EType::Evm, &self.target.clone(), || { + info!( + registered_readers = self.registered_count, + "Starting HistoricalEventCoordinator" + ); + self.started = true; + + if self.all_readers_complete() { + self.flush_buffered_events()?; + } + Ok(()) + }) } } diff --git a/crates/fhe/src/ext.rs b/crates/fhe/src/ext.rs index 20de58d4b2..4f417a4547 100644 --- a/crates/fhe/src/ext.rs +++ b/crates/fhe/src/ext.rs @@ -8,9 +8,7 @@ use crate::{Fhe, FheRepositoryFactory}; use anyhow::{anyhow, Result}; use async_trait::async_trait; use e3_data::{FromSnapshotWithParams, RepositoriesFactory, Snapshot}; -use e3_events::{ - prelude::*, BusHandle, E3Requested, EnclaveErrorType, EnclaveEvent, EnclaveEventData, -}; +use e3_events::{prelude::*, BusHandle, E3Requested, EType, EnclaveEvent, EnclaveEventData}; use e3_request::{E3Context, E3ContextSnapshot, E3Extension, TypedKey}; use e3_utils::SharedRng; use std::sync::Arc; @@ -50,10 +48,8 @@ impl E3Extension for FheExtension { } = data.clone(); let Ok(fhe_inner) = Fhe::from_encoded(¶ms, seed, self.rng.clone()) else { - self.bus.err( - EnclaveErrorType::KeyGeneration, - anyhow!(ERROR_FHE_FAILED_TO_DECODE), - ); + self.bus + .err(EType::KeyGeneration, anyhow!(ERROR_FHE_FAILED_TO_DECODE)); return; }; @@ -61,10 +57,8 @@ impl E3Extension for FheExtension { // FHE doesn't implement Checkpoint so we are going to store it manually let Ok(snapshot) = fhe.snapshot() else { - self.bus.err( - EnclaveErrorType::KeyGeneration, - anyhow!("Failed to get snapshot"), - ); + self.bus + .err(EType::KeyGeneration, anyhow!("Failed to get snapshot")); return; }; ctx.repositories().fhe(&e3_id).write(&snapshot); diff --git a/crates/keyshare/src/ext.rs b/crates/keyshare/src/ext.rs index a7ed81ca3a..a36b5a89a8 100644 --- a/crates/keyshare/src/ext.rs +++ b/crates/keyshare/src/ext.rs @@ -13,7 +13,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use e3_crypto::Cipher; use e3_data::{AutoPersist, RepositoriesFactory}; -use e3_events::{prelude::*, BusHandle, EnclaveErrorType, EnclaveEvent, EnclaveEventData}; +use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; use e3_fhe::ext::FHE_KEY; use e3_multithread::Multithread; use e3_request::{E3Context, E3ContextSnapshot, E3Extension, META_KEY}; @@ -48,10 +48,8 @@ impl E3Extension for KeyshareExtension { // Has the FHE dependency been already setup? (hint: it should have) let Some(fhe) = ctx.get_dependency(FHE_KEY) else { - self.bus.err( - EnclaveErrorType::KeyGeneration, - anyhow!(ERROR_KEYSHARE_FHE_MISSING), - ); + self.bus + .err(EType::KeyGeneration, anyhow!(ERROR_KEYSHARE_FHE_MISSING)); return; }; @@ -91,10 +89,8 @@ impl E3Extension for KeyshareExtension { // Has the FHE dependency been already setup? (hint: it should have) let Some(fhe) = ctx.get_dependency(FHE_KEY) else { - self.bus.err( - EnclaveErrorType::KeyGeneration, - anyhow!(ERROR_KEYSHARE_FHE_MISSING), - ); + self.bus + .err(EType::KeyGeneration, anyhow!(ERROR_KEYSHARE_FHE_MISSING)); return Ok(()); }; @@ -150,10 +146,8 @@ impl E3Extension for ThresholdKeyshareExtension { let e3_id = data.clone().e3_id; let party_id = data.clone().party_id; let Some(meta) = ctx.get_dependency(META_KEY) else { - self.bus.err( - EnclaveErrorType::KeyGeneration, - anyhow!(ERROR_KEYSHARE_FHE_MISSING), - ); + self.bus + .err(EType::KeyGeneration, anyhow!(ERROR_KEYSHARE_FHE_MISSING)); return; }; let repo = ctx.repositories().threshold_keyshare(&e3_id); diff --git a/crates/keyshare/src/keyshare.rs b/crates/keyshare/src/keyshare.rs index 3db9dff093..2a848d7875 100644 --- a/crates/keyshare/src/keyshare.rs +++ b/crates/keyshare/src/keyshare.rs @@ -5,12 +5,13 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::prelude::*; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context as AnyhowContext, Result}; use e3_crypto::Cipher; use e3_data::Persistable; use e3_events::{ - prelude::*, BusHandle, CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, - Die, E3RequestComplete, EnclaveErrorType, EnclaveEvent, EnclaveEventData, KeyshareCreated, + prelude::*, trap, BusHandle, CiphernodeSelected, CiphertextOutputPublished, + DecryptionshareCreated, Die, E3RequestComplete, EType, EnclaveEvent, EnclaveEventData, + KeyshareCreated, }; use e3_fhe::{DecryptCiphertext, Fhe}; use e3_utils::utility_types::ArcBytes; @@ -90,28 +91,27 @@ impl Handler for Keyshare { type Result = (); fn handle(&mut self, event: CiphernodeSelected, _: &mut actix::Context) -> Self::Result { - let CiphernodeSelected { e3_id, .. } = event; - - // generate keyshare - let Ok((secret, pubkey)) = self.fhe.generate_keyshare() else { - self.bus.err( - EnclaveErrorType::KeyGeneration, - anyhow!("Error creating Keyshare for {e3_id}"), - ); - return; - }; - - // Save secret on state - if let Err(err) = self.set_secret(secret) { - self.bus.err(EnclaveErrorType::KeyGeneration, err) - }; - - // Broadcast the KeyshareCreated message - self.bus.publish(KeyshareCreated { - pubkey, - e3_id, - node: self.address.clone(), - }); + trap(EType::KeyGeneration, &self.bus.clone(), || { + let CiphernodeSelected { e3_id, .. } = event; + + // generate keyshare + let (secret, pubkey) = self + .fhe + .generate_keyshare() + .with_context(|| format!("Error creating Keyshare for {}", e3_id))?; + + // Save secret on state + self.set_secret(secret)?; + + // Broadcast the KeyshareCreated message + self.bus.publish(KeyshareCreated { + pubkey, + e3_id, + node: self.address.clone(), + })?; + + Ok(()) + }) } } @@ -123,44 +123,32 @@ impl Handler for Keyshare { event: CiphertextOutputPublished, _: &mut actix::Context, ) -> Self::Result { - let CiphertextOutputPublished { - e3_id, - ciphertext_output, - } = event; - - let Ok(secret) = self.get_secret() else { - self.bus.err( - EnclaveErrorType::Decryption, - anyhow!("Secret not available for Keyshare for e3_id {e3_id}"), - ); - return; - }; - - let Some(ciphertext) = ciphertext_output.first() else { - self.bus.err( - EnclaveErrorType::Decryption, - anyhow!("Ciphernode output array is empty!"), - ); - return; - }; - - let Ok(decryption_share) = self.fhe.decrypt_ciphertext(DecryptCiphertext { - ciphertext: ciphertext.extract_bytes(), - unsafe_secret: secret, - }) else { - self.bus.err( - EnclaveErrorType::Decryption, - anyhow!("error decrypting ciphertext: {:?}", ciphertext_output), - ); - return; - }; - - self.bus.publish(DecryptionshareCreated { - party_id: 0, // Not used - e3_id, - decryption_share: vec![ArcBytes::from_bytes(&decryption_share)], - node: self.address.clone(), - }); + trap(EType::KeyGeneration, &self.bus.clone(), || { + let CiphertextOutputPublished { + e3_id, + ciphertext_output, + } = event; + + let secret = self.get_secret()?; + + let ciphertext = ciphertext_output + .first() + .ok_or(anyhow!("Ciphernode output array is empty!"))?; + + let decryption_share = self.fhe.decrypt_ciphertext(DecryptCiphertext { + ciphertext: ciphertext.extract_bytes(), + unsafe_secret: secret, + })?; + + self.bus.publish(DecryptionshareCreated { + party_id: 0, // Not used + e3_id, + decryption_share: vec![ArcBytes::from_bytes(&decryption_share)], + node: self.address.clone(), + })?; + + Ok(()) + }) } } diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 470cebb2e2..d79398f13a 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -535,7 +535,7 @@ impl ThresholdKeyshare { sk_sss, }), external: false, - }); + })?; Ok(()) } @@ -634,7 +634,7 @@ impl ThresholdKeyshare { pubkey: current.pk_share, e3_id, node: address, - }); + })?; Ok(()) } @@ -697,7 +697,7 @@ impl ThresholdKeyshare { }; // send the decryption share - self.bus.publish(event); + self.bus.publish(event)?; // mark as complete self.state.try_mutate(|s| { diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 21693d70cf..a061995fa2 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -15,8 +15,8 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use e3_events::{ prelude::*, BusHandle, CiphernodeSelected, CorrelationId, DocumentKind, DocumentMeta, - DocumentReceived, E3RequestComplete, E3id, EnclaveErrorType, EnclaveEvent, EnclaveEventData, - Event, PartyId, PublishDocumentRequested, ThresholdShareCreated, + DocumentReceived, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, Event, + PartyId, PublishDocumentRequested, ThresholdShareCreated, }; use e3_utils::retry::{retry_with_backoff, to_retry}; use e3_utils::ArcBytes; @@ -155,7 +155,7 @@ impl Handler for DocumentPublisher { Ok(_) => (), Err(e) => { error!(error=?e, "Could not handle publish document requested"); - bus.err(EnclaveErrorType::IO, e) + bus.err(EType::IO, e) } } }) @@ -204,7 +204,7 @@ impl Handler for DocumentPublisher { Ok(_) => (), Err(e) => { error!(error=?e, "Could not handle document published notification"); - bus.err(EnclaveErrorType::IO, e); + bus.err(EType::IO, e); } } }) @@ -283,7 +283,7 @@ pub async fn handle_document_published_notification( bus.publish(DocumentReceived { meta: event.meta, value, - }); + })?; Ok(()) } @@ -422,7 +422,8 @@ impl EventConverter { vec![], None, ); - self.bus.publish(PublishDocumentRequested::new(meta, value)); + self.bus + .publish(PublishDocumentRequested::new(meta, value))?; Ok(()) } /// Received document externally @@ -437,7 +438,7 @@ impl EventConverter { }, }; - self.bus.publish(event); + self.bus.publish(event)?; Ok(()) } } @@ -486,7 +487,7 @@ mod tests { use actix::Addr; use anyhow::{bail, Result}; use e3_events::{ - hlc::Hlc, BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EnclaveError, + BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EnclaveError, EnclaveEvent, EventBus, EventBusConfig, GetEvents, HistoryCollector, PublishDocumentRequested, TakeEvents, }; @@ -545,7 +546,7 @@ mod tests { bus.publish(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), - }); + })?; // 2. Document publisher should have asked the NetInterface to put the doc on Kademlia let Some(NetCommand::DhtPutRecord { @@ -621,7 +622,7 @@ mod tests { threshold_m: 3, threshold_n: 5, ..CiphernodeSelected::default() - }); + })?; net_evt_tx.send(NetEvent::GossipData( GossipData::DocumentPublishedNotification(DocumentPublishedNotification { @@ -681,7 +682,7 @@ mod tests { bus.publish(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), - }); + })?; for _ in 0..4 { // Expect retry @@ -733,7 +734,7 @@ mod tests { threshold_m: 3, threshold_n: 5, ..CiphernodeSelected::default() - }); + })?; // 2. Dispatch a NetEvent from the NetInterface signaling that a document was published net_evt_tx.send(NetEvent::GossipData( diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index e9e1436949..d6dc8226b3 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -10,7 +10,6 @@ use crate::E3ContextParams; use crate::E3ContextSnapshot; use crate::E3MetaExtension; use crate::RouterRepositoryFactory; -use actix::AsyncContext; use actix::Message; use actix::{Actor, Addr, Context, Handler}; use anyhow::*; @@ -22,16 +21,16 @@ use e3_data::RepositoriesFactory; use e3_data::Repository; use e3_data::Snapshot; use e3_events::prelude::*; +use e3_events::trap; use e3_events::BusHandle; use e3_events::E3RequestComplete; +use e3_events::EType; use e3_events::EnclaveEventData; -use e3_events::Shutdown; use e3_events::{E3id, EnclaveEvent, Event}; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; use std::{collections::HashMap, sync::Arc}; -use tracing::error; /// Buffers events for downstream instances to handle out-of-order event delivery. /// Events are stored in a HashMap keyed by string identifiers until they are ready @@ -151,63 +150,65 @@ impl Actor for E3Router { impl Handler for E3Router { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - // If we are shutting down then bail on anything else - if let EnclaveEventData::Shutdown(_) = msg.get_data() { - for (_, ctx) in self.contexts.iter() { - ctx.forward_message_now(&msg) + trap(EType::Event, &self.bus.clone(), || { + // If we are shutting down then bail on anything else + if let EnclaveEventData::Shutdown(_) = msg.get_data() { + for (_, ctx) in self.contexts.iter() { + ctx.forward_message_now(&msg) + } + + return Ok(()); } - return; - } - - // Only process events with e3_ids - let Some(e3_id) = msg.get_e3_id() else { - return; - }; - - // If this e3 round has already been completed then we are not going to do anything here - if self.completed.contains(&e3_id) { - error!("Received the following event to E3Id({}) despite already being completed:\n\n{:?}\n\n", e3_id, msg); - return; - } - - let repositories = self.repository().repositories(); - let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { - E3Context::from_params(E3ContextParams { - e3_id: e3_id.clone(), - repository: repositories.context(&e3_id), - extensions: self.extensions.clone(), - }) - }); - - for extension in self.extensions.iter() { - extension.on_event(context, &msg); - } + // Only process events with e3_ids + let Some(e3_id) = msg.get_e3_id() else { + return Ok(()); + }; - context.forward_message(&msg, &mut self.buffer); + // If this e3 round has already been completed then we are not going to do anything here + if self.completed.contains(&e3_id) { + return Err(anyhow!("Received the following event to E3Id({}) despite already being completed:\n\n{:?}\n\n", e3_id, msg)); + } - match msg.into_data() { - EnclaveEventData::PlaintextAggregated(_) => { - // Here we are detemining that by receiving the PlaintextAggregated event our request is - // complete and we can notify everyone. This might change as we consider other factors - // when determining if the request is complete - let event = E3RequestComplete { + let repositories = self.repository().repositories(); + let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { + E3Context::from_params(E3ContextParams { e3_id: e3_id.clone(), - }; + repository: repositories.context(&e3_id), + extensions: self.extensions.clone(), + }) + }); - // Send to bus so all other actors can react to a request being complete. - self.bus.publish(event); + for extension in self.extensions.iter() { + extension.on_event(context, &msg); } - EnclaveEventData::E3RequestComplete(_) => { - // Note this will be sent above to the children who can kill themselves based on - // the event - self.contexts.remove(&e3_id); - self.completed.insert(e3_id); + + context.forward_message(&msg, &mut self.buffer); + + match msg.into_data() { + EnclaveEventData::PlaintextAggregated(_) => { + // Here we are detemining that by receiving the PlaintextAggregated event our request is + // complete and we can notify everyone. This might change as we consider other factors + // when determining if the request is complete + let event = E3RequestComplete { + e3_id: e3_id.clone(), + }; + + // Send to bus so all other actors can react to a request being complete. + self.bus.publish(event)?; + } + EnclaveEventData::E3RequestComplete(_) => { + // Note this will be sent above to the children who can kill themselves based on + // the event + self.contexts.remove(&e3_id); + self.completed.insert(e3_id); + } + _ => (), } - _ => (), - } - self.checkpoint(); + self.checkpoint(); + Ok(()) + }); } } diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 1962c066de..8de35d2885 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -11,8 +11,8 @@ use actix::prelude::*; use e3_config::StoreKeys; use e3_data::{DataStore, RepositoriesFactory}; use e3_events::{ - prelude::*, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, EnclaveEvent, - EnclaveEventData, Shutdown, TicketGenerated, TicketId, + prelude::*, trap, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, EType, + EnclaveEvent, EnclaveEventData, Shutdown, TicketGenerated, TicketId, }; use e3_request::MetaRepositoryFactory; use tracing::info; @@ -109,11 +109,14 @@ impl Handler for CiphernodeSelector { ticket_id = tid, "Ticket generated for score sortition" ); - bus.publish(TicketGenerated { - e3_id: data.e3_id.clone(), - ticket_id: TicketId::Score(tid), - node: address.clone(), - }); + trap(EType::Sortition, &bus.clone(), || { + bus.publish(TicketGenerated { + e3_id: data.e3_id.clone(), + ticket_id: TicketId::Score(tid), + node: address.clone(), + })?; + Ok(()) + }) } } else { info!("This node is not selected"); @@ -165,16 +168,19 @@ impl Handler for CiphernodeSelector { party_id = party_id, "Node is in finalized committee, emitting CiphernodeSelected" ); - bus.publish(CiphernodeSelected { - party_id: party_id as u64, - e3_id, - threshold_m: e3_meta.threshold_m, - threshold_n: e3_meta.threshold_n, - esi_per_ct: e3_meta.esi_per_ct, - error_size: e3_meta.error_size, - params: e3_meta.params, - seed: e3_meta.seed, - }); + trap(EType::Sortition, &bus.clone(), || { + bus.publish(CiphernodeSelected { + party_id: party_id as u64, + e3_id, + threshold_m: e3_meta.threshold_m, + threshold_n: e3_meta.threshold_n, + esi_per_ct: e3_meta.esi_per_ct, + error_size: e3_meta.error_size, + params: e3_meta.params, + seed: e3_meta.seed, + })?; + Ok(()) + }) }) } } diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 3e5a8a375b..c3770080e1 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -11,8 +11,8 @@ use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, - ConfigurationUpdated, EnclaveErrorType, EnclaveEvent, OperatorActivationChanged, - PlaintextOutputPublished, Seed, TicketBalanceUpdated, + ConfigurationUpdated, EType, EnclaveEvent, OperatorActivationChanged, PlaintextOutputPublished, + Seed, TicketBalanceUpdated, }; use e3_events::{BusHandle, EnclaveEventData}; use serde::{Deserialize, Serialize}; @@ -262,7 +262,7 @@ impl Handler for Sortition { .or_insert_with(NodeState::default); Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } if let Err(err) = self.backends.try_mutate(move |mut list_map| { @@ -277,7 +277,7 @@ impl Handler for Sortition { .add(addr); Ok(list_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } info!(address = %msg.address, chain_id = chain_id, "Node added to sortition state"); @@ -297,7 +297,7 @@ impl Handler for Sortition { } Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } if let Err(err) = self.backends.try_mutate(move |mut list_map| { @@ -306,7 +306,7 @@ impl Handler for Sortition { } Ok(list_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } info!(address = %msg.address, chain_id = chain_id, "Node removed from sortition state"); @@ -336,7 +336,7 @@ impl Handler for Sortition { Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } } } @@ -363,7 +363,7 @@ impl Handler for Sortition { } Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } } } @@ -386,7 +386,7 @@ impl Handler for Sortition { ); Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } } } @@ -425,7 +425,7 @@ impl Handler for Sortition { Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } } } @@ -474,7 +474,7 @@ impl Handler for Sortition { Ok(state_map) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } } } @@ -493,7 +493,7 @@ impl Handler for Sortition { committees.insert(msg.e3_id.clone(), msg.committee.clone()); Ok(committees) }) { - self.bus.err(EnclaveErrorType::Sortition, err); + self.bus.err(EType::Sortition, err); } } } @@ -514,7 +514,7 @@ impl Handler for Sortition { backend .get_index(msg.seed, msg.size, msg.address.clone(), msg.chain_id, state) .unwrap_or_else(|err| { - bus.err(EnclaveErrorType::Sortition, err); + bus.err(EType::Sortition, err); None }) } else { diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index 632f120957..dd62af9ca5 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -197,7 +197,7 @@ mod tests { use super::*; use actix::prelude::*; use e3_data::InMemStore; - use e3_events::{hlc::Hlc, BusHandle, EventBus, EventBusConfig}; + use e3_events::{BusHandle, EventBus, EventBusConfig}; async fn mock_setup_node(address: String) -> Result { // Create mock actors for the test diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 5e40e73798..c430d24db5 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -182,7 +182,7 @@ impl AddToCommittee { self.count += 1; - self.bus.publish(evt.clone()); + self.bus.publish(evt.clone())?; Ok(evt.into()) } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 5599bd3d8e..c849726a57 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -42,7 +42,7 @@ async fn setup_score_sortition_environment( old_value: U256::ZERO, new_value: U256::from(10_000_000u64), chain_id, - }); + })?; let mut adder = AddToCommittee::new(bus, chain_id); for addr in eth_addrs { @@ -54,13 +54,13 @@ async fn setup_score_sortition_environment( new_balance: U256::from(1_000_000_000u64), reason: FixedBytes::ZERO, chain_id, - }); + })?; bus.publish(OperatorActivationChanged { operator: addr.clone(), active: true, chain_id, - }); + })?; } Ok(()) @@ -237,7 +237,7 @@ async fn test_trbfv_actor() -> Result<()> { params, }; - bus.publish(e3_requested); + bus.publish(e3_requested)?; // For score sortition, we need to wait for nodes to process E3Requested and run sortition // Since TicketGenerated is a local-only event (not shared across network), we can't collect it @@ -263,7 +263,7 @@ async fn test_trbfv_actor() -> Result<()> { e3_id: e3_id.clone(), committee, chain_id, - }); + })?; let committee_finalized_timer = Instant::now(); @@ -360,7 +360,7 @@ async fn test_trbfv_actor() -> Result<()> { e3_id: e3_id.clone(), }; - bus.publish(ciphertext_published_event.clone()); + bus.publish(ciphertext_published_event.clone())?; println!("CiphertextOutputPublished event has been dispatched!"); diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 07b15ca227..bde0c8fa65 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -129,7 +129,7 @@ async fn setup_score_sortition_environment( old_value: U256::ZERO, new_value: U256::from(10_000_000u64), chain_id, - }); + })?; let mut adder = AddToCommittee::new(bus, chain_id); for addr in eth_addrs { @@ -141,13 +141,13 @@ async fn setup_score_sortition_environment( new_balance: U256::from(1_000_000_000u64), reason: FixedBytes::ZERO, chain_id, - }); + })?; bus.publish(OperatorActivationChanged { operator: addr.clone(), active: true, chain_id, - }); + })?; } Ok(()) @@ -202,17 +202,17 @@ async fn test_public_key_aggregation_and_decryption() -> Result<()> { println!("Sending E3 event..."); // Send the computation requested event - bus.publish(e3_request_event.clone()); + bus.publish(e3_request_event.clone())?; // Test that we cannot send the same event twice - bus.publish(e3_request_event.clone()); + bus.publish(e3_request_event.clone())?; // Finalize committee with all available nodes bus.publish(CommitteeFinalized { e3_id: e3_id.clone(), committee: eth_addrs.clone(), chain_id: 1, - }); + })?; // Generate the test shares and pubkey let rng_test = create_shared_rng_from_u64(42); @@ -258,7 +258,7 @@ async fn test_public_key_aggregation_and_decryption() -> Result<()> { e3_id: e3_id.clone(), }; - bus.publish(ciphertext_published_event.clone()); + bus.publish(ciphertext_published_event.clone())?; let history = history_collector .send(TakeEvents::::new(6)) @@ -313,13 +313,13 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { seed: seed.clone(), params: ArcBytes::from_bytes(&encode_bfv_params(¶ms)), ..E3Requested::default() - }); + })?; bus.publish(CommitteeFinalized { e3_id: e3_id.clone(), committee: eth_addrs.clone(), chain_id: 1, - }); + })?; let history_collector = cn1.history().unwrap(); let error_collector = cn1.errors().unwrap(); @@ -331,7 +331,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { assert_eq!(errors.len(), 0); // SEND SHUTDOWN! - bus.publish(Shutdown); + bus.publish(Shutdown)?; // This is probably overkill but required to ensure that all the data is written sleep(Duration::from_secs(1)).await; @@ -406,7 +406,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { .map(|ct| ArcBytes::from_bytes(&ct.to_bytes())) .collect(), e3_id: e3_id.clone(), - }); + })?; let history = history_collector .send(TakeEvents::::new(5)) @@ -494,9 +494,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { ..CiphernodeSelected::default() }; - bus.publish(evt_1.clone()); - bus.publish(evt_2.clone()); - bus.publish(local_evt_3.clone()); // This is a local event which should not be broadcast to the network + bus.publish(evt_1.clone())?; + bus.publish(evt_2.clone())?; + bus.publish(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network // check the history of the event bus let history = history_collector @@ -550,13 +550,13 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { seed: seed.clone(), params: ArcBytes::from_bytes(&encode_bfv_params(¶ms)), ..E3Requested::default() - }); + })?; bus.publish(CommitteeFinalized { e3_id: E3id::new("1234", 1), committee: eth_addrs.clone(), chain_id: 1, - }); + })?; // Generate the test shares and pubkey let rng_test = create_shared_rng_from_u64(42); @@ -587,13 +587,13 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { seed: seed.clone(), params: ArcBytes::from_bytes(&encode_bfv_params(¶ms)), ..E3Requested::default() - }); + })?; bus.publish(CommitteeFinalized { e3_id: E3id::new("1234", 2), committee: eth_addrs.clone(), chain_id: 2, - }); + })?; let test_pubkey = aggregate_public_key(&generate_pk_shares( ¶ms, &crpoly, &rng_test, ð_addrs, @@ -643,7 +643,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // lets send an event from the network let _ = event_tx.send(NetEvent::GossipData(GossipData::GossipBytes( - bus.event_from(event.clone()).to_bytes()?, + bus.event_from(event.clone())?.to_bytes()?, ))); // check the history of the event bus From 355d9867438a3b6380d1750c919678b039d8472e Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 4 Dec 2025 07:55:51 +0000 Subject: [PATCH 13/27] remove wrong file --- crates/aggregator/src/test_req_res.rs | 168 -------------------------- 1 file changed, 168 deletions(-) delete mode 100644 crates/aggregator/src/test_req_res.rs diff --git a/crates/aggregator/src/test_req_res.rs b/crates/aggregator/src/test_req_res.rs deleted file mode 100644 index 5a537206b3..0000000000 --- a/crates/aggregator/src/test_req_res.rs +++ /dev/null @@ -1,168 +0,0 @@ -/** - * This is an example of how to best use actix to avoid sending stuff over async boundaries - **/ -use std::{ops::Deref, rc::Rc}; - -use actix::{Actor, Addr, AsyncContext, Context, Handler, Message, Recipient}; - -struct Alice { - bob: Addr, - result: Option, -} - -impl Actor for Alice { - type Context = Context; - fn started(&mut self, ctx: &mut Self::Context) { - self.bob - .do_send(Responder::new(Greet::new("Alice"), ctx.address())) - } -} - -impl Alice { - pub fn new(bob: &Addr) -> Self { - Self { - bob: bob.clone(), - result: None, - } - } -} - -impl Handler for Alice { - type Result = (); - fn handle(&mut self, msg: GreetResponse, _: &mut Self::Context) -> Self::Result { - self.result = Some(msg.value); - } -} - -impl Handler for Alice { - type Result = Option; - fn handle(&mut self, msg: GetInner, _: &mut Self::Context) -> Self::Result { - self.result.take() - } -} - -struct Bob { - value: Rc, -} - -impl Actor for Bob { - type Context = Context; -} - -impl Bob { - pub fn new(value: &str) -> Self { - Self { - value: Rc::new(value.into()), - } - } -} - -impl Handler> for Bob { - type Result = (); - fn handle( - &mut self, - msg: Responder, - _: &mut Self::Context, - ) -> Self::Result { - let reply = &format!("Hello {}! Regards, {}", msg.name, self.value); - - // do some sync work maybe... - // could store the responder locally - - msg.reply(GreetResponse::new(reply)); - } -} - -#[derive(Message)] -#[rtype("()")] -struct Greet { - name: String, -} - -impl Greet { - pub fn new(name: &str) -> Self { - Self { - name: name.to_owned(), - } - } -} - -#[derive(Message)] -#[rtype("()")] -struct GreetResponse { - value: String, -} - -impl GreetResponse { - pub fn new(value: &str) -> Self { - Self { - value: value.to_owned(), - } - } -} - -/////////////////////////////////////////////////////////////////////// - -#[derive(Message)] -#[rtype("Option")] -struct GetInner; - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::{Alice, Bob, GetInner}; - use actix::prelude::*; - use tokio::time::sleep; - - #[actix::test] - async fn test_things() -> anyhow::Result<()> { - let bob = Bob::new("Blurg").start(); - let alice = Alice::new(&bob).start(); - sleep(Duration::from_millis(1)).await; - alice.send(GetInner).await?.unwrap(); - - Ok(()) - } -} - -/////////////////////////////////////////////////////////////////////// - -#[derive(Message)] -#[rtype("()")] -struct Responder -where - U: Message + Send, - U::Result: Send, -{ - value: T, - sender: Recipient, -} - -impl Responder -where - U: Message + Send, - U::Result: Send, -{ - pub fn new(value: T, sender: impl Into>) -> Self { - Self { - value, - sender: sender.into(), - } - } - pub fn reply(&self, msg: U) { - let sender = &self.sender; - sender.do_send(msg); - } -} - -impl Deref for Responder -where - U: Message + Send, - U::Result: Send, -{ - type Target = T; - fn deref(&self) -> &Self::Target { - &self.value - } -} From 924061bb163eccef7ac5b9fce1fb9f1cc666f65b Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 4 Dec 2025 21:31:49 +0000 Subject: [PATCH 14/27] tidy up warnings --- crates/data/src/sled_store.rs | 4 +--- crates/entrypoint/src/helpers/datastore.rs | 2 +- crates/events/src/eventbus.rs | 1 - crates/events/src/eventbus_factory.rs | 1 - crates/evm/src/enclave_sol.rs | 2 +- crates/request/src/router.rs | 6 +----- 6 files changed, 4 insertions(+), 12 deletions(-) diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 4facb8aeca..185b30f4f5 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -7,9 +7,7 @@ use crate::{Get, Insert, InsertSync, Remove}; use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::{Context, Result}; -use e3_events::{ - get_enclave_bus_handle, prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData, -}; +use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; use once_cell::sync::Lazy; use sled::Db; use std::{ diff --git a/crates/entrypoint/src/helpers/datastore.rs b/crates/entrypoint/src/helpers/datastore.rs index 3f9677c86e..b186c64cf0 100644 --- a/crates/entrypoint/src/helpers/datastore.rs +++ b/crates/entrypoint/src/helpers/datastore.rs @@ -11,7 +11,7 @@ use anyhow::Result; use e3_config::AppConfig; use e3_data::{DataStore, InMemStore, SledDb, SledStore}; use e3_data::{Repositories, RepositoriesFactory}; -use e3_events::{get_enclave_bus_handle, BusHandle, EnclaveEvent}; +use e3_events::{get_enclave_bus_handle, BusHandle}; pub fn get_sled_store(bus: &BusHandle, db_file: &PathBuf) -> Result { Ok((&SledStore::new(bus, db_file)?).into()) diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 1e2912279a..bb6f7183b9 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -4,7 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::hlc::Hlc; use crate::traits::{ErrorEvent, Event}; use crate::{prelude::*, BusHandle, EnclaveEvent, Stored}; use actix::prelude::*; diff --git a/crates/events/src/eventbus_factory.rs b/crates/events/src/eventbus_factory.rs index 5db50b732f..f69d59ed6c 100644 --- a/crates/events/src/eventbus_factory.rs +++ b/crates/events/src/eventbus_factory.rs @@ -12,7 +12,6 @@ use std::any::TypeId; use std::collections::HashMap; use std::sync::Mutex; -use crate::hlc::Hlc; use crate::traits::Event; use crate::BusHandle; use crate::EnclaveEvent; diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 6f93c9265d..a8a6bebf30 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -12,7 +12,7 @@ use actix::Recipient; use alloy::providers::{Provider, WalletProvider}; use anyhow::Result; use e3_data::Repository; -use e3_events::{BusHandle, EnclaveEvent}; +use e3_events::BusHandle; pub struct EnclaveSol; diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index d6dc8226b3..3df1c2e1a9 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -149,7 +149,7 @@ impl Actor for E3Router { impl Handler for E3Router { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Event, &self.bus.clone(), || { // If we are shutting down then bail on anything else if let EnclaveEventData::Shutdown(_) = msg.get_data() { @@ -306,7 +306,3 @@ impl E3RouterBuilder { Ok(addr) } } - -#[derive(Message, Clone, Debug)] -#[rtype("()")] -struct EventWrapper(T); From 31d55ca804f20826379c1fa9954e2cdbda127a3d Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 4 Dec 2025 22:16:57 +0000 Subject: [PATCH 15/27] use correct error type --- crates/keyshare/src/keyshare.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/keyshare/src/keyshare.rs b/crates/keyshare/src/keyshare.rs index 2a848d7875..ff80744861 100644 --- a/crates/keyshare/src/keyshare.rs +++ b/crates/keyshare/src/keyshare.rs @@ -123,7 +123,7 @@ impl Handler for Keyshare { event: CiphertextOutputPublished, _: &mut actix::Context, ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { + trap(EType::Decryption, &self.bus.clone(), || { let CiphertextOutputPublished { e3_id, ciphertext_output, From 619d376999b14209ad80f329950ae6423cb90189 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 4 Dec 2025 22:19:07 +0000 Subject: [PATCH 16/27] fix error message --- crates/keyshare/src/ext.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/keyshare/src/ext.rs b/crates/keyshare/src/ext.rs index a36b5a89a8..39b8db852a 100644 --- a/crates/keyshare/src/ext.rs +++ b/crates/keyshare/src/ext.rs @@ -135,6 +135,9 @@ impl ThresholdKeyshareExtension { } } +const ERROR_KEYSHARE_META_MISSING: &str = + "Could not create ThresholdKeyshare because the meta instance it depends on was not set on the context."; + #[async_trait] impl E3Extension for ThresholdKeyshareExtension { fn on_event(&self, ctx: &mut E3Context, evt: &EnclaveEvent) { @@ -147,7 +150,7 @@ impl E3Extension for ThresholdKeyshareExtension { let party_id = data.clone().party_id; let Some(meta) = ctx.get_dependency(META_KEY) else { self.bus - .err(EType::KeyGeneration, anyhow!(ERROR_KEYSHARE_FHE_MISSING)); + .err(EType::KeyGeneration, anyhow!(ERROR_KEYSHARE_META_MISSING)); return; }; let repo = ctx.repositories().threshold_keyshare(&e3_id); From 02fb28af63808a8f190acc40414bfc4ea0c49736 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 00:24:52 +0000 Subject: [PATCH 17/27] test HLC events --- .../src/ciphernode_builder.rs | 2 +- crates/events/src/bus_handle.rs | 162 +++++++++++++++++- crates/events/src/enclave_event/mod.rs | 13 +- crates/events/src/eventbus.rs | 2 +- crates/events/src/eventbus_factory.rs | 2 +- crates/events/src/sequencer.rs | 9 +- crates/net/src/document_publisher.rs | 2 +- crates/test-helpers/src/ciphernode_system.rs | 2 +- 8 files changed, 178 insertions(+), 16 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 61741fc5ca..0a88d6793a 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -299,7 +299,7 @@ impl CiphernodeBuilder { }; // Get a handle from the event bus - let bus = BusHandle::new(local_bus); + let bus = BusHandle::new_from_consumer(local_bus); let addr = if let Some(addr) = self.address.clone() { info!("Using eth address = {}", addr); diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 9d7dca5ff2..6206bfd1dc 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -31,9 +31,17 @@ pub struct BusHandle { } impl BusHandle { - pub fn new(consumer: Addr>>) -> Self { + pub fn new_from_consumer(consumer: Addr>>) -> Self { let producer = Sequencer::new(&consumer).start(); let hlc = Hlc::default(); + Self::new(consumer, producer, hlc) + } + + pub fn new( + consumer: Addr>>, + producer: Addr, + hlc: Hlc, + ) -> Self { Self { consumer, producer, @@ -126,12 +134,160 @@ impl EventSubscriber> for BusHandle { impl Into for Addr> { fn into(self) -> BusHandle { - BusHandle::new(self) + BusHandle::new_from_consumer(self) } } impl Into for &Addr> { fn into(self) -> BusHandle { - BusHandle::new(self.clone()) + BusHandle::new_from_consumer(self.clone()) + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use crate::{ + hlc::Hlc, prelude::*, sequencer::Sequencer, BusHandle, EnclaveEvent, EnclaveEventData, + EventBus, TestEvent, + }; + use actix::{Actor, Handler, Message}; + use tokio::time::sleep; + fn now_micros() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() as u64 + } + + #[actix::test] + async fn test_hlc_events() -> anyhow::Result<()> { + #[derive(Message)] + #[rtype("Vec")] + struct GetEventsOrdered; + + // Setup forwarder + struct Forwarder { + dest: BusHandle, + } + impl Actor for Forwarder { + type Context = actix::Context; + } + + impl Handler for Forwarder { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + let ts = msg.get_ts(); + self.dest.publish_from_remote(msg.into_data(), ts).unwrap() + } + } + + // Setup saver + struct Saver { + events: Vec, + } + + impl Actor for Saver { + type Context = actix::Context; + } + + impl Handler for Saver { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + self.events.push(msg); + } + } + + impl Handler for Saver { + type Result = Vec; + fn handle(&mut self, _: GetEventsOrdered, _: &mut Self::Context) -> Self::Result { + self.events.clone() + } + } + + // 1. setup up two separate busses with out of sync clocks A and B. B should be 30 seconds + // faster than A. + let consumer_a = EventBus::::default().start(); + let producer_a = Sequencer::new(&consumer_a).start(); + let clock_a = Hlc::new(1).with_clock(move || now_micros().saturating_sub(30_000_000)); // Late + let bus_a = BusHandle::new(consumer_a, producer_a, clock_a); + + let consumer_b = EventBus::::default().start(); + let producer_b = Sequencer::new(&consumer_b).start(); + let clock_b = Hlc::new(2); // in sync + let bus_b = BusHandle::new(consumer_b, producer_b, clock_b); + + let consumer_c = EventBus::::default().start(); + let producer_c = Sequencer::new(&consumer_c).start(); + let clock_c = Hlc::new(3); // in sync + let bus_c = BusHandle::new(consumer_c, producer_c, clock_c); + + let forwarder = Forwarder { + dest: bus_c.clone(), + } + .start(); + + // pipe all bus_a and bus_b events to bus_c + bus_a.subscribe("*", forwarder.clone().into()); + bus_b.subscribe("*", forwarder.into()); + + // Create and subscribe the Saver to bus_c + let saver = Saver { events: vec![] }.start(); + bus_c.subscribe("*", saver.clone().into()); + + // Publish events in causal order across buses + bus_a.publish(TestEvent::new("one", 1))?; + sleep(Duration::from_millis(5)).await; // next tick + bus_b.publish(TestEvent::new("two", 2))?; + sleep(Duration::from_millis(5)).await; // next tick + bus_a.publish(TestEvent::new("three", 3))?; + sleep(Duration::from_millis(5)).await; // next tick + bus_b.publish(TestEvent::new("four", 4))?; + sleep(Duration::from_millis(50)).await; // next tick + + // Get events + let events = saver.send(GetEventsOrdered).await?; + + // Sort by HLC timestamp + let mut sorted_events = events.clone(); + sorted_events.sort_by_key(|e| e.get_ts()); + + // Extract the payloads/names in HLC-sorted order + let ordered_names: Vec<_> = sorted_events + .iter() + .filter_map(|e| match e.get_data() { + EnclaveEventData::TestEvent(e) => Some(e.msg.clone()), + _ => None, + }) + .collect(); + + // ASSERTION 1: Causal order is preserved despite clock drift + assert_eq!( + ordered_names, + vec!["one", "two", "three", "four"], + "HLC should preserve causal ordering despite 30s clock drift on bus_a" + ); + + // ASSERTION 2: All timestamps are unique (HLC guarantee) + let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); + let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); + assert_eq!( + timestamps.len(), + unique_timestamps.len(), + "All HLC timestamps should be unique" + ); + + // ASSERTION 3: Timestamps are strictly monotonically increasing when sorted + for window in timestamps.windows(2) { + assert!( + window[0] < window[1], + "HLC timestamps should be strictly increasing: {:?} should be < {:?}", + window[0], + window[1] + ); + } + + Ok(()) } } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 984158d056..14f819c355 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -156,6 +156,7 @@ pub struct EnclaveEvent { id: EventId, payload: EnclaveEventData, seq: S::Seq, + ts: u128, } impl EnclaveEvent @@ -173,6 +174,10 @@ where pub fn get_id(&self) -> EventId { self.into() } + + pub fn get_ts(&self) -> u128 { + self.ts + } } impl EnclaveEvent { @@ -180,6 +185,7 @@ impl EnclaveEvent { EnclaveEvent:: { id: self.id, payload: self.payload, + ts: self.ts, seq, } } @@ -207,6 +213,7 @@ impl Event for EnclaveEvent { fn get_data(&self) -> &EnclaveEventData { &self.payload } + fn into_data(self) -> EnclaveEventData { self.payload } @@ -219,10 +226,12 @@ impl ErrorEvent for EnclaveEvent { fn from_error(err_type: Self::ErrType, msg: impl Into) -> Self { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); + let ts = 0; // XXX: fix this EnclaveEvent { payload: payload.into(), id, seq: (), + ts, } } } @@ -316,14 +325,14 @@ impl fmt::Display for EnclaveEvent { } impl EventConstructorWithTimestamp for EnclaveEvent { - fn new_with_timestamp(data: Self::Data, _ts: u128) -> Self { + fn new_with_timestamp(data: Self::Data, ts: u128) -> Self { let payload = data.into(); let id = EventId::hash(&payload); - // hcl.receive(remote_ts)?; EnclaveEvent { id, payload, seq: (), + ts, } } } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index bb6f7183b9..92b2743968 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -413,7 +413,7 @@ impl Handler for HistoryCollector { /// Function to help with testing when we want to maintain a vec of events pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) { let consumer = EventBus::>::default().start(); - let bus: BusHandle = BusHandle::new(consumer); + let bus: BusHandle = BusHandle::new_from_consumer(consumer); let history = HistoryCollector::new().start(); bus.subscribe("*", history.clone().recipient()); (bus, history) diff --git a/crates/events/src/eventbus_factory.rs b/crates/events/src/eventbus_factory.rs index f69d59ed6c..f251d86622 100644 --- a/crates/events/src/eventbus_factory.rs +++ b/crates/events/src/eventbus_factory.rs @@ -95,5 +95,5 @@ pub fn get_error_collector() -> Addr> { pub fn get_enclave_bus_handle() -> BusHandle { let bus = get_enclave_event_bus(); - BusHandle::new(bus) + BusHandle::new_from_consumer(bus) } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index ecf60f64df..b16f07e03d 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::time::{SystemTime, UNIX_EPOCH}; + use actix::{Actor, Addr, Handler}; use crate::{EnclaveEvent, EventBus, Stored, Unstored}; @@ -42,7 +44,7 @@ mod tests { #[actix::test] async fn it_adds_seqence_numbers_to_events() -> anyhow::Result<()> { - let bus = BusHandle::new(EventBus::::default().start()); + let bus = BusHandle::new_from_consumer(EventBus::::default().start()); let history = bus.history(); let event_data = vec![ @@ -64,9 +66,4 @@ mod tests { assert_eq!(events, expected); Ok(()) } - - #[actix::test] - async fn test_hlc_events() -> anyhow::Result<()> { - Ok(()) - } } diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index a061995fa2..a0aac9e06f 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -519,7 +519,7 @@ mod tests { let guard = tracing::subscriber::set_default(subscriber); let consumer = EventBus::::new(EventBusConfig { deduplicate: true }).start(); - let bus = BusHandle::new(consumer); + let bus = BusHandle::new_from_consumer(consumer); let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); let (net_evt_tx, net_evt_rx) = broadcast::channel(100); let net_evt_rx = Arc::new(net_evt_rx); diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index dd62af9ca5..2772ffac09 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -205,7 +205,7 @@ mod tests { let bus = EventBus::::new(EventBusConfig { deduplicate: true }).start(); let history = EventBus::::history(&bus); let errors = EventBus::::error(&bus); - let bus = BusHandle::new(bus); + let bus = BusHandle::new_from_consumer(bus); Ok(CiphernodeHandle { address, From 1ab4b7bd6e684f52de50f42d4b3d3199e7751506 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 00:36:23 +0000 Subject: [PATCH 18/27] Sequenced and Unsequenced --- crates/events/src/bus_handle.rs | 44 +++++++++++++------------- crates/events/src/enclave_event/mod.rs | 30 +++++++++--------- crates/events/src/eventbus.rs | 7 ++-- crates/events/src/sequencer.rs | 12 +++---- crates/request/src/context.rs | 4 +-- 5 files changed, 48 insertions(+), 49 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 6206bfd1dc..cb2d8db26c 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -17,28 +17,28 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, Stored, - Subscribe, Unstored, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, Sequenced, + Subscribe, Unsequenced, }; #[derive(Clone, Derivative)] #[derivative(Debug)] pub struct BusHandle { - consumer: Addr>>, + consumer: Addr>>, producer: Addr, #[derivative(Debug = "ignore")] hlc: Arc, } impl BusHandle { - pub fn new_from_consumer(consumer: Addr>>) -> Self { + pub fn new_from_consumer(consumer: Addr>>) -> Self { let producer = Sequencer::new(&consumer).start(); let hlc = Hlc::default(); Self::new(consumer, producer, hlc) } pub fn new( - consumer: Addr>>, + consumer: Addr>>, producer: Addr, hlc: Hlc, ) -> Self { @@ -49,20 +49,20 @@ impl BusHandle { } } - pub fn history(&self) -> Addr>> { - EventBus::>::history(&self.consumer) + pub fn history(&self) -> Addr>> { + EventBus::>::history(&self.consumer) } pub fn producer(&self) -> &Addr { &self.producer } - pub fn consumer(&self) -> &Addr>> { + pub fn consumer(&self) -> &Addr>> { &self.consumer } } -impl EventPublisher> for BusHandle { +impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) -> Result<()> { let evt = self.event_from(data)?; self.producer.do_send(evt); @@ -75,22 +75,22 @@ impl EventPublisher> for BusHandle { Ok(()) } - fn naked_dispatch(&self, event: EnclaveEvent) { + fn naked_dispatch(&self, event: EnclaveEvent) { self.producer.do_send(event); } } -impl ErrorDispatcher> for BusHandle { +impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { let evt = self.event_from_error(err_type, error); self.producer.do_send(evt); } } -impl EventFactory> for BusHandle { - fn event_from(&self, data: impl Into) -> Result> { +impl EventFactory> for BusHandle { + fn event_from(&self, data: impl Into) -> Result> { let ts = self.hlc.tick()?; - Ok(EnclaveEvent::::new_with_timestamp( + Ok(EnclaveEvent::::new_with_timestamp( data.into(), ts.into(), )) @@ -100,31 +100,31 @@ impl EventFactory> for BusHandle { &self, data: impl Into, ts: u128, - ) -> Result> { + ) -> Result> { let ts = self.hlc.receive(&ts.into())?; - Ok(EnclaveEvent::::new_with_timestamp( + Ok(EnclaveEvent::::new_with_timestamp( data.into(), ts.into(), )) } } -impl ErrorFactory> for BusHandle { +impl ErrorFactory> for BusHandle { fn event_from_error( &self, err_type: EType, error: impl Into, - ) -> EnclaveEvent { - EnclaveEvent::::from_error(err_type, error) + ) -> EnclaveEvent { + EnclaveEvent::::from_error(err_type, error) } } -impl EventSubscriber> for BusHandle { - fn subscribe(&self, event_type: &str, recipient: Recipient>) { +impl EventSubscriber> for BusHandle { + fn subscribe(&self, event_type: &str, recipient: Recipient>) { self.consumer.do_send(Subscribe::new(event_type, recipient)) } - fn subscribe_all(&self, event_types: &[&str], recipient: Recipient>) { + fn subscribe_all(&self, event_types: &[&str], recipient: Recipient>) { for event_type in event_types.into_iter() { self.consumer .do_send(Subscribe::new(*event_type, recipient.clone())); diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 14f819c355..e5a4d586a3 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -137,22 +137,22 @@ pub trait SeqState: Clone + std::fmt::Debug + 'static { } #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Unstored; +pub struct Unsequenced; #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Stored; +pub struct Sequenced; -impl SeqState for Unstored { +impl SeqState for Unsequenced { type Seq = (); } -impl SeqState for Stored { +impl SeqState for Sequenced { type Seq = u64; } #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EnclaveEvent { +pub struct EnclaveEvent { id: EventId, payload: EnclaveEventData, seq: S::Seq, @@ -180,9 +180,9 @@ where } } -impl EnclaveEvent { - pub fn into_stored(self, seq: u64) -> EnclaveEvent { - EnclaveEvent:: { +impl EnclaveEvent { + pub fn into_stored(self, seq: u64) -> EnclaveEvent { + EnclaveEvent:: { id: self.id, payload: self.payload, ts: self.ts, @@ -192,9 +192,9 @@ impl EnclaveEvent { } #[cfg(feature = "test-helpers")] -impl EnclaveEvent { +impl EnclaveEvent { pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, time).into_stored(seq) + EnclaveEvent::::new_with_timestamp(data, time).into_stored(seq) } } @@ -219,7 +219,7 @@ impl Event for EnclaveEvent { } } -impl ErrorEvent for EnclaveEvent { +impl ErrorEvent for EnclaveEvent { type ErrType = EType; type FromError = anyhow::Error; @@ -300,16 +300,16 @@ impl_into_event_data!( ThresholdShareCreated ); -impl TryFrom<&EnclaveEvent> for EnclaveError { +impl TryFrom<&EnclaveEvent> for EnclaveError { type Error = anyhow::Error; fn try_from(value: &EnclaveEvent) -> Result { value.clone().try_into() } } -impl TryFrom> for EnclaveError { +impl TryFrom> for EnclaveError { type Error = anyhow::Error; - fn try_from(value: EnclaveEvent) -> Result { + fn try_from(value: EnclaveEvent) -> Result { if let EnclaveEventData::EnclaveError(data) = value.payload.clone() { Ok(data) } else { @@ -324,7 +324,7 @@ impl fmt::Display for EnclaveEvent { } } -impl EventConstructorWithTimestamp for EnclaveEvent { +impl EventConstructorWithTimestamp for EnclaveEvent { fn new_with_timestamp(data: Self::Data, ts: u128) -> Self { let payload = data.into(); let id = EventId::hash(&payload); diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 92b2743968..81d748b7ab 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::traits::{ErrorEvent, Event}; -use crate::{prelude::*, BusHandle, EnclaveEvent, Stored}; +use crate::{prelude::*, BusHandle, EnclaveEvent, Sequenced}; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; use std::collections::{HashMap, VecDeque}; @@ -411,8 +411,9 @@ impl Handler for HistoryCollector { ////////////////////////////////////////////////////////////////////////////// /// Function to help with testing when we want to maintain a vec of events -pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) { - let consumer = EventBus::>::default().start(); +pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) +{ + let consumer = EventBus::>::default().start(); let bus: BusHandle = BusHandle::new_from_consumer(consumer); let history = HistoryCollector::new().start(); bus.subscribe("*", history.clone().recipient()); diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index b16f07e03d..98d4d7be37 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -4,19 +4,17 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::time::{SystemTime, UNIX_EPOCH}; - use actix::{Actor, Addr, Handler}; -use crate::{EnclaveEvent, EventBus, Stored, Unstored}; +use crate::{EnclaveEvent, EventBus, Sequenced, Unsequenced}; pub struct Sequencer { - bus: Addr>>, + bus: Addr>>, seq: u64, } impl Sequencer { - pub fn new(bus: &Addr>>) -> Self { + pub fn new(bus: &Addr>>) -> Self { Self { bus: bus.clone(), seq: 0, @@ -28,9 +26,9 @@ impl Actor for Sequencer { type Context = actix::Context; } -impl Handler> for Sequencer { +impl Handler> for Sequencer { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { self.seq += 1; self.bus.do_send(msg.into_stored(self.seq)) } diff --git a/crates/request/src/context.rs b/crates/request/src/context.rs index 7d25ae756e..d45d3b44a0 100644 --- a/crates/request/src/context.rs +++ b/crates/request/src/context.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use e3_data::{ Checkpoint, FromSnapshotWithParams, Repositories, RepositoriesFactory, Repository, Snapshot, }; -use e3_events::{E3id, EnclaveEvent, Stored}; +use e3_events::{E3id, EnclaveEvent}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; @@ -91,7 +91,7 @@ impl E3Context { }); } - pub fn forward_message_now(&self, msg: &EnclaveEvent) { + pub fn forward_message_now(&self, msg: &EnclaveEvent) { self.recipients().into_iter().for_each(|(_, recipient)| { if let Some(act) = recipient { act.do_send(msg.clone()); From c804222882fec2f4d199d57c42b5caa55a046b11 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 00:40:07 +0000 Subject: [PATCH 19/27] add comment --- crates/events/src/sequencer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 98d4d7be37..8323617dd5 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -29,6 +29,7 @@ impl Actor for Sequencer { impl Handler> for Sequencer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + // NOTE: FAKE SEQUENCER FOR NOW - JUST SET THE SEQUENCE NUMBER AND UPDATE self.seq += 1; self.bus.do_send(msg.into_stored(self.seq)) } From 1c1862c1424d1c4845008ac895cee03f260449f9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 00:48:56 +0000 Subject: [PATCH 20/27] rename into_sequenced --- crates/events/src/enclave_event/mod.rs | 4 ++-- crates/events/src/sequencer.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index e5a4d586a3..37ddaefa1c 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -181,7 +181,7 @@ where } impl EnclaveEvent { - pub fn into_stored(self, seq: u64) -> EnclaveEvent { + pub fn into_sequenced(self, seq: u64) -> EnclaveEvent { EnclaveEvent:: { id: self.id, payload: self.payload, @@ -194,7 +194,7 @@ impl EnclaveEvent { #[cfg(feature = "test-helpers")] impl EnclaveEvent { pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, time).into_stored(seq) + EnclaveEvent::::new_with_timestamp(data, time).into_sequenced(seq) } } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 8323617dd5..4bc549c78d 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -31,7 +31,7 @@ impl Handler> for Sequencer { fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { // NOTE: FAKE SEQUENCER FOR NOW - JUST SET THE SEQUENCE NUMBER AND UPDATE self.seq += 1; - self.bus.do_send(msg.into_stored(self.seq)) + self.bus.do_send(msg.into_sequenced(self.seq)) } } From 38b862133ffd1b3e7e8f1d99dd12571a11a205ba Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 01:10:17 +0000 Subject: [PATCH 21/27] fix tests --- crates/events/src/enclave_event/mod.rs | 4 ++++ crates/events/src/sequencer.rs | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 37ddaefa1c..185e8a6dbc 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -196,6 +196,10 @@ impl EnclaveEvent { pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { EnclaveEvent::::new_with_timestamp(data, time).into_sequenced(seq) } + + pub fn get_seq(&self) -> u64 { + self.seq + } } impl Event for EnclaveEvent { diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 4bc549c78d..049becabe0 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -40,6 +40,9 @@ mod tests { use crate::{prelude::*, BusHandle, EnclaveEvent, EventBus, TakeEvents, TestEvent}; use actix::Actor; + fn strips_ts(event: &EnclaveEvent) -> EnclaveEvent { + EnclaveEvent::new_stored_event(event.get_data().clone(), 0, event.get_seq()) + } #[actix::test] async fn it_adds_seqence_numbers_to_events() -> anyhow::Result<()> { @@ -62,7 +65,7 @@ mod tests { .collect::>(); let events = history.send(TakeEvents::new(3)).await?; - assert_eq!(events, expected); + assert_eq!(events.iter().map(strips_ts).collect::>(), expected); Ok(()) } } From 33482c554b3351e23341ec50bcd415a18ba894cb Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 04:12:27 +0000 Subject: [PATCH 22/27] fix tests --- crates/events/src/enclave_event/mod.rs | 12 ++++++++++-- crates/events/src/sequencer.rs | 11 +++++++---- crates/tests/tests/integration_legacy.rs | 18 +++++++++--------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 185e8a6dbc..b0d2b5de85 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -180,6 +180,12 @@ where } } +impl EnclaveEvent { + pub fn get_seq(&self) -> u64 { + self.seq + } +} + impl EnclaveEvent { pub fn into_sequenced(self, seq: u64) -> EnclaveEvent { EnclaveEvent:: { @@ -193,12 +199,14 @@ impl EnclaveEvent { #[cfg(feature = "test-helpers")] impl EnclaveEvent { + /// test-helpers only utility function to create a new unsequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { EnclaveEvent::::new_with_timestamp(data, time).into_sequenced(seq) } - pub fn get_seq(&self) -> u64 { - self.seq + /// test-helpers only utility function to remove time information from an event + pub fn strip_ts(&self) -> EnclaveEvent { + EnclaveEvent::new_stored_event(self.get_data().clone(), 0, self.get_seq()) } } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 049becabe0..1a691e7f53 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -40,9 +40,6 @@ mod tests { use crate::{prelude::*, BusHandle, EnclaveEvent, EventBus, TakeEvents, TestEvent}; use actix::Actor; - fn strips_ts(event: &EnclaveEvent) -> EnclaveEvent { - EnclaveEvent::new_stored_event(event.get_data().clone(), 0, event.get_seq()) - } #[actix::test] async fn it_adds_seqence_numbers_to_events() -> anyhow::Result<()> { @@ -65,7 +62,13 @@ mod tests { .collect::>(); let events = history.send(TakeEvents::new(3)).await?; - assert_eq!(events.iter().map(strips_ts).collect::>(), expected); + assert_eq!( + events + .iter() + .map(EnclaveEvent::strip_ts) + .collect::>(), + expected + ); Ok(()) } } diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index bde0c8fa65..769f661648 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -16,6 +16,7 @@ use e3_data::InMemStore; use e3_events::BusHandle; use e3_events::EnclaveEventData; use e3_events::GetEvents; +use e3_events::Sequenced; use e3_events::{ prelude::*, CiphernodeSelected, CiphertextOutputPublished, CommitteeFinalized, ConfigurationUpdated, E3Requested, E3id, EnclaveEvent, EventBus, EventBusConfig, @@ -453,7 +454,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { NetEventTranslator::setup(&bus, &cmd_tx, &event_rx, "my-topic"); // Capture messages from output on msgs vec - let msgs: Arc>> = Arc::new(Mutex::new(Vec::new())); + let msgs: Arc>> = Arc::new(Mutex::new(Vec::new())); let msgs_loop = msgs.clone(); @@ -467,8 +468,11 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { e3_net::events::NetCommand::GossipPublish { data, .. } => Some(data), _ => None, } { - msgs_loop.lock().await.push(msg.clone()); - event_tx.send(NetEvent::GossipData(msg))?; + if let GossipData::GossipBytes(ref data) = msg { + let data: EnclaveEvent = EnclaveEvent::from_bytes(data)?; + msgs_loop.lock().await.push(data.strip_ts()); + event_tx.send(NetEvent::GossipData(msg))?; + } } // if this manages to broadcast an event to the // event bus we will expect to see an extra event on @@ -506,12 +510,8 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { assert_eq!( *msgs.lock().await, vec![ - GossipData::GossipBytes( - EnclaveEvent::new_stored_event(evt_1.clone().into(), 0, 1).to_bytes()? - ), - GossipData::GossipBytes( - EnclaveEvent::new_stored_event(evt_2.clone().into(), 0, 2).to_bytes()? - ) + EnclaveEvent::new_stored_event(evt_1.clone().into(), 0, 1), + EnclaveEvent::new_stored_event(evt_2.clone().into(), 0, 2) ], // notice no local events "NetEventTranslator did not transmit correct events to the network" ); From 6a9c82221cdc32b7ec8dbb7820db6dbcd28ff287 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 04:31:06 +0000 Subject: [PATCH 23/27] allow error to return result --- crates/events/src/bus_handle.rs | 12 ++++++++---- crates/events/src/enclave_event/mod.rs | 11 +++++++---- crates/events/src/traits.rs | 8 ++++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index cb2d8db26c..44cacd6c9f 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use actix::{Actor, Addr, Recipient}; use anyhow::Result; use derivative::Derivative; +use tracing::error; use crate::{ hlc::Hlc, @@ -82,8 +83,10 @@ impl EventPublisher> for BusHandle { impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { - let evt = self.event_from_error(err_type, error); - self.producer.do_send(evt); + match self.event_from_error(err_type, error) { + Ok(evt) => self.producer.do_send(evt), + Err(e) => error!("{e}"), + } } } @@ -114,8 +117,9 @@ impl ErrorFactory> for BusHandle { &self, err_type: EType, error: impl Into, - ) -> EnclaveEvent { - EnclaveEvent::::from_error(err_type, error) + ) -> Result> { + let ts = self.hlc.tick()?; + EnclaveEvent::::from_error(err_type, error, ts.into()) } } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index b0d2b5de85..07db49ea51 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -235,16 +235,19 @@ impl ErrorEvent for EnclaveEvent { type ErrType = EType; type FromError = anyhow::Error; - fn from_error(err_type: Self::ErrType, msg: impl Into) -> Self { + fn from_error( + err_type: Self::ErrType, + msg: impl Into, + ts: u128, + ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); - let ts = 0; // XXX: fix this - EnclaveEvent { + Ok(EnclaveEvent { payload: payload.into(), id, seq: (), ts, - } + }) } } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 12386e85e5..8f2c5fed47 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -30,7 +30,11 @@ pub trait ErrorEvent: Event { type ErrType; type FromError; - fn from_error(err_type: Self::ErrType, error: impl Into) -> Self; + fn from_error( + err_type: Self::ErrType, + error: impl Into, + ts: u128, + ) -> Result; } /// An EventFactory creates events @@ -49,7 +53,7 @@ pub trait EventFactory { /// An ErrorFactory creates errors. pub trait ErrorFactory { /// Create an error event from the given error. - fn event_from_error(&self, err_type: E::ErrType, error: impl Into) -> E; + fn event_from_error(&self, err_type: E::ErrType, error: impl Into) -> Result; } /// An EventPublisher publishes events on it's internal EventBus From b1cd3414cbfbb3917fd4f91e123091543cf8e237 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 09:23:44 +0000 Subject: [PATCH 24/27] refactor gossipdaat conversion --- crates/events/src/enclave_event/mod.rs | 10 +++++++++ crates/net/src/events.rs | 29 ++++++++++++++++++++++++-- crates/net/src/net_event_translator.rs | 21 +++++++++---------- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 07db49ea51..0f7f2b07ad 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -178,12 +178,22 @@ where pub fn get_ts(&self) -> u128 { self.ts } + + pub fn split(self) -> (EnclaveEventData, u128) { + (self.payload, self.ts) + } } impl EnclaveEvent { pub fn get_seq(&self) -> u64 { self.seq } + + pub fn clone_unsequenced(&self) -> EnclaveEvent { + let ts = self.get_ts(); + let data = self.clone().into_data(); + EnclaveEvent::new_with_timestamp(data, ts) + } } impl EnclaveEvent { diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 010e7130be..89acfd079f 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -6,8 +6,11 @@ use crate::Cid; use actix::Message; -use anyhow::{Context, Result}; -use e3_events::{CorrelationId, DocumentMeta}; +use anyhow::{bail, Context, Result}; +use e3_events::{ + CorrelationId, DocumentMeta, EnclaveEvent, EventConstructorWithTimestamp, Sequenced, + Unsequenced, +}; use e3_utils::ArcBytes; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, @@ -39,6 +42,28 @@ impl GossipData { } } +impl TryFrom> for GossipData { + type Error = anyhow::Error; + fn try_from(value: EnclaveEvent) -> Result { + let bytes = value + .clone_unsequenced() // Note serializing UNSEQUENCED + .to_bytes() + .context("Could not convert event to bytes for serialization!")?; + Ok(GossipData::GossipBytes(bytes)) + } +} + +impl TryFrom for EnclaveEvent { + type Error = anyhow::Error; + fn try_from(value: GossipData) -> Result { + let GossipData::GossipBytes(bytes) = value else { + bail!("GossipData was not the GossipBytes variant"); + }; + + Ok(EnclaveEvent::from_bytes(&bytes)?) + } +} + /// NetInterface Commands are sent to the network peer over a mspc channel #[derive(Debug)] pub enum NetCommand { diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index e6797c892e..4752c360ca 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -16,9 +16,11 @@ use anyhow::{bail, Result}; use e3_crypto::Cipher; use e3_data::Repository; use e3_events::prelude::*; +use e3_events::trap; use e3_events::BusHandle; use e3_events::EnclaveEventData; use e3_events::Event; +use e3_events::Unsequenced; use e3_events::{CorrelationId, EnclaveEvent, EventId}; use libp2p::identity::ed25519; use std::collections::HashSet; @@ -47,7 +49,7 @@ impl Actor for NetEventTranslator { /// Libp2pEvent is used to send data to the NetInterface from the NetEventTranslator #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "anyhow::Result<()>")] -struct LibP2pEvent(pub Vec); +struct LibP2pEvent(pub GossipData); impl NetEventTranslator { /// Create a new NetEventTranslator actor @@ -78,8 +80,8 @@ impl NetEventTranslator { while let Ok(event) = rx.recv().await { match event { NetEvent::GossipData(data) => { - if let GossipData::GossipBytes(payload) = data { - addr.do_send(LibP2pEvent(payload)); + if let GossipData::GossipBytes(_) = data { + addr.do_send(LibP2pEvent(data)); } } _ => (), @@ -159,14 +161,11 @@ impl NetEventTranslator { impl Handler for NetEventTranslator { type Result = anyhow::Result<()>; fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { - let LibP2pEvent(bytes) = msg; - match EnclaveEvent::from_bytes(&bytes) { - Ok(event) => { - self.bus.naked_dispatch(event.clone()); // TODO: convert to receive - self.sent_events.insert(event.into()); - } - Err(err) => error!(error=?err, "Could not create EnclaveEvent from Libp2p Bytes!"), - } + let LibP2pEvent(data) = msg; + let event: EnclaveEvent = data.try_into()?; + self.sent_events.insert(event.get_id()); + let (data, ts) = event.split(); + self.bus.publish_from_remote(data, ts)?; Ok(()) } } From bb22801aaf367465193426d11c295b71aa621d14 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 09:43:59 +0000 Subject: [PATCH 25/27] refactor handler --- .../events/src/enclave_event/enclave_error.rs | 1 + crates/net/src/net_event_translator.rs | 44 ++++++++----------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index a69b9d5707..e748298cd5 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -33,6 +33,7 @@ pub enum EType { KeyGeneration, PublickeyAggregation, IO, + Net, PlaintextAggregation, Decryption, Sortition, diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 4752c360ca..8c8ba66ff2 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -18,6 +18,7 @@ use e3_data::Repository; use e3_events::prelude::*; use e3_events::trap; use e3_events::BusHandle; +use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::Event; use e3_events::Unsequenced; @@ -171,44 +172,37 @@ impl Handler for NetEventTranslator { } impl Handler for NetEventTranslator { - type Result = ResponseFuture<()>; + type Result = (); fn handle(&mut self, event: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - let sent_events = self.sent_events.clone(); - let tx = self.tx.clone(); - let evt = event.clone(); - let topic = self.topic.clone(); - Box::pin(async move { + trap(EType::Net, &self.bus.clone(), || { + let sent_events = self.sent_events.clone(); + let tx = self.tx.clone(); + let evt = event.clone(); + let topic = self.topic.clone(); let id: EventId = evt.clone().into(); // Ignore events that should be considered local if !Self::is_forwardable_event(&evt) { trace!(evt_id=%id,"Local events should not be rebroadcast so ignoring"); - return; + return Ok(()); } // if we have seen this event before dont rebroadcast if sent_events.contains(&id) { trace!(evt_id=%id,"Have seen event before not rebroadcasting!"); - return; + return Ok(()); } + warn!("GossipPublish event: {}", event.event_type()); - match evt.to_bytes() { - Ok(data) => { - if let Err(e) = tx - .send(NetCommand::GossipPublish { - topic, - data: GossipData::GossipBytes(data), - correlation_id: CorrelationId::new(), - }) - .await - { - error!(error=?e, "Error sending bytes to libp2p: {e}"); - }; - } - Err(error) => { - error!(error=?error, "Could not convert event to bytes for serialization!") - } - } + let data: GossipData = evt.try_into()?; + + tx.blocking_send(NetCommand::GossipPublish { + topic, + data, + correlation_id: CorrelationId::new(), + })?; + + Ok(()) }) } } From 179ef2d94912ddf182df4a86367c416183c833c4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 22:53:28 +0000 Subject: [PATCH 26/27] fix tests --- crates/net/src/events.rs | 35 ++++++++++++++++++++++++ crates/net/src/net_event_translator.rs | 2 +- crates/net/src/net_interface.rs | 4 +-- crates/tests/tests/integration_legacy.rs | 18 ++++++------ 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 89acfd079f..0218cae8c1 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -245,3 +245,38 @@ where .map_err(|_| anyhow::anyhow!(format!("Timed out waiting for response from {}", debug_cmd)))?; result } + +#[cfg(test)] +mod tests { + use e3_events::{ + EnclaveEvent, EventConstructorWithTimestamp, Sequenced, TestEvent, Unsequenced, + }; + + use super::GossipData; + + #[test] + fn test_enclave_event_gossip_lifecycle() -> anyhow::Result<()> { + // event is created locally + let event: EnclaveEvent = + EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), 31415); + + // event is sequenced after bus.publish() adds a sequence number + let event: EnclaveEvent = event.into_sequenced(90210); + + // event is broadcast + let gossip_data: GossipData = event.try_into()?; + + let GossipData::GossipBytes(_) = gossip_data else { + panic!("events must only be serialized to GossipBytes"); + }; + + // received gossip data from libp2p convert to unsequenced event + let event: EnclaveEvent = gossip_data.try_into()?; + let (data, ts) = event.split(); + + assert_eq!(data, TestEvent::new("fish", 42).into()); + assert_eq!(ts, 31415); + + Ok(()) + } +} diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 8c8ba66ff2..d12abd5793 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -196,7 +196,7 @@ impl Handler for NetEventTranslator { warn!("GossipPublish event: {}", event.event_type()); let data: GossipData = evt.try_into()?; - tx.blocking_send(NetCommand::GossipPublish { + tx.try_send(NetCommand::GossipPublish { topic, data, correlation_id: CorrelationId::new(), diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index c130cc25f9..3041c63878 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -74,8 +74,8 @@ impl NetInterface { udp_port: Option, topic: &str, ) -> Result { - let (event_tx, _) = broadcast::channel(100); // TODO : tune this param - let (cmd_tx, cmd_rx) = mpsc::channel(100); // TODO : tune this param + let (event_tx, _) = broadcast::channel(1000); // TODO : tune this param + let (cmd_tx, cmd_rx) = mpsc::channel(1000); // TODO : tune this param let swarm = libp2p::SwarmBuilder::with_existing_identity(id.clone()) .with_tokio() diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 769f661648..0d52323162 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -16,7 +16,7 @@ use e3_data::InMemStore; use e3_events::BusHandle; use e3_events::EnclaveEventData; use e3_events::GetEvents; -use e3_events::Sequenced; +use e3_events::Unsequenced; use e3_events::{ prelude::*, CiphernodeSelected, CiphertextOutputPublished, CommitteeFinalized, ConfigurationUpdated, E3Requested, E3id, EnclaveEvent, EventBus, EventBusConfig, @@ -454,7 +454,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { NetEventTranslator::setup(&bus, &cmd_tx, &event_rx, "my-topic"); // Capture messages from output on msgs vec - let msgs: Arc>> = Arc::new(Mutex::new(Vec::new())); + let msgs: Arc>> = Arc::new(Mutex::new(Vec::new())); let msgs_loop = msgs.clone(); @@ -468,10 +468,11 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { e3_net::events::NetCommand::GossipPublish { data, .. } => Some(data), _ => None, } { - if let GossipData::GossipBytes(ref data) = msg { - let data: EnclaveEvent = EnclaveEvent::from_bytes(data)?; - msgs_loop.lock().await.push(data.strip_ts()); - event_tx.send(NetEvent::GossipData(msg))?; + if let GossipData::GossipBytes(ref bytes) = msg { + let event: EnclaveEvent = msg.clone().try_into().unwrap(); + let (data, _) = event.split(); + msgs_loop.lock().await.push(data); + event_tx.send(NetEvent::GossipData(msg)).unwrap(); } } // if this manages to broadcast an event to the @@ -509,10 +510,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { assert_eq!( *msgs.lock().await, - vec![ - EnclaveEvent::new_stored_event(evt_1.clone().into(), 0, 1), - EnclaveEvent::new_stored_event(evt_2.clone().into(), 0, 2) - ], // notice no local events + vec![evt_1.clone().into(), evt_2.clone().into()], // notice no local events "NetEventTranslator did not transmit correct events to the network" ); From ffd12873bf4b52c175d492944ea15846fd6a3883 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 5 Dec 2025 23:01:14 +0000 Subject: [PATCH 27/27] remove ref --- crates/tests/tests/integration_legacy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 0d52323162..be35b47dbe 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -468,7 +468,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { e3_net::events::NetCommand::GossipPublish { data, .. } => Some(data), _ => None, } { - if let GossipData::GossipBytes(ref bytes) = msg { + if let GossipData::GossipBytes(_) = msg { let event: EnclaveEvent = msg.clone().try_into().unwrap(); let (data, _) = event.split(); msgs_loop.lock().await.push(data);