From c80bbec2e56b211dea36deedabe4f0da2208fc5e Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 7 Jan 2026 05:03:32 +0000 Subject: [PATCH 001/102] add event ctx --- crates/events/src/enclave_event/mod.rs | 9 +++ crates/events/src/event_ctx.rs | 95 ++++++++++++++++++++++++++ crates/events/src/event_id.rs | 2 +- crates/events/src/lib.rs | 1 + crates/events/src/traits.rs | 2 +- 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 crates/events/src/event_ctx.rs diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 7536e30c13..64fe40917d 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -163,6 +163,15 @@ impl SeqState for Sequenced { type Seq = u64; } +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct ContextEvent { + pub data: T, + pub correlation_id: EventId, + pub causation_id: Option, + pub ts: u128, +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct EnclaveEvent { diff --git a/crates/events/src/event_ctx.rs b/crates/events/src/event_ctx.rs new file mode 100644 index 0000000000..dc0a0a20a5 --- /dev/null +++ b/crates/events/src/event_ctx.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::{fmt, ops::Deref}; + +use crate::{E3id, EventId}; + +#[derive(Clone, Copy)] +pub struct AggregateId(usize); + +impl AggregateId { + pub fn new(value: usize) -> Self { + Self(value) + } +} + +impl From> for AggregateId { + fn from(value: Option) -> Self { + if let Some(e3_id) = value { + Self::new(e3_id.chain_id() as usize) + } else { + Self::new(0) + } + } +} + +impl Deref for AggregateId { + type Target = usize; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for AggregateId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", &self.0) + } +} + +pub struct EventCtx { + id: EventId, + causation_id: EventId, + origin_id: EventId, + aggregate_id: AggregateId, + seq: u64, + ts: u128, +} + +impl EventCtx { + pub fn new( + id: EventId, + causation_id: EventId, + origin_id: EventId, + aggregate_id: AggregateId, + seq: u64, + ts: u128, + ) -> Self { + Self { + id, + causation_id, + origin_id, + aggregate_id, + seq, + ts, + } + } + + pub fn id(&self) -> EventId { + self.id + } + + pub fn causation_id(&self) -> EventId { + self.causation_id + } + + pub fn origin_id(&self) -> EventId { + self.origin_id + } + + pub fn aggregate_id(&self) -> AggregateId { + self.aggregate_id + } + + pub fn seq(&self) -> u64 { + self.seq + } + + pub fn ts(&self) -> u128 { + self.ts + } +} diff --git a/crates/events/src/event_id.rs b/crates/events/src/event_id.rs index 60b2fccdcb..cec880d5fc 100644 --- a/crates/events/src/event_id.rs +++ b/crates/events/src/event_id.rs @@ -12,7 +12,7 @@ use std::{ hash::{DefaultHasher, Hash, Hasher}, }; -#[derive(Derivative, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Derivative, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[derivative(Debug)] pub struct EventId(#[derivative(Debug(format_with = "e3_utils::formatters::hexf"))] pub [u8; 32]); diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 863ed76b54..1c600191fe 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -8,6 +8,7 @@ mod bus_handle; mod correlation_id; mod e3id; mod enclave_event; +mod event_ctx; mod event_id; mod eventbus; mod events; diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index cdae6383a1..e27706b931 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -9,7 +9,7 @@ use anyhow::Result; use std::fmt::Display; use std::hash::Hash; -use crate::{EnclaveEvent, Unsequenced}; +use crate::{EnclaveEvent, EventId, Unsequenced}; /// Trait that must be implemented by events used with EventBus pub trait Event: From 585378f2fb98db8b104c6a367ac106784afdedbf Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 7 Jan 2026 06:35:08 +0000 Subject: [PATCH 002/102] setup typed event --- crates/events/src/enclave_event/mod.rs | 10 +-- .../events/src/enclave_event/typed_event.rs | 64 +++++++++++++++++++ .../src/{event_ctx.rs => event_context.rs} | 25 +++++--- crates/events/src/lib.rs | 2 +- crates/events/src/traits.rs | 18 +++++- 5 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 crates/events/src/enclave_event/typed_event.rs rename crates/events/src/{event_ctx.rs => event_context.rs} (73%) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 64fe40917d..004480bffc 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -34,6 +34,7 @@ mod threshold_share_created; mod ticket_balance_updated; mod ticket_generated; mod ticket_submitted; +mod typed_event; pub use ciphernode_added::*; pub use ciphernode_removed::*; @@ -163,15 +164,6 @@ impl SeqState for Sequenced { type Seq = u64; } -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[rtype(result = "()")] -pub struct ContextEvent { - pub data: T, - pub correlation_id: EventId, - pub causation_id: Option, - pub ts: u128, -} - #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct EnclaveEvent { diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs new file mode 100644 index 0000000000..043ddf3d63 --- /dev/null +++ b/crates/events/src/enclave_event/typed_event.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::ops::Deref; + +use actix::Message; +use serde::{Deserialize, Serialize}; + +use crate::{ + event_context::{AggregateId, ConcreteEventCtx}, + EventContext, EventId, +}; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct TypedEvent { + inner: T, + ctx: ConcreteEventCtx, +} + +impl Deref for TypedEvent { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl EventContext for TypedEvent { + fn id(&self) -> EventId { + self.ctx.id() + } + + fn ts(&self) -> u128 { + self.ctx.ts() + } + + fn seq(&self) -> u64 { + self.ctx.seq() + } + + fn origin_id(&self) -> EventId { + self.ctx.origin_id() + } + + fn causation_id(&self) -> EventId { + self.ctx.causation_id() + } + + fn aggregate_id(&self) -> AggregateId { + self.ctx.aggregate_id() + } +} + +impl From<(T, ConcreteEventCtx)> for TypedEvent { + fn from(value: (T, ConcreteEventCtx)) -> Self { + Self { + inner: value.0, + ctx: value.1, + } + } +} diff --git a/crates/events/src/event_ctx.rs b/crates/events/src/event_context.rs similarity index 73% rename from crates/events/src/event_ctx.rs rename to crates/events/src/event_context.rs index dc0a0a20a5..a4c7c9c132 100644 --- a/crates/events/src/event_ctx.rs +++ b/crates/events/src/event_context.rs @@ -6,9 +6,11 @@ use std::{fmt, ops::Deref}; -use crate::{E3id, EventId}; +use serde::{Deserialize, Serialize}; -#[derive(Clone, Copy)] +use crate::{E3id, EventContext, EventId}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct AggregateId(usize); impl AggregateId { @@ -41,7 +43,8 @@ impl fmt::Display for AggregateId { } } -pub struct EventCtx { +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ConcreteEventCtx { id: EventId, causation_id: EventId, origin_id: EventId, @@ -50,7 +53,7 @@ pub struct EventCtx { ts: u128, } -impl EventCtx { +impl ConcreteEventCtx { pub fn new( id: EventId, causation_id: EventId, @@ -68,28 +71,30 @@ impl EventCtx { ts, } } +} - pub fn id(&self) -> EventId { +impl EventContext for ConcreteEventCtx { + fn id(&self) -> EventId { self.id } - pub fn causation_id(&self) -> EventId { + fn causation_id(&self) -> EventId { self.causation_id } - pub fn origin_id(&self) -> EventId { + fn origin_id(&self) -> EventId { self.origin_id } - pub fn aggregate_id(&self) -> AggregateId { + fn aggregate_id(&self) -> AggregateId { self.aggregate_id } - pub fn seq(&self) -> u64 { + fn seq(&self) -> u64 { self.seq } - pub fn ts(&self) -> u128 { + fn ts(&self) -> u128 { self.ts } } diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 1c600191fe..34c4c54d18 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -8,7 +8,7 @@ mod bus_handle; mod correlation_id; mod e3id; mod enclave_event; -mod event_ctx; +mod event_context; mod event_id; mod eventbus; mod events; diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index e27706b931..43883a50a3 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -9,7 +9,7 @@ use anyhow::Result; use std::fmt::Display; use std::hash::Hash; -use crate::{EnclaveEvent, EventId, Unsequenced}; +use crate::{event_context::AggregateId, EnclaveEvent, EventId, Unsequenced}; /// Trait that must be implemented by events used with EventBus pub trait Event: @@ -115,3 +115,19 @@ pub trait EventLog: Unpin + 'static { /// Read all events starting from the given sequence number (inclusive) fn read_from(&self, from: u64) -> Box)>>; } + +/// EventContext allows consumers to extract infrastructure metadata from event objects +pub trait EventContext { + /// This event id + fn id(&self) -> EventId; + /// The id of the event that directly caused this + fn causation_id(&self) -> EventId; + /// The original event that started the causal chain + fn origin_id(&self) -> EventId; + /// The aggregate id associated with the event + fn aggregate_id(&self) -> AggregateId; + /// The sequence number of the event + fn seq(&self) -> u64; + /// The timestamp for the event + fn ts(&self) -> u128; +} From cfc05761364af64e2fd838d1320a990b267544b9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 07:13:43 +0000 Subject: [PATCH 003/102] update to include option ctx --- crates/data/src/commit_log_event_log.rs | 2 +- crates/data/src/in_mem_event_log.rs | 2 +- crates/events/src/bus_handle.rs | 22 +++- crates/events/src/enclave_event/mod.rs | 65 ++++++++++- .../events/src/enclave_event/typed_event.rs | 22 ++-- crates/events/src/event_context.rs | 105 ++++++++++++++++-- crates/events/src/traits.rs | 41 +++++-- crates/net/src/events.rs | 2 +- crates/tests/tests/integration_legacy.rs | 2 +- 9 files changed, 222 insertions(+), 41 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 5905f1b7dd..46dc04dea9 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -82,7 +82,7 @@ mod tests { use tempfile::tempdir; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), 123) + EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) } #[test] diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 9b95398919..a283ded98d 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -49,7 +49,7 @@ mod tests { use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), 123) + EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) } #[test] diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index ece22e36c1..a7d831b9db 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -12,6 +12,7 @@ use derivative::Derivative; use tracing::error; use crate::{ + event_context::ConcreteEventContext, hlc::Hlc, sequencer::Sequencer, traits::{ @@ -32,6 +33,8 @@ pub struct BusHandle { /// Hlc clock used to time all events created on this BusHandle #[derivative(Debug = "ignore")] hlc: Arc, + /// Temporary context for events the bus publishes + ctx: Option>, } impl BusHandle { @@ -45,6 +48,7 @@ impl BusHandle { consumer, producer, hlc: Arc::new(hlc), + ctx: None, } } @@ -81,13 +85,13 @@ impl BusHandle { impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) -> Result<()> { - let evt = self.event_from(data)?; + let evt = self.event_from(data, self.ctx.clone())?; self.producer.do_send(evt); Ok(()) } fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { - let evt = self.event_from_remote_source(data, ts)?; + let evt = self.event_from_remote_source(data, self.ctx.clone(), ts)?; self.producer.do_send(evt); Ok(()) } @@ -99,7 +103,7 @@ impl EventPublisher> for BusHandle { impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { - match self.event_from_error(err_type, error) { + match self.event_from_error(err_type, error, self.ctx.clone()) { Ok(evt) => self.producer.do_send(evt), Err(e) => error!("{e}"), } @@ -107,10 +111,15 @@ impl ErrorDispatcher> for BusHandle { } impl EventFactory> for BusHandle { - fn event_from(&self, data: impl Into) -> Result> { + fn event_from( + &self, + data: impl Into, + ctx: Option>, + ) -> Result> { let ts = self.hlc.tick()?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), + ctx, ts.into(), )) } @@ -118,11 +127,13 @@ impl EventFactory> for BusHandle { fn event_from_remote_source( &self, data: impl Into, + ctx: Option>, ts: u128, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), + ctx, ts.into(), )) } @@ -133,9 +144,10 @@ impl ErrorFactory> for BusHandle { &self, err_type: EType, error: impl Into, + ctx: Option>, ) -> Result> { let ts = self.hlc.tick()?; - EnclaveEvent::::from_error(err_type, error, ts.into()) + EnclaveEvent::::from_error(err_type, error, ts.into(), ctx) } } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 004480bffc..8c9a0e0f91 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -70,8 +70,9 @@ pub use ticket_generated::*; pub use ticket_submitted::*; use crate::{ + event_context::{AggregateId, ConcreteEventContext}, traits::{ErrorEvent, Event, EventConstructorWithTimestamp}, - E3id, EventId, + E3id, EventContext, EventId, }; use actix::Message; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -166,11 +167,16 @@ impl SeqState for Sequenced { #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] +#[serde(bound( + serialize = "S: SeqState, S::Seq: Serialize", + deserialize = "S: SeqState, S::Seq: DeserializeOwned" +))] pub struct EnclaveEvent { id: EventId, payload: EnclaveEventData, seq: S::Seq, ts: u128, + ctx: ConcreteEventContext, } impl EnclaveEvent @@ -206,7 +212,7 @@ impl EnclaveEvent { pub fn clone_unsequenced(&self) -> EnclaveEvent { let ts = self.get_ts(); let data = self.clone().into_data(); - EnclaveEvent::new_with_timestamp(data, ts) + EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts) } } @@ -217,6 +223,7 @@ impl EnclaveEvent { payload: self.payload, ts: self.ts, seq, + ctx: self.ctx.sequence(seq), } } } @@ -225,7 +232,7 @@ impl EnclaveEvent { impl EnclaveEvent { /// test-helpers only utility function to create a new unsequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, time).into_sequenced(seq) + EnclaveEvent::::new_with_timestamp(data, None, time).into_sequenced(seq) } /// test-helpers only utility function to remove time information from an event @@ -263,14 +270,23 @@ impl ErrorEvent for EnclaveEvent { err_type: Self::ErrType, msg: impl Into, ts: u128, + mut ctx: Option>, ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); + let ctx = if let Some(ctx) = ctx.take() { + let aggregate_id = ctx.aggregate_id(); + ctx.causes(id, aggregate_id, ts) + } else { + ConcreteEventContext::::new(id, id, id, AggregateId::new(0), ts) + }; + Ok(EnclaveEvent { payload: payload.into(), id, seq: (), ts, + ctx, }) } } @@ -287,9 +303,9 @@ impl From<&EnclaveEvent> for EventId { } } -impl EnclaveEvent { +impl EnclaveEventData { pub fn get_e3_id(&self) -> Option { - match self.payload { + match self { EnclaveEventData::KeyshareCreated(ref data) => Some(data.e3_id.clone()), EnclaveEventData::E3Requested(ref data) => Some(data.e3_id.clone()), EnclaveEventData::PublicKeyAggregated(ref data) => Some(data.e3_id.clone()), @@ -312,6 +328,27 @@ impl EnclaveEvent { } } +impl EnclaveEvent { + pub fn get_e3_id(&self) -> Option { + self.payload.get_e3_id() + } +} + +pub trait WithAggregateId { + fn get_aggregate_id(&self) -> AggregateId; +} + +impl WithAggregateId for EnclaveEventData { + fn get_aggregate_id(&self) -> AggregateId { + let maybe_e3_id = self.get_e3_id(); + if let Some(e3_id) = maybe_e3_id { + AggregateId::new(e3_id.chain_id() as usize) + } else { + AggregateId::new(0) + } + } +} + impl_into_event_data!( KeyshareCreated, E3Requested, @@ -373,7 +410,12 @@ impl fmt::Display for EnclaveEvent { } impl EventConstructorWithTimestamp for EnclaveEvent { - fn new_with_timestamp(data: Self::Data, ts: u128) -> Self { + fn new_with_timestamp( + data: Self::Data, + ctx: Option>, + ts: u128, + ) -> Self { + let aggregate_id = data.get_aggregate_id(); let payload = data.into(); let id = EventId::hash(&payload); EnclaveEvent { @@ -381,6 +423,17 @@ impl EventConstructorWithTimestamp for EnclaveEvent { payload, seq: (), ts, + ctx: if let Some(ctx) = ctx { + ConcreteEventContext::new( + ctx.id(), + ctx.causation_id(), + ctx.origin_id(), + ctx.aggregate_id(), + ts, + ) + } else { + ConcreteEventContext::new(id, id, id, aggregate_id, ts) + }, } } } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 043ddf3d63..0df4c69e5c 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -10,15 +10,17 @@ use actix::Message; use serde::{Deserialize, Serialize}; use crate::{ - event_context::{AggregateId, ConcreteEventCtx}, - EventContext, EventId, + event_context::{AggregateId, ConcreteEventContext}, + EventContext, EventContextSeq, EventId, }; +use super::Sequenced; + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct TypedEvent { inner: T, - ctx: ConcreteEventCtx, + ctx: ConcreteEventContext, } impl Deref for TypedEvent { @@ -37,10 +39,6 @@ impl EventContext for TypedEvent { self.ctx.ts() } - fn seq(&self) -> u64 { - self.ctx.seq() - } - fn origin_id(&self) -> EventId { self.ctx.origin_id() } @@ -54,8 +52,14 @@ impl EventContext for TypedEvent { } } -impl From<(T, ConcreteEventCtx)> for TypedEvent { - fn from(value: (T, ConcreteEventCtx)) -> Self { +impl EventContextSeq for TypedEvent { + fn seq(&self) -> u64 { + self.ctx.seq() + } +} + +impl From<(T, ConcreteEventContext)> for TypedEvent { + fn from(value: (T, ConcreteEventContext)) -> Self { Self { inner: value.0, ctx: value.1, diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index a4c7c9c132..8d02076b9d 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -8,7 +8,7 @@ use std::{fmt, ops::Deref}; use serde::{Deserialize, Serialize}; -use crate::{E3id, EventContext, EventId}; +use crate::{E3id, EventContext, EventContextSeq, EventId, SeqState, Sequenced, Unsequenced}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct AggregateId(usize); @@ -44,22 +44,33 @@ impl fmt::Display for AggregateId { } #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct ConcreteEventCtx { +pub struct ConcreteEventContext { id: EventId, causation_id: EventId, origin_id: EventId, aggregate_id: AggregateId, - seq: u64, + seq: S::Seq, ts: u128, } -impl ConcreteEventCtx { +impl ConcreteEventContext { + /// Tracks events as they create other events + pub fn causes( + self, + id: EventId, + aggregate_id: AggregateId, + ts: u128, + ) -> ConcreteEventContext { + ConcreteEventContext::::new(id, self.id, self.origin_id, aggregate_id, ts) + } +} + +impl ConcreteEventContext { pub fn new( id: EventId, causation_id: EventId, origin_id: EventId, aggregate_id: AggregateId, - seq: u64, ts: u128, ) -> Self { Self { @@ -67,13 +78,24 @@ impl ConcreteEventCtx { causation_id, origin_id, aggregate_id, - seq, + seq: (), ts, } } + + pub fn sequence(self, value: u64) -> ConcreteEventContext { + ConcreteEventContext:: { + seq: value, + id: self.id, + causation_id: self.causation_id, + origin_id: self.origin_id, + aggregate_id: self.aggregate_id, + ts: self.ts, + } + } } -impl EventContext for ConcreteEventCtx { +impl EventContext for ConcreteEventContext { fn id(&self) -> EventId { self.id } @@ -90,11 +112,76 @@ impl EventContext for ConcreteEventCtx { self.aggregate_id } + fn ts(&self) -> u128 { + self.ts + } +} + +impl EventContextSeq for ConcreteEventContext { fn seq(&self) -> u64 { self.seq } +} - fn ts(&self) -> u128 { - self.ts +#[cfg(test)] +mod tests { + use crate::{ + event_context::{AggregateId, ConcreteEventContext}, + EventId, + }; + + #[test] + fn test_event_context_cycle() { + let mut events = vec![]; + + let one = ConcreteEventContext::new( + EventId::hash(1), + EventId::hash(1), + EventId::hash(1), + AggregateId::new(1), + 1, + ) + .sequence(1); + events.push(one.clone()); + + let two = one + .causes(EventId::hash(2), AggregateId::new(1), 2) + .sequence(2); + events.push(two.clone()); + + let three = two + .causes(EventId::hash(3), AggregateId::new(2), 3) + .sequence(3); + events.push(three.clone()); + + assert_eq!( + events, + vec![ + ConcreteEventContext { + aggregate_id: AggregateId::new(1), + seq: 1, + id: EventId::hash(1), + origin_id: EventId::hash(1), + causation_id: EventId::hash(1), + ts: 1, + }, + ConcreteEventContext { + aggregate_id: AggregateId::new(1), + seq: 2, + id: EventId::hash(2), + origin_id: EventId::hash(1), + causation_id: EventId::hash(1), + ts: 2, + }, + ConcreteEventContext { + aggregate_id: AggregateId::new(2), + seq: 3, + id: EventId::hash(3), + origin_id: EventId::hash(1), + causation_id: EventId::hash(2), + ts: 3, + }, + ] + ) } } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 43883a50a3..a412677aa4 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -9,7 +9,10 @@ use anyhow::Result; use std::fmt::Display; use std::hash::Hash; -use crate::{event_context::AggregateId, EnclaveEvent, EventId, Unsequenced}; +use crate::{ + event_context::{AggregateId, ConcreteEventContext}, + EnclaveEvent, EventId, Sequenced, Unsequenced, WithAggregateId, +}; /// Trait that must be implemented by events used with EventBus pub trait Event: @@ -18,7 +21,7 @@ pub trait Event: type Id: Hash + Eq + Clone + Unpin + Send + Sync + Display; /// Payload for the Event - type Data; + type Data: WithAggregateId; fn event_type(&self) -> String; fn event_id(&self) -> Self::Id; @@ -36,6 +39,7 @@ pub trait ErrorEvent: Event { err_type: Self::ErrType, error: impl Into, ts: u128, + ctx: Option>, ) -> Result; } @@ -44,18 +48,32 @@ pub trait EventFactory { /// Create a new event from the given event data, apply a local HLC timestamp. /// /// This method should be used for events that have originated locally. - fn event_from(&self, data: impl Into) -> Result; + fn event_from( + &self, + data: impl Into, + ctx: Option>, + ) -> Result; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering. /// /// This method should be used for events that originated from remote sources. - fn event_from_remote_source(&self, data: impl Into, ts: u128) -> Result; + fn event_from_remote_source( + &self, + data: impl Into, + ctx: Option>, + ts: u128, + ) -> Result; } /// An ErrorFactory creates errors. pub trait ErrorFactory { /// Create an error event from the given error. - fn event_from_error(&self, err_type: E::ErrType, error: impl Into) -> Result; + fn event_from_error( + &self, + err_type: E::ErrType, + error: impl Into, + ctx: Option>, + ) -> Result; } /// An EventPublisher publishes events on it's internal EventBus @@ -91,7 +109,11 @@ pub trait EventSubscriber { /// Trait to create an event with a timestamp from its associated type data pub trait EventConstructorWithTimestamp: Event + Sized { /// Create an event passing attaching a specific timestamp. - fn new_with_timestamp(data: Self::Data, ts: u128) -> Self; + fn new_with_timestamp( + data: Self::Data, + ctx: Option>, + ts: u128, + ) -> Self; } pub trait CompositeEvent: EventConstructorWithTimestamp {} @@ -126,8 +148,11 @@ pub trait EventContext { fn origin_id(&self) -> EventId; /// The aggregate id associated with the event fn aggregate_id(&self) -> AggregateId; - /// The sequence number of the event - fn seq(&self) -> u64; /// The timestamp for the event fn ts(&self) -> u128; } + +pub trait EventContextSeq { + /// The sequence number of the event + fn seq(&self) -> u64; +} diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index e042eedd1d..f49591254d 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -255,7 +255,7 @@ mod tests { fn test_enclave_event_gossip_lifecycle() -> anyhow::Result<()> { // event is created locally let event: EnclaveEvent = - EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), 31415); + EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), None, 31415); // event is sequenced after bus.publish() adds a sequence number let event: EnclaveEvent = event.into_sequenced(90210); diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index ebdcb67727..1b759142dd 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -662,7 +662,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // lets send an event from the network let _ = event_tx.send(NetEvent::GossipData(GossipData::GossipBytes( - bus.event_from(event.clone())?.to_bytes()?, + bus.event_from(event.clone(), None)?.to_bytes()?, ))); // check the history of the event bus From 38418f71e0651a8ccda80bad93a6a30ce27ef531 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 07:51:39 +0000 Subject: [PATCH 004/102] remove aggregate_id in favour of data.get_aggregate_id --- crates/events/src/bus_handle.rs | 10 +-- crates/events/src/enclave_event/mod.rs | 38 ++++------ .../events/src/enclave_event/typed_event.rs | 16 ++--- crates/events/src/event_context.rs | 71 ++++++------------- crates/events/src/traits.rs | 28 ++++---- 5 files changed, 59 insertions(+), 104 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index a7d831b9db..ab8a885758 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -12,7 +12,7 @@ use derivative::Derivative; use tracing::error; use crate::{ - event_context::ConcreteEventContext, + event_context::EventContext, hlc::Hlc, sequencer::Sequencer, traits::{ @@ -34,7 +34,7 @@ pub struct BusHandle { #[derivative(Debug = "ignore")] hlc: Arc, /// Temporary context for events the bus publishes - ctx: Option>, + ctx: Option>, } impl BusHandle { @@ -114,7 +114,7 @@ impl EventFactory> for BusHandle { fn event_from( &self, data: impl Into, - ctx: Option>, + ctx: Option>, ) -> Result> { let ts = self.hlc.tick()?; Ok(EnclaveEvent::::new_with_timestamp( @@ -127,7 +127,7 @@ impl EventFactory> for BusHandle { fn event_from_remote_source( &self, data: impl Into, - ctx: Option>, + ctx: Option>, ts: u128, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; @@ -144,7 +144,7 @@ impl ErrorFactory> for BusHandle { &self, err_type: EType, error: impl Into, - ctx: Option>, + ctx: Option>, ) -> Result> { let ts = self.hlc.tick()?; EnclaveEvent::::from_error(err_type, error, ts.into(), ctx) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 8c9a0e0f91..457c5f9f18 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -70,9 +70,9 @@ pub use ticket_generated::*; pub use ticket_submitted::*; use crate::{ - event_context::{AggregateId, ConcreteEventContext}, - traits::{ErrorEvent, Event, EventConstructorWithTimestamp}, - E3id, EventContext, EventId, + event_context::{AggregateId, EventContext}, + traits::{ErrorEvent, Event, EventConstructorWithTimestamp, EventContextAccessors}, + E3id, EventId, WithAggregateId, }; use actix::Message; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -172,11 +172,11 @@ impl SeqState for Sequenced { deserialize = "S: SeqState, S::Seq: DeserializeOwned" ))] pub struct EnclaveEvent { - id: EventId, + id: EventId, // XXX: move to context payload: EnclaveEventData, - seq: S::Seq, - ts: u128, - ctx: ConcreteEventContext, + seq: S::Seq, // XXX: move to context + ts: u128, // XXX: move to context + ctx: EventContext, } impl EnclaveEvent @@ -270,15 +270,14 @@ impl ErrorEvent for EnclaveEvent { err_type: Self::ErrType, msg: impl Into, ts: u128, - mut ctx: Option>, + mut ctx: Option>, ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); let ctx = if let Some(ctx) = ctx.take() { - let aggregate_id = ctx.aggregate_id(); - ctx.causes(id, aggregate_id, ts) + ctx.causes(id, ts) } else { - ConcreteEventContext::::new(id, id, id, AggregateId::new(0), ts) + EventContext::::new(id, id, id, ts) }; Ok(EnclaveEvent { @@ -334,10 +333,6 @@ impl EnclaveEvent { } } -pub trait WithAggregateId { - fn get_aggregate_id(&self) -> AggregateId; -} - impl WithAggregateId for EnclaveEventData { fn get_aggregate_id(&self) -> AggregateId { let maybe_e3_id = self.get_e3_id(); @@ -412,10 +407,9 @@ impl fmt::Display for EnclaveEvent { impl EventConstructorWithTimestamp for EnclaveEvent { fn new_with_timestamp( data: Self::Data, - ctx: Option>, + ctx: Option>, ts: u128, ) -> Self { - let aggregate_id = data.get_aggregate_id(); let payload = data.into(); let id = EventId::hash(&payload); EnclaveEvent { @@ -424,15 +418,9 @@ impl EventConstructorWithTimestamp for EnclaveEvent { seq: (), ts, ctx: if let Some(ctx) = ctx { - ConcreteEventContext::new( - ctx.id(), - ctx.causation_id(), - ctx.origin_id(), - ctx.aggregate_id(), - ts, - ) + EventContext::new(ctx.id(), ctx.causation_id(), ctx.origin_id(), ts) } else { - ConcreteEventContext::new(id, id, id, aggregate_id, ts) + EventContext::new(id, id, id, ts) }, } } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 0df4c69e5c..3928de798a 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -10,8 +10,8 @@ use actix::Message; use serde::{Deserialize, Serialize}; use crate::{ - event_context::{AggregateId, ConcreteEventContext}, - EventContext, EventContextSeq, EventId, + event_context::{AggregateId, EventContext}, + EventContextAccessors, EventContextSeq, EventId, }; use super::Sequenced; @@ -20,7 +20,7 @@ use super::Sequenced; #[rtype(result = "()")] pub struct TypedEvent { inner: T, - ctx: ConcreteEventContext, + ctx: EventContext, } impl Deref for TypedEvent { @@ -30,7 +30,7 @@ impl Deref for TypedEvent { } } -impl EventContext for TypedEvent { +impl EventContextAccessors for TypedEvent { fn id(&self) -> EventId { self.ctx.id() } @@ -46,10 +46,6 @@ impl EventContext for TypedEvent { fn causation_id(&self) -> EventId { self.ctx.causation_id() } - - fn aggregate_id(&self) -> AggregateId { - self.ctx.aggregate_id() - } } impl EventContextSeq for TypedEvent { @@ -58,8 +54,8 @@ impl EventContextSeq for TypedEvent { } } -impl From<(T, ConcreteEventContext)> for TypedEvent { - fn from(value: (T, ConcreteEventContext)) -> Self { +impl From<(T, EventContext)> for TypedEvent { + fn from(value: (T, EventContext)) -> Self { Self { inner: value.0, ctx: value.1, diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 8d02076b9d..09dbe26bfe 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -8,7 +8,9 @@ use std::{fmt, ops::Deref}; use serde::{Deserialize, Serialize}; -use crate::{E3id, EventContext, EventContextSeq, EventId, SeqState, Sequenced, Unsequenced}; +use crate::{ + E3id, EventContextAccessors, EventContextSeq, EventId, SeqState, Sequenced, Unsequenced, +}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct AggregateId(usize); @@ -44,58 +46,44 @@ impl fmt::Display for AggregateId { } #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct ConcreteEventContext { +pub struct EventContext { id: EventId, causation_id: EventId, origin_id: EventId, - aggregate_id: AggregateId, seq: S::Seq, ts: u128, } -impl ConcreteEventContext { +impl EventContext { /// Tracks events as they create other events - pub fn causes( - self, - id: EventId, - aggregate_id: AggregateId, - ts: u128, - ) -> ConcreteEventContext { - ConcreteEventContext::::new(id, self.id, self.origin_id, aggregate_id, ts) + pub fn causes(self, id: EventId, ts: u128) -> EventContext { + EventContext::::new(id, self.id, self.origin_id, ts) } } -impl ConcreteEventContext { - pub fn new( - id: EventId, - causation_id: EventId, - origin_id: EventId, - aggregate_id: AggregateId, - ts: u128, - ) -> Self { +impl EventContext { + pub fn new(id: EventId, causation_id: EventId, origin_id: EventId, ts: u128) -> Self { Self { id, causation_id, origin_id, - aggregate_id, seq: (), ts, } } - pub fn sequence(self, value: u64) -> ConcreteEventContext { - ConcreteEventContext:: { + pub fn sequence(self, value: u64) -> EventContext { + EventContext:: { seq: value, id: self.id, causation_id: self.causation_id, origin_id: self.origin_id, - aggregate_id: self.aggregate_id, ts: self.ts, } } } -impl EventContext for ConcreteEventContext { +impl EventContextAccessors for EventContext { fn id(&self) -> EventId { self.id } @@ -108,16 +96,12 @@ impl EventContext for ConcreteEventContext { self.origin_id } - fn aggregate_id(&self) -> AggregateId { - self.aggregate_id - } - fn ts(&self) -> u128 { self.ts } } -impl EventContextSeq for ConcreteEventContext { +impl EventContextSeq for EventContext { fn seq(&self) -> u64 { self.seq } @@ -126,7 +110,7 @@ impl EventContextSeq for ConcreteEventContext { #[cfg(test)] mod tests { use crate::{ - event_context::{AggregateId, ConcreteEventContext}, + event_context::{AggregateId, EventContext}, EventId, }; @@ -134,47 +118,34 @@ mod tests { fn test_event_context_cycle() { let mut events = vec![]; - let one = ConcreteEventContext::new( - EventId::hash(1), - EventId::hash(1), - EventId::hash(1), - AggregateId::new(1), - 1, - ) - .sequence(1); + let one = + EventContext::new(EventId::hash(1), EventId::hash(1), EventId::hash(1), 1).sequence(1); events.push(one.clone()); - let two = one - .causes(EventId::hash(2), AggregateId::new(1), 2) - .sequence(2); + let two = one.causes(EventId::hash(2), 2).sequence(2); events.push(two.clone()); - let three = two - .causes(EventId::hash(3), AggregateId::new(2), 3) - .sequence(3); + let three = two.causes(EventId::hash(3), 3).sequence(3); events.push(three.clone()); assert_eq!( events, vec![ - ConcreteEventContext { - aggregate_id: AggregateId::new(1), + EventContext { seq: 1, id: EventId::hash(1), origin_id: EventId::hash(1), causation_id: EventId::hash(1), ts: 1, }, - ConcreteEventContext { - aggregate_id: AggregateId::new(1), + EventContext { seq: 2, id: EventId::hash(2), origin_id: EventId::hash(1), causation_id: EventId::hash(1), ts: 2, }, - ConcreteEventContext { - aggregate_id: AggregateId::new(2), + EventContext { seq: 3, id: EventId::hash(3), origin_id: EventId::hash(1), diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index a412677aa4..46972e2527 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -10,8 +10,8 @@ use std::fmt::Display; use std::hash::Hash; use crate::{ - event_context::{AggregateId, ConcreteEventContext}, - EnclaveEvent, EventId, Sequenced, Unsequenced, WithAggregateId, + event_context::{AggregateId, EventContext}, + EnclaveEvent, EventId, Sequenced, Unsequenced, }; /// Trait that must be implemented by events used with EventBus @@ -39,7 +39,7 @@ pub trait ErrorEvent: Event { err_type: Self::ErrType, error: impl Into, ts: u128, - ctx: Option>, + ctx: Option>, ) -> Result; } @@ -51,7 +51,7 @@ pub trait EventFactory { fn event_from( &self, data: impl Into, - ctx: Option>, + ctx: Option>, ) -> Result; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering. @@ -60,7 +60,7 @@ pub trait EventFactory { fn event_from_remote_source( &self, data: impl Into, - ctx: Option>, + ctx: Option>, ts: u128, ) -> Result; } @@ -72,7 +72,7 @@ pub trait ErrorFactory { &self, err_type: E::ErrType, error: impl Into, - ctx: Option>, + ctx: Option>, ) -> Result; } @@ -109,11 +109,8 @@ pub trait EventSubscriber { /// Trait to create an event with a timestamp from its associated type data pub trait EventConstructorWithTimestamp: Event + Sized { /// Create an event passing attaching a specific timestamp. - fn new_with_timestamp( - data: Self::Data, - ctx: Option>, - ts: u128, - ) -> Self; + fn new_with_timestamp(data: Self::Data, ctx: Option>, ts: u128) + -> Self; } pub trait CompositeEvent: EventConstructorWithTimestamp {} @@ -139,15 +136,13 @@ pub trait EventLog: Unpin + 'static { } /// EventContext allows consumers to extract infrastructure metadata from event objects -pub trait EventContext { +pub trait EventContextAccessors { /// This event id fn id(&self) -> EventId; /// The id of the event that directly caused this fn causation_id(&self) -> EventId; /// The original event that started the causal chain fn origin_id(&self) -> EventId; - /// The aggregate id associated with the event - fn aggregate_id(&self) -> AggregateId; /// The timestamp for the event fn ts(&self) -> u128; } @@ -156,3 +151,8 @@ pub trait EventContextSeq { /// The sequence number of the event fn seq(&self) -> u64; } + +pub trait WithAggregateId { + /// Extract the aggregate id from the object + fn get_aggregate_id(&self) -> AggregateId; +} From d3a3f78bd37f2ce6a32083c498ed67b297c95d06 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 08:06:59 +0000 Subject: [PATCH 005/102] tidy up data model --- crates/events/src/bus_handle.rs | 6 +-- crates/events/src/enclave_event/mod.rs | 68 ++++++++++++++------------ crates/events/src/eventstore.rs | 4 +- crates/net/src/net_event_translator.rs | 3 +- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index ab8a885758..b7775be508 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -199,7 +199,7 @@ mod tests { impl Handler for Forwarder { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - let ts = msg.get_ts(); + let ts = msg.ts(); self.dest.publish_from_remote(msg.into_data(), ts).unwrap() } } @@ -270,7 +270,7 @@ mod tests { // Sort by HLC timestamp let mut sorted_events = events.clone(); - sorted_events.sort_by_key(|e| e.get_ts()); + sorted_events.sort_by_key(|e| e.ts()); // Extract the payloads/names in HLC-sorted order let ordered_names: Vec<_> = sorted_events @@ -289,7 +289,7 @@ mod tests { ); // ASSERTION 2: All timestamps are unique (HLC guarantee) - let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); + let timestamps: Vec<_> = sorted_events.iter().map(|e| e.ts()).collect(); let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); assert_eq!( timestamps.len(), diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 457c5f9f18..d8946590b5 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -72,7 +72,7 @@ pub use ticket_submitted::*; use crate::{ event_context::{AggregateId, EventContext}, traits::{ErrorEvent, Event, EventConstructorWithTimestamp, EventContextAccessors}, - E3id, EventId, WithAggregateId, + E3id, EventContextSeq, EventId, WithAggregateId, }; use actix::Message; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -172,10 +172,7 @@ impl SeqState for Sequenced { deserialize = "S: SeqState, S::Seq: DeserializeOwned" ))] pub struct EnclaveEvent { - id: EventId, // XXX: move to context payload: EnclaveEventData, - seq: S::Seq, // XXX: move to context - ts: u128, // XXX: move to context ctx: EventContext, } @@ -191,26 +188,40 @@ where bincode::deserialize(bytes) } - pub fn get_id(&self) -> EventId { - self.into() + pub fn split(self) -> (EnclaveEventData, u128) { + (self.payload, self.ctx.ts()) } +} - pub fn get_ts(&self) -> u128 { - self.ts +impl EventContextAccessors for EnclaveEvent { + fn causation_id(&self) -> EventId { + self.ctx.causation_id() + } + fn origin_id(&self) -> EventId { + self.ctx.origin_id() } + fn ts(&self) -> u128 { + self.ctx.ts() + } + fn id(&self) -> EventId { + self.ctx.id() + } +} - pub fn split(self) -> (EnclaveEventData, u128) { - (self.payload, self.ts) +impl EventContextSeq for EnclaveEvent { + fn seq(&self) -> u64 { + self.ctx.seq() } } impl EnclaveEvent { + // TOOD: remove pub fn get_seq(&self) -> u64 { - self.seq + self.seq() } pub fn clone_unsequenced(&self) -> EnclaveEvent { - let ts = self.get_ts(); + let ts = self.ts(); let data = self.clone().into_data(); EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts) } @@ -219,10 +230,7 @@ impl EnclaveEvent { impl EnclaveEvent { pub fn into_sequenced(self, seq: u64) -> EnclaveEvent { EnclaveEvent:: { - id: self.id, payload: self.payload, - ts: self.ts, - seq, ctx: self.ctx.sequence(seq), } } @@ -250,7 +258,7 @@ impl Event for EnclaveEvent { } fn event_id(&self) -> Self::Id { - self.get_id() + self.id() } fn get_data(&self) -> &EnclaveEventData { @@ -282,9 +290,6 @@ impl ErrorEvent for EnclaveEvent { Ok(EnclaveEvent { payload: payload.into(), - id, - seq: (), - ts, ctx, }) } @@ -292,13 +297,13 @@ impl ErrorEvent for EnclaveEvent { impl From> for EventId { fn from(value: EnclaveEvent) -> Self { - value.id + value.id() } } impl From<&EnclaveEvent> for EventId { fn from(value: &EnclaveEvent) -> Self { - value.id.clone() + value.id() } } @@ -327,12 +332,6 @@ impl EnclaveEventData { } } -impl EnclaveEvent { - pub fn get_e3_id(&self) -> Option { - self.payload.get_e3_id() - } -} - impl WithAggregateId for EnclaveEventData { fn get_aggregate_id(&self) -> AggregateId { let maybe_e3_id = self.get_e3_id(); @@ -344,6 +343,18 @@ impl WithAggregateId for EnclaveEventData { } } +impl EnclaveEvent { + pub fn get_e3_id(&self) -> Option { + self.payload.get_e3_id() + } +} + +impl WithAggregateId for EnclaveEvent { + fn get_aggregate_id(&self) -> AggregateId { + self.payload.get_aggregate_id() + } +} + impl_into_event_data!( KeyshareCreated, E3Requested, @@ -413,10 +424,7 @@ impl EventConstructorWithTimestamp for EnclaveEvent { let payload = data.into(); let id = EventId::hash(&payload); EnclaveEvent { - id, payload, - seq: (), - ts, ctx: if let Some(ctx) = ctx { EventContext::new(ctx.id(), ctx.causation_id(), ctx.origin_id(), ts) } else { diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 5218459196..d3e6e57371 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -6,7 +6,7 @@ use crate::{ events::{EventStored, StoreEventRequested}, - EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, + EventContextAccessors, EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; @@ -21,7 +21,7 @@ impl EventStore { pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { let event = msg.event; let sender = msg.sender; - let ts = event.get_ts(); + let ts = event.ts(); if let Some(_) = self.index.get(ts)? { bail!("Event already stored at timestamp {ts}!"); } diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 54126d7e13..a454144aea 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -21,6 +21,7 @@ use e3_events::BusHandle; use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::Event; +use e3_events::EventContextAccessors; use e3_events::Unsequenced; use e3_events::{CorrelationId, EnclaveEvent, EventId}; use libp2p::identity::ed25519; @@ -164,7 +165,7 @@ impl Handler for NetEventTranslator { fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { let LibP2pEvent(data) = msg; let event: EnclaveEvent = data.try_into()?; - self.sent_events.insert(event.get_id()); + self.sent_events.insert(event.id()); let (data, ts) = event.split(); self.bus.publish_from_remote(data, ts)?; Ok(()) From 3f186fd928fcefab63717381219dfc33c31fbcdb Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 08:09:27 +0000 Subject: [PATCH 006/102] add comment --- crates/events/src/enclave_event/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index d8946590b5..4997231429 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -257,6 +257,7 @@ impl Event for EnclaveEvent { self.payload.event_type() } + // TODO: remove? fn event_id(&self) -> Self::Id { self.id() } From de4dc0bd4cae6399afff2ac86013fd0bcdf7f3d7 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 21:19:05 +0000 Subject: [PATCH 007/102] refactor --- crates/events/src/enclave_event/mod.rs | 20 ++++++++------------ crates/events/src/event_context.rs | 24 +++++++++++------------- crates/events/src/traits.rs | 7 +++++-- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 4997231429..cf06d521ec 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -279,15 +279,13 @@ impl ErrorEvent for EnclaveEvent { err_type: Self::ErrType, msg: impl Into, ts: u128, - mut ctx: Option>, + ctx: Option>, ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); - let ctx = if let Some(ctx) = ctx.take() { - ctx.causes(id, ts) - } else { - EventContext::::new(id, id, id, ts) - }; + let ctx = ctx + .map(|cause| EventContext::from_cause(id, cause, ts)) + .unwrap_or_else(|| EventContext::new_origin(id, ts)); Ok(EnclaveEvent { payload: payload.into(), @@ -419,18 +417,16 @@ impl fmt::Display for EnclaveEvent { impl EventConstructorWithTimestamp for EnclaveEvent { fn new_with_timestamp( data: Self::Data, - ctx: Option>, + caused_by: Option>, ts: u128, ) -> Self { let payload = data.into(); let id = EventId::hash(&payload); EnclaveEvent { payload, - ctx: if let Some(ctx) = ctx { - EventContext::new(ctx.id(), ctx.causation_id(), ctx.origin_id(), ts) - } else { - EventContext::new(id, id, id, ts) - }, + ctx: caused_by + .map(|cause| EventContext::from_cause(id, cause, ts)) + .unwrap_or_else(|| EventContext::new_origin(id, ts)), } } } diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 09dbe26bfe..3319df17cc 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -54,13 +54,6 @@ pub struct EventContext { ts: u128, } -impl EventContext { - /// Tracks events as they create other events - pub fn causes(self, id: EventId, ts: u128) -> EventContext { - EventContext::::new(id, self.id, self.origin_id, ts) - } -} - impl EventContext { pub fn new(id: EventId, causation_id: EventId, origin_id: EventId, ts: u128) -> Self { Self { @@ -72,6 +65,14 @@ impl EventContext { } } + pub fn new_origin(id: EventId, ts: u128) -> Self { + Self::new(id, id, id, ts) + } + + pub fn from_cause(id: EventId, cause: EventContext, ts: u128) -> Self { + EventContext::new(id, cause.id(), cause.origin_id(), ts) + } + pub fn sequence(self, value: u64) -> EventContext { EventContext:: { seq: value, @@ -109,10 +110,7 @@ impl EventContextSeq for EventContext { #[cfg(test)] mod tests { - use crate::{ - event_context::{AggregateId, EventContext}, - EventId, - }; + use crate::{event_context::EventContext, EventId}; #[test] fn test_event_context_cycle() { @@ -122,10 +120,10 @@ mod tests { EventContext::new(EventId::hash(1), EventId::hash(1), EventId::hash(1), 1).sequence(1); events.push(one.clone()); - let two = one.causes(EventId::hash(2), 2).sequence(2); + let two = EventContext::from_cause(EventId::hash(2), one, 2).sequence(2); events.push(two.clone()); - let three = two.causes(EventId::hash(3), 3).sequence(3); + let three = EventContext::from_cause(EventId::hash(3), two, 3).sequence(3); events.push(three.clone()); assert_eq!( diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 46972e2527..2b0ec381f3 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -109,8 +109,11 @@ pub trait EventSubscriber { /// Trait to create an event with a timestamp from its associated type data pub trait EventConstructorWithTimestamp: Event + Sized { /// Create an event passing attaching a specific timestamp. - fn new_with_timestamp(data: Self::Data, ctx: Option>, ts: u128) - -> Self; + fn new_with_timestamp( + data: Self::Data, + caused_by: Option>, + ts: u128, + ) -> Self; } pub trait CompositeEvent: EventConstructorWithTimestamp {} From 96077dafb19558d2a533f96d9e70fbf4d704ec3c Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 21:42:23 +0000 Subject: [PATCH 008/102] tidy up methods --- crates/events/src/enclave_event/mod.rs | 20 +++++++------------ .../events/src/enclave_event/typed_event.rs | 5 +---- crates/events/src/sequencer.rs | 4 ++-- crates/events/src/traits.rs | 3 +-- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index cf06d521ec..6538635737 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -215,11 +215,6 @@ impl EventContextSeq for EnclaveEvent { } impl EnclaveEvent { - // TOOD: remove - pub fn get_seq(&self) -> u64 { - self.seq() - } - pub fn clone_unsequenced(&self) -> EnclaveEvent { let ts = self.ts(); let data = self.clone().into_data(); @@ -245,7 +240,7 @@ impl EnclaveEvent { /// test-helpers only utility function to remove time information from an event pub fn strip_ts(&self) -> EnclaveEvent { - EnclaveEvent::new_stored_event(self.get_data().clone(), 0, self.get_seq()) + EnclaveEvent::new_stored_event(self.get_data().clone(), 0, self.seq()) } } @@ -253,13 +248,12 @@ impl Event for EnclaveEvent { type Id = EventId; type Data = EnclaveEventData; - fn event_type(&self) -> String { - self.payload.event_type() + fn event_id(&self) -> Self::Id { + self.ctx.id() } - // TODO: remove? - fn event_id(&self) -> Self::Id { - self.id() + fn event_type(&self) -> String { + self.payload.event_type() } fn get_data(&self) -> &EnclaveEventData { @@ -296,13 +290,13 @@ impl ErrorEvent for EnclaveEvent { impl From> for EventId { fn from(value: EnclaveEvent) -> Self { - value.id() + value.ctx.id() } } impl From<&EnclaveEvent> for EventId { fn from(value: &EnclaveEvent) -> Self { - value.id() + value.ctx.id() } } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 3928de798a..65f75bb554 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -9,10 +9,7 @@ use std::ops::Deref; use actix::Message; use serde::{Deserialize, Serialize}; -use crate::{ - event_context::{AggregateId, EventContext}, - EventContextAccessors, EventContextSeq, EventId, -}; +use crate::{event_context::EventContext, EventContextAccessors, EventContextSeq, EventId}; use super::Sequenced; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index a521637def..ccb13b7ef9 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -8,7 +8,7 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; use crate::{ events::{CommitSnapshot, EventStored, StoreEventRequested}, - EnclaveEvent, EventBus, Sequenced, Unsequenced, + EnclaveEvent, EventBus, EventContextSeq, Sequenced, Unsequenced, }; /// Component to sequence the storage of events @@ -48,7 +48,7 @@ impl Handler for Sequencer { type Result = (); fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { let event = msg.into_event(); - let seq = event.get_seq(); + let seq = event.seq(); self.buffer.do_send(CommitSnapshot::new(seq)); self.bus.do_send(event) } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 2b0ec381f3..e1d5e90087 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -22,9 +22,8 @@ pub trait Event: /// Payload for the Event type Data: WithAggregateId; - - fn event_type(&self) -> String; fn event_id(&self) -> Self::Id; + fn event_type(&self) -> String; fn get_data(&self) -> &Self::Data; fn into_data(self) -> Self::Data; } From a7d7f1c75b52327f34453656cab05fab7926a23f Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 21:49:27 +0000 Subject: [PATCH 009/102] tidy up names --- crates/events/src/bus_handle.rs | 12 ++++++------ crates/events/src/enclave_event/mod.rs | 4 ++-- crates/events/src/traits.rs | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index b7775be508..7ea865c260 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -114,12 +114,12 @@ impl EventFactory> for BusHandle { fn event_from( &self, data: impl Into, - ctx: Option>, + caused_by: Option>, ) -> Result> { let ts = self.hlc.tick()?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), - ctx, + caused_by, ts.into(), )) } @@ -127,13 +127,13 @@ impl EventFactory> for BusHandle { fn event_from_remote_source( &self, data: impl Into, - ctx: Option>, + caused_by: Option>, ts: u128, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), - ctx, + caused_by, ts.into(), )) } @@ -144,10 +144,10 @@ impl ErrorFactory> for BusHandle { &self, err_type: EType, error: impl Into, - ctx: Option>, + caused_by: Option>, ) -> Result> { let ts = self.hlc.tick()?; - EnclaveEvent::::from_error(err_type, error, ts.into(), ctx) + EnclaveEvent::::from_error(err_type, error, ts.into(), caused_by) } } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 6538635737..e59282f5f9 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -273,11 +273,11 @@ impl ErrorEvent for EnclaveEvent { err_type: Self::ErrType, msg: impl Into, ts: u128, - ctx: Option>, + caused_by: Option>, ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); - let ctx = ctx + let ctx = caused_by .map(|cause| EventContext::from_cause(id, cause, ts)) .unwrap_or_else(|| EventContext::new_origin(id, ts)); diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index e1d5e90087..434941a18d 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -38,7 +38,7 @@ pub trait ErrorEvent: Event { err_type: Self::ErrType, error: impl Into, ts: u128, - ctx: Option>, + caused_by: Option>, ) -> Result; } @@ -50,7 +50,7 @@ pub trait EventFactory { fn event_from( &self, data: impl Into, - ctx: Option>, + caused_by: Option>, ) -> Result; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering. @@ -59,7 +59,7 @@ pub trait EventFactory { fn event_from_remote_source( &self, data: impl Into, - ctx: Option>, + caused_by: Option>, ts: u128, ) -> Result; } @@ -71,7 +71,7 @@ pub trait ErrorFactory { &self, err_type: E::ErrType, error: impl Into, - ctx: Option>, + caused_by: Option>, ) -> Result; } From 25463fe2f39047779b9c5ec027cab4ba729d61be Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 8 Jan 2026 23:58:33 +0000 Subject: [PATCH 010/102] rename producer consumer vars to be clearer --- crates/events/src/bus_handle.rs | 39 ++++++++++++------------ crates/tests/tests/integration.rs | 4 +-- crates/tests/tests/integration_legacy.rs | 2 +- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 7ea865c260..f2b3d2eaac 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -27,9 +27,9 @@ use crate::{ #[derivative(Debug, PartialEq, Eq)] pub struct BusHandle { /// EventBus that actors can consume sequenced events from - consumer: Addr>>, + event_bus: Addr>>, /// Sequencer that new events should be produced from - producer: Addr, + sequencer: Addr, /// Hlc clock used to time all events created on this BusHandle #[derivative(Debug = "ignore")] hlc: Arc, @@ -40,13 +40,13 @@ pub struct BusHandle { impl BusHandle { /// Create a new BusHandle pub fn new( - consumer: Addr>>, - producer: Addr, + event_bus: Addr>>, + sequencer: Addr, hlc: Hlc, ) -> Self { Self { - consumer, - producer, + event_bus, + sequencer, hlc: Arc::new(hlc), ctx: None, } @@ -54,17 +54,17 @@ impl BusHandle { /// Return a HistoryCollector for examining events that have passed through on the events bus pub fn history(&self) -> Addr>> { - EventBus::>::history(&self.consumer) + EventBus::>::history(&self.event_bus) } - /// Access the producer to internally dispatch am event to - pub fn producer(&self) -> &Addr { - &self.producer + /// Access the sequencer to internally dispatch am event to + pub fn sequencer(&self) -> &Addr { + &self.sequencer } - /// Access the consumer to internally subscribe to events - pub fn consumer(&self) -> &Addr>> { - &self.consumer + /// Access the event_bus to internally subscribe to events + pub fn event_bus(&self) -> &Addr>> { + &self.event_bus } /// Get a new timestamp. Note this ticks over the internal Hlc. @@ -86,25 +86,25 @@ impl BusHandle { impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) -> Result<()> { let evt = self.event_from(data, self.ctx.clone())?; - self.producer.do_send(evt); + self.sequencer.do_send(evt); Ok(()) } fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { let evt = self.event_from_remote_source(data, self.ctx.clone(), ts)?; - self.producer.do_send(evt); + self.sequencer.do_send(evt); Ok(()) } fn naked_dispatch(&self, event: EnclaveEvent) { - self.producer.do_send(event); + self.sequencer.do_send(event); } } impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { match self.event_from_error(err_type, error, self.ctx.clone()) { - Ok(evt) => self.producer.do_send(evt), + Ok(evt) => self.sequencer.do_send(evt), Err(e) => error!("{e}"), } } @@ -153,12 +153,13 @@ impl ErrorFactory> for BusHandle { impl EventSubscriber> for BusHandle { fn subscribe(&self, event_type: &str, recipient: Recipient>) { - self.consumer.do_send(Subscribe::new(event_type, recipient)) + self.event_bus + .do_send(Subscribe::new(event_type, recipient)) } fn subscribe_all(&self, event_types: &[&str], recipient: Recipient>) { for event_type in event_types.into_iter() { - self.consumer + self.event_bus .do_send(Subscribe::new(*event_type, recipient.clone())); } } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 69a556b9f1..f6696b47aa 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -185,7 +185,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_pubkey_aggregation() .with_sortition_score() .with_threshold_plaintext_aggregation() - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() .await @@ -200,7 +200,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_shared_multithread_report(&multithread_report) .with_trbfv() .with_sortition_score() - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() .await diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 1b759142dd..4232692638 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -60,7 +60,7 @@ async fn setup_local_ciphernode( let mut builder = CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_keyshare() .with_address(addr) - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() From 5cc13e8c9999eae55387dc334ef6d5326468f5e7 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 9 Jan 2026 00:18:15 +0000 Subject: [PATCH 011/102] use context manager --- crates/events/src/bus_handle.rs | 19 ++++++++++++++----- crates/events/src/traits.rs | 7 +++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index f2b3d2eaac..a5823029aa 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -19,8 +19,8 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, Sequenced, - Subscribe, Unsequenced, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, + HistoryCollector, Sequenced, Subscribe, Unsequenced, }; #[derive(Clone, Derivative)] @@ -85,13 +85,13 @@ impl BusHandle { impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) -> Result<()> { - let evt = self.event_from(data, self.ctx.clone())?; + let evt = self.event_from(data, self.get_ctx())?; self.sequencer.do_send(evt); Ok(()) } fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { - let evt = self.event_from_remote_source(data, self.ctx.clone(), ts)?; + let evt = self.event_from_remote_source(data, self.get_ctx(), ts)?; self.sequencer.do_send(evt); Ok(()) } @@ -103,7 +103,7 @@ impl EventPublisher> for BusHandle { impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { - match self.event_from_error(err_type, error, self.ctx.clone()) { + match self.event_from_error(err_type, error, self.get_ctx()) { Ok(evt) => self.sequencer.do_send(evt), Err(e) => error!("{e}"), } @@ -165,6 +165,15 @@ impl EventSubscriber> for BusHandle { } } +impl EventContextManager for BusHandle { + fn set_ctx(&mut self, value: EventContext) { + self.ctx = Some(value); + } + fn get_ctx(&self) -> Option> { + self.ctx.clone() + } +} + #[cfg(test)] mod tests { use actix::{Actor, Handler, Message}; diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 434941a18d..5d06812684 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -59,6 +59,8 @@ pub trait EventFactory { fn event_from_remote_source( &self, data: impl Into, + // NOTE: `caused_by` makes sense here as we could be sending out requests and receiving + // responses that relate to the request caused_by: Option>, ts: u128, ) -> Result; @@ -158,3 +160,8 @@ pub trait WithAggregateId { /// Extract the aggregate id from the object fn get_aggregate_id(&self) -> AggregateId; } + +pub trait EventContextManager { + fn set_ctx(&mut self, value: EventContext); + fn get_ctx(&self) -> Option>; +} From b248377ceaf2e0bf6c5de91dd24ca462eed13a84 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 9 Jan 2026 00:50:34 +0000 Subject: [PATCH 012/102] add context to persistable --- crates/data/src/persistable.rs | 13 +++++++++++++ crates/events/src/lib.rs | 1 + crates/events/src/traits.rs | 2 ++ 3 files changed, 16 insertions(+) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index e1a73c587c..bf500272fe 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -7,6 +7,7 @@ use crate::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use anyhow::*; use async_trait::async_trait; +use e3_events::{EventContext, EventContextManager, Sequenced}; use serde::{de::DeserializeOwned, Serialize}; pub trait PersistableData: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} @@ -64,6 +65,7 @@ where pub struct Persistable { data: Option, repo: Repository, + ctx: Option>, } impl Persistable @@ -75,6 +77,7 @@ where Self { data, repo: repo.clone(), + ctx: None, } } @@ -167,6 +170,16 @@ where } } +impl EventContextManager for Persistable { + fn get_ctx(&self) -> Option> { + self.ctx.clone() + } + + fn set_ctx(&mut self, value: EventContext) { + self.ctx = Some(value) + } +} + impl Snapshot for Persistable where T: PersistableData, diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 34c4c54d18..8d233f80b5 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -24,6 +24,7 @@ pub use bus_handle::*; pub use correlation_id::*; pub use e3id::*; pub use enclave_event::*; +pub use event_context::*; pub use event_id::*; pub use eventbus::*; pub use events::*; diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 5d06812684..63a26fc498 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -161,6 +161,8 @@ pub trait WithAggregateId { fn get_aggregate_id(&self) -> AggregateId; } +/// An EventContextManager hold the current event context for use in event publishing and +/// persistence management pub trait EventContextManager { fn set_ctx(&mut self, value: EventContext); fn get_ctx(&self) -> Option>; From 145f617292f929f8a4b428040c0c9b29b7042417 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 9 Jan 2026 10:53:54 +0000 Subject: [PATCH 013/102] refactor persistable --- crates/data/src/data_store.rs | 16 + crates/data/src/events.rs | 34 +- crates/data/src/persistable.rs | 412 ++++-------------- crates/data/src/repository.rs | 11 +- crates/events/src/bus_handle.rs | 4 +- crates/events/src/enclave_event/mod.rs | 10 + .../events/src/enclave_event/typed_event.rs | 20 +- crates/events/src/traits.rs | 2 +- crates/keyshare/src/threshold_keyshare.rs | 50 ++- 9 files changed, 206 insertions(+), 353 deletions(-) diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index b2df9273cf..d1b48240ab 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -99,6 +99,22 @@ impl DataStore { &self.addr } + pub fn get_recipient(&self) -> Recipient { + self.get.clone() + } + + pub fn remove_recipient(&self) -> Recipient { + self.remove.clone() + } + + pub fn insert_recipient(&self) -> Recipient { + self.insert.clone() + } + + pub fn scope_bytes(&self) -> Vec { + self.scope.clone() + } + /// Changes the scope for the data store. /// Note that if the scope does not start with a slash one is appended. /// ``` diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index 6ecfc25d40..6f36b8c53c 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -7,21 +7,47 @@ use crate::IntoKey; use actix::Message; use anyhow::Result; +use e3_events::{EventContext, Sequenced}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] -pub struct Insert(pub Vec, pub Vec); +pub struct Insert { + key: Vec, + value: Vec, + ctx: Option>, +} + impl Insert { pub fn new(key: K, value: Vec) -> Self { - Self(key.into_key(), value) + Self { + key: key.into_key(), + value, + ctx: None, + } + } + + pub fn new_with_context( + key: K, + value: Vec, + ctx: EventContext, + ) -> Self { + Self { + key: key.into_key(), + value, + ctx: Some(ctx), + } } pub fn key(&self) -> &Vec { - &self.0 + &self.key } pub fn value(&self) -> &Vec { - &self.1 + &self.value + } + + pub fn ctx(&self) -> Option<&EventContext> { + self.ctx.as_ref() } } diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index bf500272fe..d78bbbdbdc 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -3,8 +3,8 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; +use crate::{Get, Insert, Remove, Repository}; +use actix::Recipient; use anyhow::*; use async_trait::async_trait; use e3_events::{EventContext, EventContextManager, Sequenced}; @@ -36,35 +36,56 @@ impl AutoPersist for Repository where T: PersistableData, { - /// Load the data from the repository into an auto persist container async fn load(&self) -> Result> { - Ok(Persistable::load(self).await?) + Persistable::load(self.to_connector()?).await } - /// Create a new auto persist container and set some data on it to send back to the repository fn send(&self, data: Option) -> Persistable { - Persistable::new(data, self).save() + Persistable::new(data, self.to_connector().unwrap()).save() } - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the given default data async fn load_or_default(&self, default: T) -> Result> { - Ok(Persistable::load_or_default(self, default).await?) + Persistable::load_or_default(self.to_connector()?, default).await } - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the result of the callback async fn load_or_else(&self, f: F) -> Result> where F: Send + FnOnce() -> Result, { - Ok(Persistable::load_or_else(self, f).await?) + Persistable::load_or_else(self.to_connector()?, f).await } } -/// A container that automatically persists it's content every time it is mutated or changed. +/// Connector to connect to store +#[derive(Clone, Debug)] +pub struct StoreConnector { + pub key: Vec, + pub get: Recipient, + pub insert: Recipient, + pub remove: Recipient, +} + +impl StoreConnector { + pub fn new( + key: Vec, + get: Recipient, + insert: Recipient, + remove: Recipient, + ) -> Self { + Self { + key, + get, + insert, + remove, + } + } +} + +/// A container that automatically persists its content every time it is mutated or changed. #[derive(Debug)] pub struct Persistable { data: Option, - repo: Repository, + connector: StoreConnector, ctx: Option>, } @@ -72,72 +93,97 @@ impl Persistable where T: PersistableData, { - /// Create a new container with the given option data and repository - pub fn new(data: Option, repo: &Repository) -> Self { + /// Create a new container with the given data and connector + pub fn new(data: Option, connector: StoreConnector) -> Self { Self { data, - repo: repo.clone(), + connector, ctx: None, } } - /// Load data from the repository to the container - pub async fn load(repo: &Repository) -> Result { - let data = repo.read().await?; - - Ok(Self::new(data, repo)) + /// Load data from the store + pub async fn load(connector: StoreConnector) -> Result { + let data = Self::read_from_store(&connector).await?; + Ok(Self::new(data, connector)) } - /// Load the data from the repo or save and sync the given default value - pub async fn load_or_default(repo: &Repository, default: T) -> Result { - let instance = Self::new(Some(repo.read().await?.unwrap_or(default)), repo); - + /// Load the data or save and sync the given default value + pub async fn load_or_default(connector: StoreConnector, default: T) -> Result { + let data = Self::read_from_store(&connector).await?.unwrap_or(default); + let instance = Self::new(Some(data), connector); Ok(instance.save()) } - /// Load the data from the repo or save and sync the result of the given callback - pub async fn load_or_else(repo: &Repository, f: F) -> Result + /// Load the data or save and sync the result of the given callback + pub async fn load_or_else(connector: StoreConnector, f: F) -> Result where F: FnOnce() -> Result, { - let data = repo - .read() + let data = Self::read_from_store(&connector) .await? .ok_or_else(|| anyhow!("Not found")) .or_else(|_| f())?; - - let instance = Self::new(Some(data), repo); + let instance = Self::new(Some(data), connector); Ok(instance.save()) } - /// Save the data in the container to the database + async fn read_from_store(connector: &StoreConnector) -> Result> { + let Some(bytes) = connector.get.send(Get::new(&connector.key)).await? else { + return Ok(None); + }; + if bytes == [0] { + return Ok(None); + } + Ok(Some(bincode::deserialize(&bytes)?)) + } + + fn write_to_store(&self) { + let Some(ref data) = self.data else { + return; + }; + let Result::Ok(serialized) = bincode::serialize(data) else { + tracing::error!("Could not serialize value for persistable"); + return; + }; + + let msg = if let Some(ctx) = self.ctx.clone() { + Insert::new_with_context(&self.connector.key, serialized, ctx) + } else { + Insert::new(&self.connector.key, serialized) + }; + self.connector.insert.do_send(msg); + } + + /// Save the data in the container to the store pub fn save(self) -> Self { - self.checkpoint(); + self.write_to_store(); self } - /// Mutate the content if it is available or return an error if either the mutator function - /// fails or if the data has not been set. + /// Mutate the content if available or return an error pub fn try_mutate(&mut self, mutator: F) -> Result<()> where F: FnOnce(T) -> Result, { let content = self.data.clone().ok_or(anyhow!("Data has not been set"))?; self.data = Some(mutator(content)?); - self.checkpoint(); + self.write_to_store(); Ok(()) } - /// Set the data on both the persistable and the repository. + /// Set the data on both the persistable and the store pub fn set(&mut self, data: T) { self.data = Some(data); - self.checkpoint(); + self.write_to_store(); } - /// Clear the data from both the persistable and the repository. + /// Clear the data from both the persistable and the store pub fn clear(&mut self) { self.data = None; - self.clear_checkpoint(); + self.connector + .remove + .do_send(Remove::new(&self.connector.key)); } /// Get the data currently stored on the container as an Option @@ -145,29 +191,17 @@ where self.data.clone() } - /// Get the data from the container or return an error. + /// Get the data from the container or return an error pub fn try_get(&self) -> Result { self.data .clone() .ok_or(anyhow!("Data was not set on container.")) } - /// Returns true if there is data on the container and false if there is not. + /// Returns true if there is data on the container pub fn has(&self) -> bool { self.data.is_some() } - - /// Get an immutable reference to the data on the container if the data is not set on the - /// container return an error - pub fn try_with(&self, f: F) -> Result - where - F: FnOnce(&T) -> Result, - { - match &self.data { - Some(data) => f(data), - None => Err(anyhow!("Data was not set on container.")), - } - } } impl EventContextManager for Persistable { @@ -175,275 +209,7 @@ impl EventContextManager for Persistable { self.ctx.clone() } - fn set_ctx(&mut self, value: EventContext) { - self.ctx = Some(value) - } -} - -impl Snapshot for Persistable -where - T: PersistableData, -{ - type Snapshot = T; - fn snapshot(&self) -> Result { - Ok(self - .data - .clone() - .ok_or(anyhow!("No data stored on container"))?) - } -} - -impl Checkpoint for Persistable -where - T: PersistableData, -{ - fn repository(&self) -> &Repository { - &self.repo - } -} - -#[async_trait] -impl FromSnapshotWithParams for Persistable -where - T: PersistableData, -{ - type Params = Repository; - async fn from_snapshot(params: Repository, snapshot: T) -> Result { - Ok(Persistable::new(Some(snapshot), ¶ms)) - } -} - -#[cfg(test)] -mod tests { - use crate::{AutoPersist, DataStore, GetLog, InMemStore, Repository}; - use actix::{Actor, Addr}; - use anyhow::{anyhow, Result}; - - fn get_repo() -> (Repository, Addr) { - let addr = InMemStore::new(true).start(); - let store = DataStore::from(&addr).scope("/"); - let repo: Repository = Repository::new(store); - (repo, addr) - } - - #[actix::test] - async fn persistable_loads_with_default() -> Result<()> { - let (repo, addr) = get_repo::>(); - let container = repo - .clone() - .load_or_default(vec!["berlin".to_string()]) - .await?; - - assert_eq!(addr.send(GetLog).await?.len(), 1); - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_loads_with_default_override() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["berlin".to_string()]); - let container = repo - .clone() - .load_or_default(vec!["amsterdam".to_string()]) - .await?; - - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_load() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["berlin".to_string()]); - let container = repo.clone().load().await?; - - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_send() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["amsterdam".to_string()]); - let container = repo.clone().send(Some(vec!["berlin".to_string()])); - - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_mutate() -> Result<()> { - let (repo, addr) = get_repo::>(); - - let mut container = repo.clone().send(Some(vec!["berlin".to_string()])); - - container.try_mutate(|mut list| { - list.push(String::from("amsterdam")); - Ok(list) - })?; - - assert_eq!( - repo.read().await?, - Some(vec!["berlin".to_string(), "amsterdam".to_string()]) - ); - - assert_eq!(addr.send(GetLog).await?.len(), 2); - - Ok(()) - } - - #[actix::test] - async fn test_clear_persistable() -> Result<()> { - let (repo, _) = get_repo::>(); - let repo_ref = &repo; - let mut container = repo_ref.send(Some(vec!["berlin".to_string()])); - - assert!(container.has()); - container.clear(); - assert!(!container.has()); - assert_eq!(repo_ref.read().await?, None); - Ok(()) - } - - #[actix::test] - async fn test_set_persistable() -> Result<()> { - let (repo, _) = get_repo::>(); - let mut container = repo.clone().send(None); - - container.set(vec!["amsterdam".to_string()]); - - assert!(container.has()); - assert_eq!(repo.read().await?, Some(vec!["amsterdam".to_string()])); - Ok(()) - } - - #[actix::test] - async fn test_try_get_with_data() -> Result<()> { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(Some(vec!["berlin".to_string()])); - - let result = container.try_get()?; - assert_eq!(result, vec!["berlin".to_string()]); - Ok(()) - } - - #[actix::test] - async fn test_try_get_without_data() { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(None); - - assert!(container.try_get().is_err()); - } - - #[actix::test] - async fn test_try_with_success() -> Result<()> { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(Some(vec!["berlin".to_string()])); - - let length = container.try_with(|data| Ok(data.len()))?; - assert_eq!(length, 1); - Ok(()) - } - - #[actix::test] - async fn test_try_with_failure() { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(None); - - let result = container.try_with(|data| Ok(data.len())); - assert!(result.is_err()); - } - - #[actix::test] - async fn test_try_mutate_failure() { - let (repo, _) = get_repo::>(); - let mut container = repo.clone().send(None); - - let result = container.try_mutate(|mut list| { - list.push(String::from("amsterdam")); - Ok(list) - }); - assert!(result.is_err()); - } - - #[actix::test] - async fn test_mutate_with_error() -> Result<()> { - let (repo, _) = get_repo::>(); - let mut container = repo.clone().send(Some(vec!["berlin".to_string()])); - - let result = - container.try_mutate(|_| -> Result> { Err(anyhow!("Mutation failed")) }); - - assert!(result.is_err()); - // Original data should remain unchanged - assert_eq!(container.try_get()?, vec!["berlin".to_string()]); - Ok(()) - } - - #[actix::test] - async fn test_load_or_else_success_with_empty_repo() -> Result<()> { - let (repo, _) = get_repo::>(); - - let container = repo - .clone() - .load_or_else(|| Ok(vec!["amsterdam".to_string()])) - .await?; - - assert_eq!(container.try_get()?, vec!["amsterdam".to_string()]); - assert_eq!(repo.read().await?, Some(vec!["amsterdam".to_string()])); - Ok(()) - } - - #[actix::test] - async fn test_load_or_else_skips_callback_when_data_exists() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["berlin".to_string()]); - - let container = repo - .clone() - .load_or_else(|| { - panic!("This callback should not be called!"); - #[allow(unreachable_code)] - Ok(vec!["amsterdam".to_string()]) - }) - .await?; - - assert_eq!(container.try_get()?, vec!["berlin".to_string()]); - Ok(()) - } - - #[actix::test] - async fn test_load_or_else_propagates_callback_error() -> Result<()> { - let (repo, _) = get_repo::>(); - - let result = repo - .clone() - .load_or_else(|| Err(anyhow!("Failed to create default data"))) - .await; - - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("Failed to create default data")); - assert_eq!(repo.read().await?, None); - Ok(()) - } - - #[actix::test] - async fn test_load_or_else_custom_error_message() -> Result<()> { - let (repo, _) = get_repo::>(); - let error_msg = "Custom initialization error"; - - let result = repo.load_or_else(|| Err(anyhow!(error_msg))).await; - - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains(error_msg)); - Ok(()) + fn set_ctx(&mut self, value: &EventContext) { + self.ctx = Some(value.clone()) } } diff --git a/crates/data/src/repository.rs b/crates/data/src/repository.rs index 085719ce65..e0f47374bf 100644 --- a/crates/data/src/repository.rs +++ b/crates/data/src/repository.rs @@ -8,7 +8,7 @@ use std::marker::PhantomData; use anyhow::Result; -use crate::DataStore; +use crate::{DataStore, StoreConnector}; #[derive(Debug)] pub struct Repository { @@ -24,6 +24,15 @@ impl Repository { _p: PhantomData, } } + + pub fn to_connector(&self) -> Result { + Ok(StoreConnector::new( + self.store.scope_bytes(), + self.store.get_recipient(), + self.store.insert_recipient(), + self.store.remove_recipient(), + )) + } } impl From> for DataStore { diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index a5823029aa..bedf144831 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -166,8 +166,8 @@ impl EventSubscriber> for BusHandle { } impl EventContextManager for BusHandle { - fn set_ctx(&mut self, value: EventContext) { - self.ctx = Some(value); + fn set_ctx(&mut self, value: &EventContext) { + self.ctx = Some(value.clone()); } fn get_ctx(&self) -> Option> { self.ctx.clone() diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index e59282f5f9..207480c43d 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -68,6 +68,7 @@ pub use threshold_share_created::*; pub use ticket_balance_updated::*; pub use ticket_generated::*; pub use ticket_submitted::*; +pub use typed_event::*; use crate::{ event_context::{AggregateId, EventContext}, @@ -191,6 +192,10 @@ where pub fn split(self) -> (EnclaveEventData, u128) { (self.payload, self.ctx.ts()) } + + pub fn get_ctx(&self) -> &EventContext { + &self.ctx + } } impl EventContextAccessors for EnclaveEvent { @@ -220,6 +225,11 @@ impl EnclaveEvent { let data = self.clone().into_data(); EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts) } + + pub fn to_typed_event(&self, data: T) -> TypedEvent { + let ctx: EventContext = self.get_ctx().clone(); + TypedEvent::new(data, ctx) + } } impl EnclaveEvent { diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 65f75bb554..4c2a00332b 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -20,6 +20,20 @@ pub struct TypedEvent { ctx: EventContext, } +impl TypedEvent { + pub fn new(inner: T, ctx: EventContext) -> Self { + Self { inner, ctx } + } + + pub fn into_inner(self) -> T { + self.inner + } + + pub fn get_ctx(&self) -> &EventContext { + &self.ctx + } +} + impl Deref for TypedEvent { type Target = T; fn deref(&self) -> &Self::Target { @@ -51,11 +65,11 @@ impl EventContextSeq for TypedEvent { } } -impl From<(T, EventContext)> for TypedEvent { - fn from(value: (T, EventContext)) -> Self { +impl From<(T, &EventContext)> for TypedEvent { + fn from(value: (T, &EventContext)) -> Self { Self { inner: value.0, - ctx: value.1, + ctx: value.1.clone(), } } } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 63a26fc498..79ea5004bf 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -164,6 +164,6 @@ pub trait WithAggregateId { /// An EventContextManager hold the current event context for use in event publishing and /// persistence management pub trait EventContextManager { - fn set_ctx(&mut self, value: EventContext); + fn set_ctx(&mut self, value: &EventContext); fn get_ctx(&self) -> Option>; } diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index f437a11dc8..4178e11b2b 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -13,7 +13,7 @@ use e3_events::{ ComputeResponse, CorrelationId, DecryptionshareCreated, Die, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, KeyshareCreated, PartyId, ThresholdShare, ThresholdShareCollectionFailed, - ThresholdShareCreated, + ThresholdShareCreated, TypedEvent, }; use e3_fhe::create_crp; use e3_trbfv::{ @@ -406,7 +406,9 @@ impl ThresholdKeyshare { Ok(()) } - pub fn handle_compute_response(&mut self, msg: ComputeResponse) -> Result<()> { + pub fn handle_compute_response(&mut self, msg: TypedEvent) -> Result<()> { + self.bus.set_ctx(msg.get_ctx()); + self.state.set_ctx(msg.get_ctx()); match &msg.response { TrBFVResponse::GenEsiSss(_) => self.handle_gen_esi_sss_response(msg), TrBFVResponse::GenPkShareAndSkSss(_) => { @@ -425,7 +427,7 @@ impl ThresholdKeyshare { /// 1. CiphernodeSelected - Generate BFV keys and start collecting pub fn handle_ciphernode_selected( &mut self, - msg: CiphernodeSelected, + msg: TypedEvent, address: Addr, ) -> Result<()> { info!("CiphernodeSelected received."); @@ -447,7 +449,7 @@ impl ThresholdKeyshare { CollectingEncryptionKeysData { sk_bfv: sk_bfv_encrypted.clone(), pk_bfv: pk_bfv_bytes.clone(), - ciphernode_selected: msg.clone(), + ciphernode_selected: msg.into_inner(), }, )) })?; @@ -537,8 +539,8 @@ impl ThresholdKeyshare { } /// 2a. GenEsiSss result - pub fn handle_gen_esi_sss_response(&mut self, res: ComputeResponse) -> Result<()> { - let output: GenEsiSssResponse = res.try_into()?; + pub fn handle_gen_esi_sss_response(&mut self, res: TypedEvent) -> Result<()> { + let output: GenEsiSssResponse = res.into_inner().try_into()?; let esi_sss = output.esi_sss; @@ -616,8 +618,11 @@ impl ThresholdKeyshare { } /// 3a. GenPkShareAndSkSss result - pub fn handle_gen_pk_share_and_sk_sss_response(&mut self, res: ComputeResponse) -> Result<()> { - let TrBFVResponse::GenPkShareAndSkSss(output) = res.response else { + pub fn handle_gen_pk_share_and_sk_sss_response( + &mut self, + res: TypedEvent, + ) -> Result<()> { + let TrBFVResponse::GenPkShareAndSkSss(output) = res.into_inner().response else { bail!("Error extracting data from compute process") }; @@ -817,8 +822,11 @@ impl ThresholdKeyshare { } /// 5a. CalculateDecryptionKeyResponse -> KeyshareCreated - pub fn handle_calculate_decryption_key_response(&mut self, res: ComputeResponse) -> Result<()> { - let TrBFVResponse::CalculateDecryptionKey(output) = res.response else { + pub fn handle_calculate_decryption_key_response( + &mut self, + res: TypedEvent, + ) -> Result<()> { + let TrBFVResponse::CalculateDecryptionKey(output) = res.into_inner().response else { bail!("Error extracting data from compute process") }; @@ -901,9 +909,9 @@ impl ThresholdKeyshare { /// CalculateDecryptionShareResponse pub fn handle_calculate_decryption_share_response( &mut self, - res: ComputeResponse, + res: TypedEvent, ) -> Result<()> { - let msg: CalculateDecryptionShareResponse = res.try_into()?; + let msg: CalculateDecryptionShareResponse = res.into_inner().try_into()?; let state = self.state.try_get()?; let party_id = state.party_id; let node = state.address; @@ -936,8 +944,8 @@ impl ThresholdKeyshare { impl Handler for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::CiphernodeSelected(data) => ctx.notify(data), + match msg.clone().into_data() { + EnclaveEventData::CiphernodeSelected(data) => ctx.notify(msg.to_typed_event(data)), EnclaveEventData::CiphertextOutputPublished(data) => ctx.notify(data), EnclaveEventData::ThresholdShareCreated(data) => { let _ = self.handle_threshold_share_created(data, ctx.address()); @@ -946,24 +954,28 @@ impl Handler for ThresholdKeyshare { let _ = self.handle_encryption_key_created(data, ctx.address()); } EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), - EnclaveEventData::ComputeResponse(data) => ctx.notify(data), + EnclaveEventData::ComputeResponse(data) => ctx.notify(msg.to_typed_event(data)), _ => (), } } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: ComputeResponse, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_compute_response(msg) }) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: CiphernodeSelected, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_ciphernode_selected(msg, ctx.address()) }) From 47e1e7c4d445057caa1bc8fc37f2fb844d5994c5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 9 Jan 2026 11:03:45 +0000 Subject: [PATCH 014/102] remove result --- crates/data/src/persistable.rs | 8 ++++---- crates/data/src/repository.rs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index d78bbbdbdc..602eebd68f 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -37,22 +37,22 @@ where T: PersistableData, { async fn load(&self) -> Result> { - Persistable::load(self.to_connector()?).await + Persistable::load(self.to_connector()).await } fn send(&self, data: Option) -> Persistable { - Persistable::new(data, self.to_connector().unwrap()).save() + Persistable::new(data, self.to_connector()).save() } async fn load_or_default(&self, default: T) -> Result> { - Persistable::load_or_default(self.to_connector()?, default).await + Persistable::load_or_default(self.to_connector(), default).await } async fn load_or_else(&self, f: F) -> Result> where F: Send + FnOnce() -> Result, { - Persistable::load_or_else(self.to_connector()?, f).await + Persistable::load_or_else(self.to_connector(), f).await } } diff --git a/crates/data/src/repository.rs b/crates/data/src/repository.rs index e0f47374bf..4718a019b6 100644 --- a/crates/data/src/repository.rs +++ b/crates/data/src/repository.rs @@ -25,13 +25,13 @@ impl Repository { } } - pub fn to_connector(&self) -> Result { - Ok(StoreConnector::new( + pub fn to_connector(&self) -> StoreConnector { + StoreConnector::new( self.store.scope_bytes(), self.store.get_recipient(), self.store.insert_recipient(), self.store.remove_recipient(), - )) + ) } } From a9f939ddd082af5bda905d4f4a93ebd5bb7f04da Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 9 Jan 2026 11:38:35 +0000 Subject: [PATCH 015/102] add auto persist to store connector --- crates/data/src/persistable.rs | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 602eebd68f..8646fdf24e 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -37,22 +37,22 @@ where T: PersistableData, { async fn load(&self) -> Result> { - Persistable::load(self.to_connector()).await + self.to_connector().load().await } fn send(&self, data: Option) -> Persistable { - Persistable::new(data, self.to_connector()).save() + self.to_connector().send(data) } async fn load_or_default(&self, default: T) -> Result> { - Persistable::load_or_default(self.to_connector(), default).await + self.to_connector().load_or_default(default).await } async fn load_or_else(&self, f: F) -> Result> where F: Send + FnOnce() -> Result, { - Persistable::load_or_else(self.to_connector(), f).await + self.to_connector().load_or_else(f).await } } @@ -81,6 +81,31 @@ impl StoreConnector { } } +#[async_trait] +impl AutoPersist for StoreConnector +where + T: PersistableData, +{ + async fn load(&self) -> Result> { + Persistable::load(self.clone()).await + } + + fn send(&self, data: Option) -> Persistable { + Persistable::new(data, self.clone()).save() + } + + async fn load_or_default(&self, default: T) -> Result> { + Persistable::load_or_default(self.clone(), default).await + } + + async fn load_or_else(&self, f: F) -> Result> + where + F: Send + FnOnce() -> Result, + { + Persistable::load_or_else(self.clone(), f).await + } +} + /// A container that automatically persists its content every time it is mutated or changed. #[derive(Debug)] pub struct Persistable { From 72602ff322d2691c8b75d3b0e2fb9adc20633051 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 9 Jan 2026 11:42:36 +0000 Subject: [PATCH 016/102] update doc comments --- crates/data/src/persistable.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 8646fdf24e..b153630e52 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -19,13 +19,13 @@ pub trait AutoPersist where T: PersistableData, { - /// Load the data from the repository into an auto persist container + /// Load the data from the source into an auto persist container async fn load(&self) -> Result>; - /// Create a new auto persist container and set some data on it to send back to the repository + /// Create a new auto persist container and set some data on it to send back to the source fn send(&self, data: Option) -> Persistable; - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the given default data + /// Load the data from the source into an auto persist container. If there is no persisted data then persist the given default data async fn load_or_default(&self, default: T) -> Result>; - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the given default data + /// Load the data from the source into an auto persist container. If there is no persisted data then persist the given default data async fn load_or_else(&self, f: F) -> Result> where F: Send + FnOnce() -> Result; From 4d2ccea788ecf69572d384e8d132a91947619d0b Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 02:28:35 +0000 Subject: [PATCH 017/102] insert_sync_recipient() --- crates/data/src/data_store.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index d1b48240ab..349dfbe3e8 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -111,6 +111,10 @@ impl DataStore { self.insert.clone() } + pub fn insert_sync_recipient(&self) -> Recipient { + self.insert_sync.clone() + } + pub fn scope_bytes(&self) -> Vec { self.scope.clone() } From 96d034d8341f376c95c9af6640bc4ff8a64c03a2 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 04:07:27 +0000 Subject: [PATCH 018/102] update connector --- crates/data/src/data_store.rs | 25 +++++++++++++++---------- crates/data/src/persistable.rs | 16 ++++++++-------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 349dfbe3e8..d4516f858f 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -99,24 +99,29 @@ impl DataStore { &self.addr } - pub fn get_recipient(&self) -> Recipient { - self.get.clone() + /// Get a reference to the Recipient + pub fn get_recipient(&self) -> &Recipient { + &self.get } - pub fn remove_recipient(&self) -> Recipient { - self.remove.clone() + /// Get a reference to the Recipient + pub fn remove_recipient(&self) -> &Recipient { + &self.remove } - pub fn insert_recipient(&self) -> Recipient { - self.insert.clone() + /// Get a reference to the Recipient + pub fn insert_recipient(&self) -> &Recipient { + &self.insert } - pub fn insert_sync_recipient(&self) -> Recipient { - self.insert_sync.clone() + /// Get a reference to the Recipient + pub fn insert_sync_recipient(&self) -> &Recipient { + &self.insert_sync } - pub fn scope_bytes(&self) -> Vec { - self.scope.clone() + /// Get a clone of the scope bytes + pub fn scope_bytes(&self) -> &[u8] { + &self.scope } /// Changes the scope for the data store. diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index b153630e52..da50a35e54 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -67,16 +67,16 @@ pub struct StoreConnector { impl StoreConnector { pub fn new( - key: Vec, - get: Recipient, - insert: Recipient, - remove: Recipient, + key: &[u8], + get: &Recipient, + insert: &Recipient, + remove: &Recipient, ) -> Self { Self { - key, - get, - insert, - remove, + key: key.to_owned(), + get: get.clone(), + insert: insert.clone(), + remove: remove.clone(), } } } From 75f2323284226a8d9279a661f63ee8078ea9d70e Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 04:43:35 +0000 Subject: [PATCH 019/102] add aggregate_id to event context --- crates/events/src/enclave_event/mod.rs | 16 ++++-- .../events/src/enclave_event/typed_event.rs | 9 +++- crates/events/src/event_context.rs | 52 +++++++++++++++---- crates/events/src/traits.rs | 10 ++-- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 207480c43d..2b808486b1 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -211,6 +211,9 @@ impl EventContextAccessors for EnclaveEvent { fn id(&self) -> EventId { self.ctx.id() } + fn aggregate_id(&self) -> AggregateId { + self.ctx.aggregate_id() + } } impl EventContextSeq for EnclaveEvent { @@ -287,9 +290,11 @@ impl ErrorEvent for EnclaveEvent { ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); + let aggregate_id = AggregateId::new(0); // Error events use default aggregate_id + let ctx = caused_by - .map(|cause| EventContext::from_cause(id, cause, ts)) - .unwrap_or_else(|| EventContext::new_origin(id, ts)); + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)); Ok(EnclaveEvent { payload: payload.into(), @@ -424,13 +429,14 @@ impl EventConstructorWithTimestamp for EnclaveEvent { caused_by: Option>, ts: u128, ) -> Self { - let payload = data.into(); + let payload: EnclaveEventData = data.into(); let id = EventId::hash(&payload); + let aggregate_id = payload.get_aggregate_id(); EnclaveEvent { payload, ctx: caused_by - .map(|cause| EventContext::from_cause(id, cause, ts)) - .unwrap_or_else(|| EventContext::new_origin(id, ts)), + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)), } } } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 4c2a00332b..7f3da4c0ee 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -9,7 +9,10 @@ use std::ops::Deref; use actix::Message; use serde::{Deserialize, Serialize}; -use crate::{event_context::EventContext, EventContextAccessors, EventContextSeq, EventId}; +use crate::{ + event_context::{AggregateId, EventContext}, + EventContextAccessors, EventContextSeq, EventId, +}; use super::Sequenced; @@ -57,6 +60,10 @@ impl EventContextAccessors for TypedEvent { fn causation_id(&self) -> EventId { self.ctx.causation_id() } + + fn aggregate_id(&self) -> AggregateId { + self.ctx.aggregate_id() + } } impl EventContextSeq for TypedEvent { diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 3319df17cc..39375785db 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -52,25 +52,38 @@ pub struct EventContext { origin_id: EventId, seq: S::Seq, ts: u128, + aggregate_id: AggregateId, } impl EventContext { - pub fn new(id: EventId, causation_id: EventId, origin_id: EventId, ts: u128) -> Self { + pub fn new( + id: EventId, + causation_id: EventId, + origin_id: EventId, + ts: u128, + aggregate_id: AggregateId, + ) -> Self { Self { id, causation_id, origin_id, seq: (), ts, + aggregate_id, } } - pub fn new_origin(id: EventId, ts: u128) -> Self { - Self::new(id, id, id, ts) + pub fn new_origin(id: EventId, ts: u128, aggregate_id: AggregateId) -> Self { + Self::new(id, id, id, ts, aggregate_id) } - pub fn from_cause(id: EventId, cause: EventContext, ts: u128) -> Self { - EventContext::new(id, cause.id(), cause.origin_id(), ts) + pub fn from_cause( + id: EventId, + cause: EventContext, + ts: u128, + aggregate_id: AggregateId, + ) -> Self { + EventContext::new(id, cause.id(), cause.origin_id(), ts, aggregate_id) } pub fn sequence(self, value: u64) -> EventContext { @@ -80,6 +93,7 @@ impl EventContext { causation_id: self.causation_id, origin_id: self.origin_id, ts: self.ts, + aggregate_id: self.aggregate_id, } } } @@ -100,6 +114,10 @@ impl EventContextAccessors for EventContext { fn ts(&self) -> u128 { self.ts } + + fn aggregate_id(&self) -> AggregateId { + self.aggregate_id + } } impl EventContextSeq for EventContext { @@ -110,20 +128,31 @@ impl EventContextSeq for EventContext { #[cfg(test)] mod tests { - use crate::{event_context::EventContext, EventId}; + use crate::{ + event_context::{AggregateId, EventContext}, + EventId, + }; #[test] fn test_event_context_cycle() { let mut events = vec![]; - let one = - EventContext::new(EventId::hash(1), EventId::hash(1), EventId::hash(1), 1).sequence(1); + let one = EventContext::new( + EventId::hash(1), + EventId::hash(1), + EventId::hash(1), + 1, + AggregateId::new(1), + ) + .sequence(1); events.push(one.clone()); - let two = EventContext::from_cause(EventId::hash(2), one, 2).sequence(2); + let two = + EventContext::from_cause(EventId::hash(2), one, 2, AggregateId::new(1)).sequence(2); events.push(two.clone()); - let three = EventContext::from_cause(EventId::hash(3), two, 3).sequence(3); + let three = + EventContext::from_cause(EventId::hash(3), two, 3, AggregateId::new(1)).sequence(3); events.push(three.clone()); assert_eq!( @@ -135,6 +164,7 @@ mod tests { origin_id: EventId::hash(1), causation_id: EventId::hash(1), ts: 1, + aggregate_id: AggregateId::new(1), }, EventContext { seq: 2, @@ -142,6 +172,7 @@ mod tests { origin_id: EventId::hash(1), causation_id: EventId::hash(1), ts: 2, + aggregate_id: AggregateId::new(1), }, EventContext { seq: 3, @@ -149,6 +180,7 @@ mod tests { origin_id: EventId::hash(1), causation_id: EventId::hash(2), ts: 3, + aggregate_id: AggregateId::new(1), }, ] ) diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 79ea5004bf..3217b86280 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -141,14 +141,16 @@ pub trait EventLog: Unpin + 'static { /// EventContext allows consumers to extract infrastructure metadata from event objects pub trait EventContextAccessors { - /// This event id + /// The unique id for this event fn id(&self) -> EventId; - /// The id of the event that directly caused this + /// The event that caused this event to occur fn causation_id(&self) -> EventId; - /// The original event that started the causal chain + /// The root event that caused this event to occur fn origin_id(&self) -> EventId; - /// The timestamp for the event + /// The timestamp when the event occurred fn ts(&self) -> u128; + /// The aggregate id for this event + fn aggregate_id(&self) -> AggregateId; } pub trait EventContextSeq { From 61b31710a6691ad8c9c608123b119ec4b8895263 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 05:10:14 +0000 Subject: [PATCH 020/102] add hlc walltime --- crates/events/src/hlc.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index a49c077e79..b0b03dda1a 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -44,6 +44,11 @@ impl HlcTimestamp { Self { ts, counter, node } } + /// Extract wall time from a u128 timestamp without full HLC conversion + pub fn wall_time(ts: u128) -> u64 { + (ts >> 64) as u64 + } + /// Packs the HLC timestamp into a 128bit big-endian representation. /// /// Layout: From ab20fbd5c95a1dd91f7fdb6151d1ff4e0ac1dd2a Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 05:19:44 +0000 Subject: [PATCH 021/102] add writebuffer config --- crates/ciphernode-builder/src/event_system.rs | 6 +++++- crates/data/src/write_buffer.rs | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 73ad4dfba0..30a3c9f663 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -14,6 +14,7 @@ use e3_data::{ use e3_events::hlc::Hlc; use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; use once_cell::sync::OnceCell; +use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; @@ -171,7 +172,10 @@ impl EventSystem { pub fn buffer(&self) -> Addr { let buffer = self .buffer - .get_or_init(|| WriteBuffer::new().start()) + .get_or_init(|| { + let default_config = HashMap::new(); + WriteBuffer::with_config(default_config).start() + }) .clone(); self.wire_if_ready(); buffer diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index c3d39afdb0..119c19250d 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -5,13 +5,15 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler, Message, Recipient}; -use e3_events::CommitSnapshot; +use e3_events::{AggregateId, CommitSnapshot}; +use std::collections::HashMap; use crate::{Insert, InsertBatch}; pub struct WriteBuffer { dest: Option>, buffer: Vec, + config: HashMap, } impl Actor for WriteBuffer { @@ -23,6 +25,15 @@ impl WriteBuffer { Self { dest: None, buffer: Vec::new(), + config: HashMap::new(), + } + } + + pub fn with_config(config: HashMap) -> Self { + Self { + dest: None, + buffer: Vec::new(), + config, } } } From 55ee6eae773ddf808ff05d968eb75a54d47424b4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 05:44:29 +0000 Subject: [PATCH 022/102] add buffering based on aggregate_id --- crates/data/src/write_buffer.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 119c19250d..5782ee24c9 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -5,14 +5,25 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler, Message, Recipient}; -use e3_events::{AggregateId, CommitSnapshot}; +use e3_events::{AggregateId, CommitSnapshot, EventContextAccessors}; use std::collections::HashMap; use crate::{Insert, InsertBatch}; +struct AggregateBuffer { + buffer: Vec, +} + +impl AggregateBuffer { + fn new() -> Self { + Self { buffer: Vec::new() } + } +} + pub struct WriteBuffer { dest: Option>, buffer: Vec, + aggregate_buffers: HashMap, config: HashMap, } @@ -25,6 +36,7 @@ impl WriteBuffer { Self { dest: None, buffer: Vec::new(), + aggregate_buffers: HashMap::new(), config: HashMap::new(), } } @@ -33,6 +45,7 @@ impl WriteBuffer { Self { dest: None, buffer: Vec::new(), + aggregate_buffers: HashMap::new(), config, } } @@ -49,6 +62,14 @@ impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + if let Some(event_ctx) = msg.ctx() { + let aggregate_id = event_ctx.aggregate_id().clone(); + let agg_buffer = self + .aggregate_buffers + .entry(aggregate_id) + .or_insert_with(|| AggregateBuffer::new()); + agg_buffer.buffer.push(msg.clone()); + } self.buffer.push(msg); } } From bdfc7a55e0cc173955b54c5f9a2b3bd20bd54b51 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 06:18:01 +0000 Subject: [PATCH 023/102] add comments --- crates/data/src/write_buffer.rs | 8 ++++++-- crates/events/src/hlc.rs | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 5782ee24c9..d44a4a54f1 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -21,10 +21,14 @@ impl AggregateBuffer { } pub struct WriteBuffer { + /// Destination recipient for batched inserts dest: Option>, + /// Buffer for storing individual inserts buffer: Vec, + /// Per-aggregate buffers for organizing inserts aggregate_buffers: HashMap, - config: HashMap, + /// Per-aggregate wait time (microseconds) before sending inserts to destination + config: HashMap, } impl Actor for WriteBuffer { @@ -41,7 +45,7 @@ impl WriteBuffer { } } - pub fn with_config(config: HashMap) -> Self { + pub fn with_config(config: HashMap) -> Self { Self { dest: None, buffer: Vec::new(), diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index b0b03dda1a..af871dce54 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -33,8 +33,11 @@ pub enum HlcError { /// HLC timestamp #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct HlcTimestamp { + /// Physical timestamp in microseconds since UNIX epoch pub ts: u64, + /// Logical counter for same-timestamp ordering pub counter: u32, + /// Unique node identifier for tie-breaking pub node: u32, } @@ -44,9 +47,9 @@ impl HlcTimestamp { Self { ts, counter, node } } - /// Extract wall time from a u128 timestamp without full HLC conversion + /// Extract wall time from a u128 timestamp pub fn wall_time(ts: u128) -> u64 { - (ts >> 64) as u64 + Self::from_u128(ts).ts } /// Packs the HLC timestamp into a 128bit big-endian representation. @@ -178,7 +181,9 @@ pub struct Hlc { #[derive(PartialEq)] struct HlcInner { + /// Current timestamp value ts: u64, + /// Current logical counter counter: u32, } From c061694bf416f2725bc8d32a50ad305e5e6574f4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 07:37:37 +0000 Subject: [PATCH 024/102] implement time-based- batching with per-aggregate delays --- crates/data/src/write_buffer.rs | 77 ++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index d44a4a54f1..226435dcb5 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -6,7 +6,10 @@ use actix::{Actor, Handler, Message, Recipient}; use e3_events::{AggregateId, CommitSnapshot, EventContextAccessors}; -use std::collections::HashMap; +use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, +}; use crate::{Insert, InsertBatch}; @@ -23,8 +26,6 @@ impl AggregateBuffer { pub struct WriteBuffer { /// Destination recipient for batched inserts dest: Option>, - /// Buffer for storing individual inserts - buffer: Vec, /// Per-aggregate buffers for organizing inserts aggregate_buffers: HashMap, /// Per-aggregate wait time (microseconds) before sending inserts to destination @@ -39,7 +40,6 @@ impl WriteBuffer { pub fn new() -> Self { Self { dest: None, - buffer: Vec::new(), aggregate_buffers: HashMap::new(), config: HashMap::new(), } @@ -48,7 +48,6 @@ impl WriteBuffer { pub fn with_config(config: HashMap) -> Self { Self { dest: None, - buffer: Vec::new(), aggregate_buffers: HashMap::new(), config, } @@ -66,28 +65,66 @@ impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - if let Some(event_ctx) = msg.ctx() { - let aggregate_id = event_ctx.aggregate_id().clone(); - let agg_buffer = self - .aggregate_buffers - .entry(aggregate_id) - .or_insert_with(|| AggregateBuffer::new()); - agg_buffer.buffer.push(msg.clone()); - } - self.buffer.push(msg); + let aggregate_id = if let Some(event_ctx) = msg.ctx() { + event_ctx.aggregate_id().clone() + } else { + AggregateId::new(0) + }; + + let agg_buffer = self + .aggregate_buffers + .entry(aggregate_id) + .or_insert_with(|| AggregateBuffer::new()); + agg_buffer.buffer.push(msg.clone()); } } impl Handler for WriteBuffer { type Result = (); - fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, _msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_millis(); + if let Some(ref dest) = self.dest { - if !self.buffer.is_empty() { - let mut inserts = std::mem::take(&mut self.buffer); - inserts.push(Insert::new("//seq", msg.seq().to_be_bytes().to_vec())); - let batch = InsertBatch::new(inserts); - dest.do_send(batch); + let mut aggregates_to_remove = Vec::new(); + + for (aggregate_id, agg_buffer) in &mut self.aggregate_buffers { + let delay_micros = self.config.get(aggregate_id).copied().unwrap_or(0); + let delay_ms = delay_micros / 1000; + let cutoff_time = now.saturating_sub(delay_ms as u128); + + let mut expired_inserts = Vec::new(); + let mut remaining_inserts = Vec::new(); + + for insert in &agg_buffer.buffer { + if let Some(ctx) = insert.ctx() { + if ctx.ts() < cutoff_time { + expired_inserts.push(insert.clone()); + } else { + remaining_inserts.push(insert.clone()); + } + } else { + remaining_inserts.push(insert.clone()); + } + } + + if !expired_inserts.is_empty() { + let batch = InsertBatch::new(expired_inserts); + dest.do_send(batch); + } + + agg_buffer.buffer = remaining_inserts; + + if agg_buffer.buffer.is_empty() { + aggregates_to_remove.push(aggregate_id.clone()); + } + } + + for aggregate_id in aggregates_to_remove { + self.aggregate_buffers.remove(&aggregate_id); } } } From bb2f5026376e6790e6a1f8075cb8ac3ef7829522 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 08:20:42 +0000 Subject: [PATCH 025/102] extract function to be able to be unit tested --- crates/data/src/write_buffer.rs | 146 ++++++++++++++++++++++++-------- crates/events/src/traits.rs | 2 +- 2 files changed, 113 insertions(+), 35 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 226435dcb5..02f8ba8291 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -79,6 +79,46 @@ impl Handler for WriteBuffer { } } +fn process_expired_inserts( + aggregate_buffers: &HashMap, + config: &HashMap, + now: u128, +) -> (HashMap, Vec) { + let mut updated_buffers = HashMap::new(); + let mut all_expired_inserts = Vec::new(); + + for (aggregate_id, agg_buffer) in aggregate_buffers { + let delay_micros = config.get(aggregate_id).copied().unwrap_or(0); + let delay_ms = delay_micros / 1000; + let cutoff_time = now.saturating_sub(delay_ms.into()); + + let mut expired_inserts = Vec::new(); + let mut remaining_inserts = Vec::new(); + + for insert in &agg_buffer.buffer { + if let Some(ctx) = insert.ctx() { + if ctx.ts() < cutoff_time { + expired_inserts.push(insert.clone()); + } else { + remaining_inserts.push(insert.clone()); + } + } else { + remaining_inserts.push(insert.clone()); + } + } + + all_expired_inserts.extend(expired_inserts); + + if !remaining_inserts.is_empty() { + let mut new_agg_buffer = AggregateBuffer::new(); + new_agg_buffer.buffer = remaining_inserts; + updated_buffers.insert(aggregate_id.clone(), new_agg_buffer); + } + } + + (updated_buffers, all_expired_inserts) +} + impl Handler for WriteBuffer { type Result = (); @@ -89,47 +129,85 @@ impl Handler for WriteBuffer { .as_millis(); if let Some(ref dest) = self.dest { - let mut aggregates_to_remove = Vec::new(); - - for (aggregate_id, agg_buffer) in &mut self.aggregate_buffers { - let delay_micros = self.config.get(aggregate_id).copied().unwrap_or(0); - let delay_ms = delay_micros / 1000; - let cutoff_time = now.saturating_sub(delay_ms as u128); - - let mut expired_inserts = Vec::new(); - let mut remaining_inserts = Vec::new(); - - for insert in &agg_buffer.buffer { - if let Some(ctx) = insert.ctx() { - if ctx.ts() < cutoff_time { - expired_inserts.push(insert.clone()); - } else { - remaining_inserts.push(insert.clone()); - } - } else { - remaining_inserts.push(insert.clone()); - } - } - - if !expired_inserts.is_empty() { - let batch = InsertBatch::new(expired_inserts); - dest.do_send(batch); - } + let (updated_buffers, expired_inserts) = + process_expired_inserts(&self.aggregate_buffers, &self.config, now); - agg_buffer.buffer = remaining_inserts; - - if agg_buffer.buffer.is_empty() { - aggregates_to_remove.push(aggregate_id.clone()); - } + if !expired_inserts.is_empty() { + let batch = InsertBatch::new(expired_inserts); + dest.do_send(batch); } - for aggregate_id in aggregates_to_remove { - self.aggregate_buffers.remove(&aggregate_id); - } + self.aggregate_buffers = updated_buffers; } } } +#[cfg(test)] +mod tests { + use super::*; + use crate::events::Insert; + use e3_events::{EventContext, EventId}; + + #[test] + fn test_process_expired_inserts() { + let aggregate_id = AggregateId::new(1); + + // Create test inserts with different timestamps + let old_ctx = EventContext::new( + EventId::hash(1), + EventId::hash(1), + EventId::hash(1), + 500, + aggregate_id.clone(), + ) + .sequence(1); + + let new_ctx = EventContext::new( + EventId::hash(2), + EventId::hash(2), + EventId::hash(2), + 3000, + aggregate_id.clone(), + ) + .sequence(2); + + let old_insert = Insert::new_with_context("old_key", b"old_value".to_vec(), old_ctx); + let new_insert = Insert::new_with_context("new_key", b"new_value".to_vec(), new_ctx); + let insert_no_ctx = Insert::new("no_ctx_key", b"no_ctx_value".to_vec()); + + // Set up aggregate buffer with mixed inserts + let mut agg_buffer = AggregateBuffer::new(); + agg_buffer.buffer.push(old_insert.clone()); + agg_buffer.buffer.push(new_insert.clone()); + agg_buffer.buffer.push(insert_no_ctx.clone()); + + let mut aggregate_buffers = HashMap::new(); + aggregate_buffers.insert(aggregate_id.clone(), agg_buffer); + + // Set config with 1000ms delay + let mut config = HashMap::new(); + config.insert(aggregate_id.clone(), 1000000); // 1000ms in microseconds + + // Use current time of 2000ms, so old insert (1000ms) should expire, + // new insert (3000ms) and insert without context should remain + let now = 2000; + + let (updated_buffers, expired_inserts) = + process_expired_inserts(&aggregate_buffers, &config, now); + + // Verify expired inserts + assert_eq!(expired_inserts.len(), 1); + assert_eq!(expired_inserts[0], old_insert); + + // Verify remaining inserts in buffer + assert_eq!(updated_buffers.len(), 1); + let remaining_buffer = updated_buffers.get(&aggregate_id).unwrap(); + assert_eq!(remaining_buffer.buffer.len(), 2); + assert!(remaining_buffer.buffer.contains(&new_insert)); + assert!(remaining_buffer.buffer.contains(&insert_no_ctx)); + } +} + #[derive(Message)] #[rtype("()")] pub struct ForwardTo(Recipient); diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 3217b86280..54c1cbcffe 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -147,7 +147,7 @@ pub trait EventContextAccessors { fn causation_id(&self) -> EventId; /// The root event that caused this event to occur fn origin_id(&self) -> EventId; - /// The timestamp when the event occurred + /// The timestamp when the event occurred timestamp is encoded HlcTimestamp format fn ts(&self) -> u128; /// The aggregate id for this event fn aggregate_id(&self) -> AggregateId; From 22fcbb082368adabfa03fa16b16bebd3c30db9c9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 08:54:43 +0000 Subject: [PATCH 026/102] update test to ensure delay works --- crates/data/src/write_buffer.rs | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 02f8ba8291..9fa8164085 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler, Message, Recipient}; +use e3_events::hlc::HlcTimestamp; use e3_events::{AggregateId, CommitSnapshot, EventContextAccessors}; use std::{ collections::HashMap, @@ -82,22 +83,22 @@ impl Handler for WriteBuffer { fn process_expired_inserts( aggregate_buffers: &HashMap, config: &HashMap, - now: u128, + now: u64, ) -> (HashMap, Vec) { let mut updated_buffers = HashMap::new(); let mut all_expired_inserts = Vec::new(); for (aggregate_id, agg_buffer) in aggregate_buffers { let delay_micros = config.get(aggregate_id).copied().unwrap_or(0); - let delay_ms = delay_micros / 1000; - let cutoff_time = now.saturating_sub(delay_ms.into()); + let cutoff_time = now.saturating_sub(delay_micros); let mut expired_inserts = Vec::new(); let mut remaining_inserts = Vec::new(); for insert in &agg_buffer.buffer { if let Some(ctx) = insert.ctx() { - if ctx.ts() < cutoff_time { + let event_wall_time = HlcTimestamp::wall_time(ctx.ts()); + if event_wall_time < cutoff_time { expired_inserts.push(insert.clone()); } else { remaining_inserts.push(insert.clone()); @@ -126,7 +127,7 @@ impl Handler for WriteBuffer { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_else(|_| std::time::Duration::from_secs(0)) - .as_millis(); + .as_micros() as u64; if let Some(ref dest) = self.dest { let (updated_buffers, expired_inserts) = @@ -146,18 +147,22 @@ impl Handler for WriteBuffer { mod tests { use super::*; use crate::events::Insert; - use e3_events::{EventContext, EventId}; + use e3_events::{hlc::HlcTimestamp, EventContext, EventId}; #[test] fn test_process_expired_inserts() { let aggregate_id = AggregateId::new(1); - // Create test inserts with different timestamps + // Create test inserts with different timestamps (in microseconds) + // Create proper HlcTimestamps and encode them to u128 + let old_hlc = HlcTimestamp::new(500_000, 0, 1); // 0.5 seconds ago + let new_hlc = HlcTimestamp::new(3_000_000, 0, 2); // 3 seconds from epoch + let old_ctx = EventContext::new( EventId::hash(1), EventId::hash(1), EventId::hash(1), - 500, + old_hlc.into(), aggregate_id.clone(), ) .sequence(1); @@ -166,7 +171,7 @@ mod tests { EventId::hash(2), EventId::hash(2), EventId::hash(2), - 3000, + new_hlc.into(), aggregate_id.clone(), ) .sequence(2); @@ -184,13 +189,13 @@ mod tests { let mut aggregate_buffers = HashMap::new(); aggregate_buffers.insert(aggregate_id.clone(), agg_buffer); - // Set config with 1000ms delay + // Set config with 1 second delay let mut config = HashMap::new(); - config.insert(aggregate_id.clone(), 1000000); // 1000ms in microseconds + config.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds - // Use current time of 2000ms, so old insert (1000ms) should expire, - // new insert (3000ms) and insert without context should remain - let now = 2000; + // Use current time of 2 seconds, so old insert (0.5s) should expire, + // new insert (3s) and insert without context should remain + let now = 2_000_000; // 2 seconds in microseconds let (updated_buffers, expired_inserts) = process_expired_inserts(&aggregate_buffers, &config, now); From fff85a18d6a2f30a116c7b2b55643b000dc72aa8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 13:32:06 +0000 Subject: [PATCH 027/102] fix test --- crates/data/src/write_buffer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 9fa8164085..f1e9ecfa20 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -14,6 +14,7 @@ use std::{ use crate::{Insert, InsertBatch}; +#[derive(Debug)] struct AggregateBuffer { buffer: Vec, } @@ -91,7 +92,6 @@ fn process_expired_inserts( for (aggregate_id, agg_buffer) in aggregate_buffers { let delay_micros = config.get(aggregate_id).copied().unwrap_or(0); let cutoff_time = now.saturating_sub(delay_micros); - let mut expired_inserts = Vec::new(); let mut remaining_inserts = Vec::new(); @@ -104,7 +104,8 @@ fn process_expired_inserts( remaining_inserts.push(insert.clone()); } } else { - remaining_inserts.push(insert.clone()); + // If there is no context just flush it + expired_inserts.push(insert.clone()); } } @@ -132,7 +133,6 @@ impl Handler for WriteBuffer { if let Some(ref dest) = self.dest { let (updated_buffers, expired_inserts) = process_expired_inserts(&self.aggregate_buffers, &self.config, now); - if !expired_inserts.is_empty() { let batch = InsertBatch::new(expired_inserts); dest.do_send(batch); From b796ce18aa31610e44984e629fcf23dbb9e7244f Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 12 Jan 2026 15:34:24 +0000 Subject: [PATCH 028/102] fix test --- crates/data/src/write_buffer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index f1e9ecfa20..6149ec5fd7 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -193,23 +193,23 @@ mod tests { let mut config = HashMap::new(); config.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds - // Use current time of 2 seconds, so old insert (0.5s) should expire, - // new insert (3s) and insert without context should remain + // Use current time of 2 seconds, so old insert (0.5s) and insert without context should expire, + // new insert (3s) should remain let now = 2_000_000; // 2 seconds in microseconds let (updated_buffers, expired_inserts) = process_expired_inserts(&aggregate_buffers, &config, now); - // Verify expired inserts - assert_eq!(expired_inserts.len(), 1); - assert_eq!(expired_inserts[0], old_insert); + // Verify expired inserts (old insert and insert without context) + assert_eq!(expired_inserts.len(), 2); + assert!(expired_inserts.contains(&old_insert)); + assert!(expired_inserts.contains(&insert_no_ctx)); // Verify remaining inserts in buffer assert_eq!(updated_buffers.len(), 1); let remaining_buffer = updated_buffers.get(&aggregate_id).unwrap(); - assert_eq!(remaining_buffer.buffer.len(), 2); + assert_eq!(remaining_buffer.buffer.len(), 1); assert!(remaining_buffer.buffer.contains(&new_insert)); - assert!(remaining_buffer.buffer.contains(&insert_no_ctx)); } } From 8f986df2efafa0323e2d16f129f4dc42a779f8cb Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 14 Jan 2026 07:02:22 +0000 Subject: [PATCH 029/102] add staging mode to persistable --- crates/data/src/persistable.rs | 122 +++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index da50a35e54..29ed0ed0e3 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -112,6 +112,7 @@ pub struct Persistable { data: Option, connector: StoreConnector, ctx: Option>, + staging_mode: bool, } impl Persistable @@ -124,6 +125,7 @@ where data, connector, ctx: None, + staging_mode: false, } } @@ -164,6 +166,10 @@ where } fn write_to_store(&self) { + if self.staging_mode { + return; + } + let Some(ref data) = self.data else { return; }; @@ -227,6 +233,17 @@ where pub fn has(&self) -> bool { self.data.is_some() } + + /// Enter staging mode - changes held in memory only + pub fn stage(&mut self) { + self.staging_mode = true; + } + + /// Commit mode - writes current state and enables persistence + pub fn commit(&mut self) { + self.staging_mode = false; + self.write_to_store(); + } } impl EventContextManager for Persistable { @@ -238,3 +255,108 @@ impl EventContextManager for Persistable { self.ctx = Some(value.clone()) } } + +#[cfg(test)] +mod tests { + use actix::{Actor, Addr, Handler, Message}; + + use crate::{Get, Insert, Remove}; + + use super::{Persistable, StoreConnector}; + + #[derive(Debug, Clone)] + enum Evts { + Get, + Insert(Insert), + Remove, + } + + struct MockConnector { + key: Vec, + events: Vec, + } + #[derive(Message)] + #[rtype("Vec")] + struct GetEvents; + + impl Actor for MockConnector { + type Context = actix::Context; + } + + impl Handler for MockConnector { + type Result = Vec; + fn handle(&mut self, msg: GetEvents, ctx: &mut Self::Context) -> Self::Result { + self.events.clone() + } + } + + impl Handler for MockConnector { + type Result = Option>; + fn handle(&mut self, msg: Get, ctx: &mut Self::Context) -> Self::Result { + self.events.push(Evts::Get); + None + } + } + + impl Handler for MockConnector { + type Result = (); + fn handle(&mut self, msg: Insert, ctx: &mut Self::Context) -> Self::Result { + self.events.push(Evts::Insert(msg)); + } + } + + impl Handler for MockConnector { + type Result = (); + fn handle(&mut self, msg: Remove, ctx: &mut Self::Context) -> Self::Result { + self.events.push(Evts::Remove); + } + } + + impl MockConnector { + fn new(key: impl Into>) -> Self { + Self { + key: key.into(), + events: Vec::new(), + } + } + + fn to_store_connector(self) -> (Addr, StoreConnector) { + let key = self.key.clone(); + let addr = self.start(); + ( + addr.clone(), + StoreConnector::new( + &key, + &addr.clone().recipient(), + &addr.clone().recipient(), + &addr.clone().recipient(), + ), + ) + } + } + + #[actix::test] + async fn test_persistable_staging() { + let (addr, connector) = MockConnector::new(b"loc").to_store_connector(); + let mut p = Persistable::new(Some(42i32), connector); + + p.set(100); + let events = addr.send(GetEvents).await.unwrap(); + assert_eq!(events.len(), 1); + assert!( + matches!(&events[0], Evts::Insert(msg) if msg.value() == &bincode::serialize(&100i32).unwrap()) + ); + + p.stage(); + p.set(200); + let events = addr.send(GetEvents).await.unwrap(); + assert_eq!(events.len(), 1); + + p.commit(); + let events = addr.send(GetEvents).await.unwrap(); + assert_eq!(events.len(), 2); + assert!( + matches!(&events[1], Evts::Insert(msg) if msg.value() == &bincode::serialize(&200i32).unwrap()) + ); + } +} From 37e3ea0fd29664c69a2ecef350478da63c3a04c1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 15 Jan 2026 03:40:50 +0000 Subject: [PATCH 030/102] remove unnecessary nstaging --- crates/data/src/persistable.rs | 1 - crates/data/src/write_buffer.rs | 69 ++++++++++++++--------- crates/events/src/events.rs | 17 ++++-- crates/events/src/sequencer.rs | 5 +- crates/keyshare/src/threshold_keyshare.rs | 4 +- 5 files changed, 58 insertions(+), 38 deletions(-) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 29ed0ed0e3..c396c8f15d 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -242,7 +242,6 @@ where /// Commit mode - writes current state and enables persistence pub fn commit(&mut self) { self.staging_mode = false; - self.write_to_store(); } } diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 6149ec5fd7..19ad35a2f0 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -54,6 +54,45 @@ impl WriteBuffer { config, } } + + fn handle_insert(&mut self, msg: Insert) { + let aggregate_id = if let Some(event_ctx) = msg.ctx() { + event_ctx.aggregate_id().clone() + } else { + AggregateId::new(0) + }; + + let agg_buffer = self + .aggregate_buffers + .entry(aggregate_id) + .or_insert_with(|| AggregateBuffer::new()); + agg_buffer.buffer.push(msg.clone()); + } + + fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { + // Store the sequence number as an Insert message so snapshots hold the most recent event + // they were created against + self.handle_insert(Insert::new( + &format!("//aggregate_seq/{}", msg.aggregate_id()), + msg.seq().to_le_bytes().to_vec(), // Same as bincode avoiding result + )); + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_micros() as u64; + + if let Some(ref dest) = self.dest { + let (updated_buffers, expired_inserts) = + process_expired_inserts(&self.aggregate_buffers, &self.config, now); + if !expired_inserts.is_empty() { + let batch = InsertBatch::new(expired_inserts); + dest.do_send(batch); + } + + self.aggregate_buffers = updated_buffers; + } + } } impl Handler for WriteBuffer { @@ -67,17 +106,7 @@ impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - let aggregate_id = if let Some(event_ctx) = msg.ctx() { - event_ctx.aggregate_id().clone() - } else { - AggregateId::new(0) - }; - - let agg_buffer = self - .aggregate_buffers - .entry(aggregate_id) - .or_insert_with(|| AggregateBuffer::new()); - agg_buffer.buffer.push(msg.clone()); + self.handle_insert(msg) } } @@ -124,22 +153,8 @@ fn process_expired_inserts( impl Handler for WriteBuffer { type Result = (); - fn handle(&mut self, _msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|_| std::time::Duration::from_secs(0)) - .as_micros() as u64; - - if let Some(ref dest) = self.dest { - let (updated_buffers, expired_inserts) = - process_expired_inserts(&self.aggregate_buffers, &self.config, now); - if !expired_inserts.is_empty() { - let batch = InsertBatch::new(expired_inserts); - dest.do_send(batch); - } - - self.aggregate_buffers = updated_buffers; - } + fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { + self.handle_commit_snapshot(msg) } } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 96450e4921..d29f682633 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -6,20 +6,27 @@ use actix::{Message, Recipient}; -use crate::{EnclaveEvent, Sequenced, Unsequenced}; +use crate::{AggregateId, EnclaveEvent, Sequenced, Unsequenced}; /// Direct event received by the snapshot buffer in order to save snapshot to disk #[derive(Message, Debug)] #[rtype("()")] -pub struct CommitSnapshot(u64); +pub struct CommitSnapshot { + seq: u64, + aggregate_id: AggregateId, +} impl CommitSnapshot { - pub fn new(seq: u64) -> Self { - Self(seq) + pub fn new(seq: u64, aggregate_id: AggregateId) -> Self { + Self { seq, aggregate_id } } pub fn seq(&self) -> u64 { - self.0 + self.seq + } + + pub fn aggregate_id(&self) -> AggregateId { + self.aggregate_id } } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index ccb13b7ef9..552f502833 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -8,7 +8,7 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; use crate::{ events::{CommitSnapshot, EventStored, StoreEventRequested}, - EnclaveEvent, EventBus, EventContextSeq, Sequenced, Unsequenced, + EnclaveEvent, EventBus, EventContextAccessors, EventContextSeq, Sequenced, Unsequenced, }; /// Component to sequence the storage of events @@ -49,7 +49,8 @@ impl Handler for Sequencer { fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { let event = msg.into_event(); let seq = event.seq(); - self.buffer.do_send(CommitSnapshot::new(seq)); + self.buffer + .do_send(CommitSnapshot::new(seq, event.aggregate_id())); self.bus.do_send(event) } } diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 4178e11b2b..7156ff159c 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -491,7 +491,6 @@ impl ThresholdKeyshare { }, )) })?; - self.handle_gen_esi_sss_requested(GenEsiSss(current.ciphernode_selected.clone()))?; self.handle_gen_pk_share_and_sk_sss_requested(GenPkShareAndSkSss( current.ciphernode_selected, @@ -666,7 +665,7 @@ impl ThresholdKeyshare { } /// 4. SharesGenerated - Encrypt shares with BFV and publish - pub fn handle_shares_generated(&self) -> Result<()> { + pub fn handle_shares_generated(&mut self) -> Result<()> { let Some(ThresholdKeyshareState { state: KeyshareState::AggregatingDecryptionKey(AggregatingDecryptionKey { @@ -744,7 +743,6 @@ impl ThresholdKeyshare { external: false, })?; } - Ok(()) } From 2158772e3d61375cf6fa4c1dfb24975b0249c6a3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 15 Jan 2026 04:38:03 +0000 Subject: [PATCH 031/102] revert persistable change --- crates/data/src/persistable.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index c396c8f15d..29ed0ed0e3 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -242,6 +242,7 @@ where /// Commit mode - writes current state and enables persistence pub fn commit(&mut self) { self.staging_mode = false; + self.write_to_store(); } } From 22ce1c60b0141388cb31afff7d14a5c61cff99f9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 15 Jan 2026 06:00:34 +0000 Subject: [PATCH 032/102] add per-chain finalization times with optimized provider usage --- .../src/ciphernode_builder.rs | 61 +++++++++++++++++-- crates/ciphernode-builder/src/event_system.rs | 30 ++++++++- crates/config/src/chain_config.rs | 2 + 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 2c09de7996..74fb0f9435 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -294,6 +294,16 @@ impl CiphernodeBuilder { EventBus::::new(EventBusConfig { deduplicate: true }).start() } + async fn validate_chains_and_build_delays( + &self, + provider_cache: &mut ProviderCaches, + ) -> anyhow::Result> { + // Use cached providers instead of creating new ones + provider_cache + .ensure_providers_and_build_delays(&self.chains) + .await + } + pub async fn build(mut self) -> anyhow::Result { // Local bus for ciphernode events can either be forked from a bus or it can be directly // attached to a source bus @@ -336,15 +346,29 @@ impl CiphernodeBuilder { rand_eth_addr(&self.rng) }; + // Create provider cache early to use for chain validation + let mut provider_cache = ProviderCaches::new(); + + // Validate chains and build finalization delays using cached providers + let chain_delays = self + .validate_chains_and_build_delays(&mut provider_cache) + .await?; + // Get an event system instance. let event_system = if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { - EventSystem::persisted(&addr, log_path, kv_path).with_event_bus(local_bus) + EventSystem::persisted(&addr, log_path, kv_path) + .with_event_bus(local_bus) + .with_chain_delays(chain_delays) } else { if let Some(ref store) = self.in_mem_store { - EventSystem::in_mem_from_store(&addr, store).with_event_bus(local_bus) + EventSystem::in_mem_from_store(&addr, store) + .with_event_bus(local_bus) + .with_chain_delays(chain_delays) } else { - EventSystem::in_mem(&addr).with_event_bus(local_bus) + EventSystem::in_mem(&addr) + .with_event_bus(local_bus) + .with_chain_delays(chain_delays) } }; @@ -368,7 +392,6 @@ impl CiphernodeBuilder { CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) .await?; - let mut provider_cache = ProviderCaches::new(); let cipher = &self.cipher; let coordinator = HistoricalEventCoordinator::setup(bus.clone()); @@ -577,6 +600,36 @@ impl ProviderCaches { } } + /// Ensure all chains have cached providers and return chain_id → finalization_delay mapping + pub async fn ensure_providers_and_build_delays( + &mut self, + chains: &[ChainConfig], + ) -> anyhow::Result> { + let mut chain_delays = HashMap::new(); + + for chain in chains { + // Ensure provider is cached and get chain_id in one operation + let provider = self.ensure_read_provider(chain).await?; + let chain_id = provider.chain_id(); + + // Store finalization time if configured + if let Some(finalization_ms) = chain.finalization_ms { + info!( + "Chain {} (ID: {}) finalization time: {}ms", + chain.name, chain_id, finalization_ms + ); + chain_delays.insert(chain_id, finalization_ms * 1000); // ms → microseconds + } else { + info!( + "Chain {} (ID: {}) no finalization time configured", + chain.name, chain_id + ); + } + } + + Ok(chain_delays) + } + pub async fn ensure_signer( &mut self, cipher: &Cipher, diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 30a3c9f663..abc7ba1142 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -12,7 +12,9 @@ use e3_data::{ InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, }; use e3_events::hlc::Hlc; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; +use e3_events::{ + AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer, +}; use once_cell::sync::OnceCell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; @@ -82,6 +84,8 @@ pub struct EventSystem { wired: OnceCell<()>, /// Hlc override hlc: OnceCell, + /// Chain-specific finalization delays (chain_id → delay_us) + chain_delays: OnceCell>, } impl EventSystem { @@ -104,6 +108,7 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), + chain_delays: OnceCell::new(), } } @@ -121,6 +126,7 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), + chain_delays: OnceCell::new(), } } @@ -140,6 +146,7 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), + chain_delays: OnceCell::new(), } } @@ -163,6 +170,12 @@ impl EventSystem { self } + /// Add chain-specific finalization delays (chain_id → delay_us) + pub fn with_chain_delays(self, delays: HashMap) -> Self { + let _ = self.chain_delays.set(delays); + self + } + /// Get the eventbus address pub fn eventbus(&self) -> Addr> { self.eventbus.get_or_init(get_enclave_event_bus).clone() @@ -173,8 +186,19 @@ impl EventSystem { let buffer = self .buffer .get_or_init(|| { - let default_config = HashMap::new(); - WriteBuffer::with_config(default_config).start() + let config = self + .chain_delays + .get() + .map(|delays| { + delays + .iter() + .map(|(chain_id, delay_us)| { + (AggregateId::new(*chain_id as usize), *delay_us) + }) + .collect() + }) + .unwrap_or_default(); + WriteBuffer::with_config(config).start() }) .clone(); self.wire_if_ready(); diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index 6c25192767..f0b15c8dfe 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -21,6 +21,8 @@ pub struct ChainConfig { #[serde(default)] pub rpc_auth: RpcAuth, pub contracts: ContractAddresses, + #[serde(default)] + pub finalization_ms: Option, } impl ChainConfig { From e5b8f965f04a959b2971bc205c1df55c8edc126f Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 00:57:55 +0000 Subject: [PATCH 033/102] feat: add optional chain_id validation to ChainConfig - Add optional chain_id field to ChainConfig struct - Validate chain_id against provider response when specified - Provide clear error message for chain_id mismatches - Remove redundant serde(default) annotations from Option fields --- .../src/ciphernode_builder.rs | 22 +++++++++++++++---- crates/config/src/chain_config.rs | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 74fb0f9435..8e81378a86 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -610,19 +610,33 @@ impl ProviderCaches { for chain in chains { // Ensure provider is cached and get chain_id in one operation let provider = self.ensure_read_provider(chain).await?; - let chain_id = provider.chain_id(); + let actual_chain_id = provider.chain_id(); + + // Validate chain_id if specified in configuration + if let Some(expected_chain_id) = chain.chain_id { + if actual_chain_id != expected_chain_id { + return Err(anyhow::anyhow!( + "Chain '{}' validation failed: expected chain_id {}, but provider returned chain_id {}", + chain.name, expected_chain_id, actual_chain_id + )); + } + info!( + "Chain '{}' (ID: {}) chain_id validation passed", + chain.name, actual_chain_id + ); + } // Store finalization time if configured if let Some(finalization_ms) = chain.finalization_ms { info!( "Chain {} (ID: {}) finalization time: {}ms", - chain.name, chain_id, finalization_ms + chain.name, actual_chain_id, finalization_ms ); - chain_delays.insert(chain_id, finalization_ms * 1000); // ms → microseconds + chain_delays.insert(actual_chain_id, finalization_ms * 1000); // ms → microseconds } else { info!( "Chain {} (ID: {}) no finalization time configured", - chain.name, chain_id + chain.name, actual_chain_id ); } } diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index f0b15c8dfe..eb0a22e586 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -21,8 +21,8 @@ pub struct ChainConfig { #[serde(default)] pub rpc_auth: RpcAuth, pub contracts: ContractAddresses, - #[serde(default)] pub finalization_ms: Option, + pub chain_id: Option, } impl ChainConfig { From a0b3482871c9e8501f5c7ee4292948cf19a59d06 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 01:39:25 +0000 Subject: [PATCH 034/102] replace chain delays with aggregate configuration architecture --- .../src/ciphernode_builder.rs | 122 +++++++++++------- crates/ciphernode-builder/src/event_system.rs | 56 +++++--- 2 files changed, 111 insertions(+), 67 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 8e81378a86..3bf9fb1fad 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -4,6 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::event_system::AggregateConfig; use crate::{CiphernodeHandle, EventSystem}; use actix::{Actor, Addr}; use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; @@ -40,6 +41,66 @@ use rayon::ThreadPool; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; +fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> anyhow::Result<()> { + if let Some(expected_chain_id) = chain.chain_id { + if actual_chain_id != expected_chain_id { + return Err(anyhow::anyhow!( + "Chain '{}' validation failed: expected chain_id {}, but provider returned chain_id {}", + chain.name, expected_chain_id, actual_chain_id + )); + } + info!( + "Chain '{}' (ID: {}) chain_id validation passed", + chain.name, actual_chain_id + ); + } + Ok(()) +} + +fn build_delay_for_chain( + chain: &ChainConfig, + actual_chain_id: u64, +) -> Option<(e3_events::AggregateId, u64)> { + if let Some(finalization_ms) = chain.finalization_ms { + info!( + "Chain {} (ID: {}) finalization time: {}ms", + chain.name, actual_chain_id, finalization_ms + ); + let aggregate_id = e3_events::AggregateId::new(actual_chain_id as usize); + let delay_us = finalization_ms * 1000; // ms → microseconds + Some((aggregate_id, delay_us)) + } else { + info!( + "Chain {} (ID: {}) no finalization time configured", + chain.name, actual_chain_id + ); + None + } +} + +fn build_delays_from_chains<'a>( + chain_providers: &'a [(ChainConfig, EthProvider)], +) -> HashMap { + let mut delays = HashMap::new(); + + for (chain, provider) in chain_providers { + let actual_chain_id = provider.chain_id(); + + // Validate chain_id if specified in configuration + if let Err(e) = validate_chain_id(chain, actual_chain_id) { + error!("Chain validation failed: {}", e); + continue; // Skip this chain and continue with others + } + + // Add delay if configured + if let Some((aggregate_id, delay_us)) = build_delay_for_chain(chain, actual_chain_id) { + delays.insert(aggregate_id, delay_us); + } + } + + delays +} + #[derive(Clone, Debug)] enum EventSystemType { Persisted { log_path: PathBuf, kv_path: PathBuf }, @@ -294,13 +355,13 @@ impl CiphernodeBuilder { EventBus::::new(EventBusConfig { deduplicate: true }).start() } - async fn validate_chains_and_build_delays( + /// Create aggregate configuration from configured chains + async fn create_aggregate_config( &self, provider_cache: &mut ProviderCaches, - ) -> anyhow::Result> { - // Use cached providers instead of creating new ones + ) -> anyhow::Result { provider_cache - .ensure_providers_and_build_delays(&self.chains) + .build_aggregate_config_from_chains(&self.chains) .await } @@ -349,26 +410,23 @@ impl CiphernodeBuilder { // Create provider cache early to use for chain validation let mut provider_cache = ProviderCaches::new(); - // Validate chains and build finalization delays using cached providers - let chain_delays = self - .validate_chains_and_build_delays(&mut provider_cache) - .await?; + let aggregate_config = self.create_aggregate_config(&mut provider_cache).await?; // Get an event system instance. let event_system = if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { EventSystem::persisted(&addr, log_path, kv_path) .with_event_bus(local_bus) - .with_chain_delays(chain_delays) + .with_aggregate_config(aggregate_config) } else { if let Some(ref store) = self.in_mem_store { EventSystem::in_mem_from_store(&addr, store) .with_event_bus(local_bus) - .with_chain_delays(chain_delays) + .with_aggregate_config(aggregate_config.clone()) } else { EventSystem::in_mem(&addr) .with_event_bus(local_bus) - .with_chain_delays(chain_delays) + .with_aggregate_config(aggregate_config.clone()) } }; @@ -600,48 +658,18 @@ impl ProviderCaches { } } - /// Ensure all chains have cached providers and return chain_id → finalization_delay mapping - pub async fn ensure_providers_and_build_delays( + pub async fn build_aggregate_config_from_chains( &mut self, chains: &[ChainConfig], - ) -> anyhow::Result> { - let mut chain_delays = HashMap::new(); - + ) -> anyhow::Result { + let mut chain_providers = Vec::new(); for chain in chains { - // Ensure provider is cached and get chain_id in one operation let provider = self.ensure_read_provider(chain).await?; - let actual_chain_id = provider.chain_id(); - - // Validate chain_id if specified in configuration - if let Some(expected_chain_id) = chain.chain_id { - if actual_chain_id != expected_chain_id { - return Err(anyhow::anyhow!( - "Chain '{}' validation failed: expected chain_id {}, but provider returned chain_id {}", - chain.name, expected_chain_id, actual_chain_id - )); - } - info!( - "Chain '{}' (ID: {}) chain_id validation passed", - chain.name, actual_chain_id - ); - } - - // Store finalization time if configured - if let Some(finalization_ms) = chain.finalization_ms { - info!( - "Chain {} (ID: {}) finalization time: {}ms", - chain.name, actual_chain_id, finalization_ms - ); - chain_delays.insert(actual_chain_id, finalization_ms * 1000); // ms → microseconds - } else { - info!( - "Chain {} (ID: {}) no finalization time configured", - chain.name, actual_chain_id - ); - } + chain_providers.push((chain.clone(), provider)); } - Ok(chain_delays) + let delays = build_delays_from_chains(&chain_providers); + Ok(AggregateConfig::new(delays)) } pub async fn ensure_signer( diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index abc7ba1142..e702bd5e21 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -20,7 +20,30 @@ use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; -/// Hold the InMem EventStore instance and InMemStore +/// Central configuration for aggregates in the EventSystem +#[derive(Debug, Clone)] +pub struct AggregateConfig { + pub delays: HashMap, +} + +impl AggregateConfig { + /// Create a new AggregateConfig with the specified delays + pub fn new(delays: HashMap) -> Self { + Self { delays } + } + + pub fn empty() -> Self { + Self { + delays: HashMap::new(), + } + } + + pub fn with_delay(mut self, aggregate_id: AggregateId, delay_us: u64) -> Self { + self.delays.insert(aggregate_id, delay_us); + self + } +} + struct InMemBackend { eventstore: OnceCell>>, store: OnceCell>, @@ -84,8 +107,8 @@ pub struct EventSystem { wired: OnceCell<()>, /// Hlc override hlc: OnceCell, - /// Chain-specific finalization delays (chain_id → delay_us) - chain_delays: OnceCell>, + /// Central configuration for aggregates, including delays and other settings + aggregate_config: OnceCell, } impl EventSystem { @@ -108,7 +131,7 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), - chain_delays: OnceCell::new(), + aggregate_config: OnceCell::new(), } } @@ -126,7 +149,7 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), - chain_delays: OnceCell::new(), + aggregate_config: OnceCell::new(), } } @@ -146,7 +169,7 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), - chain_delays: OnceCell::new(), + aggregate_config: OnceCell::new(), } } @@ -170,9 +193,9 @@ impl EventSystem { self } - /// Add chain-specific finalization delays (chain_id → delay_us) - pub fn with_chain_delays(self, delays: HashMap) -> Self { - let _ = self.chain_delays.set(delays); + /// Add aggregate configuration including delays and other settings + pub fn with_aggregate_config(self, config: AggregateConfig) -> Self { + let _ = self.aggregate_config.set(config); self } @@ -186,19 +209,12 @@ impl EventSystem { let buffer = self .buffer .get_or_init(|| { - let config = self - .chain_delays + let delays = self + .aggregate_config .get() - .map(|delays| { - delays - .iter() - .map(|(chain_id, delay_us)| { - (AggregateId::new(*chain_id as usize), *delay_us) - }) - .collect() - }) + .map(|config| config.delays.clone()) .unwrap_or_default(); - WriteBuffer::with_config(config).start() + WriteBuffer::with_config(delays).start() }) .clone(); self.wire_if_ready(); From a02e43f250ec943870ef2c2a52703df29070c152 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 02:14:37 +0000 Subject: [PATCH 035/102] refactor: extract build_aggregate_config_from_chains to standalone function Move build_aggregate_config_from_chains and its helper functions from ProviderCaches impl to standalone functions for better modularity and reusability. The functionality remains unchanged. - Extract validate_chain_id, build_delay_for_chain, build_delays_from_chains - Create standalone build_aggregate_config_from_chains with generic provider factory - Update call site to use direct implementation - Remove old method from ProviderCaches --- .../src/ciphernode_builder.rs | 159 +++++++++--------- 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 3bf9fb1fad..c1d334ce10 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -17,7 +17,7 @@ use e3_aggregator::ext::{ use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{InMemStore, Repositories, RepositoriesFactory}; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig}; +use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig}; use e3_evm::{ helpers::{ load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, @@ -37,70 +37,9 @@ use e3_sortition::{ NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; use e3_utils::{rand_eth_addr, SharedRng}; -use rayon::ThreadPool; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; -fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> anyhow::Result<()> { - if let Some(expected_chain_id) = chain.chain_id { - if actual_chain_id != expected_chain_id { - return Err(anyhow::anyhow!( - "Chain '{}' validation failed: expected chain_id {}, but provider returned chain_id {}", - chain.name, expected_chain_id, actual_chain_id - )); - } - info!( - "Chain '{}' (ID: {}) chain_id validation passed", - chain.name, actual_chain_id - ); - } - Ok(()) -} - -fn build_delay_for_chain( - chain: &ChainConfig, - actual_chain_id: u64, -) -> Option<(e3_events::AggregateId, u64)> { - if let Some(finalization_ms) = chain.finalization_ms { - info!( - "Chain {} (ID: {}) finalization time: {}ms", - chain.name, actual_chain_id, finalization_ms - ); - let aggregate_id = e3_events::AggregateId::new(actual_chain_id as usize); - let delay_us = finalization_ms * 1000; // ms → microseconds - Some((aggregate_id, delay_us)) - } else { - info!( - "Chain {} (ID: {}) no finalization time configured", - chain.name, actual_chain_id - ); - None - } -} - -fn build_delays_from_chains<'a>( - chain_providers: &'a [(ChainConfig, EthProvider)], -) -> HashMap { - let mut delays = HashMap::new(); - - for (chain, provider) in chain_providers { - let actual_chain_id = provider.chain_id(); - - // Validate chain_id if specified in configuration - if let Err(e) = validate_chain_id(chain, actual_chain_id) { - error!("Chain validation failed: {}", e); - continue; // Skip this chain and continue with others - } - - // Add delay if configured - if let Some((aggregate_id, delay_us)) = build_delay_for_chain(chain, actual_chain_id) { - delays.insert(aggregate_id, delay_us); - } - } - - delays -} - #[derive(Clone, Debug)] enum EventSystemType { Persisted { log_path: PathBuf, kv_path: PathBuf }, @@ -360,9 +299,14 @@ impl CiphernodeBuilder { &self, provider_cache: &mut ProviderCaches, ) -> anyhow::Result { - provider_cache - .build_aggregate_config_from_chains(&self.chains) - .await + let mut chain_providers = Vec::new(); + for chain in &self.chains { + let provider = provider_cache.ensure_read_provider(chain).await?; + chain_providers.push((chain.clone(), provider)); + } + + let delays = build_delays_from_chains(&chain_providers); + Ok(AggregateConfig::new(delays)) } pub async fn build(mut self) -> anyhow::Result { @@ -649,6 +593,77 @@ struct ProviderCaches { write_provider_cache: HashMap>, } +/// Validate chain ID matches expected configuration +fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { + if let Some(expected_chain_id) = chain.chain_id { + if actual_chain_id != expected_chain_id { + return Err(anyhow::anyhow!( + "Chain '{}' validation failed: expected chain_id {}, but provider returned chain_id {}", + chain.name, expected_chain_id, actual_chain_id + )); + } + info!( + "Chain '{}' (ID: {}) chain_id validation passed", + chain.name, actual_chain_id + ); + } + Ok(()) +} + +/// Build delay configuration for a specific chain +fn build_delay_for_chain(chain: &ChainConfig, actual_chain_id: u64) -> Option<(AggregateId, u64)> { + if let Some(finalization_ms) = chain.finalization_ms { + let aggregate_id = e3_events::AggregateId::new(actual_chain_id as usize); + let delay_us = finalization_ms * 1000; // ms → microseconds + Some((aggregate_id, delay_us)) + } else { + None + } +} + +/// Build delays configuration from chain providers +fn build_delays_from_chains( + chain_providers: &[(ChainConfig, EthProvider)], +) -> HashMap { + let mut delays = HashMap::new(); + + for (chain, provider) in chain_providers { + let actual_chain_id = provider.chain_id(); + + // Validate chain_id if specified in configuration + if let Err(e) = validate_chain_id(chain, actual_chain_id) { + error!("Chain validation failed: {}", e); + continue; // Skip this chain and continue with others + } + + // Add delay if configured + if let Some((aggregate_id, delay_us)) = build_delay_for_chain(chain, actual_chain_id) { + delays.insert(aggregate_id, delay_us); + } + } + + delays +} + +/// Build aggregate configuration from chains with provider factory function +pub async fn build_aggregate_config_from_chains( + chains: &[ChainConfig], + mut provider_factory: F, +) -> anyhow::Result +where + F: FnMut(&ChainConfig) -> Fut, + Fut: std::future::Future>>, +{ + let mut chain_providers = Vec::new(); + for chain in chains { + let provider = provider_factory(chain).await?; + chain_providers.push((chain.clone(), provider)); + } + + let delays = build_delays_from_chains(&chain_providers); + Ok(AggregateConfig::new(delays)) +} + impl ProviderCaches { pub fn new() -> Self { ProviderCaches { @@ -658,20 +673,6 @@ impl ProviderCaches { } } - pub async fn build_aggregate_config_from_chains( - &mut self, - chains: &[ChainConfig], - ) -> anyhow::Result { - let mut chain_providers = Vec::new(); - for chain in chains { - let provider = self.ensure_read_provider(chain).await?; - chain_providers.push((chain.clone(), provider)); - } - - let delays = build_delays_from_chains(&chain_providers); - Ok(AggregateConfig::new(delays)) - } - pub async fn ensure_signer( &mut self, cipher: &Cipher, From a8400ae4653ce70e23ca88ef82c8b65fbc16e5ba Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 02:23:18 +0000 Subject: [PATCH 036/102] fix: resolve non-deprecation warnings - Fix unused variables in test handlers by prefixing with underscore - Move cargo-release configuration keys to proper [workspace.metadata.release] section - Remove unused standalone build_aggregate_config_from_chains function --- Cargo.toml | 2 ++ .../src/ciphernode_builder.rs | 19 ------------------- crates/data/src/persistable.rs | 8 ++++---- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4bb2a832ba..b427042042 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ exclude = [ ] resolver = "3" msrv = "1.86.0" + +[workspace.metadata.release] shared-version = true pre-release-commit-message = "chore: Release {{crate_name}} v{{version}}" pre-release-replacements = [ diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index c1d334ce10..1c9e0f07e1 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -645,25 +645,6 @@ fn build_delays_from_chains( delays } -/// Build aggregate configuration from chains with provider factory function -pub async fn build_aggregate_config_from_chains( - chains: &[ChainConfig], - mut provider_factory: F, -) -> anyhow::Result -where - F: FnMut(&ChainConfig) -> Fut, - Fut: std::future::Future>>, -{ - let mut chain_providers = Vec::new(); - for chain in chains { - let provider = provider_factory(chain).await?; - chain_providers.push((chain.clone(), provider)); - } - - let delays = build_delays_from_chains(&chain_providers); - Ok(AggregateConfig::new(delays)) -} - impl ProviderCaches { pub fn new() -> Self { ProviderCaches { diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 29ed0ed0e3..7247a252b0 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -285,14 +285,14 @@ mod tests { impl Handler for MockConnector { type Result = Vec; - fn handle(&mut self, msg: GetEvents, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, _msg: GetEvents, _ctx: &mut Self::Context) -> Self::Result { self.events.clone() } } impl Handler for MockConnector { type Result = Option>; - fn handle(&mut self, msg: Get, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, _msg: Get, _ctx: &mut Self::Context) -> Self::Result { self.events.push(Evts::Get); None } @@ -300,14 +300,14 @@ mod tests { impl Handler for MockConnector { type Result = (); - fn handle(&mut self, msg: Insert, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: Insert, _ctx: &mut Self::Context) -> Self::Result { self.events.push(Evts::Insert(msg)); } } impl Handler for MockConnector { type Result = (); - fn handle(&mut self, msg: Remove, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, _msg: Remove, _ctx: &mut Self::Context) -> Self::Result { self.events.push(Evts::Remove); } } From 842f11e70410701b77d464f1476e23b91ab506b0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 02:27:05 +0000 Subject: [PATCH 037/102] refactor: remove unused AggregateConfig methods --- crates/ciphernode-builder/src/event_system.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index e702bd5e21..5ccc974f95 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -31,17 +31,6 @@ impl AggregateConfig { pub fn new(delays: HashMap) -> Self { Self { delays } } - - pub fn empty() -> Self { - Self { - delays: HashMap::new(), - } - } - - pub fn with_delay(mut self, aggregate_id: AggregateId, delay_us: u64) -> Self { - self.delays.insert(aggregate_id, delay_us); - self - } } struct InMemBackend { From 68826703ce8abfbec75a33a59a39fa42c49e3858 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 02:32:32 +0000 Subject: [PATCH 038/102] refactor: rename delay functions to use aggregate terminology --- crates/ciphernode-builder/src/ciphernode_builder.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 1c9e0f07e1..2987cfda40 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -305,7 +305,7 @@ impl CiphernodeBuilder { chain_providers.push((chain.clone(), provider)); } - let delays = build_delays_from_chains(&chain_providers); + let delays = create_aggregate_delays(&chain_providers); Ok(AggregateConfig::new(delays)) } @@ -611,7 +611,7 @@ fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { } /// Build delay configuration for a specific chain -fn build_delay_for_chain(chain: &ChainConfig, actual_chain_id: u64) -> Option<(AggregateId, u64)> { +fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> Option<(AggregateId, u64)> { if let Some(finalization_ms) = chain.finalization_ms { let aggregate_id = e3_events::AggregateId::new(actual_chain_id as usize); let delay_us = finalization_ms * 1000; // ms → microseconds @@ -622,7 +622,7 @@ fn build_delay_for_chain(chain: &ChainConfig, actual_chain_id: u64) -> Option<(A } /// Build delays configuration from chain providers -fn build_delays_from_chains( +fn create_aggregate_delays( chain_providers: &[(ChainConfig, EthProvider)], ) -> HashMap { let mut delays = HashMap::new(); @@ -637,7 +637,7 @@ fn build_delays_from_chains( } // Add delay if configured - if let Some((aggregate_id, delay_us)) = build_delay_for_chain(chain, actual_chain_id) { + if let Some((aggregate_id, delay_us)) = create_aggregate_delay(chain, actual_chain_id) { delays.insert(aggregate_id, delay_us); } } From a1ba1e6ea9a25d0310f72737d4f5539f35108cfd Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 03:06:26 +0000 Subject: [PATCH 039/102] refactor: WriteBuffer accepts AggregateConfig instead of HashMap - Move AggregateConfig from ciphernode-builder to data crate - Update WriteBuffer struct to use AggregateConfig field - Modify WriteBuffer::with_config to accept AggregateConfig - Update EventSystem to pass AggregateConfig directly - Improve type safety and encapsulation of aggregate delay configuration --- crates/ciphernode-builder/src/event_system.rs | 25 ++++------------ crates/data/src/write_buffer.rs | 30 ++++++++++++++----- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 5ccc974f95..076ca7fa38 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -12,26 +12,13 @@ use e3_data::{ InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, }; use e3_events::hlc::Hlc; -use e3_events::{ - AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer, -}; +use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; use once_cell::sync::OnceCell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; -/// Central configuration for aggregates in the EventSystem -#[derive(Debug, Clone)] -pub struct AggregateConfig { - pub delays: HashMap, -} - -impl AggregateConfig { - /// Create a new AggregateConfig with the specified delays - pub fn new(delays: HashMap) -> Self { - Self { delays } - } -} +pub use e3_data::AggregateConfig; struct InMemBackend { eventstore: OnceCell>>, @@ -198,12 +185,12 @@ impl EventSystem { let buffer = self .buffer .get_or_init(|| { - let delays = self + let config = self .aggregate_config .get() - .map(|config| config.delays.clone()) - .unwrap_or_default(); - WriteBuffer::with_config(delays).start() + .cloned() + .unwrap_or_else(|| AggregateConfig::new(HashMap::new())); + WriteBuffer::with_config(config).start() }) .clone(); self.wire_if_ready(); diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 19ad35a2f0..0e0b9a5138 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -14,6 +14,19 @@ use std::{ use crate::{Insert, InsertBatch}; +/// Central configuration for aggregates in the WriteBuffer +#[derive(Debug, Clone)] +pub struct AggregateConfig { + pub delays: HashMap, +} + +impl AggregateConfig { + /// Create a new AggregateConfig with the specified delays + pub fn new(delays: HashMap) -> Self { + Self { delays } + } +} + #[derive(Debug)] struct AggregateBuffer { buffer: Vec, @@ -30,8 +43,8 @@ pub struct WriteBuffer { dest: Option>, /// Per-aggregate buffers for organizing inserts aggregate_buffers: HashMap, - /// Per-aggregate wait time (microseconds) before sending inserts to destination - config: HashMap, + /// Per-aggregate wait time configuration + config: AggregateConfig, } impl Actor for WriteBuffer { @@ -43,11 +56,11 @@ impl WriteBuffer { Self { dest: None, aggregate_buffers: HashMap::new(), - config: HashMap::new(), + config: AggregateConfig::new(HashMap::new()), } } - pub fn with_config(config: HashMap) -> Self { + pub fn with_config(config: AggregateConfig) -> Self { Self { dest: None, aggregate_buffers: HashMap::new(), @@ -84,7 +97,7 @@ impl WriteBuffer { if let Some(ref dest) = self.dest { let (updated_buffers, expired_inserts) = - process_expired_inserts(&self.aggregate_buffers, &self.config, now); + process_expired_inserts(&self.aggregate_buffers, &self.config.delays, now); if !expired_inserts.is_empty() { let batch = InsertBatch::new(expired_inserts); dest.do_send(batch); @@ -205,15 +218,16 @@ mod tests { aggregate_buffers.insert(aggregate_id.clone(), agg_buffer); // Set config with 1 second delay - let mut config = HashMap::new(); - config.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds + let mut delays = HashMap::new(); + delays.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds + let config = AggregateConfig::new(delays); // Use current time of 2 seconds, so old insert (0.5s) and insert without context should expire, // new insert (3s) should remain let now = 2_000_000; // 2 seconds in microseconds let (updated_buffers, expired_inserts) = - process_expired_inserts(&aggregate_buffers, &config, now); + process_expired_inserts(&aggregate_buffers, &config.delays, now); // Verify expired inserts (old insert and insert without context) assert_eq!(expired_inserts.len(), 2); From 06559a2a5638aa733c3f5e025479c6021b991bc8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 03:26:12 +0000 Subject: [PATCH 040/102] feat: add aggregate_config accessor method and use get_or_init pattern --- crates/ciphernode-builder/src/event_system.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 076ca7fa38..5f150b0cf1 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -180,16 +180,19 @@ impl EventSystem { self.eventbus.get_or_init(get_enclave_event_bus).clone() } + /// Get the aggregate configuration + pub fn aggregate_config(&self) -> AggregateConfig { + self.aggregate_config + .get_or_init(|| AggregateConfig::new(HashMap::new())) + .clone() + } + /// Get the buffer address pub fn buffer(&self) -> Addr { let buffer = self .buffer .get_or_init(|| { - let config = self - .aggregate_config - .get() - .cloned() - .unwrap_or_else(|| AggregateConfig::new(HashMap::new())); + let config = self.aggregate_config(); WriteBuffer::with_config(config).start() }) .clone(); From d081620af6a8dee9200df9e2a38be470d8197ec8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 03:54:07 +0000 Subject: [PATCH 041/102] feat: add EventStoreRouter for routing events to appropriate stores --- crates/events/src/eventstore_router.rs | 66 ++++++++++++++++++++++++++ crates/events/src/lib.rs | 2 + 2 files changed, 68 insertions(+) create mode 100644 crates/events/src/eventstore_router.rs diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs new file mode 100644 index 0000000000..aedc2692b2 --- /dev/null +++ b/crates/events/src/eventstore_router.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::eventstore::EventStore; +use crate::{ + events::StoreEventRequested, AggregateId, EventContextAccessors, EventLog, SequenceIndex, +}; +use actix::{Actor, Handler}; +use std::collections::HashMap; + +pub struct EventStoreRouter { + stores: HashMap>>, +} + +impl EventStoreRouter { + pub fn new() -> Self { + Self { + stores: HashMap::new(), + } + } + + pub fn register_store( + &mut self, + aggregate_id: AggregateId, + store: actix::Addr>, + ) { + self.stores.insert(aggregate_id, store); + } + + pub fn handle_store_event_requested( + &mut self, + msg: StoreEventRequested, + ) -> Result<(), Box> { + let aggregate_id = msg.event.aggregate_id(); + + let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { + self.stores + .get(&AggregateId::new(0)) + .expect("Default EventStore for AggregateId(0) not found") + }); + + let event = msg.event; + let sender = msg.sender; + + let forwarded_msg = StoreEventRequested::new(event, sender); + store_addr.do_send(forwarded_msg); + Ok(()) + } +} + +impl Actor for EventStoreRouter { + type Context = actix::Context; +} + +impl Handler for EventStoreRouter { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_store_event_requested(msg) { + tracing::error!("Failed to route store event request: {}", e); + } + } +} diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 8d233f80b5..17e606c8f8 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -13,6 +13,7 @@ mod event_id; mod eventbus; mod events; mod eventstore; +mod eventstore_router; pub mod hlc; mod ordered_set; pub mod prelude; @@ -29,6 +30,7 @@ pub use event_id::*; pub use eventbus::*; pub use events::*; pub use eventstore::*; +pub use eventstore_router::*; pub use ordered_set::*; pub use seed::*; pub use sequencer::*; From 4eb2c4444bee5ccc1d9976d561161487081a8440 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 04:28:35 +0000 Subject: [PATCH 042/102] refactor: add enumerate_path utility and update event system to support multiple event stores - Add enumerate_path function to insert index before file extension - Change EventStoreAddr to EventStoreAddrs to support multiple event store addresses - Update internal storage from single EventStore to Vec in both InMem and Persisted backends - Modify TryFrom implementations to handle vector of addresses - Update eventstore method to eventstores returning multiple addresses - Adjust sequencer and test code to work with new vector-based approach --- crates/ciphernode-builder/src/event_system.rs | 118 ++++++++++++++---- 1 file changed, 95 insertions(+), 23 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 5f150b0cf1..ed784802fd 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -20,8 +20,63 @@ use std::path::PathBuf; pub use e3_data::AggregateConfig; +/// Enumerates a PathBuf by inserting an index before the file extension +/// or at the end if there is no extension +/// +/// Examples: +/// - "/foo/bar/thing.pdf" -> "/foo/bar/thing.0.pdf" +/// - "/foo/bar/thing" -> "/foo/bar/thing.0" +pub fn enumerate_path(path: &PathBuf, index: usize) -> PathBuf { + if let Some(parent) = path.parent() { + if let Some(file_name) = path.file_name() { + if let Some(file_name_str) = file_name.to_str() { + if let Some(dot_pos) = file_name_str.rfind('.') { + // Has extension + let (stem, extension) = file_name_str.split_at(dot_pos); + let new_name = format!("{}.{}{}", stem, index, extension); + parent.join(new_name) + } else { + // No extension + let new_name = format!("{}.{}", file_name_str, index); + parent.join(new_name) + } + } else { + // Invalid UTF-8 in filename, append index directly + let new_name = format!("{}.{}", file_name.to_string_lossy(), index); + parent.join(new_name) + } + } else { + // Path ends with '/', just append index + path.join(format!("{}", index)) + } + } else { + // No parent, just modify the filename directly + if let Some(file_name) = path.file_name() { + if let Some(file_name_str) = file_name.to_str() { + if let Some(dot_pos) = file_name_str.rfind('.') { + // Has extension + let (stem, extension) = file_name_str.split_at(dot_pos); + let new_name = format!("{}.{}{}", stem, index, extension); + PathBuf::from(new_name) + } else { + // No extension + let new_name = format!("{}.{}", file_name_str, index); + PathBuf::from(new_name) + } + } else { + // Invalid UTF-8 in filename, append index directly + let new_name = format!("{}.{}", file_name.to_string_lossy(), index); + PathBuf::from(new_name) + } + } else { + // Empty path, just return the index as a path + PathBuf::from(format!("{}", index)) + } + } +} + struct InMemBackend { - eventstore: OnceCell>>, + eventstore: OnceCell>>>, store: OnceCell>, } @@ -29,7 +84,7 @@ struct InMemBackend { struct PersistedBackend { log_path: PathBuf, sled_path: PathBuf, - eventstore: OnceCell>>, + eventstore: OnceCell>>>, store: OnceCell>, } @@ -39,16 +94,20 @@ enum EventSystemBackend { Persisted(PersistedBackend), } -pub enum EventStoreAddr { - InMem(Addr>), - Persisted(Addr>), +pub enum EventStoreAddrs { + InMem(Vec>>), + Persisted(Vec>>), } -impl TryFrom for Addr> { +impl TryFrom for Addr> { type Error = anyhow::Error; - fn try_from(value: EventStoreAddr) -> std::result::Result { - if let EventStoreAddr::InMem(addr) = value { - Ok(addr) + fn try_from(value: EventStoreAddrs) -> std::result::Result { + if let EventStoreAddrs::InMem(mut addrs) = value { + if let Some(addr) = addrs.pop() { + Ok(addr) + } else { + Err(anyhow!("InMem event store addresses vector is empty")) + } } else { Err(anyhow!( "address was not EventStore" @@ -203,39 +262,52 @@ impl EventSystem { /// Get the sequencer address pub fn sequencer(&self) -> Result> { self.sequencer - .get_or_try_init(|| match self.eventstore()? { - EventStoreAddr::InMem(es) => { + .get_or_try_init(|| match self.eventstores()? { + EventStoreAddrs::InMem(addrs) => { + let es = addrs + .get(0) + .ok_or_else(|| anyhow!("No event stores available"))? + .clone(); Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) } - EventStoreAddr::Persisted(es) => { + EventStoreAddrs::Persisted(addrs) => { + let es = addrs + .get(0) + .ok_or_else(|| anyhow!("No event stores available"))? + .clone(); Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) } }) .cloned() } - /// Get the EventStore address - pub fn eventstore(&self) -> Result { + /// Get the EventStore addresses + pub fn eventstores(&self) -> Result { match &self.backend { EventSystemBackend::InMem(b) => { - let addr = b + let addrs = b .eventstore .get_or_init(|| { - EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()).start() + vec![ + EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()) + .start(), + ] }) .clone(); - Ok(EventStoreAddr::InMem(addr)) + Ok(EventStoreAddrs::InMem(addrs)) } EventSystemBackend::Persisted(b) => { - let addr = b + let addrs = b .eventstore .get_or_try_init(|| -> Result<_> { + // For now, create only one event store with the original sequence_index tree name + // In the future, this could be extended to create multiple stores with enumerated names let index = SledSequenceIndex::new(&b.sled_path, "sequence_index")?; let log = CommitLogEventLog::new(&b.log_path)?; - Ok(EventStore::new(index, log).start()) + Ok(vec![EventStore::new(index, log).start()]) })? .clone(); - Ok(EventStoreAddr::Persisted(addr)) + Ok(EventStoreAddrs::Persisted(addrs)) } } } @@ -417,7 +489,7 @@ mod tests { let system = EventSystem::in_mem("cn1").with_fresh_bus(); let handle = system.handle()?; let datastore = system.store()?; - let eventstore = system.eventstore()?; + let eventstores = system.eventstores()?; let listener = Listener { logs: Vec::new(), events: Vec::new(), @@ -479,8 +551,8 @@ mod tests { let logs = listener.send(GetLogs).await?; assert_eq!(logs, vec!["pink", "yellow", "red", "white"]); - // Get the in mem address for the event store - let es: Addr> = eventstore.try_into()?; + // Get the in mem address for the event store (using index 0) + let es: Addr> = eventstores.try_into()?; // Get all events after the given timestamp and send them to the listener es.do_send(GetEventsAfter::new(ts, listener.clone())); From 06210794ae92569fd3afc74be89c596f1197545c Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 05:00:21 +0000 Subject: [PATCH 043/102] feat: replace Vec with HashMap for event store addresses to support indexed access - Update InMemBackend and PersistedBackend to use HashMap instead of Vec - Add Clone derive to EventStoreAddrs enum - Implement TryFrom for SledSequenceIndex EventStoreAddrs conversion - Update eventstores() method to create indexed event stores based on aggregate config - Add path and tree name enumeration for multiple persistent event stores - Add comprehensive test for multiple event stores functionality --- crates/ciphernode-builder/src/event_system.rs | 135 +++++++++++++++--- 1 file changed, 116 insertions(+), 19 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index ed784802fd..834b153139 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -76,7 +76,7 @@ pub fn enumerate_path(path: &PathBuf, index: usize) -> PathBuf { } struct InMemBackend { - eventstore: OnceCell>>>, + eventstore: OnceCell>>>, store: OnceCell>, } @@ -84,7 +84,7 @@ struct InMemBackend { struct PersistedBackend { log_path: PathBuf, sled_path: PathBuf, - eventstore: OnceCell>>>, + eventstore: OnceCell>>>, store: OnceCell>, } @@ -94,19 +94,20 @@ enum EventSystemBackend { Persisted(PersistedBackend), } +#[derive(Clone)] pub enum EventStoreAddrs { - InMem(Vec>>), - Persisted(Vec>>), + InMem(HashMap>>), + Persisted(HashMap>>), } impl TryFrom for Addr> { type Error = anyhow::Error; fn try_from(value: EventStoreAddrs) -> std::result::Result { - if let EventStoreAddrs::InMem(mut addrs) = value { - if let Some(addr) = addrs.pop() { - Ok(addr) + if let EventStoreAddrs::InMem(addrs) = value { + if let Some(addr) = addrs.get(&0) { + Ok(addr.clone()) } else { - Err(anyhow!("InMem event store addresses vector is empty")) + Err(anyhow!("InMem event store addresses hashmap is empty")) } } else { Err(anyhow!( @@ -116,6 +117,23 @@ impl TryFrom for Addr for Addr> { + type Error = anyhow::Error; + fn try_from(value: EventStoreAddrs) -> std::result::Result { + if let EventStoreAddrs::Persisted(addrs) = value { + if let Some(addr) = addrs.get(&0) { + Ok(addr.clone()) + } else { + Err(anyhow!("Persisted event store addresses hashmap is empty")) + } + } else { + Err(anyhow!( + "address was not EventStore" + )) + } + } +} + /// EventSystem holds interconnected references to the components that manage events and /// persistence within the node. The EventSystem connects: /// @@ -265,14 +283,14 @@ impl EventSystem { .get_or_try_init(|| match self.eventstores()? { EventStoreAddrs::InMem(addrs) => { let es = addrs - .get(0) + .get(&0) .ok_or_else(|| anyhow!("No event stores available"))? .clone(); Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) } EventStoreAddrs::Persisted(addrs) => { let es = addrs - .get(0) + .get(&0) .ok_or_else(|| anyhow!("No event stores available"))? .clone(); Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) @@ -285,26 +303,50 @@ impl EventSystem { pub fn eventstores(&self) -> Result { match &self.backend { EventSystemBackend::InMem(b) => { + let config = self.aggregate_config(); + let indexes: Vec = config.delays.keys().map(|id| **id).collect(); + let indexes = if indexes.is_empty() { vec![0] } else { indexes }; + let addrs = b .eventstore .get_or_init(|| { - vec![ - EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()) - .start(), - ] + let mut eventstore_map = std::collections::HashMap::new(); + for &index in &indexes { + eventstore_map.insert( + index, + EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()) + .start(), + ); + } + eventstore_map }) .clone(); Ok(EventStoreAddrs::InMem(addrs)) } EventSystemBackend::Persisted(b) => { + let config = self.aggregate_config(); + let indexes: Vec = config.delays.keys().map(|id| **id).collect(); + let indexes = if indexes.is_empty() { vec![0] } else { indexes }; + let addrs = b .eventstore .get_or_try_init(|| -> Result<_> { - // For now, create only one event store with the original sequence_index tree name - // In the future, this could be extended to create multiple stores with enumerated names - let index = SledSequenceIndex::new(&b.sled_path, "sequence_index")?; - let log = CommitLogEventLog::new(&b.log_path)?; - Ok(vec![EventStore::new(index, log).start()]) + let mut eventstore_map = std::collections::HashMap::new(); + for &index in &indexes { + // Enumerate the log path for each eventstore + let enumerated_log_path = enumerate_path(&b.log_path, index); + // Enumerate the sequence_index tree name for each eventstore + let tree_name = if index == 0 { + "sequence_index".to_string() + } else { + format!("sequence_index.{}", index) + }; + + let index_store = SledSequenceIndex::new(&b.sled_path, &tree_name)?; + let log = CommitLogEventLog::new(&enumerated_log_path)?; + eventstore_map.insert(index, EventStore::new(index_store, log).start()); + } + Ok(eventstore_map) })? .clone(); Ok(EventStoreAddrs::Persisted(addrs)) @@ -563,4 +605,59 @@ mod tests { assert_eq!(events, vec!["yellow", "red", "white"]); Ok(()) } + + #[actix::test] + async fn test_multiple_eventstores() -> Result<()> { + use e3_events::AggregateId; + + // Create an AggregateConfig with multiple AggregateIds + let mut delays = std::collections::HashMap::new(); + delays.insert(AggregateId::new(0), 1000); // 1ms delay + delays.insert(AggregateId::new(1), 2000); // 2ms delay + delays.insert(AggregateId::new(2), 3000); // 3ms delay + let aggregate_config = AggregateConfig::new(delays); + + // Test in-memory eventstores + let system = EventSystem::in_mem("test_multi").with_aggregate_config(aggregate_config); + let eventstores = system.eventstores()?; + + // Should create 3 eventstores for 3 AggregateIds + match &eventstores { + EventStoreAddrs::InMem(addrs) => { + assert_eq!(addrs.len(), 3); + // Test that we can access the first eventstore (index 0) + assert!(addrs.contains_key(&0)); + assert!(addrs.contains_key(&1)); + assert!(addrs.contains_key(&2)); + // Test that we can access the first eventstore + let _es: Addr> = + eventstores.clone().try_into()?; + } + EventStoreAddrs::Persisted(_) => panic!("Expected InMem eventstores"), + } + + // Test persistent eventstores + let tmp = TempDir::new().unwrap(); + let persisted_system = EventSystem::persisted( + "test_persisted", + tmp.path().join("log"), + tmp.path().join("sled"), + ) + .with_aggregate_config(AggregateConfig::new(std::collections::HashMap::new())); + + let persisted_eventstores = persisted_system.eventstores()?; + + // Should create at least 1 eventstore (even with empty config) + match &persisted_eventstores { + EventStoreAddrs::Persisted(addrs) => { + assert_eq!(addrs.len(), 1); + assert!(addrs.contains_key(&0)); + let _es: Addr> = + persisted_eventstores.clone().try_into()?; + } + EventStoreAddrs::InMem(_) => panic!("Expected Persisted eventstores"), + } + + Ok(()) + } } From 679a300b91706d778e90877a64a7ad75ffbdb85e Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 05:11:29 +0000 Subject: [PATCH 044/102] refactor: move enumerate_path utility to utils crate and simplify HashMap type references --- crates/ciphernode-builder/src/event_system.rs | 64 ++----------- crates/utils/src/lib.rs | 2 + crates/utils/src/path.rs | 89 +++++++++++++++++++ 3 files changed, 96 insertions(+), 59 deletions(-) create mode 100644 crates/utils/src/path.rs diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 834b153139..4799cf2699 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -13,6 +13,7 @@ use e3_data::{ }; use e3_events::hlc::Hlc; use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; +use e3_utils::enumerate_path; use once_cell::sync::OnceCell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; @@ -20,61 +21,6 @@ use std::path::PathBuf; pub use e3_data::AggregateConfig; -/// Enumerates a PathBuf by inserting an index before the file extension -/// or at the end if there is no extension -/// -/// Examples: -/// - "/foo/bar/thing.pdf" -> "/foo/bar/thing.0.pdf" -/// - "/foo/bar/thing" -> "/foo/bar/thing.0" -pub fn enumerate_path(path: &PathBuf, index: usize) -> PathBuf { - if let Some(parent) = path.parent() { - if let Some(file_name) = path.file_name() { - if let Some(file_name_str) = file_name.to_str() { - if let Some(dot_pos) = file_name_str.rfind('.') { - // Has extension - let (stem, extension) = file_name_str.split_at(dot_pos); - let new_name = format!("{}.{}{}", stem, index, extension); - parent.join(new_name) - } else { - // No extension - let new_name = format!("{}.{}", file_name_str, index); - parent.join(new_name) - } - } else { - // Invalid UTF-8 in filename, append index directly - let new_name = format!("{}.{}", file_name.to_string_lossy(), index); - parent.join(new_name) - } - } else { - // Path ends with '/', just append index - path.join(format!("{}", index)) - } - } else { - // No parent, just modify the filename directly - if let Some(file_name) = path.file_name() { - if let Some(file_name_str) = file_name.to_str() { - if let Some(dot_pos) = file_name_str.rfind('.') { - // Has extension - let (stem, extension) = file_name_str.split_at(dot_pos); - let new_name = format!("{}.{}{}", stem, index, extension); - PathBuf::from(new_name) - } else { - // No extension - let new_name = format!("{}.{}", file_name_str, index); - PathBuf::from(new_name) - } - } else { - // Invalid UTF-8 in filename, append index directly - let new_name = format!("{}.{}", file_name.to_string_lossy(), index); - PathBuf::from(new_name) - } - } else { - // Empty path, just return the index as a path - PathBuf::from(format!("{}", index)) - } - } -} - struct InMemBackend { eventstore: OnceCell>>>, store: OnceCell>, @@ -310,7 +256,7 @@ impl EventSystem { let addrs = b .eventstore .get_or_init(|| { - let mut eventstore_map = std::collections::HashMap::new(); + let mut eventstore_map = HashMap::new(); for &index in &indexes { eventstore_map.insert( index, @@ -331,7 +277,7 @@ impl EventSystem { let addrs = b .eventstore .get_or_try_init(|| -> Result<_> { - let mut eventstore_map = std::collections::HashMap::new(); + let mut eventstore_map = HashMap::new(); for &index in &indexes { // Enumerate the log path for each eventstore let enumerated_log_path = enumerate_path(&b.log_path, index); @@ -611,7 +557,7 @@ mod tests { use e3_events::AggregateId; // Create an AggregateConfig with multiple AggregateIds - let mut delays = std::collections::HashMap::new(); + let mut delays = HashMap::new(); delays.insert(AggregateId::new(0), 1000); // 1ms delay delays.insert(AggregateId::new(1), 2000); // 2ms delay delays.insert(AggregateId::new(2), 3000); // 3ms delay @@ -643,7 +589,7 @@ mod tests { tmp.path().join("log"), tmp.path().join("sled"), ) - .with_aggregate_config(AggregateConfig::new(std::collections::HashMap::new())); + .with_aggregate_config(AggregateConfig::new(HashMap::new())); let persisted_eventstores = persisted_system.eventstores()?; diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 761c59c4f4..ec17ce7a9c 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -8,11 +8,13 @@ pub mod actix; pub mod alloy; pub mod formatters; pub mod helpers; +pub mod path; pub mod retry; pub mod utility_types; pub use actix::*; pub use alloy::*; pub use formatters::*; pub use helpers::*; +pub use path::*; pub use retry::*; pub use utility_types::*; diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs new file mode 100644 index 0000000000..a335089e6d --- /dev/null +++ b/crates/utils/src/path.rs @@ -0,0 +1,89 @@ +use std::path::PathBuf; + +/// Enumerates a PathBuf by inserting an index before the file extension +/// or at the end if there is no extension +/// +/// Examples: +/// - "/foo/bar/thing.pdf" -> "/foo/bar/thing.0.pdf" +/// - "/foo/bar/thing" -> "/foo/bar/thing.0" +pub fn enumerate_path(path: &PathBuf, index: usize) -> PathBuf { + if let Some(parent) = path.parent() { + if let Some(file_name) = path.file_name() { + if let Some(file_name_str) = file_name.to_str() { + if let Some(dot_pos) = file_name_str.rfind('.') { + // Has extension + let (stem, extension) = file_name_str.split_at(dot_pos); + let new_name = format!("{}.{}{}", stem, index, extension); + parent.join(new_name) + } else { + // No extension + let new_name = format!("{}.{}", file_name_str, index); + parent.join(new_name) + } + } else { + // Invalid UTF-8 in filename, append index directly + let new_name = format!("{}.{}", file_name.to_string_lossy(), index); + parent.join(new_name) + } + } else { + // Path ends with '/', just append index + path.join(format!("{}", index)) + } + } else { + // No parent, just modify the filename directly + if let Some(file_name) = path.file_name() { + if let Some(file_name_str) = file_name.to_str() { + if let Some(dot_pos) = file_name_str.rfind('.') { + // Has extension + let (stem, extension) = file_name_str.split_at(dot_pos); + let new_name = format!("{}.{}{}", stem, index, extension); + PathBuf::from(new_name) + } else { + // No extension + let new_name = format!("{}.{}", file_name_str, index); + PathBuf::from(new_name) + } + } else { + // Invalid UTF-8 in filename, append index directly + let new_name = format!("{}.{}", file_name.to_string_lossy(), index); + PathBuf::from(new_name) + } + } else { + // Empty path, just return the index as a path + PathBuf::from(format!("{}", index)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_enumerate_path_with_extension() { + let path = PathBuf::from("/foo/bar/thing.pdf"); + let result = enumerate_path(&path, 0); + assert_eq!(result, PathBuf::from("/foo/bar/thing.0.pdf")); + } + + #[test] + fn test_enumerate_path_without_extension() { + let path = PathBuf::from("/foo/bar/thing"); + let result = enumerate_path(&path, 5); + assert_eq!(result, PathBuf::from("/foo/bar/thing.5")); + } + + #[test] + fn test_enumerate_path_no_parent() { + let path = PathBuf::from("thing.txt"); + let result = enumerate_path(&path, 1); + assert_eq!(result, PathBuf::from("thing.1.txt")); + } + + #[test] + fn test_enumerate_path_empty() { + let path = PathBuf::from(""); + let result = enumerate_path(&path, 2); + assert_eq!(result, PathBuf::from("2")); + } +} From d14c6a93259468f4480342eb1934653d545383d7 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 05:14:13 +0000 Subject: [PATCH 045/102] headers --- crates/utils/src/path.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs index a335089e6d..163f32790f 100644 --- a/crates/utils/src/path.rs +++ b/crates/utils/src/path.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use std::path::PathBuf; /// Enumerates a PathBuf by inserting an index before the file extension From da9f9fb23b8f233c6a014407c6c137d5b54f0890 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 05:41:15 +0000 Subject: [PATCH 046/102] refactor: add eventstore router with caching and rename eventstore fields - Add cached_eventstores field for idempotency and performance - Add eventstore_router() method to create configured routers - Rename eventstore field to eventstores for consistency - Clean up imports by using Addr directly --- crates/ciphernode-builder/src/event_system.rs | 140 +++++++++++------- crates/events/src/eventstore_router.rs | 8 +- 2 files changed, 88 insertions(+), 60 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 4799cf2699..cafd187bb5 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -12,7 +12,10 @@ use e3_data::{ InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, }; use e3_events::hlc::Hlc; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; +use e3_events::{ + AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, + Sequencer, StoreEventRequested, +}; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; use std::collections::HashMap; @@ -22,7 +25,7 @@ use std::path::PathBuf; pub use e3_data::AggregateConfig; struct InMemBackend { - eventstore: OnceCell>>>, + eventstores: OnceCell>>>, store: OnceCell>, } @@ -30,7 +33,7 @@ struct InMemBackend { struct PersistedBackend { log_path: PathBuf, sled_path: PathBuf, - eventstore: OnceCell>>>, + eventstores: OnceCell>>>, store: OnceCell>, } @@ -108,6 +111,8 @@ pub struct EventSystem { hlc: OnceCell, /// Central configuration for aggregates, including delays and other settings aggregate_config: OnceCell, + /// Cached EventStoreAddrs for idempotency + cached_eventstores: OnceCell, } impl EventSystem { @@ -121,7 +126,7 @@ impl EventSystem { Self { node_id: EventSystem::node_id(node_id), backend: EventSystemBackend::InMem(InMemBackend { - eventstore: OnceCell::new(), + eventstores: OnceCell::new(), store: OnceCell::new(), }), buffer: OnceCell::new(), @@ -131,6 +136,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), + cached_eventstores: OnceCell::new(), } } @@ -139,7 +145,7 @@ impl EventSystem { Self { node_id: EventSystem::node_id(node_id), backend: EventSystemBackend::InMem(InMemBackend { - eventstore: OnceCell::new(), + eventstores: OnceCell::new(), store: OnceCell::from(store.to_owned()), }), buffer: OnceCell::new(), @@ -149,6 +155,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), + cached_eventstores: OnceCell::new(), } } @@ -159,7 +166,7 @@ impl EventSystem { backend: EventSystemBackend::Persisted(PersistedBackend { log_path, sled_path, - eventstore: OnceCell::new(), + eventstores: OnceCell::new(), store: OnceCell::new(), }), buffer: OnceCell::new(), @@ -169,6 +176,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), + cached_eventstores: OnceCell::new(), } } @@ -247,55 +255,79 @@ impl EventSystem { /// Get the EventStore addresses pub fn eventstores(&self) -> Result { - match &self.backend { - EventSystemBackend::InMem(b) => { - let config = self.aggregate_config(); - let indexes: Vec = config.delays.keys().map(|id| **id).collect(); - let indexes = if indexes.is_empty() { vec![0] } else { indexes }; - - let addrs = b - .eventstore - .get_or_init(|| { - let mut eventstore_map = HashMap::new(); - for &index in &indexes { - eventstore_map.insert( - index, - EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()) - .start(), - ); - } - eventstore_map - }) - .clone(); - Ok(EventStoreAddrs::InMem(addrs)) + self.cached_eventstores + .get_or_try_init(|| { + match &self.backend { + EventSystemBackend::InMem(b) => { + let config = self.aggregate_config(); + let indexes: Vec = config.delays.keys().map(|id| **id).collect(); + let indexes = if indexes.is_empty() { vec![0] } else { indexes }; + + let addrs = b + .eventstores + .get_or_init(|| { + let mut eventstore_map = HashMap::new(); + for &index in &indexes { + eventstore_map.insert( + index, + EventStore::new( + InMemSequenceIndex::new(), + InMemEventLog::new(), + ) + .start(), + ); + } + eventstore_map + }) + .clone(); + Ok(EventStoreAddrs::InMem(addrs)) + } + EventSystemBackend::Persisted(b) => { + let config = self.aggregate_config(); + let indexes: Vec = config.delays.keys().map(|id| **id).collect(); + let indexes = if indexes.is_empty() { vec![0] } else { indexes }; + + let addrs = b + .eventstores + .get_or_try_init(|| -> Result<_> { + let mut eventstore_map = HashMap::new(); + for &index in &indexes { + // Enumerate the log path for each eventstore + let enumerated_log_path = enumerate_path(&b.log_path, index); + let tree_name = format!("sequence_index.{}", index); + let index_store = + SledSequenceIndex::new(&b.sled_path, &tree_name)?; + let log = CommitLogEventLog::new(&enumerated_log_path)?; + eventstore_map + .insert(index, EventStore::new(index_store, log).start()); + } + Ok(eventstore_map) + })? + .clone(); + Ok(EventStoreAddrs::Persisted(addrs)) + } + } + }) + .cloned() + } + + /// Get an EventStoreRouter + pub fn eventstore_router(&self) -> Result> { + let eventstores = self.eventstores()?; + match eventstores { + EventStoreAddrs::InMem(addrs) => { + let mut router = EventStoreRouter::new(); + for (index, addr) in addrs { + router.register_store(AggregateId::new(index), addr); + } + Ok(router.start().recipient()) } - EventSystemBackend::Persisted(b) => { - let config = self.aggregate_config(); - let indexes: Vec = config.delays.keys().map(|id| **id).collect(); - let indexes = if indexes.is_empty() { vec![0] } else { indexes }; - - let addrs = b - .eventstore - .get_or_try_init(|| -> Result<_> { - let mut eventstore_map = HashMap::new(); - for &index in &indexes { - // Enumerate the log path for each eventstore - let enumerated_log_path = enumerate_path(&b.log_path, index); - // Enumerate the sequence_index tree name for each eventstore - let tree_name = if index == 0 { - "sequence_index".to_string() - } else { - format!("sequence_index.{}", index) - }; - - let index_store = SledSequenceIndex::new(&b.sled_path, &tree_name)?; - let log = CommitLogEventLog::new(&enumerated_log_path)?; - eventstore_map.insert(index, EventStore::new(index_store, log).start()); - } - Ok(eventstore_map) - })? - .clone(); - Ok(EventStoreAddrs::Persisted(addrs)) + EventStoreAddrs::Persisted(addrs) => { + let mut router = EventStoreRouter::new(); + for (index, addr) in addrs { + router.register_store(AggregateId::new(index), addr); + } + Ok(router.start().recipient()) } } } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index aedc2692b2..eae975a224 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -8,7 +8,7 @@ use crate::eventstore::EventStore; use crate::{ events::StoreEventRequested, AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; -use actix::{Actor, Handler}; +use actix::{Actor, Addr, Handler}; use std::collections::HashMap; pub struct EventStoreRouter { @@ -22,11 +22,7 @@ impl EventStoreRouter { } } - pub fn register_store( - &mut self, - aggregate_id: AggregateId, - store: actix::Addr>, - ) { + pub fn register_store(&mut self, aggregate_id: AggregateId, store: Addr>) { self.stores.insert(aggregate_id, store); } From 20a69e238426fac87c52889ddfa6e81eb19e2fea Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 05:42:00 +0000 Subject: [PATCH 047/102] remove cached prefix from eventstores for simplicity --- crates/ciphernode-builder/src/event_system.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index cafd187bb5..c2f6c0332c 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -112,7 +112,7 @@ pub struct EventSystem { /// Central configuration for aggregates, including delays and other settings aggregate_config: OnceCell, /// Cached EventStoreAddrs for idempotency - cached_eventstores: OnceCell, + eventstores: OnceCell, } impl EventSystem { @@ -136,7 +136,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), - cached_eventstores: OnceCell::new(), + eventstores: OnceCell::new(), } } @@ -155,7 +155,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), - cached_eventstores: OnceCell::new(), + eventstores: OnceCell::new(), } } @@ -176,7 +176,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), - cached_eventstores: OnceCell::new(), + eventstores: OnceCell::new(), } } @@ -255,7 +255,7 @@ impl EventSystem { /// Get the EventStore addresses pub fn eventstores(&self) -> Result { - self.cached_eventstores + self.eventstores .get_or_try_init(|| { match &self.backend { EventSystemBackend::InMem(b) => { From 2b0f1c17012ae63e993e43061a4f5dd5dfb13ae1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 05:45:19 +0000 Subject: [PATCH 048/102] refactor: move indexed ID logic to AggregateConfig - Add indexed_ids() method to AggregateConfig for getting IDs with default fallback - Replace manual index extraction and empty check in event system - Centralize the logic for handling empty delay configurations --- crates/ciphernode-builder/src/event_system.rs | 6 ++---- crates/data/src/write_buffer.rs | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index c2f6c0332c..5f7cdfdab2 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -260,8 +260,7 @@ impl EventSystem { match &self.backend { EventSystemBackend::InMem(b) => { let config = self.aggregate_config(); - let indexes: Vec = config.delays.keys().map(|id| **id).collect(); - let indexes = if indexes.is_empty() { vec![0] } else { indexes }; + let indexes = config.indexed_ids(); let addrs = b .eventstores @@ -284,8 +283,7 @@ impl EventSystem { } EventSystemBackend::Persisted(b) => { let config = self.aggregate_config(); - let indexes: Vec = config.delays.keys().map(|id| **id).collect(); - let indexes = if indexes.is_empty() { vec![0] } else { indexes }; + let indexes = config.indexed_ids(); let addrs = b .eventstores diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 0e0b9a5138..d6d0341e45 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -25,6 +25,16 @@ impl AggregateConfig { pub fn new(delays: HashMap) -> Self { Self { delays } } + + /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured + pub fn indexed_ids(&self) -> Vec { + let indexes: Vec = self.delays.keys().map(|id| **id).collect(); + if indexes.is_empty() { + vec![0] + } else { + indexes + } + } } #[derive(Debug)] From 66478cdd3189b3dfe066e5f08c5af06b0d0e4a59 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 06:15:19 +0000 Subject: [PATCH 049/102] refactor: simplify sequencer initialization using eventstore router --- crates/ciphernode-builder/src/event_system.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 5f7cdfdab2..445aa5134d 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -234,21 +234,9 @@ impl EventSystem { /// Get the sequencer address pub fn sequencer(&self) -> Result> { self.sequencer - .get_or_try_init(|| match self.eventstores()? { - EventStoreAddrs::InMem(addrs) => { - let es = addrs - .get(&0) - .ok_or_else(|| anyhow!("No event stores available"))? - .clone(); - Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) - } - EventStoreAddrs::Persisted(addrs) => { - let es = addrs - .get(&0) - .ok_or_else(|| anyhow!("No event stores available"))? - .clone(); - Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) - } + .get_or_try_init(|| { + let router = self.eventstore_router()?; + Ok(Sequencer::new(&self.eventbus(), router, self.buffer()).start()) }) .cloned() } From 2cfded82566b73004bb3518f355efb17c668053d Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 07:43:50 +0000 Subject: [PATCH 050/102] refactor: simplify aggregate config creation and improve error handling - Simplify create_aggregate_delay to always return a value instead of Option - Improve error propagation in create_aggregate_delays by returning Result - Add to_usize() method to AggregateId for cleaner ID conversion - Ensure AggregateId 0 is always present in AggregateConfig with delay 0 - Remove verbose chain_id validation logging --- .../src/ciphernode_builder.rs | 43 +++++++------------ crates/data/src/write_buffer.rs | 13 +++--- crates/events/src/event_context.rs | 4 ++ 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 2987cfda40..8a49a4023e 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -298,14 +298,14 @@ impl CiphernodeBuilder { async fn create_aggregate_config( &self, provider_cache: &mut ProviderCaches, - ) -> anyhow::Result { + ) -> Result { let mut chain_providers = Vec::new(); for chain in &self.chains { let provider = provider_cache.ensure_read_provider(chain).await?; - chain_providers.push((chain.clone(), provider)); + chain_providers.push((chain.clone(), provider.chain_id())); } - let delays = create_aggregate_delays(&chain_providers); + let delays = create_aggregate_delays(&chain_providers)?; Ok(AggregateConfig::new(delays)) } @@ -602,47 +602,34 @@ fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { chain.name, expected_chain_id, actual_chain_id )); } - info!( - "Chain '{}' (ID: {}) chain_id validation passed", - chain.name, actual_chain_id - ); } Ok(()) } /// Build delay configuration for a specific chain -fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> Option<(AggregateId, u64)> { - if let Some(finalization_ms) = chain.finalization_ms { - let aggregate_id = e3_events::AggregateId::new(actual_chain_id as usize); - let delay_us = finalization_ms * 1000; // ms → microseconds - Some((aggregate_id, delay_us)) - } else { - None - } +fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, u64) { + let aggregate_id = AggregateId::new(actual_chain_id as usize); + let finalization_ms = chain.finalization_ms.unwrap_or(0); + let delay_us = finalization_ms * 1000; // ms → microseconds + (aggregate_id, delay_us) } /// Build delays configuration from chain providers fn create_aggregate_delays( - chain_providers: &[(ChainConfig, EthProvider)], -) -> HashMap { + chain_providers: &[(ChainConfig, u64)], +) -> Result> { let mut delays = HashMap::new(); - for (chain, provider) in chain_providers { - let actual_chain_id = provider.chain_id(); - + for (chain, actual_chain_id) in chain_providers.into_iter().cloned() { // Validate chain_id if specified in configuration - if let Err(e) = validate_chain_id(chain, actual_chain_id) { - error!("Chain validation failed: {}", e); - continue; // Skip this chain and continue with others - } + validate_chain_id(&chain, actual_chain_id)?; // Add delay if configured - if let Some((aggregate_id, delay_us)) = create_aggregate_delay(chain, actual_chain_id) { - delays.insert(aggregate_id, delay_us); - } + let (aggregate_id, delay_us) = create_aggregate_delay(&chain, actual_chain_id); + delays.insert(aggregate_id, delay_us); } - delays + Ok(delays) } impl ProviderCaches { diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index d6d0341e45..c144b33403 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -22,18 +22,17 @@ pub struct AggregateConfig { impl AggregateConfig { /// Create a new AggregateConfig with the specified delays - pub fn new(delays: HashMap) -> Self { + pub fn new(mut delays: HashMap) -> Self { + // Always handle AggregatId of 0 with a delay of 0 + if let None = delays.get(&AggregateId::new(0)) { + delays.insert(AggregateId::new(0), 0); + } Self { delays } } /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured pub fn indexed_ids(&self) -> Vec { - let indexes: Vec = self.delays.keys().map(|id| **id).collect(); - if indexes.is_empty() { - vec![0] - } else { - indexes - } + self.delays.keys().map(|id| id.to_usize()).collect() } } diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 39375785db..6cb5b9bafa 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -19,6 +19,10 @@ impl AggregateId { pub fn new(value: usize) -> Self { Self(value) } + + pub fn to_usize(&self) -> usize { + self.0 + } } impl From> for AggregateId { From 3d220471c6001a2348e4e7479a2a6ea390b8c49a Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 16 Jan 2026 23:46:48 +0000 Subject: [PATCH 051/102] chore: add setup to build script --- scripts/run-crisp-test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/run-crisp-test.sh b/scripts/run-crisp-test.sh index 402e542c85..5f94042b91 100755 --- a/scripts/run-crisp-test.sh +++ b/scripts/run-crisp-test.sh @@ -6,4 +6,12 @@ echo "Press any key to continue or Ctrl+C to cancel..." read -rm -rf * && git reset --hard HEAD && git submodule update --init --recursive && pnpm install && pnpm build && cd examples/CRISP && pnpm test:e2e "$@" +rm -rf * && \ + git reset --hard HEAD && \ + git submodule update --init --recursive && \ + pnpm install && \ + cargo build && \ + pnpm build && \ + cd examples/CRISP && \ + pnpm dev:setup && \ + pnpm test:e2e "$@" From d8761c73f9cfa75c8ff21d2dff1d570dbe976e62 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 17 Jan 2026 04:54:17 +0000 Subject: [PATCH 052/102] feat: add GetAggregateEventsAfter message handler to EventStoreRouter --- crates/events/src/eventstore_router.rs | 42 +++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index eae975a224..78fcd59b1e 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -5,11 +5,15 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::eventstore::EventStore; +use crate::ReceiveEvents; use crate::{ - events::StoreEventRequested, AggregateId, EventContextAccessors, EventLog, SequenceIndex, + events::{GetEventsAfter, StoreEventRequested}, + AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; -use actix::{Actor, Addr, Handler}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use anyhow::Result; use std::collections::HashMap; +use tracing::error; pub struct EventStoreRouter { stores: HashMap>>, @@ -26,10 +30,7 @@ impl EventStoreRouter { self.stores.insert(aggregate_id, store); } - pub fn handle_store_event_requested( - &mut self, - msg: StoreEventRequested, - ) -> Result<(), Box> { + pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { let aggregate_id = msg.event.aggregate_id(); let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { @@ -45,6 +46,16 @@ impl EventStoreRouter { store_addr.do_send(forwarded_msg); Ok(()) } + + pub fn handle_get_events_after(&mut self, msg: GetAggregateEventsAfter) -> Result<()> { + for (aggregate_id, ts) in msg.ts { + if let Some(store_addr) = self.stores.get(&aggregate_id) { + let get_events_msg = GetEventsAfter::new(ts, msg.sender.clone()); + store_addr.do_send(get_events_msg); + } + } + Ok(()) + } } impl Actor for EventStoreRouter { @@ -56,7 +67,24 @@ impl Handler for EventStoreR fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { if let Err(e) = self.handle_store_event_requested(msg) { - tracing::error!("Failed to route store event request: {}", e); + error!("Failed to route store event request: {}", e); + } + } +} + +impl Handler for EventStoreRouter { + type Result = (); + + fn handle(&mut self, msg: GetAggregateEventsAfter, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_get_events_after(msg) { + error!("Failed to route get events after request: {}", e); } } } + +#[derive(Message, Debug)] +#[rtype("()")] +pub struct GetAggregateEventsAfter { + pub ts: HashMap, + pub sender: Recipient, +} From b0f4dc9c709c7d57e05f80469020be3bc51a06ad Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 17 Jan 2026 05:19:51 +0000 Subject: [PATCH 053/102] Refactor EventStoreRouter to accept stores in constructor Simplify EventStoreRouter initialization by accepting a HashMap of stores directly in the constructor instead of requiring individual registration calls. Remove unused AggregateId import from event_system.rs. --- crates/ciphernode-builder/src/event_system.rs | 14 ++++---------- crates/events/src/eventstore_router.rs | 14 ++++++-------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 445aa5134d..848d155fb0 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -13,8 +13,8 @@ use e3_data::{ }; use e3_events::hlc::Hlc; use e3_events::{ - AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, - Sequencer, StoreEventRequested, + BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, Sequencer, + StoreEventRequested, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -302,17 +302,11 @@ impl EventSystem { let eventstores = self.eventstores()?; match eventstores { EventStoreAddrs::InMem(addrs) => { - let mut router = EventStoreRouter::new(); - for (index, addr) in addrs { - router.register_store(AggregateId::new(index), addr); - } + let router = EventStoreRouter::new(addrs); Ok(router.start().recipient()) } EventStoreAddrs::Persisted(addrs) => { - let mut router = EventStoreRouter::new(); - for (index, addr) in addrs { - router.register_store(AggregateId::new(index), addr); - } + let router = EventStoreRouter::new(addrs); Ok(router.start().recipient()) } } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 78fcd59b1e..4c6760fcbc 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -20,14 +20,12 @@ pub struct EventStoreRouter { } impl EventStoreRouter { - pub fn new() -> Self { - Self { - stores: HashMap::new(), - } - } - - pub fn register_store(&mut self, aggregate_id: AggregateId, store: Addr>) { - self.stores.insert(aggregate_id, store); + pub fn new(stores: HashMap>>) -> Self { + let stores = stores + .into_iter() + .map(|(index, addr)| (AggregateId::new(index), addr)) + .collect(); + Self { stores } } pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { From 7997a2d4855eef825e467738bae9a72051873a17 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 17 Jan 2026 05:27:12 +0000 Subject: [PATCH 054/102] remove qualifier --- crates/events/src/eventstore_router.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 4c6760fcbc..1e172da565 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use tracing::error; pub struct EventStoreRouter { - stores: HashMap>>, + stores: HashMap>>, } impl EventStoreRouter { From 7814cdbb4c05313e49533c7c060ffa91a1e33cce Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 17 Jan 2026 05:40:47 +0000 Subject: [PATCH 055/102] feat: add trap_fut function to handle async error trapping --- .../events/src/enclave_event/enclave_error.rs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 241ce8f563..e5332abfeb 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -6,7 +6,11 @@ use actix::Message; use serde::{Deserialize, Serialize}; -use std::fmt::{self, Display}; +use std::{ + fmt::{self, Display}, + future::Future, + pin::Pin, +}; use crate::{BusHandle, ErrorDispatcher}; @@ -71,3 +75,21 @@ where Err(e) => bus.err(err_type, e), } } + +/// Function to accept a future that resolves to a result. If result is an Err variant it is trapped and +/// sent to the bus as an ErrorEvent +pub fn trap_fut( + err_type: EType, + bus: &BusHandle, + fut: F, +) -> Pin + Send>> +where + F: Future> + Send + 'static, +{ + let bus = bus.clone(); + Box::pin(async move { + if let Err(e) = fut.await { + bus.err(err_type, e); + } + }) +} From b81ced89d77e01d4f8b14863ee499baee386e3ea Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 17 Jan 2026 09:41:57 +0000 Subject: [PATCH 056/102] Refactor EventStoreRouter methods to return typed router addresses - Split eventstore_router into specific methods for InMem and Persisted backends - Add in_mem_eventstore_router() and persisted_eventstore_router() methods - Update existing eventstore_router() to use the new specific methods - Add comprehensive tests for the new methods --- crates/ciphernode-builder/src/event_system.rs | 71 ++++++++++++++++--- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 848d155fb0..8f010ce1d2 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -297,18 +297,38 @@ impl EventSystem { .cloned() } - /// Get an EventStoreRouter + /// Get an EventStoreRouter for InMem backend + pub fn in_mem_eventstore_router( + &self, + ) -> Result>> { + let eventstores = self.eventstores()?; + if let EventStoreAddrs::InMem(addrs) = eventstores { + let router = EventStoreRouter::new(addrs); + Ok(router.start()) + } else { + Err(anyhow!("Expected InMem backend but got Persisted")) + } + } + + /// Get an EventStoreRouter for Persisted backend + pub fn persisted_eventstore_router( + &self, + ) -> Result>> { + let eventstores = self.eventstores()?; + if let EventStoreAddrs::Persisted(addrs) = eventstores { + let router = EventStoreRouter::new(addrs); + Ok(router.start()) + } else { + Err(anyhow!("Expected Persisted backend but got InMem")) + } + } + + /// Get an EventStoreRouter Recipient pub fn eventstore_router(&self) -> Result> { let eventstores = self.eventstores()?; match eventstores { - EventStoreAddrs::InMem(addrs) => { - let router = EventStoreRouter::new(addrs); - Ok(router.start().recipient()) - } - EventStoreAddrs::Persisted(addrs) => { - let router = EventStoreRouter::new(addrs); - Ok(router.start().recipient()) - } + EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), + EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), } } @@ -564,6 +584,39 @@ mod tests { Ok(()) } + #[actix::test] + async fn test_specific_eventstore_routers() -> Result<()> { + // Test in-memory eventstore router + let system = EventSystem::in_mem("test_in_mem"); + let router = system.in_mem_eventstore_router()?; + // Verify we can call methods on the router address + let _recipient: Recipient = router.clone().recipient(); + + // Test persistent eventstore router + let tmp = TempDir::new().unwrap(); + let persisted_system = EventSystem::persisted( + "test_persisted", + tmp.path().join("log"), + tmp.path().join("sled"), + ); + let persisted_router = persisted_system.persisted_eventstore_router()?; + // Verify we can call methods on the router address + let _recipient: Recipient = persisted_router.clone().recipient(); + + // Test that wrong backend type returns error + let in_mem_system = EventSystem::in_mem("test_wrong"); + assert!(in_mem_system.persisted_eventstore_router().is_err()); + + let persisted_system = EventSystem::persisted( + "test_wrong2", + tmp.path().join("log2"), + tmp.path().join("sled2"), + ); + assert!(persisted_system.in_mem_eventstore_router().is_err()); + + Ok(()) + } + #[actix::test] async fn test_multiple_eventstores() -> Result<()> { use e3_events::AggregateId; From 90affd1564fdd232de1e009c3808af4da8d38dbc Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 17 Jan 2026 11:38:13 +0000 Subject: [PATCH 057/102] Refactor EventSystem: remove TryFrom implementations and rename eventstores field - Remove TryFrom implementations for EventStoreAddrs conversions - Rename eventstores field to eventstore_addrs for clarity - Update eventstores() method to eventstore_addrs() - Simplify test code by removing redundant TryFrom usage - Fix pattern matching in eventstore_router() method - Remove unused test for specific eventstore routers --- crates/ciphernode-builder/src/event_system.rs | 149 ++++-------------- 1 file changed, 35 insertions(+), 114 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 8f010ce1d2..9444d2ce09 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -49,40 +49,6 @@ pub enum EventStoreAddrs { Persisted(HashMap>>), } -impl TryFrom for Addr> { - type Error = anyhow::Error; - fn try_from(value: EventStoreAddrs) -> std::result::Result { - if let EventStoreAddrs::InMem(addrs) = value { - if let Some(addr) = addrs.get(&0) { - Ok(addr.clone()) - } else { - Err(anyhow!("InMem event store addresses hashmap is empty")) - } - } else { - Err(anyhow!( - "address was not EventStore" - )) - } - } -} - -impl TryFrom for Addr> { - type Error = anyhow::Error; - fn try_from(value: EventStoreAddrs) -> std::result::Result { - if let EventStoreAddrs::Persisted(addrs) = value { - if let Some(addr) = addrs.get(&0) { - Ok(addr.clone()) - } else { - Err(anyhow!("Persisted event store addresses hashmap is empty")) - } - } else { - Err(anyhow!( - "address was not EventStore" - )) - } - } -} - /// EventSystem holds interconnected references to the components that manage events and /// persistence within the node. The EventSystem connects: /// @@ -112,7 +78,7 @@ pub struct EventSystem { /// Central configuration for aggregates, including delays and other settings aggregate_config: OnceCell, /// Cached EventStoreAddrs for idempotency - eventstores: OnceCell, + eventstore_addrs: OnceCell, } impl EventSystem { @@ -136,7 +102,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), - eventstores: OnceCell::new(), + eventstore_addrs: OnceCell::new(), } } @@ -155,7 +121,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), - eventstores: OnceCell::new(), + eventstore_addrs: OnceCell::new(), } } @@ -176,7 +142,7 @@ impl EventSystem { wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), - eventstores: OnceCell::new(), + eventstore_addrs: OnceCell::new(), } } @@ -242,8 +208,8 @@ impl EventSystem { } /// Get the EventStore addresses - pub fn eventstores(&self) -> Result { - self.eventstores + pub fn eventstore_addrs(&self) -> Result { + self.eventstore_addrs .get_or_try_init(|| { match &self.backend { EventSystemBackend::InMem(b) => { @@ -301,7 +267,7 @@ impl EventSystem { pub fn in_mem_eventstore_router( &self, ) -> Result>> { - let eventstores = self.eventstores()?; + let eventstores = self.eventstore_addrs()?; if let EventStoreAddrs::InMem(addrs) = eventstores { let router = EventStoreRouter::new(addrs); Ok(router.start()) @@ -314,7 +280,7 @@ impl EventSystem { pub fn persisted_eventstore_router( &self, ) -> Result>> { - let eventstores = self.eventstores()?; + let eventstores = self.eventstore_addrs()?; if let EventStoreAddrs::Persisted(addrs) = eventstores { let router = EventStoreRouter::new(addrs); Ok(router.start()) @@ -325,8 +291,8 @@ impl EventSystem { /// Get an EventStoreRouter Recipient pub fn eventstore_router(&self) -> Result> { - let eventstores = self.eventstores()?; - match eventstores { + let eventstores = self.eventstore_addrs()?; + match &eventstores { EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), } @@ -418,7 +384,7 @@ mod tests { use e3_events::prelude::*; use e3_events::EnclaveEventData; - use e3_events::GetEventsAfter; + use e3_events::ReceiveEvents; use e3_events::TestEvent; use tempfile::TempDir; @@ -509,7 +475,6 @@ mod tests { let system = EventSystem::in_mem("cn1").with_fresh_bus(); let handle = system.handle()?; let datastore = system.store()?; - let eventstores = system.eventstores()?; let listener = Listener { logs: Vec::new(), events: Vec::new(), @@ -528,12 +493,6 @@ mod tests { // NOTE: Eventual consistency // Store should not have data set on it until event has been published - // There is an argument we should instead delay reads until the event has been stored but - // this would: - // a. Promote poor patterns of sharing data through persistence - // b. Add a large amount of complexity to batching Get operations - // For now we allow this inconsistency under the assumption that data is written for - // snapshot storage exclusively. // Let's check the eventual consistency all data points should be none... assert_eq!(datastore.scope("/foo/name").read::().await?, None); @@ -571,11 +530,18 @@ mod tests { let logs = listener.send(GetLogs).await?; assert_eq!(logs, vec!["pink", "yellow", "red", "white"]); - // Get the in mem address for the event store (using index 0) - let es: Addr> = eventstores.try_into()?; + // Get the in mem eventstore router + let router = system.in_mem_eventstore_router()?; - // Get all events after the given timestamp and send them to the listener - es.do_send(GetEventsAfter::new(ts, listener.clone())); + // Get all events after the given timestamp using the router + use e3_events::{AggregateId, GetAggregateEventsAfter}; + let mut ts_map = HashMap::new(); + ts_map.insert(AggregateId::new(0), ts); + let get_events_msg = GetAggregateEventsAfter { + ts: ts_map, + sender: listener.clone().into(), + }; + router.do_send(get_events_msg); sleep(Duration::from_millis(100)).await; // Pull the events off the listsner since the timestamp @@ -584,39 +550,6 @@ mod tests { Ok(()) } - #[actix::test] - async fn test_specific_eventstore_routers() -> Result<()> { - // Test in-memory eventstore router - let system = EventSystem::in_mem("test_in_mem"); - let router = system.in_mem_eventstore_router()?; - // Verify we can call methods on the router address - let _recipient: Recipient = router.clone().recipient(); - - // Test persistent eventstore router - let tmp = TempDir::new().unwrap(); - let persisted_system = EventSystem::persisted( - "test_persisted", - tmp.path().join("log"), - tmp.path().join("sled"), - ); - let persisted_router = persisted_system.persisted_eventstore_router()?; - // Verify we can call methods on the router address - let _recipient: Recipient = persisted_router.clone().recipient(); - - // Test that wrong backend type returns error - let in_mem_system = EventSystem::in_mem("test_wrong"); - assert!(in_mem_system.persisted_eventstore_router().is_err()); - - let persisted_system = EventSystem::persisted( - "test_wrong2", - tmp.path().join("log2"), - tmp.path().join("sled2"), - ); - assert!(persisted_system.in_mem_eventstore_router().is_err()); - - Ok(()) - } - #[actix::test] async fn test_multiple_eventstores() -> Result<()> { use e3_events::AggregateId; @@ -630,22 +563,16 @@ mod tests { // Test in-memory eventstores let system = EventSystem::in_mem("test_multi").with_aggregate_config(aggregate_config); - let eventstores = system.eventstores()?; + let Ok(EventStoreAddrs::InMem(addrs)) = system.eventstore_addrs() else { + panic!("Expected InMem event store addrs"); + }; // Should create 3 eventstores for 3 AggregateIds - match &eventstores { - EventStoreAddrs::InMem(addrs) => { - assert_eq!(addrs.len(), 3); - // Test that we can access the first eventstore (index 0) - assert!(addrs.contains_key(&0)); - assert!(addrs.contains_key(&1)); - assert!(addrs.contains_key(&2)); - // Test that we can access the first eventstore - let _es: Addr> = - eventstores.clone().try_into()?; - } - EventStoreAddrs::Persisted(_) => panic!("Expected InMem eventstores"), - } + assert_eq!(addrs.len(), 3); + // Test that we can access the first eventstore (index 0) + assert!(addrs.contains_key(&0)); + assert!(addrs.contains_key(&1)); + assert!(addrs.contains_key(&2)); // Test persistent eventstores let tmp = TempDir::new().unwrap(); @@ -656,18 +583,12 @@ mod tests { ) .with_aggregate_config(AggregateConfig::new(HashMap::new())); - let persisted_eventstores = persisted_system.eventstores()?; + let Ok(EventStoreAddrs::Persisted(addrs)) = persisted_system.eventstore_addrs() else { + panic!("Expected Persisted event store addrs"); + }; - // Should create at least 1 eventstore (even with empty config) - match &persisted_eventstores { - EventStoreAddrs::Persisted(addrs) => { - assert_eq!(addrs.len(), 1); - assert!(addrs.contains_key(&0)); - let _es: Addr> = - persisted_eventstores.clone().try_into()?; - } - EventStoreAddrs::InMem(_) => panic!("Expected Persisted eventstores"), - } + assert_eq!(addrs.len(), 1); + assert!(addrs.contains_key(&0)); Ok(()) } From f4ad5f46ed461bf259c4773fcb278af5b7cdeb7b Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 18 Jan 2026 00:55:13 +0000 Subject: [PATCH 058/102] feat: implement event sync via libp2p request-response protocol --- Cargo.lock | 33 +++ Cargo.toml | 2 + crates/ciphernode-builder/src/event_system.rs | 7 +- .../src/enclave_event/evm_sync_events.rs | 29 +++ crates/events/src/enclave_event/mod.rs | 14 +- .../src/enclave_event/net_sync_events.rs | 29 +++ .../events/src/enclave_event/sync_request.rs | 24 ++ crates/events/src/events.rs | 40 ++- crates/events/src/eventstore.rs | 8 +- crates/events/src/eventstore_router.rs | 30 ++- crates/net/Cargo.toml | 1 + crates/net/src/events.rs | 60 ++++- crates/net/src/lib.rs | 1 + crates/net/src/net_interface.rs | 106 +++++++- crates/net/src/net_sync_manager.rs | 228 ++++++++++++++++++ crates/utils/src/helpers.rs | 46 +++- 16 files changed, 625 insertions(+), 33 deletions(-) create mode 100644 crates/events/src/enclave_event/evm_sync_events.rs create mode 100644 crates/events/src/enclave_event/net_sync_events.rs create mode 100644 crates/events/src/enclave_event/sync_request.rs create mode 100644 crates/net/src/net_sync_manager.rs diff --git a/Cargo.lock b/Cargo.lock index 9ac07831d7..c09b90bb90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2057,6 +2057,15 @@ dependencies = [ "serde", ] +[[package]] +name = "cbor4ii" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "472931dd4dfcc785075b09be910147f9c6258883fc4591d0dac6116392b2daa6" +dependencies = [ + "serde", +] + [[package]] name = "cc" version = "1.2.49" @@ -3177,6 +3186,7 @@ dependencies = [ "futures", "hex", "libp2p", + "rand 0.8.5", "serde", "sha2", "tokio", @@ -4950,6 +4960,7 @@ dependencies = [ "libp2p-metrics", "libp2p-ping", "libp2p-quic", + "libp2p-request-response", "libp2p-swarm", "libp2p-tcp", "libp2p-upnp", @@ -5210,6 +5221,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "libp2p-request-response" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1356c9e376a94a75ae830c42cdaea3d4fe1290ba409a22c809033d1b7dcab0a6" +dependencies = [ + "async-trait", + "cbor4ii", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.8.5", + "serde", + "smallvec", + "tracing", + "void", + "web-time", +] + [[package]] name = "libp2p-swarm" version = "0.45.1" diff --git a/Cargo.toml b/Cargo.toml index b427042042..077a0cd2c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -182,6 +182,8 @@ libp2p = { version = "=0.54.1", features = [ "ping", "quic", "tokio", + "request-response", + "cbor" ]} zeroize = "=1.8.1" diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 9444d2ce09..b296053527 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -383,6 +383,7 @@ mod tests { use actix::Message; use e3_events::prelude::*; + use e3_events::CorrelationId; use e3_events::EnclaveEventData; use e3_events::ReceiveEvents; @@ -537,10 +538,8 @@ mod tests { use e3_events::{AggregateId, GetAggregateEventsAfter}; let mut ts_map = HashMap::new(); ts_map.insert(AggregateId::new(0), ts); - let get_events_msg = GetAggregateEventsAfter { - ts: ts_map, - sender: listener.clone().into(), - }; + let get_events_msg = + GetAggregateEventsAfter::new(CorrelationId::new(), ts_map, listener.clone().into()); router.do_send(get_events_msg); sleep(Duration::from_millis(100)).await; diff --git a/crates/events/src/enclave_event/evm_sync_events.rs b/crates/events/src/enclave_event/evm_sync_events.rs new file mode 100644 index 0000000000..8ac1c941f2 --- /dev/null +++ b/crates/events/src/enclave_event/evm_sync_events.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use super::{EnclaveEvent, Unsequenced}; + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct EvmSyncEvents { + pub events: Vec>, +} + +impl EvmSyncEvents { + pub fn new(events: Vec>) -> Self { + Self { events } + } +} + +impl Display for EvmSyncEvents { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 2b808486b1..8d7aff7900 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -21,13 +21,16 @@ mod e3_requested; mod enclave_error; mod encryption_key_collection_failed; mod encryption_key_created; +mod evm_sync_events; mod keyshare_created; +mod net_sync_events; mod operator_activation_changed; mod plaintext_aggregated; mod plaintext_output_published; mod publickey_aggregated; mod publish_document; mod shutdown; +mod sync_request; mod test_event; mod threshold_share_collection_failed; mod threshold_share_created; @@ -54,7 +57,9 @@ use e3_utils::{colorize, Color}; pub use enclave_error::*; pub use encryption_key_collection_failed::*; pub use encryption_key_created::*; +pub use evm_sync_events::*; pub use keyshare_created::*; +pub use net_sync_events::*; pub use operator_activation_changed::*; pub use plaintext_aggregated::*; pub use plaintext_output_published::*; @@ -62,6 +67,7 @@ pub use publickey_aggregated::*; pub use publish_document::*; pub use shutdown::*; use strum::IntoStaticStr; +pub use sync_request::*; pub use test_event::*; pub use threshold_share_collection_failed::*; pub use threshold_share_created::*; @@ -128,6 +134,9 @@ pub enum EnclaveEventData { ComputeRequest(ComputeRequest), ComputeResponse(ComputeResponse), ComputeRequestError(ComputeRequestError), + SyncRequest(SyncRequest), + NetSyncEvents(NetSyncEvents), + EvmSyncEvents(EvmSyncEvents), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -395,7 +404,10 @@ impl_into_event_data!( ThresholdShareCollectionFailed, ComputeRequest, ComputeResponse, - ComputeRequestError + ComputeRequestError, + SyncRequest, + NetSyncEvents, + EvmSyncEvents ); impl TryFrom<&EnclaveEvent> for EnclaveError { diff --git a/crates/events/src/enclave_event/net_sync_events.rs b/crates/events/src/enclave_event/net_sync_events.rs new file mode 100644 index 0000000000..1fb25bf18f --- /dev/null +++ b/crates/events/src/enclave_event/net_sync_events.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use super::{EnclaveEvent, Unsequenced}; + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct NetSyncEvents { + pub events: Vec>, +} + +impl NetSyncEvents { + pub fn new(events: Vec>) -> Self { + Self { events } + } +} + +impl Display for NetSyncEvents { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/sync_request.rs b/crates/events/src/enclave_event/sync_request.rs new file mode 100644 index 0000000000..45f27686a9 --- /dev/null +++ b/crates/events/src/enclave_event/sync_request.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use crate::AggregateId; + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncRequest { + // TODO: this should be the event to trigger evm sync too + pub since: Vec<(AggregateId, u128)>, +} + +impl Display for SyncRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index d29f682633..5fd9ae190d 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -6,7 +6,7 @@ use actix::{Message, Recipient}; -use crate::{AggregateId, EnclaveEvent, Sequenced, Unsequenced}; +use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; /// Direct event received by the snapshot buffer in order to save snapshot to disk #[derive(Message, Debug)] @@ -54,29 +54,53 @@ impl StoreEventRequested { #[derive(Message, Debug)] #[rtype("()")] pub struct GetEventsAfter { - pub ts: u128, - pub sender: Recipient, + correlation_id: CorrelationId, + ts: u128, + sender: Recipient, } impl GetEventsAfter { - pub fn new(ts: u128, sender: impl Into>) -> Self { + pub fn new( + correlation_id: CorrelationId, + ts: u128, + sender: impl Into>, + ) -> Self { Self { + correlation_id, ts, sender: sender.into(), } } + + pub fn ts(&self) -> u128 { + self.ts + } + + pub fn id(&self) -> CorrelationId { + self.correlation_id + } + + pub fn sender(&self) -> &Recipient { + &self.sender + } } #[derive(Message, Debug)] #[rtype("()")] -pub struct ReceiveEvents(Vec>); +pub struct ReceiveEvents { + id: CorrelationId, + events: Vec>, +} impl ReceiveEvents { - pub fn new(events: Vec) -> Self { - Self(events) + pub fn new(id: CorrelationId, events: Vec) -> Self { + Self { id, events } } pub fn events(&self) -> &Vec { - &self.0 + &self.events + } + pub fn id(&self) -> CorrelationId { + self.id } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index d3e6e57371..ce593df7dd 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -33,8 +33,9 @@ impl EventStore { pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { // if there are no events after the timestamp return an empty vector - let Some(seq) = self.index.seek(msg.ts)? else { - msg.sender.try_send(ReceiveEvents::new(vec![]))?; + let Some(seq) = self.index.seek(msg.ts())? else { + msg.sender() + .try_send(ReceiveEvents::new(msg.id(), vec![]))?; return Ok(()); }; // read and return the events @@ -43,7 +44,8 @@ impl EventStore { .read_from(seq) .map(|(s, e)| e.into_sequenced(s)) .collect::>(); - msg.sender.try_send(ReceiveEvents::new(evts))?; + + msg.sender().try_send(ReceiveEvents::new(msg.id(), evts))?; Ok(()) } } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 1e172da565..4d9a6b3321 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -5,11 +5,11 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::eventstore::EventStore; -use crate::ReceiveEvents; use crate::{ events::{GetEventsAfter, StoreEventRequested}, AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; +use crate::{CorrelationId, ReceiveEvents}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; use std::collections::HashMap; @@ -46,9 +46,10 @@ impl EventStoreRouter { } pub fn handle_get_events_after(&mut self, msg: GetAggregateEventsAfter) -> Result<()> { - for (aggregate_id, ts) in msg.ts { + for (aggregate_id, ts) in msg.ts() { if let Some(store_addr) = self.stores.get(&aggregate_id) { - let get_events_msg = GetEventsAfter::new(ts, msg.sender.clone()); + let get_events_msg = + GetEventsAfter::new(msg.id(), ts.to_owned(), msg.sender.clone()); store_addr.do_send(get_events_msg); } } @@ -83,6 +84,29 @@ impl Handler for EventSt #[derive(Message, Debug)] #[rtype("()")] pub struct GetAggregateEventsAfter { + pub correlation_id: CorrelationId, pub ts: HashMap, pub sender: Recipient, } + +impl GetAggregateEventsAfter { + pub fn new( + correlation_id: CorrelationId, + ts: HashMap, + sender: Recipient, + ) -> Self { + Self { + correlation_id, + ts, + sender, + } + } + + pub fn id(&self) -> CorrelationId { + self.correlation_id + } + + pub fn ts(&self) -> &HashMap { + &self.ts + } +} diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml index 8fb6f8ae49..af340f2ead 100644 --- a/crates/net/Cargo.toml +++ b/crates/net/Cargo.toml @@ -20,6 +20,7 @@ e3-data = { workspace = true } e3-utils = { workspace = true } hex = { workspace = true } libp2p = { workspace = true } +rand = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } tokio = { workspace = true } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index f49591254d..d394c80448 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -7,15 +7,17 @@ use crate::Cid; use actix::Message; use anyhow::{bail, Context, Result}; -use e3_events::{CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; -use e3_utils::ArcBytes; +use e3_events::{AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; +use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, kad::{store, GetRecordError, PutRecordError}, + request_response::{InboundRequestId, ResponseChannel}, swarm::{dial_opts::DialOpts, ConnectionId, DialError}, }; use serde::{Deserialize, Serialize}; use std::{ + collections::HashMap, hash::Hash, sync::Arc, time::{Duration, Instant}, @@ -61,6 +63,33 @@ impl TryFrom for EnclaveEvent { } } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SyncRequestValue { + pub since: HashMap, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SyncResponseValue { + pub events: Vec, +} + +#[derive(Message, Clone, Debug)] +#[rtype("()")] +pub struct SyncRequestReceived { + pub request_id: InboundRequestId, + pub value: SyncRequestValue, + pub channel: OnceTake>, +} + +#[derive(Message, Clone, Debug)] +#[rtype("()")] +pub struct OutgoingSyncRequestSucceeded { + pub value: SyncResponseValue, +} + +#[derive(Debug, Clone)] +pub struct OutgoingSyncRequestFailed; + /// NetInterface Commands are sent to the network peer over a mspc channel #[derive(Debug)] pub enum NetCommand { @@ -86,6 +115,14 @@ pub enum NetCommand { }, /// Shutdown signal Shutdown, + /// Called from the syning node to request libp2p events from a random peer node starting + /// from the given timestamp. + OutgoingSyncRequest { value: SyncRequestValue }, + /// Send libp2p events back to a peer that requested a sync. + SyncResponse { + value: SyncResponseValue, + channel: OnceTake>, + }, } impl NetCommand { @@ -117,9 +154,13 @@ pub enum NetEvent { message_id: MessageId, }, /// There was an error Dialing a peer - DialError { error: Arc }, + DialError { + error: Arc, + }, /// A connection was established to a peer - ConnectionEstablished { connection_id: ConnectionId }, + ConnectionEstablished { + connection_id: ConnectionId, + }, /// There was an error creating a connection OutgoingConnectionError { connection_id: ConnectionId, @@ -147,7 +188,16 @@ pub enum NetEvent { error: PutOrStoreError, }, /// GossipSubscribed - GossipSubscribed { count: usize, topic: TopicHash }, + GossipSubscribed { + count: usize, + topic: TopicHash, + }, + /// A peer node is requesting gossipsub events since the given timestamp. + /// Use the provided channel to send a `SyncResponse + SyncRequestReceived(SyncRequestReceived), + /// Received gossipsub events from a peer in response to a `SyncRequest`. + OutgoingSyncRequestSucceeded(OutgoingSyncRequestSucceeded), + OutgoingSyncRequestFailed(OutgoingSyncRequestFailed), } #[derive(Clone, Debug)] diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index 0b9868cd08..eb7c49065a 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -10,6 +10,7 @@ mod document_publisher; pub mod events; mod net_event_translator; mod net_interface; +mod net_sync_manager; mod repo; pub use cid::Cid; diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 67ada100f0..698a1f9838 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -4,14 +4,14 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; +use anyhow::{bail, Result}; use e3_events::CorrelationId; -use e3_utils::ArcBytes; +use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ connection_limits::{self, ConnectionLimits}, futures::StreamExt, gossipsub, - identify::{self, Behaviour as IdentifyBehaviour}, + identify::{Behaviour as IdentifyBehaviour, Config as IdentifyConfig}, identity::Keypair, kad::{ self, @@ -19,15 +19,21 @@ use libp2p::{ Behaviour as KademliaBehaviour, Config as KademliaConfig, GetRecordOk, QueryId, QueryResult, Quorum, Record, RecordKey, }, + request_response::{ + self, cbor::Behaviour as CborRequestResponse, Event as RequestResponseEvent, + Message as RequestResponseMessage, ProtocolSupport, ResponseChannel, + }, swarm::{dial_opts::DialOpts, NetworkBehaviour, SwarmEvent}, StreamProtocol, Swarm, }; +use rand::prelude::IteratorRandom; use std::sync::atomic::AtomicBool; use std::{ collections::HashMap, sync::{atomic::Ordering, Arc}, time::Instant, }; + use std::{io::Error, time::Duration}; use tokio::{select, sync::broadcast, sync::mpsc}; use tracing::{debug, error, info, trace, warn}; @@ -36,7 +42,10 @@ const PROTOCOL_NAME: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0"); const MAX_KADEMLIA_PAYLOAD_MB: usize = 10; const MAX_GOSSIP_MSG_SIZE_KB: usize = 700; -use crate::events::{GossipData, NetCommand}; +use crate::events::{ + GossipData, NetCommand, OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, + SyncResponseValue, +}; use crate::events::{NetEvent, PutOrStoreError}; use crate::{dialer::dial_peers, Cid}; @@ -46,6 +55,7 @@ pub struct NodeBehaviour { kademlia: KademliaBehaviour, connection_limits: connection_limits::Behaviour, identify: IdentifyBehaviour, + sync: CborRequestResponse, } /// Manage the peer to peer connection. This struct wraps a libp2p Swarm and enables communication @@ -167,8 +177,8 @@ fn create_behaviour( ) -> std::result::Result> { let peer_id = key.public().to_peer_id(); let connection_limits = connection_limits::Behaviour::new(ConnectionLimits::default()); - let identify_config = IdentifyBehaviour::new( - identify::Config::new("/enclave/0.0.1".into(), key.public()) + let identify = IdentifyBehaviour::new( + IdentifyConfig::new("/enclave/0.0.1".into(), key.public()) .with_interval(Duration::from_secs(60)), ); @@ -183,7 +193,13 @@ fn create_behaviour( gossipsub::MessageAuthenticity::Signed(key.clone()), gossipsub_config, )?; - + let sync = CborRequestResponse::::new( + [( + StreamProtocol::new("/enclave/sync/0.0.1"), + ProtocolSupport::Full, + )], + request_response::Config::default(), + ); let mut config = KademliaConfig::new(PROTOCOL_NAME); config .set_max_packet_size(MAX_KADEMLIA_PAYLOAD_MB * 1024 * 1024) @@ -203,7 +219,8 @@ fn create_behaviour( gossipsub, kademlia, connection_limits, - identify: identify_config, + identify, + sync, }) } @@ -349,9 +366,11 @@ async fn process_swarm_event( let gossip_data = GossipData::from_bytes(&message.data)?; event_tx.send(NetEvent::GossipData(gossip_data))?; } + SwarmEvent::NewListenAddr { address, .. } => { trace!("Local node is listening on {address}"); } + SwarmEvent::Behaviour(NodeBehaviourEvent::Gossipsub(gossipsub::Event::Subscribed { peer_id, topic, @@ -360,6 +379,34 @@ async fn process_swarm_event( let count = swarm.behaviour().gossipsub.mesh_peers(&topic).count(); event_tx.send(NetEvent::GossipSubscribed { count, topic })?; } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { + message: + RequestResponseMessage::Request { + request, + channel, + request_id, + }, + .. + })) => { + // received a request for events + event_tx.send(NetEvent::SyncRequestReceived(SyncRequestReceived { + request_id, + channel: OnceTake::new(channel), + value: request, + }))?; + } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { + message: RequestResponseMessage::Response { response, .. }, + .. + })) => { + // received a response to a request for events + event_tx.send(NetEvent::OutgoingSyncRequestSucceeded( + OutgoingSyncRequestSucceeded { value: response }, + ))?; + } + unknown => { trace!("Unknown event: {:?}", unknown); } @@ -405,6 +452,8 @@ async fn process_swarm_command( key, } => handle_get_record(swarm, correlator, correlation_id, key), NetCommand::Shutdown => handle_shutdown(swarm, shutdown_flag), + NetCommand::OutgoingSyncRequest { value } => handle_outgoing_sync_request(swarm, value), + NetCommand::SyncResponse { value, channel } => handle_sync_response(swarm, channel, value), } } @@ -530,6 +579,47 @@ fn handle_shutdown( Ok(()) } +fn handle_outgoing_sync_request( + swarm: &mut Swarm, + value: SyncRequestValue, +) -> Result<()> { + // TODO: + // This is a first pass. + // Lots of stuff to work through here: + // How can I know events are correct? + // How can I trust this peer? + // Can I validate events with another peer? + // Should I use an OrderedSet with a hash and request the hash from a second peer? + + // Pick a random peer + let Some(peer) = swarm + .connected_peers() + .choose(&mut rand::thread_rng()) + .copied() + else { + bail!("No peer found on swarm!") + }; + + // Request events + swarm.behaviour_mut().sync.send_request(&peer, value); + Ok(()) +} + +fn handle_sync_response( + swarm: &mut Swarm, + channel: OnceTake>, + value: SyncResponseValue, +) -> Result<()> { + let channel = channel.try_take()?; + match swarm.behaviour_mut().sync.send_response(channel, value) { + Ok(_) => (), + Err(_res) => { + // TODO: report failure + } + } + Ok(()) +} + /// This correlates query_id and correlation_id. #[derive(Clone)] struct Correlator { diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs new file mode 100644 index 0000000000..ce6c13c9c9 --- /dev/null +++ b/crates/net/src/net_sync_manager.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; +use anyhow::{anyhow, bail, Result}; +use e3_events::{ + prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, + EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEvents, ReceiveEvents, SyncRequest, + Unsequenced, +}; +use e3_utils::{retry_with_backoff, to_retry, OnceTake}; +use futures::TryFutureExt; +use libp2p::request_response::ResponseChannel; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tokio::sync::{broadcast, mpsc}; +use tracing::debug; + +use crate::events::{ + call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, + SyncRequestReceived, SyncRequestValue, SyncResponseValue, +}; + +pub struct NetSyncManager { + /// Enclave EventBus + bus: BusHandle, + /// NetCommand sender to forward commands to the NetInterface + tx: mpsc::Sender, + /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so + /// that we do not do excessive resubscribes without actually listening for events. + rx: Arc>, + eventstore: Recipient, + requests: HashMap>>, +} + +impl NetSyncManager { + pub fn new( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + eventstore: Recipient, + ) -> Self { + Self { + bus: bus.clone(), + tx: tx.clone(), + rx: rx.clone(), + + eventstore, + requests: HashMap::new(), + } + } + + pub fn setup( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + eventstore: Recipient, + ) -> Addr { + let mut events = rx.resubscribe(); + let addr = Self::new(bus, tx, rx, eventstore).start(); + + // Forward from NetEvent + tokio::spawn({ + debug!("Spawning event receive loop!"); + let addr = addr.clone(); + async move { + while let Ok(event) = events.recv().await { + debug!("Received event {:?}", event); + match event { + NetEvent::OutgoingSyncRequestSucceeded(value) => addr.do_send(value), + NetEvent::SyncRequestReceived(value) => addr.do_send(value), + _ => (), + } + } + } + }); + + addr + } +} + +impl Actor for NetSyncManager { + type Context = actix::Context; +} + +/// Event broadcast from event bus +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + match msg.into_data() { + EnclaveEventData::SyncRequest(data) => ctx.notify(data), + _ => (), + } + } +} + +/// SyncRequest is called on start up to fetch remote events +impl Handler for NetSyncManager { + type Result = ResponseFuture<()>; + fn handle(&mut self, msg: SyncRequest, _: &mut Self::Context) -> Self::Result { + trap_fut( + EType::Net, + &self.bus.clone(), + handle_sync_request_event(self.tx.clone(), self.rx.clone(), self.bus.clone(), msg), + ) + } +} + +/// We have received the sync response from the remote peer +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: OutgoingSyncRequestSucceeded, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.clone(), || { + self.bus.publish(NetSyncEvents { + events: msg + .value + .events + .iter() + .cloned() + .map(|data| data.try_into()) + .collect::>>>()?, + })?; + + Ok(()) + }); + } +} + +/// We have received a sync request from a remote peer +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: SyncRequestReceived, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus, || { + let id = CorrelationId::new(); + self.requests.insert(id, msg.channel); + self.eventstore.try_send(GetAggregateEventsAfter::new( + id, + msg.value.since, + ctx.address().recipient(), + ))?; + Ok(()) + }); + } +} + +/// Receive Events from EventStore +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.clone(), || { + let Some(channel) = self.requests.get(&msg.id()) else { + bail!("request not found with {}", msg.id()); + }; + + self.tx.try_send(NetCommand::SyncResponse { + value: SyncResponseValue { + events: msg + .events() + .into_iter() + .cloned() + .map(|ev| ev.try_into()) + .collect::>()?, + }, + channel: channel.to_owned(), + })?; + + Ok(()) + }) + } +} + +const SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +async fn sync_request( + net_cmds: mpsc::Sender, + net_events: Arc>, + since: HashMap, +) -> Result { + call_and_await_response( + net_cmds, + net_events, + NetCommand::OutgoingSyncRequest { + value: SyncRequestValue { since }, + }, + |e| match e.clone() { + NetEvent::OutgoingSyncRequestSucceeded(value) => Some(Ok(value)), + NetEvent::OutgoingSyncRequestFailed(error) => { + Some(Err(anyhow!("Outgoing sync request failed: {:?}", error))) + } + _ => None, + }, + SYNC_REQUEST_TIMEOUT, + ) + .await +} + +async fn handle_sync_request_event( + net_cmds: mpsc::Sender, + net_events: Arc>, + bus: BusHandle, + event: SyncRequest, +) -> Result<()> { + let value = retry_with_backoff( + || { + sync_request( + net_cmds.clone(), + net_events.clone(), + event.since.clone().into_iter().collect(), + ) + .map_err(to_retry) + }, + 4, + 1000, + ) + .await?; + + bus.publish(NetSyncEvents::new( + value + .value + .events + .into_iter() + .map(|data| data.try_into()) + .collect::>>>()?, + ))?; + Ok(()) +} diff --git a/crates/utils/src/helpers.rs b/crates/utils/src/helpers.rs index 77d09936ab..9380d01abf 100644 --- a/crates/utils/src/helpers.rs +++ b/crates/utils/src/helpers.rs @@ -3,7 +3,10 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; pub fn to_ordered_vec(source: HashMap) -> Vec where @@ -18,3 +21,44 @@ where // Extract to Vec of ThresholdShares in order pairs.into_iter().map(|(_, value)| value).collect() } + +/// A cloneable wrapper that allows a non-cloneable value to be shared and taken exactly once. +/// +/// Useful for passing oneshot channels or other single-use items through cloneable contexts. +/// +/// # Example +/// ``` +/// use e3_utils::OnceTake; +/// +/// let (tx, rx) = tokio::sync::oneshot::channel::(); +/// let once = OnceTake::new(tx); +/// let cloned = once.clone(); +/// cloned.take().unwrap().send(42).unwrap(); +/// assert!(once.take().is_none()); // already taken +/// ``` +#[derive(Debug)] +pub struct OnceTake(Arc>>); + +impl OnceTake { + /// Wraps an item so it can be cloned and later taken once. + pub fn new(item: T) -> Self { + Self(Arc::new(Mutex::new(Some(item)))) + } + + /// Takes the item, returning `None` if already taken. + pub fn take(&self) -> Option { + self.0.lock().unwrap().take() + } + + /// Takes the item, returning an error if already taken. + pub fn try_take(&self) -> anyhow::Result { + self.take() + .ok_or_else(|| anyhow::anyhow!("Item already taken.")) + } +} + +impl Clone for OnceTake { + fn clone(&self) -> Self { + OnceTake(Arc::clone(&self.0)) + } +} From 47c5882e92df5cfb55d07aa47397118e7b2bd83f Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 19 Jan 2026 00:51:33 +0000 Subject: [PATCH 059/102] add e3-sync crate with basic sync actor and bootstrap handler --- Cargo.lock | 7 +++++++ Cargo.toml | 2 ++ crates/sync/Cargo.toml | 11 +++++++++++ crates/sync/src/lib.rs | 3 +++ crates/sync/src/sync.rs | 20 ++++++++++++++++++++ 5 files changed, 43 insertions(+) create mode 100644 crates/sync/Cargo.toml create mode 100644 crates/sync/src/lib.rs create mode 100644 crates/sync/src/sync.rs diff --git a/Cargo.lock b/Cargo.lock index fa79435a59..f2bbfa62de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3269,6 +3269,13 @@ dependencies = [ "tokio", ] +[[package]] +name = "e3-sync" +version = "0.1.7" +dependencies = [ + "actix", +] + [[package]] name = "e3-test-helpers" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index 077a0cd2c3..574f655dfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/sdk", "crates/sortition", "crates/support-scripts", + "crates/sync", "crates/test-helpers", "crates/tests", "crates/trbfv", @@ -90,6 +91,7 @@ e3-compute-provider = { version = "0.1.7", path = "./crates/compute-provider" } e3-sortition = { version = "0.1.7", path = "./crates/sortition" } e3-program-server = { version = "0.1.7", path = "./crates/program-server" } e3-support-scripts = { version = "0.1.7", path = "./crates/support-scripts" } +e3-sync = { version = "0.1.7", path = "./crates/sync" } e3-test-helpers = { version = "0.1.7", path = "./crates/test-helpers" } e3-tests = { version = "0.1.7", path = "./crates/tests" } e3-trbfv = { version = "0.1.7", path = "./crates/trbfv" } diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml new file mode 100644 index 0000000000..a6bfa1bcf5 --- /dev/null +++ b/crates/sync/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "e3-sync" +version.workspace = true +edition.workspace = true +license.workspace = true +description.workspace = true +repository = "https://github.com/gnosisguild/enclave/crates/sync" + +[dependencies] +actix.workspace = true + diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs new file mode 100644 index 0000000000..2ff846775a --- /dev/null +++ b/crates/sync/src/lib.rs @@ -0,0 +1,3 @@ +mod sync; + +pub use sync::*; diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs new file mode 100644 index 0000000000..fad172d36b --- /dev/null +++ b/crates/sync/src/sync.rs @@ -0,0 +1,20 @@ +use actix::{Actor, Handler, Message}; + +struct Sync; + +impl Sync {} + +impl Actor for Sync { + type Context = actix::Context; +} + +impl Handler for Sync { + type Result = (); + fn handle(&mut self, msg: Bootstrap, ctx: &mut Self::Context) -> Self::Result { + // Publish SyncStart + } +} + +#[derive(Message)] +#[rtype("()")] +pub struct Bootstrap; From 0ae231d37359c29a9347a13750233be66861e83e Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 19 Jan 2026 00:52:44 +0000 Subject: [PATCH 060/102] add header --- crates/sync/src/sync.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index fad172d36b..82e0fa2c3a 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Actor, Handler, Message}; struct Sync; From e0022d4cf23abcee4cec6331b747bdca928f7356 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 20 Jan 2026 17:24:56 +0000 Subject: [PATCH 061/102] add header --- crates/sync/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index 2ff846775a..07b21af2e0 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + mod sync; pub use sync::*; From 2598912348c60540427c73ddb1e5140bff7911f3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 20 Jan 2026 18:38:17 +0000 Subject: [PATCH 062/102] add sync to Dockerfile --- crates/Dockerfile | 1 + crates/evm/src/event_reader.rs | 10 +++++++++- examples/CRISP/server/Dockerfile | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/Dockerfile b/crates/Dockerfile index 482f5fd2bf..4ad1dbcc6d 100644 --- a/crates/Dockerfile +++ b/crates/Dockerfile @@ -69,6 +69,7 @@ COPY crates/safe/Cargo.toml ./safe/Cargo.toml COPY crates/sdk/Cargo.toml ./sdk/Cargo.toml COPY crates/sortition/Cargo.toml ./sortition/Cargo.toml COPY crates/support-scripts/Cargo.toml ./support-scripts/Cargo.toml +COPY crates/sync/Cargo.toml ./sync/Cargo.toml COPY crates/test-helpers/Cargo.toml ./test-helpers/Cargo.toml COPY crates/tests/Cargo.toml ./tests/Cargo.toml COPY crates/trbfv/Cargo.toml ./trbfv/Cargo.toml diff --git a/crates/evm/src/event_reader.rs b/crates/evm/src/event_reader.rs index 39d2035253..b07b67de6e 100644 --- a/crates/evm/src/event_reader.rs +++ b/crates/evm/src/event_reader.rs @@ -85,7 +85,7 @@ pub struct EvmEventReader

{ /// Event bus for error propagation only bus: BusHandle, /// The auto persistable state of the event reader - state: Persistable, + state: Persistable, // XXX: would be best to avoid persistable state outside of domain /// The RPC URL for the provider rpc_url: String, } @@ -183,6 +183,9 @@ impl Actor for EvmEventReader

{ } } +// TODO: split this up into: +// 1. historical request (will finish) +// 2. current listener (run indefinitely) #[instrument(name = "evm_event_reader", skip_all)] async fn stream_from_evm( provider: EthProvider

, @@ -300,6 +303,8 @@ impl Handler for EvmEventReader

} } +// XXX: Sync should handle the tracking of the last block based on what has been stored in the +// eventlog impl Handler for EvmEventReader

{ type Result = (); @@ -311,6 +316,9 @@ impl Handler for EvmEventReader< } EnclaveEvmEvent::Event { event, block } => { + // XXX: Should not need state + // 1. deduplication already happens at the event bus + // 2. cursor is kept by sync match self.state.try_mutate(|mut state| { let temp_wrapped = EnclaveEvmEvent::Event { event: event.clone(), diff --git a/examples/CRISP/server/Dockerfile b/examples/CRISP/server/Dockerfile index 1421dd3e49..bd72ee2a17 100644 --- a/examples/CRISP/server/Dockerfile +++ b/examples/CRISP/server/Dockerfile @@ -83,6 +83,7 @@ COPY crates/request/Cargo.toml crates/request/Cargo.toml COPY crates/sdk/Cargo.toml crates/sdk/Cargo.toml COPY crates/sortition/Cargo.toml crates/sortition/Cargo.toml COPY crates/support-scripts/Cargo.toml crates/support-scripts/Cargo.toml +COPY crates/sync/Cargo.toml crates/sync/Cargo.toml COPY crates/test-helpers/Cargo.toml crates/test-helpers/Cargo.toml COPY crates/tests/Cargo.toml crates/tests/Cargo.toml COPY crates/trbfv/Cargo.toml crates/trbfv/Cargo.toml From 8b5702c9297b937155db95cf5b61f6693171e8a7 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 21 Jan 2026 09:01:50 +0000 Subject: [PATCH 063/102] Add Log event --- crates/evm/src/event_reader.rs | 8 ++++++++ crates/evm/src/historical_event_coordinator.rs | 1 + 2 files changed, 9 insertions(+) diff --git a/crates/evm/src/event_reader.rs b/crates/evm/src/event_reader.rs index b07b67de6e..8d431dd9e2 100644 --- a/crates/evm/src/event_reader.rs +++ b/crates/evm/src/event_reader.rs @@ -34,6 +34,12 @@ pub enum EnclaveEvmEvent { event: EnclaveEventData, block: Option, }, + /// Raw log data from the provider + Log { + data: LogData, + topic: Option, + chain_id: u64, + }, } impl EnclaveEvmEvent { @@ -355,6 +361,8 @@ impl Handler for EvmEventReader< Err(err) => self.bus.err(EType::Evm, err), } } + + _ => (), } } } diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs index 5b02e33b89..db50fc9ba8 100644 --- a/crates/evm/src/historical_event_coordinator.rs +++ b/crates/evm/src/historical_event_coordinator.rs @@ -118,6 +118,7 @@ impl Handler for HistoricalEventCoordinator { } Ok(()) } + _ => Ok(()), }) } } From 9b3a409b3595514ca28030aae42a76d11ac93def Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 22 Jan 2026 04:56:27 +0000 Subject: [PATCH 064/102] rename evm event reader to evm interface --- crates/evm/src/bonding_registry_sol.rs | 10 +++--- crates/evm/src/ciphernode_registry_sol.rs | 12 +++---- crates/evm/src/enclave_sol.rs | 4 +-- crates/evm/src/enclave_sol_reader.rs | 10 +++--- .../src/{event_reader.rs => evm_interface.rs} | 34 +++++++++---------- .../evm/src/historical_event_coordinator.rs | 2 +- crates/evm/src/lib.rs | 4 +-- crates/evm/src/repo.rs | 14 ++++---- crates/evm/tests/integration.rs | 18 +++++----- 9 files changed, 54 insertions(+), 54 deletions(-) rename crates/evm/src/{event_reader.rs => evm_interface.rs} (93%) diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index 20652f3554..d7ed6db941 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - event_reader::EvmEventReaderState, helpers::EthProvider, EnclaveEvmEvent, EvmEventReader, + evm_interface::EvmInterfaceState, helpers::EthProvider, EnclaveEvmEvent, EvmInterface, }; use actix::{Addr, Recipient}; use alloy::{ @@ -148,14 +148,14 @@ impl BondingRegistrySolReader { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, - ) -> Result>> + ) -> Result>> where P: Provider + Clone + 'static, { - let addr = EvmEventReader::attach( + let addr = EvmInterface::attach( provider, extractor, contract_address, @@ -182,7 +182,7 @@ impl BondingRegistrySol { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, ) -> Result<()> diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 5beebc8f0f..4c8d9de6e8 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -5,9 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - event_reader::EvmEventReaderState, + evm_interface::EvmInterfaceState, helpers::{send_tx_with_retry, EthProvider}, - EnclaveEvmEvent, EvmEventReader, + EnclaveEvmEvent, EvmInterface, }; use actix::prelude::*; use alloy::{ @@ -223,14 +223,14 @@ impl CiphernodeRegistrySolReader { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, - ) -> Result>> + ) -> Result>> where P: Provider + Clone + 'static, { - let addr = EvmEventReader::attach( + let addr = EvmInterface::attach( provider, extractor, contract_address, @@ -557,7 +557,7 @@ impl CiphernodeRegistrySol { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, ) -> Result<()> diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index a8a6bebf30..47f3d24b3a 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -6,7 +6,7 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - event_reader::EvmEventReaderState, helpers::EthProvider, EnclaveEvmEvent, + evm_interface::EvmInterfaceState, helpers::EthProvider, EnclaveEvmEvent, }; use actix::Recipient; use alloy::providers::{Provider, WalletProvider}; @@ -23,7 +23,7 @@ impl EnclaveSol { read_provider: EthProvider, write_provider: EthProvider, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, ) -> Result<()> diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index fa1f6f8f67..29c7d76038 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::event_reader::EvmEventReaderState; +use crate::evm_interface::EvmInterfaceState; use crate::helpers::EthProvider; -use crate::{EnclaveEvmEvent, EvmEventReader}; +use crate::{EnclaveEvmEvent, EvmInterface}; use actix::{Addr, Recipient}; use alloy::primitives::{LogData, B256}; use alloy::providers::Provider; @@ -110,14 +110,14 @@ impl EnclaveSolReader { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, - ) -> Result>> + ) -> Result>> where P: Provider + Clone + 'static, { - let addr = EvmEventReader::attach( + let addr = EvmInterface::attach( provider, extractor, contract_address, diff --git a/crates/evm/src/event_reader.rs b/crates/evm/src/evm_interface.rs similarity index 93% rename from crates/evm/src/event_reader.rs rename to crates/evm/src/evm_interface.rs index 8d431dd9e2..2183c1c770 100644 --- a/crates/evm/src/event_reader.rs +++ b/crates/evm/src/evm_interface.rs @@ -54,25 +54,25 @@ impl EnclaveEvmEvent { pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; -pub struct EvmEventReaderParams

{ +pub struct EvmInterfaceParams

{ provider: EthProvider

, extractor: ExtractorFn, contract_address: Address, start_block: Option, processor: Recipient, bus: BusHandle, - state: Persistable, + state: Persistable, rpc_url: String, } #[derive(Default, serde::Serialize, serde::Deserialize, Clone)] -pub struct EvmEventReaderState { +pub struct EvmInterfaceState { pub ids: HashSet, pub last_block: Option, } /// Connects to Enclave.sol converting EVM events to EnclaveEvents -pub struct EvmEventReader

{ +pub struct EvmInterface

{ /// The alloy provider provider: Option>, /// The contract address @@ -91,13 +91,13 @@ pub struct EvmEventReader

{ /// Event bus for error propagation only bus: BusHandle, /// The auto persistable state of the event reader - state: Persistable, // XXX: would be best to avoid persistable state outside of domain + state: Persistable, // XXX: would be best to avoid persistable state outside of domain /// The RPC URL for the provider rpc_url: String, } -impl EvmEventReader

{ - pub fn new(params: EvmEventReaderParams

) -> Self { +impl EvmInterface

{ + pub fn new(params: EvmInterfaceParams

) -> Self { let (shutdown_tx, shutdown_rx) = oneshot::channel(); Self { contract_address: params.contract_address, @@ -120,15 +120,15 @@ impl EvmEventReader

{ start_block: Option, processor: &Recipient, bus: &BusHandle, - repository: &Repository, + repository: &Repository, rpc_url: String, ) -> Result> { let sync_state = repository .clone() - .load_or_default(EvmEventReaderState::default()) + .load_or_default(EvmInterfaceState::default()) .await?; - let params = EvmEventReaderParams { + let params = EvmInterfaceParams { provider, extractor, contract_address: contract_address.parse()?, @@ -139,7 +139,7 @@ impl EvmEventReader

{ rpc_url, }; - let addr = EvmEventReader::new(params).start(); + let addr = EvmInterface::new(params).start(); processor.do_send(EnclaveEvmEvent::RegisterReader); @@ -148,7 +148,7 @@ impl EvmEventReader

{ } } -impl Actor for EvmEventReader

{ +impl Actor for EvmInterface

{ type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { @@ -192,11 +192,11 @@ impl Actor for EvmEventReader

{ // TODO: split this up into: // 1. historical request (will finish) // 2. current listener (run indefinitely) -#[instrument(name = "evm_event_reader", skip_all)] +#[instrument(name = "evm_interface", skip_all)] async fn stream_from_evm( provider: EthProvider

, contract_address: &Address, - reader_addr: Addr>, + reader_addr: Addr>, extractor: fn(&LogData, Option<&B256>, u64) -> Option, mut shutdown: oneshot::Receiver<()>, start_block: Option, @@ -297,7 +297,7 @@ fn is_local_node(rpc_url: &str) -> bool { rpc_url.contains("localhost") || rpc_url.contains("127.0.0.1") } -impl Handler for EvmEventReader

{ +impl Handler for EvmInterface

{ type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { @@ -311,10 +311,10 @@ impl Handler for EvmEventReader

// XXX: Sync should handle the tracking of the last block based on what has been stored in the // eventlog -impl Handler for EvmEventReader

{ +impl Handler for EvmInterface

{ type Result = (); - #[instrument(name = "evm_event_reader", skip_all)] + #[instrument(name = "evm_interface", skip_all)] fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { match msg { EnclaveEvmEvent::RegisterReader | EnclaveEvmEvent::HistoricalSyncComplete => { diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs index db50fc9ba8..4dc4243d28 100644 --- a/crates/evm/src/historical_event_coordinator.rs +++ b/crates/evm/src/historical_event_coordinator.rs @@ -20,7 +20,7 @@ struct BufferedEvent { #[rtype(result = "()")] pub struct CoordinatorStart; -/// Coordinates historical replay across all EvmEventReaders. +/// Coordinates historical replay across all EvmInterfaces. /// Buffers historical events, then sorts + publishes once all readers finish. pub struct HistoricalEventCoordinator { /// Count of readers that have registered diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index f864b89f8c..328512928d 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -9,7 +9,7 @@ mod ciphernode_registry_sol; mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; -mod event_reader; +mod evm_interface; pub mod helpers; mod historical_event_coordinator; mod repo; @@ -21,7 +21,7 @@ pub use ciphernode_registry_sol::{ pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use event_reader::{EnclaveEvmEvent, EvmEventReader, EvmEventReaderState, ExtractorFn}; +pub use evm_interface::{EnclaveEvmEvent, EvmInterface, EvmInterfaceState, ExtractorFn}; pub use helpers::send_tx_with_retry; pub use historical_event_coordinator::{CoordinatorStart, HistoricalEventCoordinator}; pub use repo::*; diff --git a/crates/evm/src/repo.rs b/crates/evm/src/repo.rs index 0455c09f04..827df69e18 100644 --- a/crates/evm/src/repo.rs +++ b/crates/evm/src/repo.rs @@ -7,7 +7,7 @@ use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use crate::EvmEventReaderState; +use crate::EvmInterfaceState; pub trait EthPrivateKeyRepositoryFactory { fn eth_private_key(&self) -> Repository>; @@ -20,21 +20,21 @@ impl EthPrivateKeyRepositoryFactory for Repositories { } pub trait EnclaveSolReaderRepositoryFactory { - fn enclave_sol_reader(&self, chain_id: u64) -> Repository; + fn enclave_sol_reader(&self, chain_id: u64) -> Repository; } impl EnclaveSolReaderRepositoryFactory for Repositories { - fn enclave_sol_reader(&self, chain_id: u64) -> Repository { + fn enclave_sol_reader(&self, chain_id: u64) -> Repository { Repository::new(self.store.scope(StoreKeys::enclave_sol_reader(chain_id))) } } pub trait CiphernodeRegistryReaderRepositoryFactory { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; + fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; } impl CiphernodeRegistryReaderRepositoryFactory for Repositories { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { + fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(StoreKeys::ciphernode_registry_reader(chain_id)), @@ -43,11 +43,11 @@ impl CiphernodeRegistryReaderRepositoryFactory for Repositories { } pub trait BondingRegistryReaderRepositoryFactory { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository; + fn bonding_registry_reader(&self, chain_id: u64) -> Repository; } impl BondingRegistryReaderRepositoryFactory for Repositories { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository { + fn bonding_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(StoreKeys::bonding_registry_reader(chain_id)), diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 385f890781..b4a54141d7 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -20,7 +20,7 @@ use e3_entrypoint::helpers::datastore::get_in_mem_store; use e3_events::{ prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, HistoryCollector, Shutdown, TestEvent, }; -use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmEventReader, HistoricalEventCoordinator}; +use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmInterface, HistoricalEventCoordinator}; use std::time::Duration; use tokio::time::sleep; @@ -89,7 +89,7 @@ async fn evm_reader() -> Result<()> { let coordinator = HistoricalEventCoordinator::setup(bus.clone()); let processor = coordinator.clone().recipient(); - EvmEventReader::attach( + EvmInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -172,7 +172,7 @@ async fn ensure_historical_events() -> Result<()> { .await?; } - EvmEventReader::attach( + EvmInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -249,7 +249,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { .await?; } - let addr1 = EvmEventReader::attach( + let addr1 = EvmInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -291,7 +291,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { let msgs = get_msgs(&history_collector).await?; assert_eq!(msgs, ["before", "online", "live", "events"]); - let _ = EvmEventReader::attach( + let _ = EvmInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -358,7 +358,7 @@ async fn coordinator_single_reader() -> Result<()> { .await?; } - EvmEventReader::attach( + EvmInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -450,7 +450,7 @@ async fn coordinator_multiple_readers() -> Result<()> { .watch() .await?; - EvmEventReader::attach( + EvmInterface::attach( provider.clone(), test_event_extractor, &contract1.address().to_string(), @@ -462,7 +462,7 @@ async fn coordinator_multiple_readers() -> Result<()> { ) .await?; - EvmEventReader::attach( + EvmInterface::attach( provider.clone(), test_event_extractor, &contract2.address().to_string(), @@ -509,7 +509,7 @@ async fn coordinator_no_historical_events() -> Result<()> { let coordinator = HistoricalEventCoordinator::setup(bus.clone()); let processor = coordinator.clone().recipient(); - EvmEventReader::attach( + EvmInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), From 302d7cf1a3042116251e5f27f3e9cea2118435b5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 22 Jan 2026 05:12:33 +0000 Subject: [PATCH 065/102] rename events --- ..._events.rs => evm_sync_events_received.rs} | 6 ++-- crates/events/src/enclave_event/mod.rs | 30 +++++++++---------- ..._events.rs => net_sync_events_received.rs} | 6 ++-- ..._request.rs => outgoing_sync_requested.rs} | 4 +-- crates/net/src/net_sync_manager.rs | 16 +++++----- 5 files changed, 31 insertions(+), 31 deletions(-) rename crates/events/src/enclave_event/{evm_sync_events.rs => evm_sync_events_received.rs} (86%) rename crates/events/src/enclave_event/{net_sync_events.rs => net_sync_events_received.rs} (86%) rename crates/events/src/enclave_event/{sync_request.rs => outgoing_sync_requested.rs} (88%) diff --git a/crates/events/src/enclave_event/evm_sync_events.rs b/crates/events/src/enclave_event/evm_sync_events_received.rs similarity index 86% rename from crates/events/src/enclave_event/evm_sync_events.rs rename to crates/events/src/enclave_event/evm_sync_events_received.rs index 8ac1c941f2..0fcf1e3347 100644 --- a/crates/events/src/enclave_event/evm_sync_events.rs +++ b/crates/events/src/enclave_event/evm_sync_events_received.rs @@ -12,17 +12,17 @@ use super::{EnclaveEvent, Unsequenced}; #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EvmSyncEvents { +pub struct EvmSyncEventsReceived { pub events: Vec>, } -impl EvmSyncEvents { +impl EvmSyncEventsReceived { pub fn new(events: Vec>) -> Self { Self { events } } } -impl Display for EvmSyncEvents { +impl Display for EvmSyncEventsReceived { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 8d7aff7900..10c23241ef 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -21,16 +21,16 @@ mod e3_requested; mod enclave_error; mod encryption_key_collection_failed; mod encryption_key_created; -mod evm_sync_events; +mod evm_sync_events_received; mod keyshare_created; -mod net_sync_events; +mod net_sync_events_received; mod operator_activation_changed; +mod outgoing_sync_requested; mod plaintext_aggregated; mod plaintext_output_published; mod publickey_aggregated; mod publish_document; mod shutdown; -mod sync_request; mod test_event; mod threshold_share_collection_failed; mod threshold_share_created; @@ -57,17 +57,17 @@ use e3_utils::{colorize, Color}; pub use enclave_error::*; pub use encryption_key_collection_failed::*; pub use encryption_key_created::*; -pub use evm_sync_events::*; +pub use evm_sync_events_received::*; pub use keyshare_created::*; -pub use net_sync_events::*; +pub use net_sync_events_received::*; pub use operator_activation_changed::*; +pub use outgoing_sync_requested::*; pub use plaintext_aggregated::*; pub use plaintext_output_published::*; pub use publickey_aggregated::*; pub use publish_document::*; pub use shutdown::*; use strum::IntoStaticStr; -pub use sync_request::*; pub use test_event::*; pub use threshold_share_collection_failed::*; pub use threshold_share_created::*; @@ -131,12 +131,12 @@ pub enum EnclaveEventData { EncryptionKeyCreated(EncryptionKeyCreated), EncryptionKeyCollectionFailed(EncryptionKeyCollectionFailed), ThresholdShareCollectionFailed(ThresholdShareCollectionFailed), - ComputeRequest(ComputeRequest), - ComputeResponse(ComputeResponse), - ComputeRequestError(ComputeRequestError), - SyncRequest(SyncRequest), - NetSyncEvents(NetSyncEvents), - EvmSyncEvents(EvmSyncEvents), + ComputeRequest(ComputeRequest), // ComputeRequested + ComputeResponse(ComputeResponse), // ComputeResponseReceived + ComputeRequestError(ComputeRequestError), // ComputeRequestFailed + OutgoingSyncRequested(OutgoingSyncRequested), + NetSyncEventsReceived(NetSyncEventsReceived), + EvmSyncEventsReceived(EvmSyncEventsReceived), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -405,9 +405,9 @@ impl_into_event_data!( ComputeRequest, ComputeResponse, ComputeRequestError, - SyncRequest, - NetSyncEvents, - EvmSyncEvents + OutgoingSyncRequested, + NetSyncEventsReceived, + EvmSyncEventsReceived ); impl TryFrom<&EnclaveEvent> for EnclaveError { diff --git a/crates/events/src/enclave_event/net_sync_events.rs b/crates/events/src/enclave_event/net_sync_events_received.rs similarity index 86% rename from crates/events/src/enclave_event/net_sync_events.rs rename to crates/events/src/enclave_event/net_sync_events_received.rs index 1fb25bf18f..eefe1828fb 100644 --- a/crates/events/src/enclave_event/net_sync_events.rs +++ b/crates/events/src/enclave_event/net_sync_events_received.rs @@ -12,17 +12,17 @@ use super::{EnclaveEvent, Unsequenced}; #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct NetSyncEvents { +pub struct NetSyncEventsReceived { pub events: Vec>, } -impl NetSyncEvents { +impl NetSyncEventsReceived { pub fn new(events: Vec>) -> Self { Self { events } } } -impl Display for NetSyncEvents { +impl Display for NetSyncEventsReceived { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/sync_request.rs b/crates/events/src/enclave_event/outgoing_sync_requested.rs similarity index 88% rename from crates/events/src/enclave_event/sync_request.rs rename to crates/events/src/enclave_event/outgoing_sync_requested.rs index 45f27686a9..d9a89b6996 100644 --- a/crates/events/src/enclave_event/sync_request.rs +++ b/crates/events/src/enclave_event/outgoing_sync_requested.rs @@ -12,12 +12,12 @@ use crate::AggregateId; #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct SyncRequest { +pub struct OutgoingSyncRequested { // TODO: this should be the event to trigger evm sync too pub since: Vec<(AggregateId, u128)>, } -impl Display for SyncRequest { +impl Display for OutgoingSyncRequested { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index ce6c13c9c9..d876b1bfaa 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -8,8 +8,8 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEvents, ReceiveEvents, SyncRequest, - Unsequenced, + EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, + ReceiveEvents, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake}; use futures::TryFutureExt; @@ -90,16 +90,16 @@ impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::SyncRequest(data) => ctx.notify(data), + EnclaveEventData::OutgoingSyncRequested(data) => ctx.notify(data), _ => (), } } } /// SyncRequest is called on start up to fetch remote events -impl Handler for NetSyncManager { +impl Handler for NetSyncManager { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: SyncRequest, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: OutgoingSyncRequested, _: &mut Self::Context) -> Self::Result { trap_fut( EType::Net, &self.bus.clone(), @@ -113,7 +113,7 @@ impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: OutgoingSyncRequestSucceeded, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { - self.bus.publish(NetSyncEvents { + self.bus.publish(NetSyncEventsReceived { events: msg .value .events @@ -200,7 +200,7 @@ async fn handle_sync_request_event( net_cmds: mpsc::Sender, net_events: Arc>, bus: BusHandle, - event: SyncRequest, + event: OutgoingSyncRequested, ) -> Result<()> { let value = retry_with_backoff( || { @@ -216,7 +216,7 @@ async fn handle_sync_request_event( ) .await?; - bus.publish(NetSyncEvents::new( + bus.publish(NetSyncEventsReceived::new( value .value .events From 79f93a8b361c496097d4dbb4fd1e623290abf0d3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 22 Jan 2026 07:43:27 +0000 Subject: [PATCH 066/102] rename EvmInterface -> EvmReadInterface --- crates/evm/src/bonding_registry_sol.rs | 10 +++---- crates/evm/src/ciphernode_registry_sol.rs | 12 ++++---- crates/evm/src/enclave_sol.rs | 4 +-- crates/evm/src/enclave_sol_reader.rs | 10 +++---- crates/evm/src/evm_interface.rs | 30 +++++++++---------- crates/evm/src/evm_launch_coordinator.rs | 2 ++ .../evm/src/historical_event_coordinator.rs | 2 +- crates/evm/src/lib.rs | 2 +- crates/evm/src/repo.rs | 14 ++++----- crates/evm/tests/integration.rs | 20 +++++++------ 10 files changed, 55 insertions(+), 51 deletions(-) create mode 100644 crates/evm/src/evm_launch_coordinator.rs diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index d7ed6db941..c70de97a15 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - evm_interface::EvmInterfaceState, helpers::EthProvider, EnclaveEvmEvent, EvmInterface, + evm_interface::EvmReadInterfaceState, helpers::EthProvider, EnclaveEvmEvent, EvmReadInterface, }; use actix::{Addr, Recipient}; use alloy::{ @@ -148,14 +148,14 @@ impl BondingRegistrySolReader { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, - ) -> Result>> + ) -> Result>> where P: Provider + Clone + 'static, { - let addr = EvmInterface::attach( + let addr = EvmReadInterface::attach( provider, extractor, contract_address, @@ -182,7 +182,7 @@ impl BondingRegistrySol { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, ) -> Result<()> diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 4c8d9de6e8..b1e234e948 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -5,9 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - evm_interface::EvmInterfaceState, + evm_interface::EvmReadInterfaceState, helpers::{send_tx_with_retry, EthProvider}, - EnclaveEvmEvent, EvmInterface, + EnclaveEvmEvent, EvmReadInterface, }; use actix::prelude::*; use alloy::{ @@ -223,14 +223,14 @@ impl CiphernodeRegistrySolReader { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, - ) -> Result>> + ) -> Result>> where P: Provider + Clone + 'static, { - let addr = EvmInterface::attach( + let addr = EvmReadInterface::attach( provider, extractor, contract_address, @@ -557,7 +557,7 @@ impl CiphernodeRegistrySol { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, ) -> Result<()> diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 47f3d24b3a..431a64650f 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -6,7 +6,7 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - evm_interface::EvmInterfaceState, helpers::EthProvider, EnclaveEvmEvent, + evm_interface::EvmReadInterfaceState, helpers::EthProvider, EnclaveEvmEvent, }; use actix::Recipient; use alloy::providers::{Provider, WalletProvider}; @@ -23,7 +23,7 @@ impl EnclaveSol { read_provider: EthProvider, write_provider: EthProvider, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, ) -> Result<()> diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index 29c7d76038..935ccfcc06 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::evm_interface::EvmInterfaceState; +use crate::evm_interface::EvmReadInterfaceState; use crate::helpers::EthProvider; -use crate::{EnclaveEvmEvent, EvmInterface}; +use crate::{EnclaveEvmEvent, EvmReadInterface}; use actix::{Addr, Recipient}; use alloy::primitives::{LogData, B256}; use alloy::providers::Provider; @@ -110,14 +110,14 @@ impl EnclaveSolReader { bus: &BusHandle, provider: EthProvider

, contract_address: &str, - repository: &Repository, + repository: &Repository, start_block: Option, rpc_url: String, - ) -> Result>> + ) -> Result>> where P: Provider + Clone + 'static, { - let addr = EvmInterface::attach( + let addr = EvmReadInterface::attach( provider, extractor, contract_address, diff --git a/crates/evm/src/evm_interface.rs b/crates/evm/src/evm_interface.rs index 2183c1c770..bb5f61b4cf 100644 --- a/crates/evm/src/evm_interface.rs +++ b/crates/evm/src/evm_interface.rs @@ -54,25 +54,25 @@ impl EnclaveEvmEvent { pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; -pub struct EvmInterfaceParams

{ +pub struct EvmReadInterfaceParams

{ provider: EthProvider

, extractor: ExtractorFn, contract_address: Address, start_block: Option, processor: Recipient, bus: BusHandle, - state: Persistable, + state: Persistable, rpc_url: String, } #[derive(Default, serde::Serialize, serde::Deserialize, Clone)] -pub struct EvmInterfaceState { +pub struct EvmReadInterfaceState { pub ids: HashSet, pub last_block: Option, } /// Connects to Enclave.sol converting EVM events to EnclaveEvents -pub struct EvmInterface

{ +pub struct EvmReadInterface

{ /// The alloy provider provider: Option>, /// The contract address @@ -91,13 +91,13 @@ pub struct EvmInterface

{ /// Event bus for error propagation only bus: BusHandle, /// The auto persistable state of the event reader - state: Persistable, // XXX: would be best to avoid persistable state outside of domain + state: Persistable, // XXX: would be best to avoid persistable state outside of domain /// The RPC URL for the provider rpc_url: String, } -impl EvmInterface

{ - pub fn new(params: EvmInterfaceParams

) -> Self { +impl EvmReadInterface

{ + pub fn new(params: EvmReadInterfaceParams

) -> Self { let (shutdown_tx, shutdown_rx) = oneshot::channel(); Self { contract_address: params.contract_address, @@ -120,15 +120,15 @@ impl EvmInterface

{ start_block: Option, processor: &Recipient, bus: &BusHandle, - repository: &Repository, + repository: &Repository, rpc_url: String, ) -> Result> { let sync_state = repository .clone() - .load_or_default(EvmInterfaceState::default()) + .load_or_default(EvmReadInterfaceState::default()) .await?; - let params = EvmInterfaceParams { + let params = EvmReadInterfaceParams { provider, extractor, contract_address: contract_address.parse()?, @@ -139,7 +139,7 @@ impl EvmInterface

{ rpc_url, }; - let addr = EvmInterface::new(params).start(); + let addr = EvmReadInterface::new(params).start(); processor.do_send(EnclaveEvmEvent::RegisterReader); @@ -148,7 +148,7 @@ impl EvmInterface

{ } } -impl Actor for EvmInterface

{ +impl Actor for EvmReadInterface

{ type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { @@ -196,7 +196,7 @@ impl Actor for EvmInterface

{ async fn stream_from_evm( provider: EthProvider

, contract_address: &Address, - reader_addr: Addr>, + reader_addr: Addr>, extractor: fn(&LogData, Option<&B256>, u64) -> Option, mut shutdown: oneshot::Receiver<()>, start_block: Option, @@ -297,7 +297,7 @@ fn is_local_node(rpc_url: &str) -> bool { rpc_url.contains("localhost") || rpc_url.contains("127.0.0.1") } -impl Handler for EvmInterface

{ +impl Handler for EvmReadInterface

{ type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { @@ -311,7 +311,7 @@ impl Handler for EvmInterface

{ // XXX: Sync should handle the tracking of the last block based on what has been stored in the // eventlog -impl Handler for EvmInterface

{ +impl Handler for EvmReadInterface

{ type Result = (); #[instrument(name = "evm_interface", skip_all)] diff --git a/crates/evm/src/evm_launch_coordinator.rs b/crates/evm/src/evm_launch_coordinator.rs new file mode 100644 index 0000000000..085d522d56 --- /dev/null +++ b/crates/evm/src/evm_launch_coordinator.rs @@ -0,0 +1,2 @@ +// Configured with Addr for +pub struct EvmLaunchCoordinator {} diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs index 4dc4243d28..8d1d49fd32 100644 --- a/crates/evm/src/historical_event_coordinator.rs +++ b/crates/evm/src/historical_event_coordinator.rs @@ -20,7 +20,7 @@ struct BufferedEvent { #[rtype(result = "()")] pub struct CoordinatorStart; -/// Coordinates historical replay across all EvmInterfaces. +/// Coordinates historical replay across all EvmReadInterfaces. /// Buffers historical events, then sorts + publishes once all readers finish. pub struct HistoricalEventCoordinator { /// Count of readers that have registered diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 328512928d..42b8564dde 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -21,7 +21,7 @@ pub use ciphernode_registry_sol::{ pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use evm_interface::{EnclaveEvmEvent, EvmInterface, EvmInterfaceState, ExtractorFn}; +pub use evm_interface::{EnclaveEvmEvent, EvmReadInterface, EvmReadInterfaceState, ExtractorFn}; pub use helpers::send_tx_with_retry; pub use historical_event_coordinator::{CoordinatorStart, HistoricalEventCoordinator}; pub use repo::*; diff --git a/crates/evm/src/repo.rs b/crates/evm/src/repo.rs index 827df69e18..1fc79ae011 100644 --- a/crates/evm/src/repo.rs +++ b/crates/evm/src/repo.rs @@ -7,7 +7,7 @@ use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use crate::EvmInterfaceState; +use crate::EvmReadInterfaceState; pub trait EthPrivateKeyRepositoryFactory { fn eth_private_key(&self) -> Repository>; @@ -20,21 +20,21 @@ impl EthPrivateKeyRepositoryFactory for Repositories { } pub trait EnclaveSolReaderRepositoryFactory { - fn enclave_sol_reader(&self, chain_id: u64) -> Repository; + fn enclave_sol_reader(&self, chain_id: u64) -> Repository; } impl EnclaveSolReaderRepositoryFactory for Repositories { - fn enclave_sol_reader(&self, chain_id: u64) -> Repository { + fn enclave_sol_reader(&self, chain_id: u64) -> Repository { Repository::new(self.store.scope(StoreKeys::enclave_sol_reader(chain_id))) } } pub trait CiphernodeRegistryReaderRepositoryFactory { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; + fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; } impl CiphernodeRegistryReaderRepositoryFactory for Repositories { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { + fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(StoreKeys::ciphernode_registry_reader(chain_id)), @@ -43,11 +43,11 @@ impl CiphernodeRegistryReaderRepositoryFactory for Repositories { } pub trait BondingRegistryReaderRepositoryFactory { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository; + fn bonding_registry_reader(&self, chain_id: u64) -> Repository; } impl BondingRegistryReaderRepositoryFactory for Repositories { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository { + fn bonding_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(StoreKeys::bonding_registry_reader(chain_id)), diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index b4a54141d7..84b8afb073 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -20,7 +20,9 @@ use e3_entrypoint::helpers::datastore::get_in_mem_store; use e3_events::{ prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, HistoryCollector, Shutdown, TestEvent, }; -use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmInterface, HistoricalEventCoordinator}; +use e3_evm::{ + helpers::EthProvider, CoordinatorStart, EvmReadInterface, HistoricalEventCoordinator, +}; use std::time::Duration; use tokio::time::sleep; @@ -89,7 +91,7 @@ async fn evm_reader() -> Result<()> { let coordinator = HistoricalEventCoordinator::setup(bus.clone()); let processor = coordinator.clone().recipient(); - EvmInterface::attach( + EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -172,7 +174,7 @@ async fn ensure_historical_events() -> Result<()> { .await?; } - EvmInterface::attach( + EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -249,7 +251,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { .await?; } - let addr1 = EvmInterface::attach( + let addr1 = EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -291,7 +293,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { let msgs = get_msgs(&history_collector).await?; assert_eq!(msgs, ["before", "online", "live", "events"]); - let _ = EvmInterface::attach( + let _ = EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -358,7 +360,7 @@ async fn coordinator_single_reader() -> Result<()> { .await?; } - EvmInterface::attach( + EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), @@ -450,7 +452,7 @@ async fn coordinator_multiple_readers() -> Result<()> { .watch() .await?; - EvmInterface::attach( + EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract1.address().to_string(), @@ -462,7 +464,7 @@ async fn coordinator_multiple_readers() -> Result<()> { ) .await?; - EvmInterface::attach( + EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract2.address().to_string(), @@ -509,7 +511,7 @@ async fn coordinator_no_historical_events() -> Result<()> { let coordinator = HistoricalEventCoordinator::setup(bus.clone()); let processor = coordinator.clone().recipient(); - EvmInterface::attach( + EvmReadInterface::attach( provider.clone(), test_event_extractor, &contract.address().to_string(), From b7b9bf7c5d64fd674def757c29090cb8f5432cf5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 24 Jan 2026 23:03:01 +0000 Subject: [PATCH 067/102] refactor evm events --- .../src/ciphernode_builder.rs | 56 +-- crates/events/src/bus_handle.rs | 7 +- crates/events/src/enclave_event/mod.rs | 14 +- .../events/src/enclave_event/sync_effect.rs | 27 ++ crates/events/src/enclave_event/sync_end.rs | 26 ++ crates/events/src/enclave_event/sync_start.rs | 49 +++ crates/events/src/hlc.rs | 1 + crates/events/src/traits.rs | 2 + crates/evm/src/bonding_registry_sol.rs | 73 +--- crates/evm/src/ciphernode_registry_sol.rs | 59 +-- crates/evm/src/enclave_sol.rs | 27 +- crates/evm/src/enclave_sol_reader.rs | 43 +- crates/evm/src/events.rs | 38 ++ crates/evm/src/evm_interface.rs | 368 ------------------ crates/evm/src/evm_launch_coordinator.rs | 116 +++++- crates/evm/src/evm_read_interface.rs | 220 +++++++++++ crates/evm/src/evm_reader.rs | 60 +++ crates/evm/src/evm_router.rs | 71 ++++ crates/evm/src/evm_sync_processor.rs | 98 +++++ .../evm/src/historical_event_coordinator.rs | 15 +- crates/evm/src/lib.rs | 13 +- crates/evm/tests/integration.rs | 1 - 22 files changed, 785 insertions(+), 599 deletions(-) create mode 100644 crates/events/src/enclave_event/sync_effect.rs create mode 100644 crates/events/src/enclave_event/sync_end.rs create mode 100644 crates/events/src/enclave_event/sync_start.rs create mode 100644 crates/evm/src/events.rs delete mode 100644 crates/evm/src/evm_interface.rs create mode 100644 crates/evm/src/evm_read_interface.rs create mode 100644 crates/evm/src/evm_reader.rs create mode 100644 crates/evm/src/evm_router.rs create mode 100644 crates/evm/src/evm_sync_processor.rs diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 8a49a4023e..757015b84b 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -23,11 +23,11 @@ use e3_evm::{ load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, ProviderConfig, }, - BondingRegistryReaderRepositoryFactory, BondingRegistrySol, - CiphernodeRegistryReaderRepositoryFactory, CiphernodeRegistrySol, CoordinatorStart, EnclaveSol, - EnclaveSolReader, EnclaveSolReaderRepositoryFactory, EthPrivateKeyRepositoryFactory, - HistoricalEventCoordinator, + BondingRegistryReaderRepositoryFactory, CiphernodeRegistryReaderRepositoryFactory, + CiphernodeRegistrySol, CoordinatorStart, EnclaveSol, EnclaveSolReader, + EnclaveSolReaderRepositoryFactory, EthPrivateKeyRepositoryFactory, HistoricalEventCoordinator, }; +use e3_evm::{BondingRegistrySolReader, EvmLaunchCoordinator}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; @@ -406,65 +406,38 @@ impl CiphernodeBuilder { .iter() .filter(|chain| chain.enabled.unwrap_or(true)) { + let read_provider = provider_cache.ensure_read_provider(chain).await?; + let mut launcher = EvmLaunchCoordinator::builder(&bus, read_provider); if self.contract_components.enclave { - let read_provider = provider_cache.ensure_read_provider(chain).await?; let write_provider = provider_cache .ensure_write_provider(&repositories, chain, cipher) .await?; - EnclaveSol::attach( + let addr = EnclaveSol::attach( &processor, &bus, - read_provider.clone(), write_provider.clone(), &chain.contracts.enclave.address(), - &repositories.enclave_sol_reader(read_provider.chain_id()), - chain.contracts.enclave.deploy_block(), - chain.rpc_url.clone(), ) .await?; + launcher = launcher.with_contract(chain.contracts.enclave.address(), addr)?; + // TODO: add to launch coordinator } if self.contract_components.enclave_reader { let read_provider = provider_cache.ensure_read_provider(chain).await?; - EnclaveSolReader::attach( - &processor, - &bus, - read_provider.clone(), - &chain.contracts.enclave.address(), - &repositories.enclave_sol_reader(read_provider.chain_id()), - chain.contracts.enclave.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; + EnclaveSolReader::setup(&processor); } if self.contract_components.bonding_registry { let read_provider = provider_cache.ensure_read_provider(chain).await?; - BondingRegistrySol::attach( - &processor, - &bus, - read_provider.clone(), - &chain.contracts.bonding_registry.address(), - &repositories.bonding_registry_reader(read_provider.chain_id()), - chain.contracts.bonding_registry.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; + let addr = BondingRegistrySolReader::setup(&processor); + // TODO: add to launch coordinator } if self.contract_components.ciphernode_registry { let read_provider = provider_cache.ensure_read_provider(chain).await?; - CiphernodeRegistrySol::attach( - &processor, - &bus, - read_provider.clone(), - &chain.contracts.ciphernode_registry.address(), - &repositories.ciphernode_registry_reader(read_provider.chain_id()), - chain.contracts.ciphernode_registry.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; - + let addr = CiphernodeRegistrySol::attach(&processor); + // TODO: add to launch coordinator match provider_cache .ensure_write_provider(&repositories, chain, cipher) .await @@ -490,6 +463,7 @@ impl CiphernodeBuilder { ), } } + launcher.build(); } // We start after all readers have registered diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index bedf144831..b2c785df1d 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -20,7 +20,7 @@ use crate::{ EventSubscriber, }, EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, - HistoryCollector, Sequenced, Subscribe, Unsequenced, + HistoryCollector, Sequenced, Subscribe, Unsequenced, Unsubscribe, }; #[derive(Clone, Derivative)] @@ -163,6 +163,11 @@ impl EventSubscriber> for BusHandle { .do_send(Subscribe::new(*event_type, recipient.clone())); } } + + fn unsubscribe(&self, event_type: &str, recipient: Recipient>) { + self.event_bus + .do_send(Unsubscribe::new(event_type, recipient)); + } } impl EventContextManager for BusHandle { diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 10c23241ef..d1d630c8fa 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -31,6 +31,9 @@ mod plaintext_output_published; mod publickey_aggregated; mod publish_document; mod shutdown; +mod sync_effect; +mod sync_end; +mod sync_start; mod test_event; mod threshold_share_collection_failed; mod threshold_share_created; @@ -68,6 +71,9 @@ pub use publickey_aggregated::*; pub use publish_document::*; pub use shutdown::*; use strum::IntoStaticStr; +pub use sync_effect::*; +pub use sync_end::*; +pub use sync_start::*; pub use test_event::*; pub use threshold_share_collection_failed::*; pub use threshold_share_created::*; @@ -137,6 +143,9 @@ pub enum EnclaveEventData { OutgoingSyncRequested(OutgoingSyncRequested), NetSyncEventsReceived(NetSyncEventsReceived), EvmSyncEventsReceived(EvmSyncEventsReceived), + SyncStart(SyncStart), + SyncEffect(SyncEffect), + SyncEnd(SyncEnd), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -407,7 +416,10 @@ impl_into_event_data!( ComputeRequestError, OutgoingSyncRequested, NetSyncEventsReceived, - EvmSyncEventsReceived + EvmSyncEventsReceived, + SyncStart, + SyncEffect, + SyncEnd ); impl TryFrom<&EnclaveEvent> for EnclaveError { diff --git a/crates/events/src/enclave_event/sync_effect.rs b/crates/events/src/enclave_event/sync_effect.rs new file mode 100644 index 0000000000..b503422933 --- /dev/null +++ b/crates/events/src/enclave_event/sync_effect.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +/// Dispatched from the Sync actor once the effect events are to be run but before buffered events are to +/// be dispatched +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncEffect; + +impl SyncEffect { + pub fn new() -> Self { + Self {} + } +} + +impl Display for SyncEffect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/sync_end.rs b/crates/events/src/enclave_event/sync_end.rs new file mode 100644 index 0000000000..400ba9f5e2 --- /dev/null +++ b/crates/events/src/enclave_event/sync_end.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +/// Dispatched once the sync process is complete and live listening should continue +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncEnd; + +impl SyncEnd { + pub fn new() -> Self { + Self {} + } +} + +impl Display for SyncEnd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs new file mode 100644 index 0000000000..2df094ac69 --- /dev/null +++ b/crates/events/src/enclave_event/sync_start.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Message, Recipient}; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use super::EnclaveEventData; + +/// This is a processed EnclaveEvmEvent +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncEvmEvent { + data: EnclaveEventData, + block: u64, +} + +impl SyncEvmEvent { + pub fn new(data: EnclaveEventData, block: u64) -> Self { + Self { data, block } + } +} + +/// Dispatched by the Sync actor when initial data is read and the sync process needs to be started +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncStart { + #[serde(skip)] + pub sender: Option>, // Must be option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented +} + +impl SyncStart { + pub fn new(sender: Recipient) -> Self { + Self { + sender: Some(sender), + } + } +} + +impl Display for SyncStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index af871dce54..21b98e2655 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -4,6 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use alloy::rpc::types::Log; use rand::Rng; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::{Arc, Mutex}; diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 54c1cbcffe..d677342969 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -105,6 +105,8 @@ pub trait EventSubscriber { fn subscribe(&self, event_type: &str, recipient: Recipient); /// Subscribe the recipient to events matching any of the given event types fn subscribe_all(&self, event_types: &[&str], recipient: Recipient); + /// Subscribe the recipient to events matching the given event type + fn unsubscribe(&self, event_type: &str, recipient: Recipient); } /// Trait to create an event with a timestamp from its associated type data diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index c70de97a15..a54f948ca1 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -4,20 +4,15 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{ - evm_interface::EvmReadInterfaceState, helpers::EthProvider, EnclaveEvmEvent, EvmReadInterface, -}; -use actix::{Addr, Recipient}; +use crate::{events::EvmEventProcessor, evm_reader::EvmReader}; +use actix::{Actor, Addr}; use alloy::{ primitives::{LogData, B256}, - providers::Provider, sol, sol_types::SolEvent, }; -use anyhow::Result; -use e3_data::Repository; -use e3_events::{BusHandle, EnclaveEventData}; -use tracing::{error, info, trace}; +use e3_events::EnclaveEventData; +use tracing::{error, trace}; sol!( #[sol(rpc)] @@ -141,64 +136,8 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< /// Connects to BondingRegistry.sol converting EVM events to EnclaveEvents pub struct BondingRegistrySolReader; - impl BondingRegistrySolReader { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result>> - where - P: Provider + Clone + 'static, - { - let addr = EvmReadInterface::attach( - provider, - extractor, - contract_address, - start_block, - processor, - bus, - repository, - rpc_url, - ) - .await?; - - info!(address=%contract_address, "BondingRegistrySolReader is listening to address"); - - Ok(addr) - } -} - -/// Wrapper for a reader -pub struct BondingRegistrySol; - -impl BondingRegistrySol { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result<()> - where - P: Provider + Clone + 'static, - { - BondingRegistrySolReader::attach( - processor, - bus, - provider, - contract_address, - repository, - start_block, - rpc_url, - ) - .await?; - Ok(()) + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmReader::new(next, extractor).start() } } diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index b1e234e948..fba7022e5b 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -5,9 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - evm_interface::EvmReadInterfaceState, + events::{EnclaveEvmEvent, EvmEventProcessor}, + evm_reader::EvmReader, helpers::{send_tx_with_retry, EthProvider}, - EnclaveEvmEvent, EvmReadInterface, }; use actix::prelude::*; use alloy::{ @@ -18,7 +18,6 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; -use e3_data::Repository; use e3_events::{ prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, EnclaveEvent, EnclaveEventData, EventSubscriber, OrderedSet, PublicKeyAggregated, Seed, @@ -218,33 +217,8 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result>> - where - P: Provider + Clone + 'static, - { - let addr = EvmReadInterface::attach( - provider, - extractor, - contract_address, - start_block, - processor, - bus, - repository, - rpc_url, - ) - .await?; - - info!(address=%contract_address, "CiphernodeRegistrySolReader is listening to address"); - - Ok(addr) + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmReader::new(next, extractor).start() } } @@ -552,29 +526,8 @@ pub async fn publish_committee_to_registry( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result<()> - where - P: Provider + Clone + 'static, - { - CiphernodeRegistrySolReader::attach( - processor, - bus, - provider, - contract_address, - repository, - start_block, - rpc_url, - ) - .await?; - Ok(()) + pub fn attach(processor: &Recipient) -> Addr { + CiphernodeRegistrySolReader::setup(processor) } pub async fn attach_writer

( diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 431a64650f..3e65e397a3 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -6,44 +6,29 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - evm_interface::EvmReadInterfaceState, helpers::EthProvider, EnclaveEvmEvent, + events::EnclaveEvmEvent, evm_reader::EvmReader, helpers::EthProvider, }; -use actix::Recipient; +use actix::{Addr, Recipient}; use alloy::providers::{Provider, WalletProvider}; use anyhow::Result; -use e3_data::Repository; use e3_events::BusHandle; pub struct EnclaveSol; impl EnclaveSol { - pub async fn attach( + pub async fn attach( processor: &Recipient, bus: &BusHandle, - read_provider: EthProvider, write_provider: EthProvider, contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result<()> + ) -> Result> where - R: Provider + Clone + 'static, W: Provider + WalletProvider + Clone + 'static, { - EnclaveSolReader::attach( - processor, - bus, - read_provider, - contract_address, - repository, - start_block, - rpc_url, - ) - .await?; + let addr = EnclaveSolReader::setup(processor); EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; - Ok(()) + Ok(addr) } } diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index 935ccfcc06..117b44e28f 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -4,19 +4,15 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::evm_interface::EvmReadInterfaceState; -use crate::helpers::EthProvider; -use crate::{EnclaveEvmEvent, EvmReadInterface}; -use actix::{Addr, Recipient}; +use crate::events::EvmEventProcessor; +use crate::evm_reader::EvmReader; +use actix::{Actor, Addr}; use alloy::primitives::{LogData, B256}; -use alloy::providers::Provider; use alloy::{sol, sol_types::SolEvent}; -use anyhow::Result; -use e3_data::Repository; -use e3_events::{BusHandle, E3id, EnclaveEventData}; +use e3_events::{E3id, EnclaveEventData}; use e3_utils::utility_types::ArcBytes; use num_bigint::BigUint; -use tracing::{error, info, trace}; +use tracing::{error, trace}; sol!( #[sol(rpc)] @@ -105,32 +101,7 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< pub struct EnclaveSolReader; impl EnclaveSolReader { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result>> - where - P: Provider + Clone + 'static, - { - let addr = EvmReadInterface::attach( - provider, - extractor, - contract_address, - start_block, - processor, - bus, - repository, - rpc_url, - ) - .await?; - - info!(address=%contract_address, "EnclaveSolReader is listening to address"); - - Ok(addr) + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmReader::new(next, extractor).start() } } diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs new file mode 100644 index 0000000000..ffa7518099 --- /dev/null +++ b/crates/evm/src/events.rs @@ -0,0 +1,38 @@ +use actix::{Message, Recipient}; +use alloy::rpc::types::Log; +use e3_events::{EnclaveEventData, EventId}; +use serde::{Deserialize, Serialize}; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub enum EnclaveEvmEvent { + /// Register a reader with the coordinator before it starts processing + RegisterReader, + /// Signal that this reader has completed historical sync + HistoricalSyncComplete, + /// An actual event from the blockchain + Event(EvmEvent), + /// Raw log data from the provider + Log(EvmLog), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmEvent { + pub payload: EnclaveEventData, + pub block: u64, + pub ts: u128, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmLog { + pub log: Log, + pub chain_id: u64, +} + +impl EnclaveEvmEvent { + pub fn get_id(&self) -> EventId { + EventId::hash(self.clone()) + } +} + +pub type EvmEventProcessor = Recipient; diff --git a/crates/evm/src/evm_interface.rs b/crates/evm/src/evm_interface.rs deleted file mode 100644 index bb5f61b4cf..0000000000 --- a/crates/evm/src/evm_interface.rs +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::helpers::EthProvider; -use actix::prelude::*; -use actix::{Addr, Recipient}; -use alloy::eips::BlockNumberOrTag; -use alloy::primitives::Address; -use alloy::primitives::{LogData, B256}; -use alloy::providers::Provider; -use alloy::rpc::types::Filter; -use anyhow::{anyhow, Result}; -use e3_data::{AutoPersist, Persistable, Repository}; -use e3_events::{prelude::*, EType, EnclaveEvent, EnclaveEventData, EventId}; -use e3_events::{BusHandle, Event}; -use futures_util::stream::StreamExt; -use std::collections::HashSet; -use tokio::select; -use tokio::sync::oneshot; -use tracing::{error, info, instrument, trace, warn}; - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[rtype(result = "()")] -pub enum EnclaveEvmEvent { - /// Register a reader with the coordinator before it starts processing - RegisterReader, - /// Signal that this reader has completed historical sync - HistoricalSyncComplete, - /// An actual event from the blockchain - Event { - event: EnclaveEventData, - block: Option, - }, - /// Raw log data from the provider - Log { - data: LogData, - topic: Option, - chain_id: u64, - }, -} - -impl EnclaveEvmEvent { - pub fn new(event: EnclaveEventData, block: Option) -> Self { - Self::Event { event, block } - } - - pub fn get_id(&self) -> EventId { - EventId::hash(self.clone()) - } -} - -pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; - -pub struct EvmReadInterfaceParams

{ - provider: EthProvider

, - extractor: ExtractorFn, - contract_address: Address, - start_block: Option, - processor: Recipient, - bus: BusHandle, - state: Persistable, - rpc_url: String, -} - -#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] -pub struct EvmReadInterfaceState { - pub ids: HashSet, - pub last_block: Option, -} - -/// Connects to Enclave.sol converting EVM events to EnclaveEvents -pub struct EvmReadInterface

{ - /// The alloy provider - provider: Option>, - /// The contract address - contract_address: Address, - /// The Extractor function to determine which events to extract and convert to EnclaveEvents - extractor: ExtractorFn, - /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used - /// internally. You should send the Shutdown signal to the reader directly or via the EventBus - shutdown_rx: Option>, - /// The sender for the shutdown signal this is only used internally - shutdown_tx: Option>, - /// The block that processing should start from - start_block: Option, - /// Processor to forward events an actor - processor: Recipient, - /// Event bus for error propagation only - bus: BusHandle, - /// The auto persistable state of the event reader - state: Persistable, // XXX: would be best to avoid persistable state outside of domain - /// The RPC URL for the provider - rpc_url: String, -} - -impl EvmReadInterface

{ - pub fn new(params: EvmReadInterfaceParams

) -> Self { - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - Self { - contract_address: params.contract_address, - provider: Some(params.provider), - extractor: params.extractor, - shutdown_rx: Some(shutdown_rx), - shutdown_tx: Some(shutdown_tx), - start_block: params.start_block, - processor: params.processor, - bus: params.bus, - state: params.state, - rpc_url: params.rpc_url, - } - } - - pub async fn attach( - provider: EthProvider

, - extractor: ExtractorFn, - contract_address: &str, - start_block: Option, - processor: &Recipient, - bus: &BusHandle, - repository: &Repository, - rpc_url: String, - ) -> Result> { - let sync_state = repository - .clone() - .load_or_default(EvmReadInterfaceState::default()) - .await?; - - let params = EvmReadInterfaceParams { - provider, - extractor, - contract_address: contract_address.parse()?, - start_block, - processor: processor.clone(), - bus: bus.clone(), - state: sync_state, - rpc_url, - }; - - let addr = EvmReadInterface::new(params).start(); - - processor.do_send(EnclaveEvmEvent::RegisterReader); - - bus.subscribe("Shutdown", addr.clone().into()); - Ok(addr) - } -} - -impl Actor for EvmReadInterface

{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - let reader_addr = ctx.address(); - let bus = self.bus.clone(); - - let Some(provider) = self.provider.take() else { - error!("Could not start event reader as provider has already been used."); - return; - }; - - let extractor = self.extractor; - let Some(shutdown) = self.shutdown_rx.take() else { - bus.err(EType::Evm, anyhow!("shutdown already called")); - return; - }; - - let contract_address = self.contract_address; - let start_block = self.start_block; - let rpc_url = self.rpc_url.clone(); - - ctx.spawn( - async move { - stream_from_evm( - provider, - &contract_address, - reader_addr.clone(), - extractor, - shutdown, - start_block, - &bus, - rpc_url, - ) - .await - } - .into_actor(self), - ); - } -} - -// TODO: split this up into: -// 1. historical request (will finish) -// 2. current listener (run indefinitely) -#[instrument(name = "evm_interface", skip_all)] -async fn stream_from_evm( - provider: EthProvider

, - contract_address: &Address, - reader_addr: Addr>, - extractor: fn(&LogData, Option<&B256>, u64) -> Option, - mut shutdown: oneshot::Receiver<()>, - start_block: Option, - bus: &BusHandle, - rpc_url: String, -) { - let chain_id = provider.chain_id(); - let provider_ref = provider.provider(); - - if start_block.unwrap_or(0) == 0 && !is_local_node(&rpc_url) { - error!( - "Querying from block 0 on a non-local node ({}) without a specific start_block is not allowed.", - rpc_url - ); - bus.err( - EType::Evm, - anyhow!( - "Misconfiguration: Attempted to query historical events from genesis on a non-local node. \ - Please specify a `start_block` for contract address {contract_address} on chain {chain_id} using rpc {rpc_url}" - ) - ); - return; - } - - let historical_filter = Filter::new() - .address(*contract_address) - .from_block(start_block.unwrap_or(0)); - let current_filter = Filter::new() - .address(*contract_address) - .from_block(BlockNumberOrTag::Latest); - - // Historical events - match provider_ref.get_logs(&historical_filter).await { - Ok(historical_logs) => { - info!("Fetched {} historical events", historical_logs.len()); - for log in historical_logs { - let block_number = log.block_number; - if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { - trace!("Processing historical log"); - reader_addr.do_send(EnclaveEvmEvent::new(event, block_number)); - } - } - - reader_addr.do_send(EnclaveEvmEvent::HistoricalSyncComplete); - } - Err(e) => { - error!("Failed to fetch historical events: {}", e); - bus.err(EType::Evm, anyhow!(e)); - return; - } - } - - info!("Subscribing to live events"); - match provider_ref.subscribe_logs(¤t_filter).await { - Ok(subscription) => { - let id: B256 = subscription.local_id().clone(); - let mut stream = subscription.into_stream(); - - loop { - select! { - maybe_log = stream.next() => { - match maybe_log { - Some(log) => { - let block_number = log.block_number; - trace!("Received log from EVM"); - - let Some(event) = extractor(log.data(), log.topic0(), chain_id) else { - trace!("Unknown log from EVM. This will happen from time to time."); - continue; - }; - - trace!("Extracted EVM Event: {:?}", event); - reader_addr.do_send(EnclaveEvmEvent::new(event, block_number)); - } - None => break, // Stream ended - } - } - _ = &mut shutdown => { - info!("Received shutdown signal, stopping EVM stream"); - match provider_ref.unsubscribe(id).await { - Ok(_) => info!("Unsubscribed successfully from EVM event stream"), - Err(err) => error!("Cannot unsubscribe from EVM event stream: {}", err), - }; - break; - } - } - } - } - Err(e) => { - bus.err(EType::Evm, anyhow!("{}", e)); - } - } - - info!("Exiting stream loop"); -} - -fn is_local_node(rpc_url: &str) -> bool { - rpc_url.contains("localhost") || rpc_url.contains("127.0.0.1") -} - -impl Handler for EvmReadInterface

{ - type Result = (); - - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - if let EnclaveEventData::Shutdown(_) = msg.into_data() { - if let Some(shutdown) = self.shutdown_tx.take() { - let _ = shutdown.send(()); - } - } - } -} - -// XXX: Sync should handle the tracking of the last block based on what has been stored in the -// eventlog -impl Handler for EvmReadInterface

{ - type Result = (); - - #[instrument(name = "evm_interface", skip_all)] - fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - match msg { - EnclaveEvmEvent::RegisterReader | EnclaveEvmEvent::HistoricalSyncComplete => { - self.processor.do_send(msg); - } - - EnclaveEvmEvent::Event { event, block } => { - // XXX: Should not need state - // 1. deduplication already happens at the event bus - // 2. cursor is kept by sync - match self.state.try_mutate(|mut state| { - let temp_wrapped = EnclaveEvmEvent::Event { - event: event.clone(), - block, - }; - let event_id = temp_wrapped.get_id(); - - trace!("Processing event: {}", event_id); - trace!("Cache length: {}", state.ids.len()); - - if state.ids.contains(&event_id) { - warn!( - "Event id {} has already been seen and was not forwarded", - &event_id - ); - return Ok(state); - } - - let event_type = event.event_type(); - - self.processor.do_send(EnclaveEvmEvent::Event { - event: event.clone(), - block, - }); - - // Save processed IDs - trace!("Storing event(EVM) in cache {}({})", event_type, event_id); - state.ids.insert(event_id); - state.last_block = block; - - Ok(state) - }) { - Ok(_) => (), - Err(err) => self.bus.err(EType::Evm, err), - } - } - - _ => (), - } - } -} diff --git a/crates/evm/src/evm_launch_coordinator.rs b/crates/evm/src/evm_launch_coordinator.rs index 085d522d56..06a17ed5ff 100644 --- a/crates/evm/src/evm_launch_coordinator.rs +++ b/crates/evm/src/evm_launch_coordinator.rs @@ -1,2 +1,116 @@ +use crate::evm_router::EvmRouter; +use crate::helpers::EthProvider; +use crate::EvmReadInterface; +use crate::{events::EvmEventProcessor, evm_read_interface::Filters}; +use actix::{Actor, ActorContext, Addr, AsyncContext, Handler}; +use alloy::providers::Provider; +use alloy_primitives::Address; +use anyhow::Context; +use anyhow::Result; +use e3_events::{ + trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, SyncStart, +}; +use std::collections::HashMap; + // Configured with Addr for -pub struct EvmLaunchCoordinator {} +pub struct EvmLaunchCoordinator

{ + provider: Option>, + routing_table: HashMap, + bus: BusHandle, +} + +impl

EvmLaunchCoordinator

+where + P: Provider + Clone + 'static, +{ + pub fn builder(bus: &BusHandle, provider: EthProvider

) -> EvmLaunchCoordinatorBuilder

{ + EvmLaunchCoordinatorBuilder { + routing_table: HashMap::new(), + bus: bus.clone(), + provider, + } + } + + fn filters(&self, start_block: Option) -> Filters { + let addresses = self.routing_table.keys().cloned().collect(); + Filters::new(addresses, start_block) + } + + fn bootstrap_reader(&mut self, _event: SyncStart) -> Result<()> { + // Setup upstream router + // The routing table holds addresses for upstream processors + let next = EvmRouter::setup(self.routing_table.clone()); + + // Setup read interface + EvmReadInterface::attach( + self.provider.take().context("Cannot call setup twice!")?, + &next.into(), + &self.bus, + self.filters(None), + ); + + Ok(()) + } +} + +impl

Actor for EvmLaunchCoordinator

+where + P: Provider + Clone + 'static, +{ + type Context = actix::Context; +} + +impl

Handler for EvmLaunchCoordinator

+where + P: Provider + Clone + 'static, +{ + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || { + if let EnclaveEventData::SyncStart(event) = msg.into_data() { + // Run the setup process + self.bootstrap_reader(event)?; + + // We now don't need the launcher and can kill it now + self.bus.unsubscribe("SyncStart", ctx.address().into()); + ctx.stop(); + } + Ok(()) + }) + } +} + +pub struct EvmLaunchCoordinatorBuilder

{ + provider: EthProvider

, + routing_table: HashMap, + bus: BusHandle, +} + +impl

EvmLaunchCoordinatorBuilder

+where + P: Provider + Clone + 'static, +{ + pub fn with_contract( + mut self, + address: impl AsRef, + dest: impl Into, + ) -> Result { + let address: Address = address.as_ref().parse().context("invalid address")?; + self.routing_table.insert(address, dest.into()); + Ok(self) + } + + pub fn build(self) -> Addr> { + let routing_table = self.routing_table; + let addr = EvmLaunchCoordinator { + routing_table, + provider: Some(self.provider), + bus: self.bus.clone(), + } + .start(); + + self.bus.subscribe("SyncStart", addr.clone().recipient()); + + addr + } +} diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs new file mode 100644 index 0000000000..654b75dd29 --- /dev/null +++ b/crates/evm/src/evm_read_interface.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; +use crate::helpers::EthProvider; +use actix::prelude::*; +use actix::{Addr, Recipient}; +use alloy::eips::BlockNumberOrTag; +use alloy::primitives::{LogData, B256}; +use alloy::providers::Provider; +use alloy::rpc::types::Filter; +use alloy_primitives::Address; +use anyhow::anyhow; +use e3_events::{prelude::*, EType, EnclaveEvent, EnclaveEventData, EventId}; +use e3_events::{BusHandle, Event}; +use futures_util::stream::StreamExt; +use std::collections::HashSet; +use tokio::select; +use tokio::sync::oneshot; +use tracing::{error, info, instrument}; + +pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; + +pub struct EvmReadInterfaceParams

{ + provider: EthProvider

, + processor: Recipient, + bus: BusHandle, + filters: Filters, +} + +#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] +pub struct EvmReadInterfaceState { + pub ids: HashSet, + pub last_block: Option, +} + +#[derive(Clone, Default)] +pub struct Filters { + historical: Filter, + current: Filter, +} +impl Filters { + pub fn new(addresses: Vec

, start_block: Option) -> Self { + let historical = Filter::new() + .address(addresses.clone()) + .from_block(start_block.unwrap_or(0)); + let current = Filter::new() + .address(addresses) + .from_block(BlockNumberOrTag::Latest); + + Self { + historical, + current, + } + } +} + +/// Connects to Enclave.sol converting EVM events to EnclaveEvents +pub struct EvmReadInterface

{ + /// The alloy provider + provider: Option>, + /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used + /// internally. You should send the Shutdown signal to the reader directly or via the EventBus + shutdown_rx: Option>, + /// The sender for the shutdown signal this is only used internally + shutdown_tx: Option>, + /// Processor to forward events an actor + processor: EvmEventProcessor, + /// Event bus for error propagation only + bus: BusHandle, + /// Filters to configure when to seek from + filters: Filters, +} + +impl EvmReadInterface

{ + pub fn new(params: EvmReadInterfaceParams

) -> Self { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + Self { + provider: Some(params.provider), + shutdown_rx: Some(shutdown_rx), + shutdown_tx: Some(shutdown_tx), + processor: params.processor, + bus: params.bus, + filters: params.filters, + } + } + + pub fn attach( + provider: EthProvider

, + processor: &Recipient, + bus: &BusHandle, + filters: Filters, + ) -> Addr { + let params = EvmReadInterfaceParams { + provider, + processor: processor.clone(), + bus: bus.clone(), + filters, + }; + + let addr = EvmReadInterface::new(params).start(); + + processor.do_send(EnclaveEvmEvent::RegisterReader); + + bus.subscribe("Shutdown", addr.clone().into()); + addr + } +} + +impl Actor for EvmReadInterface

{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + // let reader_addr = ctx.address(); + let bus = self.bus.clone(); + let processor = self.processor.clone(); + let filters = self.filters.clone(); + + let Some(provider) = self.provider.take() else { + error!("Could not start event reader as provider has already been used."); + return; + }; + + // let extractor = self.extractor; + let Some(shutdown) = self.shutdown_rx.take() else { + bus.err(EType::Evm, anyhow!("shutdown already called")); + return; + }; + + ctx.spawn( + async move { stream_from_evm(provider, processor, shutdown, &bus, filters).await } + .into_actor(self), + ); + } +} + +// TODO: split this up into: +// 1. historical request (will finish) +// 2. current listener (run indefinitely) +#[instrument(name = "evm_interface", skip_all)] +async fn stream_from_evm( + provider: EthProvider

, + processor: EvmEventProcessor, + mut shutdown: oneshot::Receiver<()>, + bus: &BusHandle, + filters: Filters, +) { + let chain_id = provider.chain_id(); + let provider_ref = provider.provider(); + + // Historical events + match provider_ref.get_logs(&filters.historical).await { + Ok(historical_logs) => { + info!("Fetched {} historical events", historical_logs.len()); + for log in historical_logs { + processor.do_send(EnclaveEvmEvent::Log(EvmLog { log, chain_id })) + } + + processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete); + } + Err(e) => { + error!("Failed to fetch historical events: {}", e); + bus.err(EType::Evm, anyhow!(e)); + return; + } + } + + info!("Subscribing to live events"); + match provider_ref.subscribe_logs(&filters.current).await { + Ok(subscription) => { + let id: B256 = subscription.local_id().clone(); + let mut stream = subscription.into_stream(); + + loop { + select! { + maybe_log = stream.next() => { + match maybe_log { + Some(log) => { + processor.do_send(EnclaveEvmEvent::Log(EvmLog { log, chain_id })) + } + None => break, // Stream ended + } + } + _ = &mut shutdown => { + info!("Received shutdown signal, stopping EVM stream"); + match provider_ref.unsubscribe(id).await { + Ok(_) => info!("Unsubscribed successfully from EVM event stream"), + Err(err) => error!("Cannot unsubscribe from EVM event stream: {}", err), + }; + break; + } + } + } + } + Err(e) => { + bus.err(EType::Evm, anyhow!("{}", e)); + } + } + + info!("Exiting stream loop"); +} + +fn is_local_node(rpc_url: &str) -> bool { + rpc_url.contains("localhost") || rpc_url.contains("127.0.0.1") +} + +impl Handler for EvmReadInterface

{ + type Result = (); + + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::Shutdown(_) = msg.into_data() { + if let Some(shutdown) = self.shutdown_tx.take() { + let _ = shutdown.send(()); + } + } + } +} diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_reader.rs new file mode 100644 index 0000000000..e6a659211c --- /dev/null +++ b/crates/evm/src/evm_reader.rs @@ -0,0 +1,60 @@ +use actix::{Actor, Handler}; +use e3_events::{hlc::HlcTimestamp, EnclaveEventData}; + +use crate::{ + events::{EnclaveEvmEvent, EvmEvent, EvmEventProcessor, EvmLog}, + ExtractorFn, +}; + +pub struct EvmReader { + next: EvmEventProcessor, + extractor: ExtractorFn, +} + +impl Actor for EvmReader { + type Context = actix::Context; +} + +impl EvmReader { + pub fn new(next: &EvmEventProcessor, extractor: ExtractorFn) -> Self { + Self { + next: next.clone(), + extractor, + } + } +} + +impl Handler for EvmReader { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { + let EnclaveEvmEvent::Log(EvmLog { log, chain_id }) = msg.clone() else { + return; + }; + let extractor = self.extractor; + + if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { + let err = "Log should always have metadata because we listen to non-pending blocks. If you are seeing this it is likely because there is an issue with how we are subscribing to blocks"; + let block = log.block_number.expect(err); + let block_timestamp = log.block_timestamp.expect(err); + let log_index = log.log_index.expect(err); + let ts = from_log_chain_id_to_ts(block_timestamp, log_index, chain_id); + self.next.do_send(EnclaveEvmEvent::Event(EvmEvent { + payload: event, + block, + ts, + })) + } + } +} + +fn from_log_chain_id_to_ts(block_timestamp: u64, log_index: u64, chain_id: u64) -> u128 { + let ts = block_timestamp.saturating_mul(1_000_000); + + // Use log_index as counter (orders logs within same block) + let counter = log_index as u32; + + // Use transaction_index as node (or chain_id if you have it) + let node = chain_id as u32; + + HlcTimestamp::new(ts, counter, node).into() +} diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs new file mode 100644 index 0000000000..396c24f6eb --- /dev/null +++ b/crates/evm/src/evm_router.rs @@ -0,0 +1,71 @@ +use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; +use actix::{Actor, Addr, Handler}; +use alloy_primitives::Address; +use std::collections::HashMap; + +/// Directs EnclaveEvmEvent::Log events to the correct upstream processors. Drops all other event +/// types +pub struct EvmRouter { + routing_table: HashMap, +} + +impl EvmRouter { + pub fn new(routing_table: HashMap) -> Self { + Self { routing_table } + } + + pub fn setup(routing_table: HashMap) -> Addr { + let addr = Self::new(routing_table).start(); + addr + } +} + +impl Actor for EvmRouter { + type Context = actix::Context; +} + +impl Handler for EvmRouter { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + let EnclaveEvmEvent::Log(EvmLog { log, .. }) = msg.clone() else { + return; + }; + let Some(dest) = self.routing_table.get(&log.address()) else { + return; + }; + + dest.do_send(msg); + } +} + +pub struct EvmHub { + nexts: Vec, +} + +impl EvmHub { + pub fn new(nexts: Vec) -> Self { + Self { nexts } + } + + pub fn setup(nexts: Vec) -> Addr { + let addr = Self::new(nexts).start(); + addr + } +} + +impl Actor for EvmHub { + type Context = actix::Context; +} + +impl Handler for EvmHub { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + let EnclaveEvmEvent::Log { .. } = msg.clone() else { + return; + }; + + for next in self.nexts.clone() { + next.do_send(msg.clone()); + } + } +} diff --git a/crates/evm/src/evm_sync_processor.rs b/crates/evm/src/evm_sync_processor.rs new file mode 100644 index 0000000000..f6a8d7a4e0 --- /dev/null +++ b/crates/evm/src/evm_sync_processor.rs @@ -0,0 +1,98 @@ +use actix::Recipient; +use actix::{Actor, Handler}; +use anyhow::Context; +use anyhow::Result; +use e3_events::{trap, BusHandle, EnclaveEvent, EnclaveEventData, SyncEnd, SyncStart}; +use e3_events::{EType, SyncEvmEvent}; +use e3_events::{Event, EventPublisher}; + +use crate::events::EnclaveEvmEvent; + +pub struct EvmSyncProcessor { + bus: BusHandle, + status: SyncStatus, +} + +#[derive(Clone)] +enum SyncStatus { + Init, + Syncing(Recipient), + Buffering(Vec), + Live, +} + +impl EvmSyncProcessor { + pub fn new(bus: &BusHandle) -> Self { + Self { + bus: bus.clone(), + status: SyncStatus::Init, + } + } + + fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { + let sender = msg.sender.context("No sender on SyncStart Message")?; + self.status = SyncStatus::Syncing(sender); + Ok(()) + } + + fn handle_sync_end(&mut self, msg: SyncEnd) { + self.status = SyncStatus::Live; + } + + fn handle_evm_event(&mut self, msg: EnclaveEvmEvent) -> Result<()> { + match msg { + EnclaveEvmEvent::HistoricalSyncComplete => { + self.handle_historical_sync_complete()?; + Ok(()) + } + EnclaveEvmEvent::Event(event) => { + self.handle_receive_evm_event(event.payload,event.block)?; + Ok(()) + } + _ => panic!("EvmSyncProcessor is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), + } + } + + fn handle_historical_sync_complete(&mut self) -> Result<()> { + self.status = SyncStatus::Buffering(vec![]); + Ok(()) + } + + fn handle_receive_evm_event(&mut self, event: EnclaveEventData, block: u64) -> Result<()> { + match &mut self.status { + SyncStatus::Buffering(buffer) => buffer.push(SyncEvmEvent::new(event, block)), + SyncStatus::Syncing(sender) => sender.do_send(SyncEvmEvent::new(event, block)), + SyncStatus::Live => { + self.bus + .publish_from_remote(event, 0 /*convert block or whatever to ts*/)?; + } + _ => (), + }; + Ok(()) + } +} + +impl Actor for EvmSyncProcessor { + type Context = actix::Context; +} + +impl Handler for EvmSyncProcessor { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || { + match msg.into_data() { + EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, + EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e), + _ => (), + } + Ok(()) + }) + } +} + +impl Handler for EvmSyncProcessor { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) + } +} diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs index 8d1d49fd32..0e3924f445 100644 --- a/crates/evm/src/historical_event_coordinator.rs +++ b/crates/evm/src/historical_event_coordinator.rs @@ -4,11 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::EnclaveEvmEvent; use actix::prelude::*; use e3_events::{prelude::*, trap, BusHandle, EType, EnclaveEventData}; use tracing::info; +use crate::events::{EnclaveEvmEvent, EvmEvent}; + #[derive(Clone)] struct BufferedEvent { block: u64, @@ -108,13 +109,15 @@ impl Handler for HistoricalEventCoordinator { Ok(()) } - EnclaveEvmEvent::Event { event, block } => { + EnclaveEvmEvent::Event(event) => { if !self.started || !self.all_readers_complete() { - if let Some(block) = block { - self.buffered_events.push(BufferedEvent { block, event }); - } + let block = event.block; + self.buffered_events.push(BufferedEvent { + block, + event: event.payload, + }); } else { - self.target.publish(event)?; + self.target.publish(event.payload)?; } Ok(()) } diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 42b8564dde..9120e52eed 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -9,19 +9,26 @@ mod ciphernode_registry_sol; mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; -mod evm_interface; +mod events; +mod evm_launch_coordinator; +mod evm_read_interface; +mod evm_reader; +mod evm_router; +mod evm_sync_processor; pub mod helpers; mod historical_event_coordinator; mod repo; -pub use bonding_registry_sol::{BondingRegistrySol, BondingRegistrySolReader}; +pub use bonding_registry_sol::BondingRegistrySolReader; pub use ciphernode_registry_sol::{ CiphernodeRegistrySol, CiphernodeRegistrySolReader, CiphernodeRegistrySolWriter, }; pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use evm_interface::{EnclaveEvmEvent, EvmReadInterface, EvmReadInterfaceState, ExtractorFn}; +pub use evm_launch_coordinator::*; +pub use evm_read_interface::{EvmReadInterface, EvmReadInterfaceState, ExtractorFn}; +pub use evm_sync_processor::*; pub use helpers::send_tx_with_retry; pub use historical_event_coordinator::{CoordinatorStart, HistoricalEventCoordinator}; pub use repo::*; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 84b8afb073..1381c8ae18 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -90,7 +90,6 @@ async fn evm_reader() -> Result<()> { let coordinator = HistoricalEventCoordinator::setup(bus.clone()); let processor = coordinator.clone().recipient(); - EvmReadInterface::attach( provider.clone(), test_event_extractor, From ec5322beb82e41c6e629f2f4e537de8364154855 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 25 Jan 2026 05:05:22 +0000 Subject: [PATCH 068/102] evm events are working --- Cargo.lock | 2 + .../src/ciphernode_builder.rs | 242 ++++++------------ crates/ciphernode-builder/src/lib.rs | 2 + .../ciphernode-builder/src/provider_caches.rs | 131 ++++++++++ .../events/src/enclave_event/enclave_error.rs | 1 + crates/events/src/enclave_event/sync_start.rs | 55 +++- crates/events/src/eventstore.rs | 1 + crates/events/src/lib.rs | 2 + crates/events/src/sync.rs | 18 ++ crates/evm/Cargo.toml | 2 + crates/evm/src/enclave_sol.rs | 6 +- crates/evm/src/events.rs | 30 ++- crates/evm/src/evm_hub.rs | 96 +++++++ crates/evm/src/evm_launch_coordinator.rs | 232 ++++++++--------- crates/evm/src/evm_read_interface.rs | 27 +- crates/evm/src/evm_reader.rs | 40 +-- crates/evm/src/evm_router.rs | 130 +++++++--- crates/evm/src/evm_sync_processor.rs | 98 ------- .../evm/src/historical_event_coordinator.rs | 146 ----------- crates/evm/src/lib.rs | 20 +- crates/evm/src/one_shot_runnner.rs | 86 +++++++ crates/evm/src/sync_gateway.rs | 165 ++++++++++++ crates/evm/src/sync_start_extractor.rs | 28 ++ crates/evm/tests/integration.rs | 128 ++++++--- 24 files changed, 1041 insertions(+), 647 deletions(-) create mode 100644 crates/ciphernode-builder/src/provider_caches.rs create mode 100644 crates/events/src/sync.rs create mode 100644 crates/evm/src/evm_hub.rs delete mode 100644 crates/evm/src/evm_sync_processor.rs delete mode 100644 crates/evm/src/historical_event_coordinator.rs create mode 100644 crates/evm/src/one_shot_runnner.rs create mode 100644 crates/evm/src/sync_gateway.rs create mode 100644 crates/evm/src/sync_start_extractor.rs diff --git a/Cargo.lock b/Cargo.lock index f2bbfa62de..42c05df132 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3012,6 +3012,7 @@ dependencies = [ "e3-data", "e3-entrypoint", "e3-events", + "e3-evm", "e3-sortition", "e3-utils", "futures-util", @@ -3019,6 +3020,7 @@ dependencies = [ "serde", "tokio", "tracing", + "tracing-subscriber", "url", "zeroize", ] diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 757015b84b..e2c66f29bd 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -5,9 +5,8 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::event_system::AggregateConfig; -use crate::{CiphernodeHandle, EventSystem}; +use crate::{CiphernodeHandle, EventSystem, ProviderCache}; use actix::{Actor, Addr}; -use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; use anyhow::Result; use derivative::Derivative; use e3_aggregator::ext::{ @@ -16,18 +15,10 @@ use e3_aggregator::ext::{ }; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; -use e3_data::{InMemStore, Repositories, RepositoriesFactory}; +use e3_data::{InMemStore, RepositoriesFactory}; use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig}; -use e3_evm::{ - helpers::{ - load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, - ProviderConfig, - }, - BondingRegistryReaderRepositoryFactory, CiphernodeRegistryReaderRepositoryFactory, - CiphernodeRegistrySol, CoordinatorStart, EnclaveSol, EnclaveSolReader, - EnclaveSolReaderRepositoryFactory, EthPrivateKeyRepositoryFactory, HistoricalEventCoordinator, -}; -use e3_evm::{BondingRegistrySolReader, EvmLaunchCoordinator}; +use e3_evm::{BondingRegistrySolReader, EnclaveSolWriter, EvmChainGateway}; +use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; @@ -297,7 +288,7 @@ impl CiphernodeBuilder { /// Create aggregate configuration from configured chains async fn create_aggregate_config( &self, - provider_cache: &mut ProviderCaches, + provider_cache: &mut ProviderCache, ) -> Result { let mut chain_providers = Vec::new(); for chain in &self.chains { @@ -352,8 +343,7 @@ impl CiphernodeBuilder { }; // Create provider cache early to use for chain validation - let mut provider_cache = ProviderCaches::new(); - + let mut provider_cache = ProviderCache::new(); let aggregate_config = self.create_aggregate_config(&mut provider_cache).await?; // Get an event system instance. @@ -376,8 +366,12 @@ impl CiphernodeBuilder { let bus = event_system.handle()?; let store = event_system.store()?; + let cipher = &self.cipher; + let repositories = Arc::new(store.repositories()); - let repositories = store.repositories(); + // Now we add write support as store depends on event system + let mut provider_cache = + provider_cache.with_write_support(Arc::clone(cipher), Arc::clone(&repositories)); // Use the configured backend directly let default_backend = self.sortition_backend.clone(); @@ -394,80 +388,80 @@ impl CiphernodeBuilder { CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) .await?; - let cipher = &self.cipher; - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - // TODO: gather an async handle from the event readers that closes when they shutdown and - // join it with the network manager joinhandle below - for chain in self - .chains - .iter() - .filter(|chain| chain.enabled.unwrap_or(true)) - { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - let mut launcher = EvmLaunchCoordinator::builder(&bus, read_provider); - if self.contract_components.enclave { - let write_provider = provider_cache - .ensure_write_provider(&repositories, chain, cipher) - .await?; - let addr = EnclaveSol::attach( - &processor, - &bus, - write_provider.clone(), - &chain.contracts.enclave.address(), - ) - .await?; - launcher = launcher.with_contract(chain.contracts.enclave.address(), addr)?; - // TODO: add to launch coordinator - } - - if self.contract_components.enclave_reader { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - EnclaveSolReader::setup(&processor); - } - - if self.contract_components.bonding_registry { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - let addr = BondingRegistrySolReader::setup(&processor); - // TODO: add to launch coordinator - } - - if self.contract_components.ciphernode_registry { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - let addr = CiphernodeRegistrySol::attach(&processor); - // TODO: add to launch coordinator - match provider_cache - .ensure_write_provider(&repositories, chain, cipher) - .await - { - Ok(write_provider) => { - let _writer = CiphernodeRegistrySol::attach_writer( - &bus, - write_provider.clone(), - &chain.contracts.ciphernode_registry.address(), - self.pubkey_agg, - ) - .await?; - info!("CiphernodeRegistrySolWriter attached for publishing committees"); - - if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { - info!("Attaching CommitteeFinalizer for score sortition"); - e3_aggregator::CommitteeFinalizer::attach(&bus); - } - } - Err(e) => error!( - "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", - e - ), - } - } - launcher.build(); - } - - // We start after all readers have registered - coordinator.do_send(CoordinatorStart); + // Sync processor + // All contract event processors will forward their parsed events here. + // let next = EvmChainGateway::setup(&bus).into(); + // + // // TODO: gather an async handle from the event readers that closes when they shutdown and + // // join it with the network manager joinhandle below + // for chain in self + // .chains + // .iter() + // .filter(|chain| chain.enabled.unwrap_or(true)) + // { + // // We create a launch coordinator for each chain + // // This waits for the SyncStart message before creating the EvmInterface which inturn + // // will begin the historical stream events followed by the live stream of events. + // // We need to do this so that the interface knows when to stream from. + // // Once it has created the EvmInterface it will kill itself + // let mut read_launcher = EvmLaunchCoordinator::builder( + // &bus, + // &provider_cache.ensure_read_provider(chain).await?, + // ); + // + // if self.contract_components.enclave { + // let contract_address = chain.contracts.enclave.address(); + // let write_provider = provider_cache.ensure_write_provider(chain).await?; + // + // EnclaveSolWriter::attach(&bus, write_provider.clone(), &contract_address).await?; + // + // read_launcher.with_contract(contract_address, EnclaveSolReader::setup(&next))?; + // } + // + // if self.contract_components.enclave_reader { + // let contract_address = chain.contracts.enclave.address(); + // + // read_launcher.with_contract(contract_address, EnclaveSolReader::setup(&next))?; + // } + // + // if self.contract_components.bonding_registry { + // let contract_address = chain.contracts.bonding_registry.address(); + // read_launcher + // .with_contract(contract_address, BondingRegistrySolReader::setup(&next))?; + // } + // + // if self.contract_components.ciphernode_registry { + // let contract_address = chain.contracts.ciphernode_registry.address(); + // read_launcher + // .with_contract(contract_address, CiphernodeRegistrySol::attach(&next))?; + // + // match provider_cache + // .ensure_write_provider(chain) + // .await + // { + // Ok(write_provider) => { + // let _writer = CiphernodeRegistrySol::attach_writer( + // &bus, + // write_provider.clone(), + // &contract_address, + // self.pubkey_agg, + // ) + // .await?; + // info!("CiphernodeRegistrySolWriter attached for publishing committees"); + // + // if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { + // info!("Attaching CommitteeFinalizer for score sortition"); + // e3_aggregator::CommitteeFinalizer::attach(&bus); + // } + // } + // Err(e) => error!( + // "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", + // e + // ), + // } + // } + // let _ = read_launcher.build(); + // } // E3 specific setup let mut e3_builder = E3Router::builder(&bus, store.clone()); @@ -559,14 +553,6 @@ impl CiphernodeBuilder { } } -/// Struct to cache modules required during the ciphernode construction so that providers are only -/// constructed once. -struct ProviderCaches { - signer_cache: Option>, - read_provider_cache: HashMap>, - write_provider_cache: HashMap>, -} - /// Validate chain ID matches expected configuration fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { if let Some(expected_chain_id) = chain.chain_id { @@ -605,61 +591,3 @@ fn create_aggregate_delays( Ok(delays) } - -impl ProviderCaches { - pub fn new() -> Self { - ProviderCaches { - signer_cache: None, - read_provider_cache: HashMap::new(), - write_provider_cache: HashMap::new(), - } - } - - pub async fn ensure_signer( - &mut self, - cipher: &Cipher, - repositories: &Repositories, - ) -> Result> { - if let Some(ref cache) = self.signer_cache { - return Ok(cache.clone()); - } - - let signer = load_signer_from_repository(repositories.eth_private_key(), cipher).await?; - self.signer_cache = Some(signer.clone()); - Ok(signer) - } - - pub async fn ensure_read_provider( - &mut self, - chain: &ChainConfig, - ) -> Result> { - if let Some(cache) = self.read_provider_cache.get(chain) { - return Ok(cache.clone()); - } - let rpc_url = chain.rpc_url()?; - let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); - let read_provider = provider_config.create_readonly_provider().await?; - self.read_provider_cache - .insert(chain.clone(), read_provider.clone()); - Ok(read_provider) - } - - pub async fn ensure_write_provider( - &mut self, - repositories: &Repositories, - chain: &ChainConfig, - cipher: &Cipher, - ) -> Result> { - if let Some(cache) = self.write_provider_cache.get(chain) { - return Ok(cache.clone()); - } - - let signer = self.ensure_signer(cipher, repositories).await?; - let rpc_url = chain.rpc_url()?; - let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); - let write_provider = provider_config.create_signer_provider(&signer).await?; - self.write_provider_cache - .insert(chain.clone(), write_provider.clone()); - Ok(write_provider) - } -} diff --git a/crates/ciphernode-builder/src/lib.rs b/crates/ciphernode-builder/src/lib.rs index c77952b744..7d803ab3ce 100644 --- a/crates/ciphernode-builder/src/lib.rs +++ b/crates/ciphernode-builder/src/lib.rs @@ -8,7 +8,9 @@ mod ciphernode; mod ciphernode_builder; mod event_system; mod eventbus_factory; +mod provider_caches; pub use ciphernode::*; pub use ciphernode_builder::*; pub use event_system::*; pub use eventbus_factory::*; +pub use provider_caches::*; diff --git a/crates/ciphernode-builder/src/provider_caches.rs b/crates/ciphernode-builder/src/provider_caches.rs new file mode 100644 index 0000000000..6068512c96 --- /dev/null +++ b/crates/ciphernode-builder/src/provider_caches.rs @@ -0,0 +1,131 @@ +use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; +use anyhow::Result; +use e3_config::chain_config::ChainConfig; +use e3_crypto::Cipher; +use e3_data::Repositories; +use e3_evm::helpers::{ + load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, + ProviderConfig, +}; +use e3_evm::EthPrivateKeyRepositoryFactory; +use std::collections::HashMap; +use std::sync::Arc; + +// Typestate marker types +pub struct ReadOnly; + +pub struct WriteEnabled { + cipher: Arc, + repositories: Arc, +} + +/// Struct to cache modules required during the ciphernode construction so that providers are only +/// constructed once. +pub struct ProviderCache { + signer_cache: Option>, + read_provider_cache: HashMap>, + write_provider_cache: HashMap>, + state: State, +} + +impl ProviderCache { + pub fn new() -> Self { + ProviderCache { + signer_cache: None, + read_provider_cache: HashMap::new(), + write_provider_cache: HashMap::new(), + state: ReadOnly, + } + } + + pub fn from_single_read_provider( + chain: ChainConfig, + provider: EthProvider, + ) -> Self { + ProviderCache { + signer_cache: None, + read_provider_cache: HashMap::from([(chain, provider)]), + write_provider_cache: HashMap::new(), + state: ReadOnly, + } + } + + /// Configure the cache with cipher and repositories to enable write provider support. + pub fn with_write_support( + self, + cipher: Arc, + repositories: Arc, + ) -> ProviderCache { + ProviderCache { + signer_cache: self.signer_cache, + read_provider_cache: self.read_provider_cache, + write_provider_cache: self.write_provider_cache, + state: WriteEnabled { + cipher, + repositories, + }, + } + } +} + +impl Default for ProviderCache { + fn default() -> Self { + Self::new() + } +} + +impl ProviderCache { + pub async fn ensure_read_provider( + &mut self, + chain: &ChainConfig, + ) -> Result> { + if let Some(cache) = self.read_provider_cache.get(chain) { + return Ok(cache.clone()); + } + + let rpc_url = chain.rpc_url()?; + let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); + let read_provider = provider_config.create_readonly_provider().await?; + + self.read_provider_cache + .insert(chain.clone(), read_provider.clone()); + + Ok(read_provider) + } +} + +impl ProviderCache { + pub async fn ensure_signer(&mut self) -> Result> { + if let Some(ref cache) = self.signer_cache { + return Ok(cache.clone()); + } + + let signer = load_signer_from_repository( + self.state.repositories.eth_private_key(), + &self.state.cipher, + ) + .await?; + + self.signer_cache = Some(signer.clone()); + Ok(signer) + } + + pub async fn ensure_write_provider( + &mut self, + chain: &ChainConfig, + ) -> Result> { + if let Some(cache) = self.write_provider_cache.get(chain) { + return Ok(cache.clone()); + } + + let signer = self.ensure_signer().await?; + let rpc_url = chain.rpc_url()?; + let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); + let write_provider = provider_config.create_signer_provider(&signer).await?; + + self.write_provider_cache + .insert(chain.clone(), write_provider.clone()); + + Ok(write_provider) + } +} diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index e5332abfeb..5f9100ded3 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -41,6 +41,7 @@ pub enum EType { PlaintextAggregation, Decryption, Sortition, + Sync, Data, Event, Computation, diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 2df094ac69..dd913c46a7 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -4,23 +4,37 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use super::EnclaveEventData; +use crate::SyncEvmEvent; use actix::{Message, Recipient}; use serde::{Deserialize, Serialize}; -use std::fmt::{self, Display}; - -use super::EnclaveEventData; +use std::{ + collections::HashMap, + fmt::{self, Display}, +}; -/// This is a processed EnclaveEvmEvent +/// This is a processed EvmEvent specifically typed for the Sync actor #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct SyncEvmEvent { +pub struct EvmEvent { data: EnclaveEventData, block: u64, + chain_id: u64, + ts: u128, } -impl SyncEvmEvent { - pub fn new(data: EnclaveEventData, block: u64) -> Self { - Self { data, block } +impl EvmEvent { + pub fn new(data: EnclaveEventData, block: u64, ts: u128, chain_id: u64) -> Self { + Self { + data, + block, + ts, + chain_id, + } + } + + pub fn split(self) -> (EnclaveEventData, u128, u64) { + (self.data, self.ts, self.block) } } @@ -28,18 +42,37 @@ impl SyncEvmEvent { #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct SyncStart { + /// The start block information for chains + pub evm_init_info: Vec<(u64, Option)>, // HashMap cannot derive Hash #[serde(skip)] - pub sender: Option>, // Must be option to allow serde deserialize on + pub sender: Option>, // Must be Option to allow serde deserialize on // EnclaveEvent as Default is required to be // implemented } impl SyncStart { - pub fn new(sender: Recipient) -> Self { + pub fn new( + sender: impl Into>, + evm_init_info: HashMap>, + ) -> Self { Self { - sender: Some(sender), + sender: Some(sender.into()), + evm_init_info: evm_init_info.into_iter().collect(), } } + + pub fn get_evm_init_for(&self, chain_id: u64) -> Option { + self.evm_init_info + .iter() + .find_map(|(ch_id, value)| { + if ch_id == &chain_id { + Some(value.clone()) + } else { + None + } + }) + .unwrap_or(None) + } } impl Display for SyncStart { diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index ce593df7dd..239bd68cc0 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -49,6 +49,7 @@ impl EventStore { Ok(()) } } + impl EventStore { pub fn new(index: I, log: L) -> Self { Self { index, log } diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 17e606c8f8..b9221447d7 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -19,6 +19,7 @@ mod ordered_set; pub mod prelude; mod seed; mod sequencer; +mod sync; mod traits; pub use bus_handle::*; @@ -34,4 +35,5 @@ pub use eventstore_router::*; pub use ordered_set::*; pub use seed::*; pub use sequencer::*; +pub use sync::*; pub use traits::*; diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs new file mode 100644 index 0000000000..1872f85af6 --- /dev/null +++ b/crates/events/src/sync.rs @@ -0,0 +1,18 @@ +use crate::EvmEvent; +use actix::Message; +use serde::{Deserialize, Serialize}; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub enum SyncEvmEvent { + /// Signal that this reader has completed historical sync + HistoricalSyncComplete(u64), + /// An actual event from the blockchain + Event(EvmEvent), +} + +impl From for SyncEvmEvent { + fn from(event: EvmEvent) -> SyncEvmEvent { + SyncEvmEvent::Event(event) + } +} diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index a230390952..ea1b9183c8 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -29,6 +29,8 @@ url = { workspace = true } zeroize = { workspace = true } [dev-dependencies] +e3-evm = { workspace = true } e3-entrypoint = { workspace = true } e3-ciphernode-builder = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } +tracing-subscriber = { workspace = true } diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 3e65e397a3..233ff16dd2 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -6,9 +6,9 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - events::EnclaveEvmEvent, evm_reader::EvmReader, helpers::EthProvider, + events::EvmEventProcessor, evm_reader::EvmReader, helpers::EthProvider, }; -use actix::{Addr, Recipient}; +use actix::Addr; use alloy::providers::{Provider, WalletProvider}; use anyhow::Result; use e3_events::BusHandle; @@ -17,7 +17,7 @@ pub struct EnclaveSol; impl EnclaveSol { pub async fn attach( - processor: &Recipient, + processor: &EvmEventProcessor, bus: &BusHandle, write_provider: EthProvider, contract_address: &str, diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index ffa7518099..b96d6fc74f 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -1,34 +1,42 @@ use actix::{Message, Recipient}; use alloy::rpc::types::Log; -use e3_events::{EnclaveEventData, EventId}; +use alloy_primitives::Address; +use e3_events::{EventId, EvmEvent}; use serde::{Deserialize, Serialize}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub enum EnclaveEvmEvent { - /// Register a reader with the coordinator before it starts processing - RegisterReader, /// Signal that this reader has completed historical sync - HistoricalSyncComplete, + HistoricalSyncComplete(u64), /// An actual event from the blockchain Event(EvmEvent), /// Raw log data from the provider Log(EvmLog), } -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct EvmEvent { - pub payload: EnclaveEventData, - pub block: u64, - pub ts: u128, -} - #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct EvmLog { pub log: Log, pub chain_id: u64, } +#[cfg(test)] +impl EvmLog { + pub fn test_log(address: Address, chain_id: u64) -> EvmLog { + EvmLog { + log: Log { + inner: alloy_primitives::Log { + address, + ..Default::default() + }, + ..Default::default() + }, + chain_id, + } + } +} + impl EnclaveEvmEvent { pub fn get_id(&self) -> EventId { EventId::hash(self.clone()) diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs new file mode 100644 index 0000000000..a4d6613c2c --- /dev/null +++ b/crates/evm/src/evm_hub.rs @@ -0,0 +1,96 @@ +use actix::{Actor, Addr, Handler}; + +use crate::events::{EnclaveEvmEvent, EvmEventProcessor}; + +pub struct EvmHub { + nexts: Vec, +} + +impl EvmHub { + pub fn new(nexts: Vec) -> Self { + Self { nexts } + } + + pub fn setup(nexts: Vec) -> Addr { + let addr = Self::new(nexts).start(); + addr + } +} + +impl Actor for EvmHub { + type Context = actix::Context; +} + +impl Handler for EvmHub { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + let EnclaveEvmEvent::Log { .. } = msg.clone() else { + return; + }; + + for next in self.nexts.clone() { + next.do_send(msg.clone()); + } + } +} + +#[cfg(test)] +mod tests { + use crate::events::EvmLog; + + use super::*; + use actix::prelude::*; + use alloy::primitives::address; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + use std::time::Duration; + use tokio::time::sleep; + + #[actix::test] + async fn test_evm_hub_forwards_log_events_to_all_processors() { + // Arrange + let call_count = Arc::new(AtomicUsize::new(0)); + + // Create mock processors that track invocations + let count1 = call_count.clone(); + let count2 = call_count.clone(); + + let processor1 = TestProcessor { call_count: count1 }.start(); + let processor2 = TestProcessor { call_count: count2 }.start(); + + let hub = EvmHub::setup(vec![ + processor1.clone().recipient(), + processor2.clone().recipient(), + ]); + + let log_event = EnclaveEvmEvent::Log(EvmLog::test_log( + address!("0x1111111111111111111111111111111111111111"), + 1, + )); + + hub.send(log_event).await.unwrap(); + + sleep(Duration::from_millis(10)).await; + // Assert + assert_eq!(call_count.load(Ordering::SeqCst), 2); + } + + // Helper test actor + struct TestProcessor { + call_count: Arc, + } + + impl Actor for TestProcessor { + type Context = Context; + } + + impl Handler for TestProcessor { + type Result = (); + + fn handle(&mut self, _msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + self.call_count.fetch_add(1, Ordering::SeqCst); + } + } +} diff --git a/crates/evm/src/evm_launch_coordinator.rs b/crates/evm/src/evm_launch_coordinator.rs index 06a17ed5ff..9e97eeba15 100644 --- a/crates/evm/src/evm_launch_coordinator.rs +++ b/crates/evm/src/evm_launch_coordinator.rs @@ -1,116 +1,116 @@ -use crate::evm_router::EvmRouter; -use crate::helpers::EthProvider; -use crate::EvmReadInterface; -use crate::{events::EvmEventProcessor, evm_read_interface::Filters}; -use actix::{Actor, ActorContext, Addr, AsyncContext, Handler}; -use alloy::providers::Provider; -use alloy_primitives::Address; -use anyhow::Context; -use anyhow::Result; -use e3_events::{ - trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, SyncStart, -}; -use std::collections::HashMap; - -// Configured with Addr for -pub struct EvmLaunchCoordinator

{ - provider: Option>, - routing_table: HashMap, - bus: BusHandle, -} - -impl

EvmLaunchCoordinator

-where - P: Provider + Clone + 'static, -{ - pub fn builder(bus: &BusHandle, provider: EthProvider

) -> EvmLaunchCoordinatorBuilder

{ - EvmLaunchCoordinatorBuilder { - routing_table: HashMap::new(), - bus: bus.clone(), - provider, - } - } - - fn filters(&self, start_block: Option) -> Filters { - let addresses = self.routing_table.keys().cloned().collect(); - Filters::new(addresses, start_block) - } - - fn bootstrap_reader(&mut self, _event: SyncStart) -> Result<()> { - // Setup upstream router - // The routing table holds addresses for upstream processors - let next = EvmRouter::setup(self.routing_table.clone()); - - // Setup read interface - EvmReadInterface::attach( - self.provider.take().context("Cannot call setup twice!")?, - &next.into(), - &self.bus, - self.filters(None), - ); - - Ok(()) - } -} - -impl

Actor for EvmLaunchCoordinator

-where - P: Provider + Clone + 'static, -{ - type Context = actix::Context; -} - -impl

Handler for EvmLaunchCoordinator

-where - P: Provider + Clone + 'static, -{ - type Result = (); - fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.clone(), || { - if let EnclaveEventData::SyncStart(event) = msg.into_data() { - // Run the setup process - self.bootstrap_reader(event)?; - - // We now don't need the launcher and can kill it now - self.bus.unsubscribe("SyncStart", ctx.address().into()); - ctx.stop(); - } - Ok(()) - }) - } -} - -pub struct EvmLaunchCoordinatorBuilder

{ - provider: EthProvider

, - routing_table: HashMap, - bus: BusHandle, -} - -impl

EvmLaunchCoordinatorBuilder

-where - P: Provider + Clone + 'static, -{ - pub fn with_contract( - mut self, - address: impl AsRef, - dest: impl Into, - ) -> Result { - let address: Address = address.as_ref().parse().context("invalid address")?; - self.routing_table.insert(address, dest.into()); - Ok(self) - } - - pub fn build(self) -> Addr> { - let routing_table = self.routing_table; - let addr = EvmLaunchCoordinator { - routing_table, - provider: Some(self.provider), - bus: self.bus.clone(), - } - .start(); - - self.bus.subscribe("SyncStart", addr.clone().recipient()); - - addr - } -} +// use crate::evm_router::EvmRouter; +// use crate::helpers::EthProvider; +// use crate::EvmReadInterface; +// use crate::{events::EvmEventProcessor, evm_read_interface::Filters}; +// use actix::{Actor, ActorContext, Addr, AsyncContext, Handler}; +// use alloy::providers::Provider; +// use alloy_primitives::Address; +// use anyhow::Context; +// use anyhow::Result; +// use e3_events::{ +// trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, SyncStart, +// }; +// use std::collections::HashMap; +// +// // Configured with Addr for +// pub struct EvmLaunchCoordinator

{ +// provider: Option>, +// routing_table: HashMap, +// bus: BusHandle, +// } +// +// impl

EvmLaunchCoordinator

+// where +// P: Provider + Clone + 'static, +// { +// pub fn builder(bus: &BusHandle, provider: &EthProvider

) -> EvmLaunchCoordinatorBuilder

{ +// EvmLaunchCoordinatorBuilder { +// routing_table: HashMap::new(), +// bus: bus.clone(), +// provider: provider.clone(), +// } +// } +// +// fn filters(&self, start_block: Option) -> Filters { +// let addresses = self.routing_table.keys().cloned().collect(); +// Filters::new(addresses, start_block) +// } +// +// fn bootstrap_reader(&mut self, _event: SyncStart) -> Result<()> { +// // Setup upstream router +// // The routing table holds addresses for upstream processors +// let next = EvmRouter::setup(self.routing_table.clone()); +// +// // Setup read interface +// EvmReadInterface::attach( +// self.provider.take().context("Cannot call setup twice!")?, +// &next.into(), +// &self.bus, +// self.filters(None), +// ); +// +// Ok(()) +// } +// } +// +// impl

Actor for EvmLaunchCoordinator

+// where +// P: Provider + Clone + 'static, +// { +// type Context = actix::Context; +// } +// +// impl

Handler for EvmLaunchCoordinator

+// where +// P: Provider + Clone + 'static, +// { +// type Result = (); +// fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { +// trap(EType::Evm, &self.bus.clone(), || { +// if let EnclaveEventData::SyncStart(event) = msg.into_data() { +// // Run the setup process +// self.bootstrap_reader(event)?; +// +// // We now don't need the launcher and can kill it now +// self.bus.unsubscribe("SyncStart", ctx.address().into()); +// ctx.stop(); +// } +// Ok(()) +// }) +// } +// } +// +// pub struct EvmLaunchCoordinatorBuilder

{ +// provider: EthProvider

, +// routing_table: HashMap, +// bus: BusHandle, +// } +// +// impl

EvmLaunchCoordinatorBuilder

+// where +// P: Provider + Clone + 'static, +// { +// pub fn with_contract( +// &mut self, +// address: impl AsRef, +// dest: impl Into, +// ) -> Result<()> { +// let address: Address = address.as_ref().parse().context("invalid address")?; +// self.routing_table.insert(address, dest.into()); +// Ok(()) +// } +// +// pub fn build(self) -> Addr> { +// let routing_table = self.routing_table; +// let addr = EvmLaunchCoordinator { +// routing_table, +// provider: Some(self.provider), +// bus: self.bus.clone(), +// } +// .start(); +// +// self.bus.subscribe("SyncStart", addr.clone().recipient()); +// +// addr +// } +// } diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index 654b75dd29..4f56f7180c 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -14,10 +14,10 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy_primitives::Address; use anyhow::anyhow; -use e3_events::{prelude::*, EType, EnclaveEvent, EnclaveEventData, EventId}; -use e3_events::{BusHandle, Event}; +use e3_events::{BusHandle, ErrorDispatcher, Event, EventSubscriber}; +use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; use futures_util::stream::StreamExt; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use tokio::select; use tokio::sync::oneshot; use tracing::{error, info, instrument}; @@ -42,6 +42,7 @@ pub struct Filters { historical: Filter, current: Filter, } + impl Filters { pub fn new(addresses: Vec

, start_block: Option) -> Self { let historical = Filter::new() @@ -56,6 +57,11 @@ impl Filters { current, } } + + pub fn from_routing_table(table: &HashMap, start_block: Option) -> Self { + let addresses: Vec
= table.keys().cloned().collect(); + Self::new(addresses, start_block) + } } /// Connects to Enclave.sol converting EVM events to EnclaveEvents @@ -88,23 +94,21 @@ impl EvmReadInterface

{ } } - pub fn attach( - provider: EthProvider

, - processor: &Recipient, + pub fn setup( + provider: &EthProvider

, + next: &Recipient, bus: &BusHandle, filters: Filters, ) -> Addr { let params = EvmReadInterfaceParams { - provider, - processor: processor.clone(), + provider: provider.clone(), + processor: next.clone(), bus: bus.clone(), filters, }; let addr = EvmReadInterface::new(params).start(); - processor.do_send(EnclaveEvmEvent::RegisterReader); - bus.subscribe("Shutdown", addr.clone().into()); addr } @@ -158,8 +162,6 @@ async fn stream_from_evm( for log in historical_logs { processor.do_send(EnclaveEvmEvent::Log(EvmLog { log, chain_id })) } - - processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete); } Err(e) => { error!("Failed to fetch historical events: {}", e); @@ -167,6 +169,7 @@ async fn stream_from_evm( return; } } + processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete(chain_id)); info!("Subscribing to live events"); match provider_ref.subscribe_logs(&filters.current).await { diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_reader.rs index e6a659211c..a8c86a4aa5 100644 --- a/crates/evm/src/evm_reader.rs +++ b/crates/evm/src/evm_reader.rs @@ -1,8 +1,9 @@ use actix::{Actor, Handler}; -use e3_events::{hlc::HlcTimestamp, EnclaveEventData}; +use e3_events::{hlc::HlcTimestamp, EnclaveEventData, EvmEvent}; +use tracing::info; use crate::{ - events::{EnclaveEvmEvent, EvmEvent, EvmEventProcessor, EvmLog}, + events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, ExtractorFn, }; @@ -27,22 +28,25 @@ impl EvmReader { impl Handler for EvmReader { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { - let EnclaveEvmEvent::Log(EvmLog { log, chain_id }) = msg.clone() else { - return; - }; - let extractor = self.extractor; - - if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { - let err = "Log should always have metadata because we listen to non-pending blocks. If you are seeing this it is likely because there is an issue with how we are subscribing to blocks"; - let block = log.block_number.expect(err); - let block_timestamp = log.block_timestamp.expect(err); - let log_index = log.log_index.expect(err); - let ts = from_log_chain_id_to_ts(block_timestamp, log_index, chain_id); - self.next.do_send(EnclaveEvmEvent::Event(EvmEvent { - payload: event, - block, - ts, - })) + match msg.clone() { + EnclaveEvmEvent::Log(EvmLog { log, chain_id }) => { + let extractor = self.extractor; + + if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { + let err = "Log should always have metadata because we listen to non-pending blocks. If you are seeing this it is likely because there is an issue with how we are subscribing to blocks"; + let block = log.block_number.expect(err); + let block_timestamp = log.block_timestamp.expect(err); + let log_index = log.log_index.expect(err); + let ts = from_log_chain_id_to_ts(block_timestamp, log_index, chain_id); + self.next.do_send(EnclaveEvmEvent::Event(EvmEvent::new( + event, block, ts, chain_id, + ))) + } + } + EnclaveEvmEvent::HistoricalSyncComplete(chain_id) => self + .next + .do_send(EnclaveEvmEvent::HistoricalSyncComplete(chain_id)), + _ => (), } } } diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 396c24f6eb..886fc8fac3 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -2,21 +2,35 @@ use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; use actix::{Actor, Addr, Handler}; use alloy_primitives::Address; use std::collections::HashMap; +use tracing::error; /// Directs EnclaveEvmEvent::Log events to the correct upstream processors. Drops all other event /// types pub struct EvmRouter { routing_table: HashMap, + fallback: Option, } impl EvmRouter { - pub fn new(routing_table: HashMap) -> Self { - Self { routing_table } + pub fn new() -> Self { + Self { + routing_table: HashMap::new(), + fallback: None, + } + } + + pub fn add_route(mut self, address: Address, dest: &EvmEventProcessor) -> Self { + self.routing_table.insert(address, dest.clone()); + self } - pub fn setup(routing_table: HashMap) -> Addr { - let addr = Self::new(routing_table).start(); - addr + pub fn add_fallback(mut self, fallback: &EvmEventProcessor) -> Self { + self.fallback = Some(fallback.clone()); + self + } + + pub fn get_routing_table(&self) -> &HashMap { + &self.routing_table } } @@ -27,45 +41,91 @@ impl Actor for EvmRouter { impl Handler for EvmRouter { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { - let EnclaveEvmEvent::Log(EvmLog { log, .. }) = msg.clone() else { - return; - }; - let Some(dest) = self.routing_table.get(&log.address()) else { - return; - }; - - dest.do_send(msg); + match msg.clone() { + // Take all log events and route them + EnclaveEvmEvent::Log(EvmLog { log, .. }) => { + if let Some(dest) = self.routing_table.get(&log.address()) { + dest.do_send(msg); + } else { + error!( + "Could not find a route for log with address = {:?}", + log.address() + ) + } + } + _ => { + if let Some(fallback) = self.fallback.clone() { + fallback.do_send(msg) + } + } + } } } -pub struct EvmHub { - nexts: Vec, -} +#[cfg(test)] +mod tests { + use super::*; + use actix::prelude::*; + use alloy_primitives::address; + use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, + }; + use tokio::time::sleep; -impl EvmHub { - pub fn new(nexts: Vec) -> Self { - Self { nexts } + struct TestProcessor(Arc); + + impl Actor for TestProcessor { + type Context = Context; } - pub fn setup(nexts: Vec) -> Addr { - let addr = Self::new(nexts).start(); - addr + impl Handler for TestProcessor { + type Result = (); + fn handle(&mut self, _msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + self.0.fetch_add(1, Ordering::SeqCst); + } } -} -impl Actor for EvmHub { - type Context = actix::Context; -} + #[actix::test] + async fn test_evm_router_routes_log_to_correct_processor() { + let received_count = Arc::new(AtomicUsize::new(0)); + let processor_addr = TestProcessor(received_count.clone()).start(); + let addr = address!("0x1111111111111111111111111111111111111111"); + let test_log = EvmLog::test_log(addr, 1); + let test_address = test_log.log.address(); -impl Handler for EvmHub { - type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { - let EnclaveEvmEvent::Log { .. } = msg.clone() else { - return; - }; + let router = EvmRouter::new() + .add_route(test_address, &processor_addr.recipient()) + .start(); - for next in self.nexts.clone() { - next.do_send(msg.clone()); - } + router.do_send(EnclaveEvmEvent::Log(test_log)); + + sleep(Duration::from_millis(10)).await; + + assert_eq!(received_count.load(Ordering::SeqCst), 1); + } + + #[actix::test] + async fn test_evm_router_ignores_log_with_unknown_address() { + let received_count = Arc::new(AtomicUsize::new(0)); + let processor_addr = TestProcessor(received_count.clone()).start(); + + let router_addr = address!("0x1111111111111111111111111111111111111111"); + let log_addr = address!("0x2222222222222222222222222222222222222222"); + + let test_log = EvmLog::test_log(log_addr, 1); + + let router = EvmRouter::new() + .add_route(router_addr, &processor_addr.recipient()) + .start(); + + router.do_send(EnclaveEvmEvent::Log(test_log)); + + sleep(Duration::from_millis(10)).await; + + assert_eq!(received_count.load(Ordering::SeqCst), 0); } } diff --git a/crates/evm/src/evm_sync_processor.rs b/crates/evm/src/evm_sync_processor.rs deleted file mode 100644 index f6a8d7a4e0..0000000000 --- a/crates/evm/src/evm_sync_processor.rs +++ /dev/null @@ -1,98 +0,0 @@ -use actix::Recipient; -use actix::{Actor, Handler}; -use anyhow::Context; -use anyhow::Result; -use e3_events::{trap, BusHandle, EnclaveEvent, EnclaveEventData, SyncEnd, SyncStart}; -use e3_events::{EType, SyncEvmEvent}; -use e3_events::{Event, EventPublisher}; - -use crate::events::EnclaveEvmEvent; - -pub struct EvmSyncProcessor { - bus: BusHandle, - status: SyncStatus, -} - -#[derive(Clone)] -enum SyncStatus { - Init, - Syncing(Recipient), - Buffering(Vec), - Live, -} - -impl EvmSyncProcessor { - pub fn new(bus: &BusHandle) -> Self { - Self { - bus: bus.clone(), - status: SyncStatus::Init, - } - } - - fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { - let sender = msg.sender.context("No sender on SyncStart Message")?; - self.status = SyncStatus::Syncing(sender); - Ok(()) - } - - fn handle_sync_end(&mut self, msg: SyncEnd) { - self.status = SyncStatus::Live; - } - - fn handle_evm_event(&mut self, msg: EnclaveEvmEvent) -> Result<()> { - match msg { - EnclaveEvmEvent::HistoricalSyncComplete => { - self.handle_historical_sync_complete()?; - Ok(()) - } - EnclaveEvmEvent::Event(event) => { - self.handle_receive_evm_event(event.payload,event.block)?; - Ok(()) - } - _ => panic!("EvmSyncProcessor is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), - } - } - - fn handle_historical_sync_complete(&mut self) -> Result<()> { - self.status = SyncStatus::Buffering(vec![]); - Ok(()) - } - - fn handle_receive_evm_event(&mut self, event: EnclaveEventData, block: u64) -> Result<()> { - match &mut self.status { - SyncStatus::Buffering(buffer) => buffer.push(SyncEvmEvent::new(event, block)), - SyncStatus::Syncing(sender) => sender.do_send(SyncEvmEvent::new(event, block)), - SyncStatus::Live => { - self.bus - .publish_from_remote(event, 0 /*convert block or whatever to ts*/)?; - } - _ => (), - }; - Ok(()) - } -} - -impl Actor for EvmSyncProcessor { - type Context = actix::Context; -} - -impl Handler for EvmSyncProcessor { - type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.clone(), || { - match msg.into_data() { - EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, - EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e), - _ => (), - } - Ok(()) - }) - } -} - -impl Handler for EvmSyncProcessor { - type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) - } -} diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs deleted file mode 100644 index 0e3924f445..0000000000 --- a/crates/evm/src/historical_event_coordinator.rs +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::prelude::*; -use e3_events::{prelude::*, trap, BusHandle, EType, EnclaveEventData}; -use tracing::info; - -use crate::events::{EnclaveEvmEvent, EvmEvent}; - -#[derive(Clone)] -struct BufferedEvent { - block: u64, - event: EnclaveEventData, -} - -/// Message to start forwarding buffered events after all readers have registered -#[derive(Message)] -#[rtype(result = "()")] -pub struct CoordinatorStart; - -/// Coordinates historical replay across all EvmReadInterfaces. -/// Buffers historical events, then sorts + publishes once all readers finish. -pub struct HistoricalEventCoordinator { - /// Count of readers that have registered - registered_count: usize, - /// Count of readers that have completed historical sync - completed_count: usize, - /// Buffered events during historical sync - buffered_events: Vec, - /// Target to forward events to (typically EventBus) - target: BusHandle, - /// Whether we've started forwarding (after Start message) - started: bool, -} - -impl HistoricalEventCoordinator { - pub fn new(target: BusHandle) -> Self { - Self { - registered_count: 0, - completed_count: 0, - buffered_events: Vec::new(), - target, - started: false, - } - } - - pub fn setup(target: BusHandle) -> Addr { - Self::new(target).start() - } - - fn all_readers_complete(&self) -> bool { - self.registered_count > 0 && self.registered_count == self.completed_count - } - - fn flush_buffered_events(&mut self) -> anyhow::Result<()> { - // Ordering by block number. But we should also consider the tx_index and log_index. - self.buffered_events.sort_by_key(|e| e.block); - - let count = self.buffered_events.len(); - for BufferedEvent { event, .. } in self.buffered_events.drain(..) { - self.target.publish(event)?; - } - - info!( - "HistoricalEventCoordinator: replay complete, published {} ordered events", - count - ); - Ok(()) - } -} - -impl Actor for HistoricalEventCoordinator { - type Context = Context; - - fn started(&mut self, _ctx: &mut Self::Context) { - info!("HistoricalEventCoordinator started"); - } -} - -impl Handler for HistoricalEventCoordinator { - type Result = (); - - fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.target.clone(), || match msg { - EnclaveEvmEvent::RegisterReader => { - self.registered_count += 1; - info!( - total_registered = self.registered_count, - "Reader registered with coordinator" - ); - Ok(()) - } - - EnclaveEvmEvent::HistoricalSyncComplete => { - self.completed_count += 1; - info!( - completed = self.completed_count, - total_registered = self.registered_count, - "Reader completed historical sync" - ); - - if self.started && self.all_readers_complete() { - info!("All readers completed historical sync, flushing buffered events"); - self.flush_buffered_events()?; - } - Ok(()) - } - - EnclaveEvmEvent::Event(event) => { - if !self.started || !self.all_readers_complete() { - let block = event.block; - self.buffered_events.push(BufferedEvent { - block, - event: event.payload, - }); - } else { - self.target.publish(event.payload)?; - } - Ok(()) - } - _ => Ok(()), - }) - } -} - -impl Handler for HistoricalEventCoordinator { - type Result = (); - - fn handle(&mut self, _msg: CoordinatorStart, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.target.clone(), || { - info!( - registered_readers = self.registered_count, - "Starting HistoricalEventCoordinator" - ); - self.started = true; - - if self.all_readers_complete() { - self.flush_buffered_events()?; - } - Ok(()) - }) - } -} diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 9120e52eed..dd653bcb1c 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -10,14 +10,16 @@ mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; mod events; +mod evm_hub; mod evm_launch_coordinator; mod evm_read_interface; mod evm_reader; mod evm_router; -mod evm_sync_processor; pub mod helpers; -mod historical_event_coordinator; +mod one_shot_runnner; mod repo; +mod sync_gateway; +mod sync_start_extractor; pub use bonding_registry_sol::BondingRegistrySolReader; pub use ciphernode_registry_sol::{ @@ -26,9 +28,13 @@ pub use ciphernode_registry_sol::{ pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use evm_launch_coordinator::*; -pub use evm_read_interface::{EvmReadInterface, EvmReadInterfaceState, ExtractorFn}; -pub use evm_sync_processor::*; -pub use helpers::send_tx_with_retry; -pub use historical_event_coordinator::{CoordinatorStart, HistoricalEventCoordinator}; +pub use events::*; +pub use evm_hub::*; +pub use evm_read_interface::*; +pub use evm_reader::*; +pub use evm_router::*; +pub use helpers::*; +pub use one_shot_runnner::*; pub use repo::*; +pub use sync_gateway::*; +pub use sync_start_extractor::*; diff --git a/crates/evm/src/one_shot_runnner.rs b/crates/evm/src/one_shot_runnner.rs new file mode 100644 index 0000000000..fd7f8fb604 --- /dev/null +++ b/crates/evm/src/one_shot_runnner.rs @@ -0,0 +1,86 @@ +use std::marker::PhantomData; + +use actix::prelude::*; +use anyhow::Result; +use tracing::error; +pub struct OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static, + M: Message + 'static, +{ + task: Option, + _marker: PhantomData, +} + +impl OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static + Unpin, + M: Message + 'static + Unpin, +{ + pub fn new(task: F) -> Self { + Self { + task: Some(task), + _marker: PhantomData, + } + } + pub fn setup(task: F) -> Addr { + Self::new(task).start() + } +} + +impl Actor for OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static + Unpin, + M: Message + 'static + Unpin, +{ + type Context = Context; +} + +impl Handler for OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static + Unpin, + M: Message + 'static + Unpin, +{ + type Result = (); + + fn handle(&mut self, msg: M, _ctx: &mut Self::Context) -> Self::Result { + if let Some(task) = self.task.take() { + match task(msg) { + Ok(_) => (), + Err(e) => error!("{e}"), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[derive(Message)] + #[rtype(result = "()")] + struct TestMessage(usize); + + #[actix::test] + async fn test_one_shot_runner() { + let call_count = Arc::new(AtomicUsize::new(0)); + let received_value = Arc::new(AtomicUsize::new(0)); + let call_count_clone = call_count.clone(); + let received_value_clone = received_value.clone(); + + let runner = OneShotRunner::new(move |msg: TestMessage| { + call_count_clone.fetch_add(1, Ordering::SeqCst); + received_value_clone.store(msg.0, Ordering::SeqCst); + Ok(()) + }); + + let addr = runner.start(); + addr.send(TestMessage(42)).await.unwrap(); + addr.send(TestMessage(99)).await.unwrap(); + + assert_eq!(received_value.load(Ordering::SeqCst), 42); + assert_eq!(call_count.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/evm/src/sync_gateway.rs b/crates/evm/src/sync_gateway.rs new file mode 100644 index 0000000000..a25ee8a8f8 --- /dev/null +++ b/crates/evm/src/sync_gateway.rs @@ -0,0 +1,165 @@ +use crate::events::EnclaveEvmEvent; +use actix::{Actor, Handler}; +use actix::{Addr, Recipient}; +use anyhow::Result; +use anyhow::{bail, Context}; +use e3_events::{ + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, SyncEnd, SyncEvmEvent, + SyncStart, +}; +use e3_events::{EType, EvmEvent}; +use e3_events::{Event, EventPublisher}; + +/// The chain gateway +pub struct EvmChainGateway { + bus: BusHandle, + status: SyncStatus, +} + +#[derive(Clone, Debug)] +enum SyncStatus { + /// Intial State + Init, + /// After SyncStart we forward all events to SyncActor + ForwardToSyncActor(Option>), + /// Once the chain has completed historical sync then we buffer all "live" events until sync is + /// complete + BufferUntilLive(Vec), + /// Forward all events directly to the bus + Live, +} + +impl Default for SyncStatus { + fn default() -> Self { + Self::Init + } +} + +impl SyncStatus { + pub fn forward_to_sync_actor(&mut self, sender: Recipient) -> Result<()> { + let Self::Init = self else { + bail!( + "Cannot change state to ForwardToSyncActor when state is {:?}", + self + ); + }; + + *self = SyncStatus::ForwardToSyncActor(Some(sender)); + Ok(()) + } + + pub fn buffer_until_live(&mut self) -> Result> { + let Self::ForwardToSyncActor(sender) = self else { + bail!( + "Cannot change state to BufferUntilLive when state is {:?}", + self + ); + }; + let sender = std::mem::take(sender).context("Cannot call buffer_until_live twice")?; + *self = SyncStatus::BufferUntilLive(vec![]); + Ok(sender) + } + + pub fn live(&mut self) -> Result> { + let Self::BufferUntilLive(buffer) = self else { + bail!("Cannot change state to Live when state is {:?}", self); + }; + let buffer = std::mem::take(buffer); + *self = SyncStatus::Live; + Ok(buffer) + } +} + +impl EvmChainGateway { + pub fn new(bus: &BusHandle) -> Self { + Self { + bus: bus.clone(), + status: SyncStatus::default(), + } + } + + pub fn setup(bus: &BusHandle) -> Addr { + let addr = Self::new(bus).start(); + bus.subscribe_all(&["SyncStart", "SyncEnd"], addr.clone().recipient()); + addr + } + + fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { + // Received a SyncStart event from the event bus. Get the sender within that event and forward + // all events to that actor + let sender = msg.sender.context("No sender on SyncStart Message")?; + self.status.forward_to_sync_actor(sender)?; + Ok(()) + } + + fn handle_sync_end(&mut self, _: SyncEnd) -> Result<()> { + let buffer = self.status.live()?; + for evt in buffer { + self.publish_evm_event(evt)?; + } + Ok(()) + } + + fn publish_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + let (data, ts, _) = msg.split(); + self.bus.publish_from_remote(data, ts)?; + Ok(()) + } + + fn handle_evm_event(&mut self, msg: EnclaveEvmEvent) -> Result<()> { + match msg { + EnclaveEvmEvent::HistoricalSyncComplete(chain_id) => { + self.handle_historical_sync_complete(chain_id)?; + Ok(()) + } + EnclaveEvmEvent::Event(event) => { + self.handle_receive_evm_event(event)?; + Ok(()) + } + _ => panic!("EvmChainGateway is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), + } + } + + fn handle_historical_sync_complete(&mut self, chain_id: u64) -> Result<()> { + let sender = self.status.buffer_until_live()?; + sender.do_send(SyncEvmEvent::HistoricalSyncComplete(chain_id)); + Ok(()) + } + + fn handle_receive_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + match &mut self.status { + SyncStatus::BufferUntilLive(buffer) => buffer.push(msg), + SyncStatus::ForwardToSyncActor(Some(sync_actor)) => { + sync_actor.do_send(msg.into()); + } + SyncStatus::Live => self.publish_evm_event(msg)?, + _ => (), + }; + Ok(()) + } +} + +impl Actor for EvmChainGateway { + type Context = actix::Context; +} + +impl Handler for EvmChainGateway { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || { + match msg.into_data() { + EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, + EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e)?, + _ => (), + } + Ok(()) + }) + } +} + +impl Handler for EvmChainGateway { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) + } +} diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs new file mode 100644 index 0000000000..f2700c21a0 --- /dev/null +++ b/crates/evm/src/sync_start_extractor.rs @@ -0,0 +1,28 @@ +use actix::{Actor, Addr, Handler, Recipient}; +use e3_events::{EnclaveEvent, EnclaveEventData, Event, SyncStart}; + +pub struct SyncStartExtractor { + dest: Recipient, +} + +impl SyncStartExtractor { + pub fn new(dest: impl Into>) -> Self { + Self { dest: dest.into() } + } + + pub fn setup(dest: impl Into>) -> Addr { + Self::new(dest).start() + } +} +impl Actor for SyncStartExtractor { + type Context = actix::Context; +} + +impl Handler for SyncStartExtractor { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::SyncStart(evt) = msg.into_data() { + self.dest.do_send(evt) + } + } +} diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 1381c8ae18..9862a9cc37 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::Addr; +use actix::{Actor, Addr, Handler}; use alloy::{ node_bindings::Anvil, primitives::{FixedBytes, LogData}, @@ -15,16 +15,17 @@ use alloy::{ }; use anyhow::Result; use e3_ciphernode_builder::EventSystem; -use e3_data::Repository; -use e3_entrypoint::helpers::datastore::get_in_mem_store; use e3_events::{ - prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, HistoryCollector, Shutdown, TestEvent, + prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, GetEvents, + HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, }; use e3_evm::{ - helpers::EthProvider, CoordinatorStart, EvmReadInterface, HistoricalEventCoordinator, + helpers::EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmReader, + EvmRouter, Filters, OneShotRunner, SyncStartExtractor, }; -use std::time::Duration; +use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::time::sleep; +use tracing_subscriber::{fmt, EnvFilter}; sol!( #[sol(rpc)] @@ -54,6 +55,14 @@ fn test_event_extractor( } } +struct TestEventParser; + +impl TestEventParser { + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmReader::new(next, test_event_extractor).start() + } +} + async fn get_msgs(history_collector: &Addr>) -> Result> { let history = history_collector .send(GetEvents::::new()) @@ -69,41 +78,96 @@ async fn get_msgs(history_collector: &Addr>) -> R Ok(msgs) } +struct FakeSyncActor { + bus: BusHandle, +} + +impl Actor for FakeSyncActor { + type Context = actix::Context; +} + +impl FakeSyncActor { + pub fn setup(bus: &BusHandle) -> Addr { + Self { bus: bus.clone() }.start() + } +} + +impl Handler for FakeSyncActor { + type Result = (); + fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Sync, &self.bus.clone(), || { + match msg { + SyncEvmEvent::Event(evt) => (), // self.buffer.push(evt) - sort historical event + SyncEvmEvent::HistoricalSyncComplete(evt) => self.bus.publish(SyncEnd::new())?, + }; + Ok(()) + }) + } +} + #[actix::test] async fn evm_reader() -> Result<()> { + let _guard = tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new("info")) + .with_test_writer() + .finish(), + ); + // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; let rpc_url = anvil.ws_endpoint(); // Get RPC URL - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) // Use RPC URL - .await?, - ) - .await?; + let provider = Arc::new( + EthProvider::new( + ProviderBuilder::new() + .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) + .connect_ws(WsConnect::new(rpc_url.clone())) // Use RPC URL + .await?, + ) + .await?, + ); let contract = EmitLogs::deploy(provider.provider()).await?; + let chain_id = provider.chain_id(); let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); + let contract_address = contract.address().to_string(); + + // Simulates the setup for a single chain + let gateway = EvmChainGateway::setup(&bus); + let sync = FakeSyncActor::setup(&bus); + let runner = SyncStartExtractor::setup(OneShotRunner::setup({ + let bus = bus.clone(); + let provider = provider.clone(); + let gateway = gateway.clone(); + move |msg: SyncStart| { + let info = msg.get_evm_init_for(chain_id); + let gateway = gateway.recipient(); + let router = EvmRouter::new() + // add new route per contract + .add_route( + contract_address.parse()?, + &TestEventParser::setup(&gateway).recipient(), + ) + .add_fallback(&gateway); + + let filters = Filters::from_routing_table(router.get_routing_table(), info); + let router = router.start(); + EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); + Ok(()) + } + })); - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; + bus.subscribe("SyncStart", runner.recipient()); - coordinator.do_send(CoordinatorStart); + // SyncStart holds initialization information such as start block and earliest event + // This should trigger all chains to start to sync + let mut evm_info = HashMap::new(); + evm_info.insert(chain_id, None); + bus.publish(SyncStart::new(sync, evm_info))?; + sleep(Duration::from_secs(1)).await; contract .setValue("hello".to_string()) .send() @@ -118,14 +182,12 @@ async fn evm_reader() -> Result<()> { .watch() .await?; - sleep(Duration::from_millis(1)).await; + sleep(Duration::from_secs(1)).await; let history = history_collector .send(GetEvents::::new()) .await?; - assert_eq!(history.len(), 2); - let msgs: Vec<_> = history .into_iter() .filter_map(|evt| match evt.into_data() { @@ -138,7 +200,7 @@ async fn evm_reader() -> Result<()> { Ok(()) } - +/* #[actix::test] async fn ensure_historical_events() -> Result<()> { // Create a WS provider @@ -542,4 +604,4 @@ async fn coordinator_no_historical_events() -> Result<()> { assert_eq!(msgs, ["live1", "live2"]); Ok(()) -} +}*/ From 9011ac92a3934b98df6503c61ebaefac8313d79d Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 25 Jan 2026 13:23:29 +0000 Subject: [PATCH 069/102] move fake sync actor out of the way --- crates/evm/tests/integration.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 9862a9cc37..ef56b41474 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -133,10 +133,10 @@ async fn evm_reader() -> Result<()> { let bus = system.handle()?; let history_collector = bus.history(); let contract_address = contract.address().to_string(); + let sync = FakeSyncActor::setup(&bus); // Simulates the setup for a single chain let gateway = EvmChainGateway::setup(&bus); - let sync = FakeSyncActor::setup(&bus); let runner = SyncStartExtractor::setup(OneShotRunner::setup({ let bus = bus.clone(); let provider = provider.clone(); @@ -158,7 +158,6 @@ async fn evm_reader() -> Result<()> { Ok(()) } })); - bus.subscribe("SyncStart", runner.recipient()); // SyncStart holds initialization information such as start block and earliest event From 0c250e39e79e88be3aa44dca53065a481972cccf Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 01:45:38 +0000 Subject: [PATCH 070/102] create chain builder --- crates/ciphernode-builder/src/evm_system.rs | 65 +++++++++++++++++++++ crates/ciphernode-builder/src/lib.rs | 2 + crates/evm/tests/integration.rs | 35 ++++------- 3 files changed, 77 insertions(+), 25 deletions(-) create mode 100644 crates/ciphernode-builder/src/evm_system.rs diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs new file mode 100644 index 0000000000..90e5969567 --- /dev/null +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -0,0 +1,65 @@ +use actix::Actor; +use alloy::{primitives::Address, providers::Provider}; +use anyhow::Result; +use e3_events::{BusHandle, EventSubscriber, SyncStart}; +use e3_evm::{ + EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, + OneShotRunner, SyncStartExtractor, +}; + +pub trait RouteFn: FnOnce(EvmEventProcessor) -> (Address, EvmEventProcessor) + Send {} +impl RouteFn for F where F: FnOnce(EvmEventProcessor) -> (Address, EvmEventProcessor) + Send {} + +type RouteFactory = Box; + +// Build the event system for a single chain +pub struct EvmSystemChainBuilder

{ + provider: EthProvider

, + bus: BusHandle, + chain_id: u64, + route_factories: Vec, +} + +impl EvmSystemChainBuilder

{ + pub fn new(bus: &BusHandle, provider: &EthProvider

, chain_id: u64) -> Self { + Self { + bus: bus.clone(), + provider: provider.clone(), + chain_id, + route_factories: Vec::new(), + } + } + + pub fn with_route(mut self, route_fn: F) -> Self { + self.route_factories.push(Box::new(route_fn)); + self + } + + pub fn build(self) { + let gateway = EvmChainGateway::setup(&self.bus); + let runner = SyncStartExtractor::setup(OneShotRunner::setup({ + let bus = self.bus.clone(); + let provider = self.provider.clone(); + let gateway = gateway.clone(); + let chain_id = self.chain_id; + let route_factories = self.route_factories; + move |msg: SyncStart| { + let info = msg.get_evm_init_for(chain_id); + let gateway = gateway.recipient(); + let mut router = EvmRouter::new(); + + for route_fn in route_factories { + let (address, processor) = route_fn(gateway.clone()); + router = router.add_route(address, &processor); + } + + router = router.add_fallback(&gateway); + let filters = Filters::from_routing_table(router.get_routing_table(), info); + let router = router.start(); + EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); + Ok(()) + } + })); + self.bus.subscribe("SyncStart", runner.recipient()); + } +} diff --git a/crates/ciphernode-builder/src/lib.rs b/crates/ciphernode-builder/src/lib.rs index 7d803ab3ce..70a5a3079d 100644 --- a/crates/ciphernode-builder/src/lib.rs +++ b/crates/ciphernode-builder/src/lib.rs @@ -8,9 +8,11 @@ mod ciphernode; mod ciphernode_builder; mod event_system; mod eventbus_factory; +mod evm_system; mod provider_caches; pub use ciphernode::*; pub use ciphernode_builder::*; pub use event_system::*; pub use eventbus_factory::*; +pub use evm_system::*; pub use provider_caches::*; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index ef56b41474..050792c4c0 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -14,7 +14,7 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; -use e3_ciphernode_builder::EventSystem; +use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, @@ -135,30 +135,15 @@ async fn evm_reader() -> Result<()> { let contract_address = contract.address().to_string(); let sync = FakeSyncActor::setup(&bus); - // Simulates the setup for a single chain - let gateway = EvmChainGateway::setup(&bus); - let runner = SyncStartExtractor::setup(OneShotRunner::setup({ - let bus = bus.clone(); - let provider = provider.clone(); - let gateway = gateway.clone(); - move |msg: SyncStart| { - let info = msg.get_evm_init_for(chain_id); - let gateway = gateway.recipient(); - let router = EvmRouter::new() - // add new route per contract - .add_route( - contract_address.parse()?, - &TestEventParser::setup(&gateway).recipient(), - ) - .add_fallback(&gateway); - - let filters = Filters::from_routing_table(router.get_routing_table(), info); - let router = router.start(); - EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); - Ok(()) - } - })); - bus.subscribe("SyncStart", runner.recipient()); + let contract_address = contract_address.parse()?; + EvmSystemChainBuilder::new(&bus, &provider, chain_id) + .with_route(move |upstream| { + ( + contract_address, + TestEventParser::setup(&upstream).recipient(), + ) + }) + .build(); // SyncStart holds initialization information such as start block and earliest event // This should trigger all chains to start to sync From 5d1675f67e21f8ae849155934801d436f060ab4f Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 02:17:57 +0000 Subject: [PATCH 071/102] events are now in order --- Cargo.lock | 1 + crates/ciphernode-builder/src/evm_system.rs | 5 +- crates/events/src/enclave_event/sync_start.rs | 16 +- crates/evm/Cargo.toml | 1 + crates/evm/src/events.rs | 60 ++- crates/evm/src/evm_read_interface.rs | 18 +- crates/evm/src/evm_reader.rs | 9 +- crates/evm/src/fix_historical_order.rs | 125 ++++++ crates/evm/src/lib.rs | 2 + crates/evm/src/sync_gateway.rs | 17 +- crates/evm/tests/integration.rs | 406 ++---------------- 11 files changed, 269 insertions(+), 391 deletions(-) create mode 100644 crates/evm/src/fix_historical_order.rs diff --git a/Cargo.lock b/Cargo.lock index 42c05df132..556cf57bbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3006,6 +3006,7 @@ dependencies = [ "anyhow", "async-trait", "base64", + "bloom", "e3-ciphernode-builder", "e3-config", "e3-crypto", diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 90e5969567..e0201f73de 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -1,10 +1,9 @@ use actix::Actor; use alloy::{primitives::Address, providers::Provider}; -use anyhow::Result; use e3_events::{BusHandle, EventSubscriber, SyncStart}; use e3_evm::{ EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, - OneShotRunner, SyncStartExtractor, + FixHistoricalOrder, OneShotRunner, SyncStartExtractor, }; pub trait RouteFn: FnOnce(EvmEventProcessor) -> (Address, EvmEventProcessor) + Send {} @@ -36,7 +35,7 @@ impl EvmSystemChainBuilder

{ } pub fn build(self) { - let gateway = EvmChainGateway::setup(&self.bus); + let gateway = FixHistoricalOrder::setup(EvmChainGateway::setup(&self.bus)); let runner = SyncStartExtractor::setup(OneShotRunner::setup({ let bus = self.bus.clone(); let provider = self.provider.clone(); diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index dd913c46a7..525a4868a7 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use super::EnclaveEventData; -use crate::SyncEvmEvent; +use crate::{CorrelationId, SyncEvmEvent}; use actix::{Message, Recipient}; use serde::{Deserialize, Serialize}; use std::{ @@ -21,11 +21,19 @@ pub struct EvmEvent { block: u64, chain_id: u64, ts: u128, + id: CorrelationId, } impl EvmEvent { - pub fn new(data: EnclaveEventData, block: u64, ts: u128, chain_id: u64) -> Self { + pub fn new( + id: CorrelationId, + data: EnclaveEventData, + block: u64, + ts: u128, + chain_id: u64, + ) -> Self { Self { + id, data, block, ts, @@ -36,6 +44,10 @@ impl EvmEvent { pub fn split(self) -> (EnclaveEventData, u128, u64) { (self.data, self.ts, self.block) } + + pub fn get_id(&self) -> CorrelationId { + self.id + } } /// Dispatched by the Sync actor when initial data is read and the sync process needs to be started diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index ea1b9183c8..34bab45f69 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -14,6 +14,7 @@ alloy-primitives = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } +bloom = { workspace = true } e3-crypto = { workspace = true } e3-config = { workspace = true } e3-data = { workspace = true } diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index b96d6fc74f..bfddb0b162 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -1,29 +1,76 @@ use actix::{Message, Recipient}; use alloy::rpc::types::Log; -use alloy_primitives::Address; -use e3_events::{EventId, EvmEvent}; +use e3_events::{CorrelationId, EvmEvent}; use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct HistoricalSyncComplete { + pub chain_id: u64, + pub prev_event: Option, + pub id: CorrelationId, +} + +impl HistoricalSyncComplete { + pub fn new(chain_id: u64, prev_event: Option) -> Self { + let id = CorrelationId::new(); + Self { + id, + chain_id, + prev_event, + } + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub enum EnclaveEvmEvent { /// Signal that this reader has completed historical sync - HistoricalSyncComplete(u64), + HistoricalSyncComplete(HistoricalSyncComplete), /// An actual event from the blockchain Event(EvmEvent), /// Raw log data from the provider Log(EvmLog), } +impl EnclaveEvmEvent { + pub fn get_id(&self) -> CorrelationId { + match self { + EnclaveEvmEvent::HistoricalSyncComplete(e) => e.get_id(), + EnclaveEvmEvent::Log(e) => e.get_id(), + EnclaveEvmEvent::Event(e) => e.get_id(), + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct EvmLog { + pub id: CorrelationId, pub log: Log, pub chain_id: u64, } +impl EvmLog { + pub fn new(log: Log, chain_id: u64) -> Self { + let id = CorrelationId::new(); + Self { log, chain_id, id } + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } +} + +#[cfg(test)] +use alloy_primitives::Address; + #[cfg(test)] impl EvmLog { pub fn test_log(address: Address, chain_id: u64) -> EvmLog { + let id = CorrelationId::new(); EvmLog { log: Log { inner: alloy_primitives::Log { @@ -33,14 +80,9 @@ impl EvmLog { ..Default::default() }, chain_id, + id, } } } -impl EnclaveEvmEvent { - pub fn get_id(&self) -> EventId { - EventId::hash(self.clone()) - } -} - pub type EvmEventProcessor = Recipient; diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index 4f56f7180c..7ab371aa9d 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -6,6 +6,7 @@ use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; use crate::helpers::EthProvider; +use crate::HistoricalSyncComplete; use actix::prelude::*; use actix::{Addr, Recipient}; use alloy::eips::BlockNumberOrTag; @@ -14,7 +15,7 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy_primitives::Address; use anyhow::anyhow; -use e3_events::{BusHandle, ErrorDispatcher, Event, EventSubscriber}; +use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscriber}; use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; @@ -96,7 +97,7 @@ impl EvmReadInterface

{ pub fn setup( provider: &EthProvider

, - next: &Recipient, + next: &EvmEventProcessor, bus: &BusHandle, filters: Filters, ) -> Addr { @@ -154,13 +155,15 @@ async fn stream_from_evm( ) { let chain_id = provider.chain_id(); let provider_ref = provider.provider(); - + let mut last_id: Option = None; // Historical events match provider_ref.get_logs(&filters.historical).await { Ok(historical_logs) => { info!("Fetched {} historical events", historical_logs.len()); for log in historical_logs { - processor.do_send(EnclaveEvmEvent::Log(EvmLog { log, chain_id })) + let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id)); + last_id = Some(evt.get_id()); + processor.do_send(evt) } } Err(e) => { @@ -169,7 +172,10 @@ async fn stream_from_evm( return; } } - processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete(chain_id)); + + processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + HistoricalSyncComplete::new(chain_id, last_id), + )); info!("Subscribing to live events"); match provider_ref.subscribe_logs(&filters.current).await { @@ -182,7 +188,7 @@ async fn stream_from_evm( maybe_log = stream.next() => { match maybe_log { Some(log) => { - processor.do_send(EnclaveEvmEvent::Log(EvmLog { log, chain_id })) + processor.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id))) } None => break, // Stream ended } diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_reader.rs index a8c86a4aa5..4eb1f72049 100644 --- a/crates/evm/src/evm_reader.rs +++ b/crates/evm/src/evm_reader.rs @@ -29,7 +29,7 @@ impl Handler for EvmReader { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { match msg.clone() { - EnclaveEvmEvent::Log(EvmLog { log, chain_id }) => { + EnclaveEvmEvent::Log(EvmLog { log, chain_id, id }) => { let extractor = self.extractor; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { @@ -39,13 +39,12 @@ impl Handler for EvmReader { let log_index = log.log_index.expect(err); let ts = from_log_chain_id_to_ts(block_timestamp, log_index, chain_id); self.next.do_send(EnclaveEvmEvent::Event(EvmEvent::new( - event, block, ts, chain_id, + // note we use the id from the log event above! + id, event, block, ts, chain_id, ))) } } - EnclaveEvmEvent::HistoricalSyncComplete(chain_id) => self - .next - .do_send(EnclaveEvmEvent::HistoricalSyncComplete(chain_id)), + hist @ EnclaveEvmEvent::HistoricalSyncComplete(..) => self.next.do_send(hist), _ => (), } } diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs new file mode 100644 index 0000000000..640cc639f3 --- /dev/null +++ b/crates/evm/src/fix_historical_order.rs @@ -0,0 +1,125 @@ +use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; +use actix::{Actor, Addr, Handler}; +use bloom::{BloomFilter, ASMS}; + +pub struct FixHistoricalOrder { + dest: EvmEventProcessor, + pending_sync_complete: Option, + seen_ids: BloomFilter, +} + +impl FixHistoricalOrder { + pub fn new(dest: impl Into) -> Self { + Self { + dest: dest.into(), + pending_sync_complete: None, + seen_ids: BloomFilter::with_rate(0.001, 10_000), + } + } + + pub fn setup(dest: impl Into) -> Addr { + Self::new(dest).start() + } + + fn send_pending(&mut self) { + if let Some(EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { + prev_event: Some(ref id), + .. + })) = self.pending_sync_complete + { + if self.seen_ids.contains(id) { + self.dest + .do_send(self.pending_sync_complete.take().unwrap()); + } + } + } +} + +impl Actor for FixHistoricalOrder { + type Context = actix::Context; +} + +impl Handler for FixHistoricalOrder { + type Result = (); + + fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + match msg { + none_hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { + prev_event: None, + .. + }) => { + self.dest.do_send(none_hist); + } + hist @ EnclaveEvmEvent::HistoricalSyncComplete(..) => { + self.pending_sync_complete = Some(hist); + } + other => { + self.seen_ids.insert(&other.get_id()); + self.dest.do_send(other); + } + } + self.send_pending(); + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use crate::EvmLog; + + use super::*; + use actix::prelude::*; + use alloy_primitives::Address; + use tokio::{sync::mpsc, time::sleep}; + + struct Collector(mpsc::UnboundedSender); + + impl Actor for Collector { + type Context = Context; + } + + impl Handler for Collector { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + let _ = self.0.send(msg); + } + } + + #[actix::test] + async fn test_reorders_sync_complete_after_referenced_event() { + let (tx, mut rx) = mpsc::unbounded_channel(); + let fix = FixHistoricalOrder::setup(Collector(tx).start()); + + let log_1 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 1)); + let log_2 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 2)); + let log_3 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 3)); + + let sync_complete = EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete::new( + 1, + Some(log_3.get_id()), + )); + + // Send logs 1, 2, 3 + fix.send(log_1.clone()).await.unwrap(); + // Send sync complete FIRST (out of order - references log_3 which hasn't been seen) + fix.send(sync_complete.clone()).await.unwrap(); + fix.send(log_2.clone()).await.unwrap(); + fix.send(log_3.clone()).await.unwrap(); + + sleep(Duration::from_secs(1)).await; + + // Collect results + let mut received = vec![]; + while let Ok(msg) = rx.try_recv() { + received.push(msg); + } + + // The sync complete should have been held until log_3 was seen + assert_eq!(received.len(), 4); + assert_eq!(received[0], log_1); + assert_eq!(received[1], log_2); + assert_eq!(received[2], log_3); + assert_eq!(received[3], sync_complete); + } +} diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index dd653bcb1c..c768a83206 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -15,6 +15,7 @@ mod evm_launch_coordinator; mod evm_read_interface; mod evm_reader; mod evm_router; +mod fix_historical_order; pub mod helpers; mod one_shot_runnner; mod repo; @@ -33,6 +34,7 @@ pub use evm_hub::*; pub use evm_read_interface::*; pub use evm_reader::*; pub use evm_router::*; +pub use fix_historical_order::*; pub use helpers::*; pub use one_shot_runnner::*; pub use repo::*; diff --git a/crates/evm/src/sync_gateway.rs b/crates/evm/src/sync_gateway.rs index a25ee8a8f8..cfe2c62965 100644 --- a/crates/evm/src/sync_gateway.rs +++ b/crates/evm/src/sync_gateway.rs @@ -1,14 +1,16 @@ use crate::events::EnclaveEvmEvent; +use crate::HistoricalSyncComplete; use actix::{Actor, Handler}; use actix::{Addr, Recipient}; use anyhow::Result; use anyhow::{bail, Context}; use e3_events::{ - trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, SyncEnd, SyncEvmEvent, - SyncStart, + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventId, EventSubscriber, SyncEnd, + SyncEvmEvent, SyncStart, }; use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; +use tracing::info; /// The chain gateway pub struct EvmChainGateway { @@ -45,6 +47,7 @@ impl SyncStatus { }; *self = SyncStatus::ForwardToSyncActor(Some(sender)); + info!("Changed to ForwardToSyncActor"); Ok(()) } @@ -57,6 +60,7 @@ impl SyncStatus { }; let sender = std::mem::take(sender).context("Cannot call buffer_until_live twice")?; *self = SyncStatus::BufferUntilLive(vec![]); + info!("Changed to BufferUntilLive"); Ok(sender) } @@ -66,6 +70,7 @@ impl SyncStatus { }; let buffer = std::mem::take(buffer); *self = SyncStatus::Live; + info!("Changed to Live"); Ok(buffer) } } @@ -108,8 +113,8 @@ impl EvmChainGateway { fn handle_evm_event(&mut self, msg: EnclaveEvmEvent) -> Result<()> { match msg { - EnclaveEvmEvent::HistoricalSyncComplete(chain_id) => { - self.handle_historical_sync_complete(chain_id)?; + EnclaveEvmEvent::HistoricalSyncComplete(e) => { + self.handle_historical_sync_complete(e)?; Ok(()) } EnclaveEvmEvent::Event(event) => { @@ -120,9 +125,9 @@ impl EvmChainGateway { } } - fn handle_historical_sync_complete(&mut self, chain_id: u64) -> Result<()> { + fn handle_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { let sender = self.status.buffer_until_live()?; - sender.do_send(SyncEvmEvent::HistoricalSyncComplete(chain_id)); + sender.do_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id)); Ok(()) } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 050792c4c0..27b98e21f5 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -19,12 +19,10 @@ use e3_events::{ prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, }; -use e3_evm::{ - helpers::EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmReader, - EvmRouter, Filters, OneShotRunner, SyncStartExtractor, -}; +use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmReader}; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::time::sleep; +use tracing::subscriber::DefaultGuard; use tracing_subscriber::{fmt, EnvFilter}; sol!( @@ -80,6 +78,7 @@ async fn get_msgs(history_collector: &Addr>) -> R struct FakeSyncActor { bus: BusHandle, + buffer: Vec, } impl Actor for FakeSyncActor { @@ -88,7 +87,11 @@ impl Actor for FakeSyncActor { impl FakeSyncActor { pub fn setup(bus: &BusHandle) -> Addr { - Self { bus: bus.clone() }.start() + Self { + bus: bus.clone(), + buffer: Vec::new(), + } + .start() } } @@ -97,22 +100,34 @@ impl Handler for FakeSyncActor { fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { match msg { - SyncEvmEvent::Event(evt) => (), // self.buffer.push(evt) - sort historical event - SyncEvmEvent::HistoricalSyncComplete(evt) => self.bus.publish(SyncEnd::new())?, + // Buffer events as the sync actor receives them + SyncEvmEvent::Event(event) => self.buffer.push(event), + // When we hear that sync is complete send all events on chain then publish SyncEnd + SyncEvmEvent::HistoricalSyncComplete(_) => { + for evt in self.buffer.drain(..) { + let (data, ts, _) = evt.split(); + self.bus.publish_from_remote(data, ts)?; + } + self.bus.publish(SyncEnd::new())?; + } }; Ok(()) }) } } -#[actix::test] -async fn evm_reader() -> Result<()> { - let _guard = tracing::subscriber::set_default( +fn add_tracing() -> DefaultGuard { + tracing::subscriber::set_default( fmt() .with_env_filter(EnvFilter::new("info")) .with_test_writer() .finish(), - ); + ) +} + +#[actix::test] +async fn evm_reader() -> Result<()> { + let _guard = add_tracing(); // Create a WS provider // NOTE: Anvil must be available on $PATH @@ -128,14 +143,13 @@ async fn evm_reader() -> Result<()> { .await?, ); let contract = EmitLogs::deploy(provider.provider()).await?; - let chain_id = provider.chain_id(); let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); - let contract_address = contract.address().to_string(); - let sync = FakeSyncActor::setup(&bus); - let contract_address = contract_address.parse()?; + let chain_id = provider.chain_id(); + let contract_address = contract.address().clone(); + let sync = FakeSyncActor::setup(&bus); EvmSystemChainBuilder::new(&bus, &provider, chain_id) .with_route(move |upstream| { ( @@ -184,9 +198,10 @@ async fn evm_reader() -> Result<()> { Ok(()) } -/* #[actix::test] async fn ensure_historical_events() -> Result<()> { + let _guard = add_tracing(); + // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; @@ -199,17 +214,14 @@ async fn ensure_historical_events() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; + let contract_address = contract.address().clone(); + let chain_id = provider.chain_id(); let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let historical_msgs = vec!["these", "are", "historical", "events"]; let live_events = vec!["these", "events", "are", "live"]; - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - for msg in historical_msgs.clone() { contract .setValue(msg.to_string()) @@ -219,19 +231,20 @@ async fn ensure_historical_events() -> Result<()> { .await?; } - EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; + sleep(Duration::from_millis(1)).await; - coordinator.do_send(CoordinatorStart); + let sync = FakeSyncActor::setup(&bus); + EvmSystemChainBuilder::new(&bus, &provider, chain_id) + .with_route(move |upstream| { + ( + contract_address, + TestEventParser::setup(&upstream).recipient(), + ) + }) + .build(); + let mut evm_info = HashMap::new(); + evm_info.insert(chain_id, None); + bus.publish(SyncStart::new(sync, evm_info))?; for msg in live_events.clone() { contract @@ -250,8 +263,6 @@ async fn ensure_historical_events() -> Result<()> { .send(GetEvents::::new()) .await?; - assert_eq!(history.len(), 8); - let msgs: Vec<_> = history .into_iter() .filter_map(|evt| match evt.into_data() { @@ -264,328 +275,3 @@ async fn ensure_historical_events() -> Result<()> { Ok(()) } - -#[actix::test] -async fn ensure_resume_after_shutdown() -> Result<()> { - // Create a WS provider - // NOTE: Anvil must be available on $PATH - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); // Get RPC URL - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) // Use RPC URL - .await?, - ) - .await?; - let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - for msg in ["before", "online"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - let addr1 = EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - - for msg in ["live", "events"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - // Ensure shutdown doesn't cause event to be lost. - sleep(Duration::from_millis(10)).await; - addr1 - .send(EnclaveEvent::new_stored_event(Shutdown.into(), 4321, 42)) - .await?; - - for msg in ["these", "are", "not", "lost"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(10)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs, ["before", "online", "live", "events"]); - - let _ = EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - sleep(Duration::from_millis(10)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!( - msgs, - ["before", "online", "live", "events", "these", "are", "not", "lost"] - ); - - for msg in ["resumed", "data"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(10)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!( - msgs, - ["before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data"] - ); - - Ok(()) -} - -#[actix::test] -async fn coordinator_single_reader() -> Result<()> { - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) - .await?, - ) - .await?; - let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - for msg in ["historical1", "historical2", "historical3"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - sleep(Duration::from_millis(100)).await; - - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs, ["historical1", "historical2", "historical3"]); - - for msg in ["live1", "live2"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(100)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!( - msgs, - [ - "historical1", - "historical2", - "historical3", - "live1", - "live2" - ] - ); - - Ok(()) -} - -#[actix::test] -async fn coordinator_multiple_readers() -> Result<()> { - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) - .await?, - ) - .await?; - - let contract1 = EmitLogs::deploy(provider.provider()).await?; - let contract2 = EmitLogs::deploy(provider.provider()).await?; - - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository1 = Repository::new(get_in_mem_store()); - let repository2 = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - contract1 - .setValue("contract1_msg1".to_string()) - .send() - .await? - .watch() - .await?; - contract2 - .setValue("contract2_msg1".to_string()) - .send() - .await? - .watch() - .await?; - contract1 - .setValue("contract1_msg2".to_string()) - .send() - .await? - .watch() - .await?; - contract2 - .setValue("contract2_msg2".to_string()) - .send() - .await? - .watch() - .await?; - - EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract1.address().to_string(), - None, - &processor, - &bus, - &repository1, - rpc_url.clone(), - ) - .await?; - - EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract2.address().to_string(), - None, - &processor, - &bus, - &repository2, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - - // Wait for historical events to be processed - sleep(Duration::from_millis(200)).await; - - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs.len(), 4); - assert!(msgs.contains(&"contract1_msg1".to_string())); - assert!(msgs.contains(&"contract2_msg1".to_string())); - assert!(msgs.contains(&"contract1_msg2".to_string())); - assert!(msgs.contains(&"contract2_msg2".to_string())); - - Ok(()) -} - -#[actix::test] -async fn coordinator_no_historical_events() -> Result<()> { - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) - .await?, - ) - .await?; - let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - EvmReadInterface::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - sleep(Duration::from_millis(50)).await; - - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs.len(), 0); - - for msg in ["live1", "live2"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(100)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs, ["live1", "live2"]); - - Ok(()) -}*/ From 321224493ca463258b93c5ed5be3a171a3f4271a Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:27:06 +0000 Subject: [PATCH 072/102] test sync_gateway --- crates/evm/src/sync_gateway.rs | 120 +++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 12 deletions(-) diff --git a/crates/evm/src/sync_gateway.rs b/crates/evm/src/sync_gateway.rs index cfe2c62965..cd6c4875e1 100644 --- a/crates/evm/src/sync_gateway.rs +++ b/crates/evm/src/sync_gateway.rs @@ -5,12 +5,11 @@ use actix::{Addr, Recipient}; use anyhow::Result; use anyhow::{bail, Context}; use e3_events::{ - trap, BusHandle, EnclaveEvent, EnclaveEventData, EventId, EventSubscriber, SyncEnd, - SyncEvmEvent, SyncStart, + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, SyncEnd, SyncEvmEvent, + SyncStart, }; use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; -use tracing::info; /// The chain gateway pub struct EvmChainGateway { @@ -21,7 +20,7 @@ pub struct EvmChainGateway { #[derive(Clone, Debug)] enum SyncStatus { /// Intial State - Init, + Init(Vec), // Include a buffer to hold events that arrive too early /// After SyncStart we forward all events to SyncActor ForwardToSyncActor(Option>), /// Once the chain has completed historical sync then we buffer all "live" events until sync is @@ -33,22 +32,25 @@ enum SyncStatus { impl Default for SyncStatus { fn default() -> Self { - Self::Init + Self::Init(Vec::new()) } } impl SyncStatus { - pub fn forward_to_sync_actor(&mut self, sender: Recipient) -> Result<()> { - let Self::Init = self else { + pub fn forward_to_sync_actor( + &mut self, + sender: Recipient, + ) -> Result> { + let Self::Init(buffer) = self else { bail!( "Cannot change state to ForwardToSyncActor when state is {:?}", self ); }; + let buffer = std::mem::take(buffer); *self = SyncStatus::ForwardToSyncActor(Some(sender)); - info!("Changed to ForwardToSyncActor"); - Ok(()) + Ok(buffer) } pub fn buffer_until_live(&mut self) -> Result> { @@ -60,7 +62,6 @@ impl SyncStatus { }; let sender = std::mem::take(sender).context("Cannot call buffer_until_live twice")?; *self = SyncStatus::BufferUntilLive(vec![]); - info!("Changed to BufferUntilLive"); Ok(sender) } @@ -70,7 +71,6 @@ impl SyncStatus { }; let buffer = std::mem::take(buffer); *self = SyncStatus::Live; - info!("Changed to Live"); Ok(buffer) } } @@ -93,7 +93,11 @@ impl EvmChainGateway { // Received a SyncStart event from the event bus. Get the sender within that event and forward // all events to that actor let sender = msg.sender.context("No sender on SyncStart Message")?; - self.status.forward_to_sync_actor(sender)?; + let mut buffer = self.status.forward_to_sync_actor(sender)?; + // Drain any events that were buffered early + for evt in buffer.drain(..) { + self.handle_receive_evm_event(evt)?; + } Ok(()) } @@ -138,6 +142,7 @@ impl EvmChainGateway { sync_actor.do_send(msg.into()); } SyncStatus::Live => self.publish_evm_event(msg)?, + SyncStatus::Init(buffer) => buffer.push(msg), _ => (), }; Ok(()) @@ -168,3 +173,94 @@ impl Handler for EvmChainGateway { trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) } } + +#[cfg(test)] +mod tests { + use super::*; + use e3_ciphernode_builder::EventSystem; + + use e3_events::{CorrelationId, TestEvent}; + use std::collections::HashMap; + use tokio::sync::mpsc; + + struct SyncEventCollector { + tx: mpsc::UnboundedSender, + } + + impl Actor for SyncEventCollector { + type Context = actix::Context; + } + + impl Handler for SyncEventCollector { + type Result = (); + fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) { + let _ = self.tx.send(msg); + } + } + + #[actix::test] + async fn test_evm_chain_gateway() { + let system = EventSystem::new("test").with_fresh_bus(); + let bus = system.handle().unwrap(); + + let (tx, mut rx) = mpsc::unbounded_channel(); + let collector = SyncEventCollector { tx }.start(); + + let addr = EvmChainGateway::setup(&bus); + + let chain_id = 1u64; + + // SyncStart: Init -> ForwardToSyncActor + bus.publish(SyncStart::new(collector.clone(), HashMap::new())) + .unwrap(); + + // Send EVM event while forwarding - should reach collector + let evm_event = EvmEvent::new( + CorrelationId::new(), + TestEvent { + msg: "test".to_string(), + entropy: 1, + } + .into(), + 100, + 12345, + chain_id, + ); + // This will actually arrive earlier than SyncStart but aught to be buffered + addr.do_send(EnclaveEvmEvent::Event(evm_event)); + + let received = rx.recv().await.unwrap(); + assert!(matches!(received, SyncEvmEvent::Event(_))); + + // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive + addr.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + HistoricalSyncComplete::new(chain_id, None), + )); + + let received = rx.recv().await.unwrap(); + assert!(matches!(received, SyncEvmEvent::HistoricalSyncComplete(_))); + + // Send EVM event while buffering - should be buffered (not received) + let buffered_event = EvmEvent::new( + CorrelationId::new(), + TestEvent { + msg: "buffered".to_string(), + entropy: 2, + } + .into(), + 101, + 12346, + chain_id, + ); + addr.do_send(EnclaveEvmEvent::Event(buffered_event)); + + // SyncEnd: BufferUntilLive -> Live (publishes buffered events to bus) + bus.publish(SyncEnd::new()).unwrap(); + + // Allow time for async message processing + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // Verify no more messages were sent to collector (buffered events go to bus, not collector) + assert!(rx.try_recv().is_err()); + } +} From 9d65a1afb35c0897b6d6b52d233487a5fc9151a0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:30:55 +0000 Subject: [PATCH 073/102] rename sync_gateway to evm_chain_gateway and update module exports --- crates/evm/src/{sync_gateway.rs => evm_chain_gateway.rs} | 4 +++- crates/evm/src/lib.rs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) rename crates/evm/src/{sync_gateway.rs => evm_chain_gateway.rs} (97%) diff --git a/crates/evm/src/sync_gateway.rs b/crates/evm/src/evm_chain_gateway.rs similarity index 97% rename from crates/evm/src/sync_gateway.rs rename to crates/evm/src/evm_chain_gateway.rs index cd6c4875e1..f767aaa100 100644 --- a/crates/evm/src/sync_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -11,12 +11,14 @@ use e3_events::{ use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; -/// The chain gateway +/// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. +/// It coordinates event flow between these components. pub struct EvmChainGateway { bus: BusHandle, status: SyncStatus, } +/// This state machine coordinates the function of the EvmChainGateway #[derive(Clone, Debug)] enum SyncStatus { /// Intial State diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index c768a83206..bf4d333438 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -10,6 +10,7 @@ mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; mod events; +mod evm_chain_gateway; mod evm_hub; mod evm_launch_coordinator; mod evm_read_interface; @@ -19,7 +20,6 @@ mod fix_historical_order; pub mod helpers; mod one_shot_runnner; mod repo; -mod sync_gateway; mod sync_start_extractor; pub use bonding_registry_sol::BondingRegistrySolReader; @@ -30,6 +30,7 @@ pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; pub use events::*; +pub use evm_chain_gateway::*; pub use evm_hub::*; pub use evm_read_interface::*; pub use evm_reader::*; @@ -38,5 +39,4 @@ pub use fix_historical_order::*; pub use helpers::*; pub use one_shot_runnner::*; pub use repo::*; -pub use sync_gateway::*; pub use sync_start_extractor::*; From e7ed923a059f1698bd0f6166ff5551ccd55c7375 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:34:57 +0000 Subject: [PATCH 074/102] remove launch coordinator --- crates/evm/src/evm_launch_coordinator.rs | 116 ----------------------- 1 file changed, 116 deletions(-) delete mode 100644 crates/evm/src/evm_launch_coordinator.rs diff --git a/crates/evm/src/evm_launch_coordinator.rs b/crates/evm/src/evm_launch_coordinator.rs deleted file mode 100644 index 9e97eeba15..0000000000 --- a/crates/evm/src/evm_launch_coordinator.rs +++ /dev/null @@ -1,116 +0,0 @@ -// use crate::evm_router::EvmRouter; -// use crate::helpers::EthProvider; -// use crate::EvmReadInterface; -// use crate::{events::EvmEventProcessor, evm_read_interface::Filters}; -// use actix::{Actor, ActorContext, Addr, AsyncContext, Handler}; -// use alloy::providers::Provider; -// use alloy_primitives::Address; -// use anyhow::Context; -// use anyhow::Result; -// use e3_events::{ -// trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, SyncStart, -// }; -// use std::collections::HashMap; -// -// // Configured with Addr for -// pub struct EvmLaunchCoordinator

{ -// provider: Option>, -// routing_table: HashMap, -// bus: BusHandle, -// } -// -// impl

EvmLaunchCoordinator

-// where -// P: Provider + Clone + 'static, -// { -// pub fn builder(bus: &BusHandle, provider: &EthProvider

) -> EvmLaunchCoordinatorBuilder

{ -// EvmLaunchCoordinatorBuilder { -// routing_table: HashMap::new(), -// bus: bus.clone(), -// provider: provider.clone(), -// } -// } -// -// fn filters(&self, start_block: Option) -> Filters { -// let addresses = self.routing_table.keys().cloned().collect(); -// Filters::new(addresses, start_block) -// } -// -// fn bootstrap_reader(&mut self, _event: SyncStart) -> Result<()> { -// // Setup upstream router -// // The routing table holds addresses for upstream processors -// let next = EvmRouter::setup(self.routing_table.clone()); -// -// // Setup read interface -// EvmReadInterface::attach( -// self.provider.take().context("Cannot call setup twice!")?, -// &next.into(), -// &self.bus, -// self.filters(None), -// ); -// -// Ok(()) -// } -// } -// -// impl

Actor for EvmLaunchCoordinator

-// where -// P: Provider + Clone + 'static, -// { -// type Context = actix::Context; -// } -// -// impl

Handler for EvmLaunchCoordinator

-// where -// P: Provider + Clone + 'static, -// { -// type Result = (); -// fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { -// trap(EType::Evm, &self.bus.clone(), || { -// if let EnclaveEventData::SyncStart(event) = msg.into_data() { -// // Run the setup process -// self.bootstrap_reader(event)?; -// -// // We now don't need the launcher and can kill it now -// self.bus.unsubscribe("SyncStart", ctx.address().into()); -// ctx.stop(); -// } -// Ok(()) -// }) -// } -// } -// -// pub struct EvmLaunchCoordinatorBuilder

{ -// provider: EthProvider

, -// routing_table: HashMap, -// bus: BusHandle, -// } -// -// impl

EvmLaunchCoordinatorBuilder

-// where -// P: Provider + Clone + 'static, -// { -// pub fn with_contract( -// &mut self, -// address: impl AsRef, -// dest: impl Into, -// ) -> Result<()> { -// let address: Address = address.as_ref().parse().context("invalid address")?; -// self.routing_table.insert(address, dest.into()); -// Ok(()) -// } -// -// pub fn build(self) -> Addr> { -// let routing_table = self.routing_table; -// let addr = EvmLaunchCoordinator { -// routing_table, -// provider: Some(self.provider), -// bus: self.bus.clone(), -// } -// .start(); -// -// self.bus.subscribe("SyncStart", addr.clone().recipient()); -// -// addr -// } -// } From 38f7d4c3abc29a612d42d3af0871601f7ebeb36c Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:38:09 +0000 Subject: [PATCH 075/102] fix headers --- crates/ciphernode-builder/src/evm_system.rs | 6 ++++++ crates/ciphernode-builder/src/provider_caches.rs | 6 ++++++ crates/events/src/sync.rs | 6 ++++++ crates/evm/src/evm_chain_gateway.rs | 6 ++++++ crates/evm/src/evm_hub.rs | 6 ++++++ crates/evm/src/evm_reader.rs | 7 ++++++- crates/evm/src/evm_router.rs | 6 ++++++ crates/evm/src/fix_historical_order.rs | 6 ++++++ crates/evm/src/one_shot_runnner.rs | 8 +++++++- crates/evm/src/sync_start_extractor.rs | 6 ++++++ 10 files changed, 61 insertions(+), 2 deletions(-) diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index e0201f73de..56d0153331 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::Actor; use alloy::{primitives::Address, providers::Provider}; use e3_events::{BusHandle, EventSubscriber, SyncStart}; diff --git a/crates/ciphernode-builder/src/provider_caches.rs b/crates/ciphernode-builder/src/provider_caches.rs index 6068512c96..628a7e525a 100644 --- a/crates/ciphernode-builder/src/provider_caches.rs +++ b/crates/ciphernode-builder/src/provider_caches.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; use anyhow::Result; use e3_config::chain_config::ChainConfig; diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index 1872f85af6..02ead07af4 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::EvmEvent; use actix::Message; use serde::{Deserialize, Serialize}; diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index f767aaa100..16badbbdef 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::events::EnclaveEvmEvent; use crate::HistoricalSyncComplete; use actix::{Actor, Handler}; diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs index a4d6613c2c..9e98f4344f 100644 --- a/crates/evm/src/evm_hub.rs +++ b/crates/evm/src/evm_hub.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Actor, Addr, Handler}; use crate::events::{EnclaveEvmEvent, EvmEventProcessor}; diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_reader.rs index 4eb1f72049..ba4927fcb6 100644 --- a/crates/evm/src/evm_reader.rs +++ b/crates/evm/src/evm_reader.rs @@ -1,6 +1,11 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Actor, Handler}; use e3_events::{hlc::HlcTimestamp, EnclaveEventData, EvmEvent}; -use tracing::info; use crate::{ events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 886fc8fac3..254949da27 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; use actix::{Actor, Addr, Handler}; use alloy_primitives::Address; diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index 640cc639f3..7416c122cc 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; use actix::{Actor, Addr, Handler}; use bloom::{BloomFilter, ASMS}; diff --git a/crates/evm/src/one_shot_runnner.rs b/crates/evm/src/one_shot_runnner.rs index fd7f8fb604..6a6576774d 100644 --- a/crates/evm/src/one_shot_runnner.rs +++ b/crates/evm/src/one_shot_runnner.rs @@ -1,8 +1,14 @@ -use std::marker::PhantomData; +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. use actix::prelude::*; use anyhow::Result; +use std::marker::PhantomData; use tracing::error; + pub struct OneShotRunner where F: FnOnce(M) -> Result<()> + 'static, diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs index f2700c21a0..5cd2a52ee3 100644 --- a/crates/evm/src/sync_start_extractor.rs +++ b/crates/evm/src/sync_start_extractor.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Actor, Addr, Handler, Recipient}; use e3_events::{EnclaveEvent, EnclaveEventData, Event, SyncStart}; From 15b358c6a740b8b8a9d685889c87a91759f21660 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:39:03 +0000 Subject: [PATCH 076/102] remove bad package --- crates/evm/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index bf4d333438..ad62c276a5 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -12,7 +12,6 @@ mod enclave_sol_writer; mod events; mod evm_chain_gateway; mod evm_hub; -mod evm_launch_coordinator; mod evm_read_interface; mod evm_reader; mod evm_router; From 9d07673a27f8f8c31ffd263595486708426b4432 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:40:21 +0000 Subject: [PATCH 077/102] add header --- crates/evm/src/events.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index bfddb0b162..09133e890f 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Message, Recipient}; use alloy::rpc::types::Log; use e3_events::{CorrelationId, EvmEvent}; From a3933d827a6008ca97bdb55dbaa475e4f32d34de Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 06:59:08 +0000 Subject: [PATCH 078/102] apply EvmSystem to CiphernodeBuilder --- .../src/ciphernode_builder.rs | 153 +++++++++--------- crates/ciphernode-builder/src/evm_system.rs | 5 +- crates/evm/tests/integration.rs | 8 +- 3 files changed, 87 insertions(+), 79 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index e2c66f29bd..cc44d2c8aa 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::event_system::AggregateConfig; -use crate::{CiphernodeHandle, EventSystem, ProviderCache}; +use crate::{CiphernodeHandle, EventSystem, EvmSystemChainBuilder, ProviderCache}; use actix::{Actor, Addr}; use anyhow::Result; use derivative::Derivative; @@ -390,78 +390,85 @@ impl CiphernodeBuilder { // Sync processor // All contract event processors will forward their parsed events here. - // let next = EvmChainGateway::setup(&bus).into(); - // - // // TODO: gather an async handle from the event readers that closes when they shutdown and - // // join it with the network manager joinhandle below - // for chain in self - // .chains - // .iter() - // .filter(|chain| chain.enabled.unwrap_or(true)) - // { - // // We create a launch coordinator for each chain - // // This waits for the SyncStart message before creating the EvmInterface which inturn - // // will begin the historical stream events followed by the live stream of events. - // // We need to do this so that the interface knows when to stream from. - // // Once it has created the EvmInterface it will kill itself - // let mut read_launcher = EvmLaunchCoordinator::builder( - // &bus, - // &provider_cache.ensure_read_provider(chain).await?, - // ); - // - // if self.contract_components.enclave { - // let contract_address = chain.contracts.enclave.address(); - // let write_provider = provider_cache.ensure_write_provider(chain).await?; - // - // EnclaveSolWriter::attach(&bus, write_provider.clone(), &contract_address).await?; - // - // read_launcher.with_contract(contract_address, EnclaveSolReader::setup(&next))?; - // } - // - // if self.contract_components.enclave_reader { - // let contract_address = chain.contracts.enclave.address(); - // - // read_launcher.with_contract(contract_address, EnclaveSolReader::setup(&next))?; - // } - // - // if self.contract_components.bonding_registry { - // let contract_address = chain.contracts.bonding_registry.address(); - // read_launcher - // .with_contract(contract_address, BondingRegistrySolReader::setup(&next))?; - // } - // - // if self.contract_components.ciphernode_registry { - // let contract_address = chain.contracts.ciphernode_registry.address(); - // read_launcher - // .with_contract(contract_address, CiphernodeRegistrySol::attach(&next))?; - // - // match provider_cache - // .ensure_write_provider(chain) - // .await - // { - // Ok(write_provider) => { - // let _writer = CiphernodeRegistrySol::attach_writer( - // &bus, - // write_provider.clone(), - // &contract_address, - // self.pubkey_agg, - // ) - // .await?; - // info!("CiphernodeRegistrySolWriter attached for publishing committees"); - // - // if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { - // info!("Attaching CommitteeFinalizer for score sortition"); - // e3_aggregator::CommitteeFinalizer::attach(&bus); - // } - // } - // Err(e) => error!( - // "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", - // e - // ), - // } - // } - // let _ = read_launcher.build(); - // } + + // TODO: gather an async handle from the event readers that closes when they shutdown and + // join it with the network manager joinhandle below + for chain in self + .chains + .iter() + .filter(|chain| chain.enabled.unwrap_or(true)) + { + let provider = provider_cache.ensure_read_provider(chain).await?; + + let mut system = EvmSystemChainBuilder::new(&bus, &provider); + + if self.contract_components.enclave { + let contract_address = chain.contracts.enclave.address(); + let write_provider = provider_cache.ensure_write_provider(chain).await?; + + EnclaveSolWriter::attach(&bus, write_provider.clone(), &contract_address).await?; + let contract_address = contract_address.parse()?; + system = system.with_contract(move |next| { + (contract_address, EnclaveSolReader::setup(&next).recipient()) + }); + } + + if self.contract_components.enclave_reader { + let contract_address = chain.contracts.enclave.address().parse()?; + + system = system.with_contract(move |next| { + (contract_address, EnclaveSolReader::setup(&next).recipient()) + }); + } + + if self.contract_components.bonding_registry { + let contract_address = chain.contracts.bonding_registry.address().parse()?; + system = system.with_contract(move |next| { + ( + contract_address, + BondingRegistrySolReader::setup(&next).recipient(), + ) + }); + } + + if self.contract_components.ciphernode_registry { + let contract_address_str = chain.contracts.ciphernode_registry.address(); + let contract_address = contract_address_str.parse()?; + + system = system.with_contract(move |next| { + ( + contract_address, + CiphernodeRegistrySol::attach(&next).recipient(), + ) + }); + + match provider_cache + .ensure_write_provider(chain) + .await + { + Ok(write_provider) => { + let _writer = CiphernodeRegistrySol::attach_writer( + &bus, + write_provider.clone(), + &contract_address_str, + self.pubkey_agg, + ) + .await?; + info!("CiphernodeRegistrySolWriter attached for publishing committees"); + + if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { + info!("Attaching CommitteeFinalizer for score sortition"); + e3_aggregator::CommitteeFinalizer::attach(&bus); + } + } + Err(e) => error!( + "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", + e + ), + } + } + system.build(); + } // E3 specific setup let mut e3_builder = E3Router::builder(&bus, store.clone()); diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 56d0153331..ca25ebff2c 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -26,7 +26,8 @@ pub struct EvmSystemChainBuilder

{ } impl EvmSystemChainBuilder

{ - pub fn new(bus: &BusHandle, provider: &EthProvider

, chain_id: u64) -> Self { + pub fn new(bus: &BusHandle, provider: &EthProvider

) -> Self { + let chain_id = provider.chain_id(); Self { bus: bus.clone(), provider: provider.clone(), @@ -35,7 +36,7 @@ impl EvmSystemChainBuilder

{ } } - pub fn with_route(mut self, route_fn: F) -> Self { + pub fn with_contract(mut self, route_fn: F) -> Self { self.route_factories.push(Box::new(route_fn)); self } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 27b98e21f5..8dc2ba27ea 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -150,8 +150,8 @@ async fn evm_reader() -> Result<()> { let chain_id = provider.chain_id(); let contract_address = contract.address().clone(); let sync = FakeSyncActor::setup(&bus); - EvmSystemChainBuilder::new(&bus, &provider, chain_id) - .with_route(move |upstream| { + EvmSystemChainBuilder::new(&bus, &provider) + .with_contract(move |upstream| { ( contract_address, TestEventParser::setup(&upstream).recipient(), @@ -234,8 +234,8 @@ async fn ensure_historical_events() -> Result<()> { sleep(Duration::from_millis(1)).await; let sync = FakeSyncActor::setup(&bus); - EvmSystemChainBuilder::new(&bus, &provider, chain_id) - .with_route(move |upstream| { + EvmSystemChainBuilder::new(&bus, &provider) + .with_contract(move |upstream| { ( contract_address, TestEventParser::setup(&upstream).recipient(), From b6b68da63790cd58dd1f526465c8ead960f7a05b Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 07:41:13 +0000 Subject: [PATCH 079/102] add scaffolding to sync --- Cargo.lock | 1 + crates/sync/Cargo.toml | 2 +- crates/sync/src/sync.rs | 23 ++++++++++++++++++++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7de2866a7b..f436ae3c05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3355,6 +3355,7 @@ name = "e3-sync" version = "0.1.7" dependencies = [ "actix", + "e3-events", ] [[package]] diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index a6bfa1bcf5..40e03aeda8 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -8,4 +8,4 @@ repository = "https://github.com/gnosisguild/enclave/crates/sync" [dependencies] actix.workspace = true - +e3-events.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 82e0fa2c3a..3da65cc515 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -4,20 +4,37 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Handler, Message}; +use std::collections::HashMap; -struct Sync; +use actix::{Actor, AsyncContext, Handler, Message}; +use e3_events::{trap, BusHandle, EventPublisher, SyncEvmEvent, SyncStart}; + +struct Sync { + bus: BusHandle, +} impl Sync {} impl Actor for Sync { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.notify(Bootstrap); + } +} + +impl Handler for Sync { + type Result = (); + fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result {} } impl Handler for Sync { type Result = (); fn handle(&mut self, msg: Bootstrap, ctx: &mut Self::Context) -> Self::Result { - // Publish SyncStart + trap(e3_events::EType::Sync, &self.bus.clone(), || { + // Fetch snapshot state + self.bus + .publish(SyncStart::new(ctx.address(), HashMap::new())) + }) } } From 1d99a5b7ff472fc2ec859d5490e0617bcf8315d8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 10:39:55 +0000 Subject: [PATCH 080/102] add scaffolding to sync --- .husky/pre-commit | 1 - crates/ciphernode-builder/src/sync_builder.rs | 24 +++++++++++++++++++ crates/config/src/contract.rs | 15 ++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) delete mode 100644 .husky/pre-commit create mode 100644 crates/ciphernode-builder/src/sync_builder.rs diff --git a/.husky/pre-commit b/.husky/pre-commit deleted file mode 100644 index cb2c84d5c3..0000000000 --- a/.husky/pre-commit +++ /dev/null @@ -1 +0,0 @@ -pnpm lint-staged diff --git a/crates/ciphernode-builder/src/sync_builder.rs b/crates/ciphernode-builder/src/sync_builder.rs new file mode 100644 index 0000000000..cebe58fd1f --- /dev/null +++ b/crates/ciphernode-builder/src/sync_builder.rs @@ -0,0 +1,24 @@ +use std::collections::HashMap; + +use alloy::primitives::Address; +use e3_config::chain_config::ChainConfig; + +type ChainId = u64; +type DeployBlock = u64; + +pub struct SyncBuilder { + config: HashMap>, +} + +impl SyncBuilder { + pub fn with_chain(&mut self, chain: &ChainConfig) { + let contracts = chain.contracts.contracts(); + let mut map = HashMap::new(); + for contract in contracts { + let key = contract.address(); + let value = contract.deploy_block(); + map.insert(key, value); + } + self.config.insert(chain, map); + } +} diff --git a/crates/config/src/contract.rs b/crates/config/src/contract.rs index cf8115d978..c1e8dc0560 100644 --- a/crates/config/src/contract.rs +++ b/crates/config/src/contract.rs @@ -42,3 +42,18 @@ pub struct ContractAddresses { pub e3_program: Option, pub fee_token: Option, } + +impl ContractAddresses { + pub fn contracts(&self) -> Vec<&Contract> { + [ + Some(&self.enclave), + Some(&self.ciphernode_registry), + Some(&self.bonding_registry), + self.e3_program.as_ref(), + self.fee_token.as_ref(), + ] + .into_iter() + .flatten() + .collect() + } +} From a19f73ce321e608cf5a8c5d78cc39a79028cbd88 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 26 Jan 2026 10:42:13 +0000 Subject: [PATCH 081/102] header --- crates/ciphernode-builder/src/sync_builder.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/ciphernode-builder/src/sync_builder.rs b/crates/ciphernode-builder/src/sync_builder.rs index cebe58fd1f..5a32836399 100644 --- a/crates/ciphernode-builder/src/sync_builder.rs +++ b/crates/ciphernode-builder/src/sync_builder.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use std::collections::HashMap; use alloy::primitives::Address; From c8063c702ac1ccb7b57e54809caa3ebef39948a9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 05:52:56 +0000 Subject: [PATCH 082/102] add evm event config with deploy block validation - breaking change to sync initialization --- Cargo.lock | 1 + .../src/ciphernode_builder.rs | 3 - crates/ciphernode-builder/src/evm_system.rs | 5 +- crates/ciphernode-builder/src/sync_builder.rs | 19 +-- crates/config/src/chain_config.rs | 29 ++++ crates/config/src/rpc.rs | 135 +++++++++++++----- crates/events/src/enclave_event/sync_start.rs | 39 +++-- crates/events/src/sync.rs | 41 ++++++ crates/evm/src/evm_chain_gateway.rs | 6 +- crates/evm/src/evm_read_interface.rs | 6 +- crates/evm/tests/integration.rs | 12 +- crates/sync/Cargo.toml | 1 + crates/sync/src/sync.rs | 34 +++-- 13 files changed, 241 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f436ae3c05..e5e37dc965 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3355,6 +3355,7 @@ name = "e3-sync" version = "0.1.7" dependencies = [ "actix", + "anyhow", "e3-events", ] diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index cc44d2c8aa..f87208fe63 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -388,9 +388,6 @@ impl CiphernodeBuilder { CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) .await?; - // Sync processor - // All contract event processors will forward their parsed events here. - // TODO: gather an async handle from the event readers that closes when they shutdown and // join it with the network manager joinhandle below for chain in self diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index ca25ebff2c..02c37286ff 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -50,7 +50,7 @@ impl EvmSystemChainBuilder

{ let chain_id = self.chain_id; let route_factories = self.route_factories; move |msg: SyncStart| { - let info = msg.get_evm_init_for(chain_id); + let config = msg.get_evm_config(chain_id)?; let gateway = gateway.recipient(); let mut router = EvmRouter::new(); @@ -60,7 +60,8 @@ impl EvmSystemChainBuilder

{ } router = router.add_fallback(&gateway); - let filters = Filters::from_routing_table(router.get_routing_table(), info); + let filters = + Filters::from_routing_table(router.get_routing_table(), config.deploy_block()); let router = router.start(); EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); Ok(()) diff --git a/crates/ciphernode-builder/src/sync_builder.rs b/crates/ciphernode-builder/src/sync_builder.rs index 5a32836399..7d46bb7c78 100644 --- a/crates/ciphernode-builder/src/sync_builder.rs +++ b/crates/ciphernode-builder/src/sync_builder.rs @@ -7,24 +7,19 @@ use std::collections::HashMap; use alloy::primitives::Address; +use anyhow::Result; use e3_config::chain_config::ChainConfig; - +use e3_events::EvmEventConfig; type ChainId = u64; -type DeployBlock = u64; +type DeployBlock = Option; pub struct SyncBuilder { - config: HashMap>, + config: EvmEventConfig, } impl SyncBuilder { - pub fn with_chain(&mut self, chain: &ChainConfig) { - let contracts = chain.contracts.contracts(); - let mut map = HashMap::new(); - for contract in contracts { - let key = contract.address(); - let value = contract.deploy_block(); - map.insert(key, value); - } - self.config.insert(chain, map); + pub fn with_chain(&mut self, chain_id: u64, chain: ChainConfig) -> Result<()> { + self.config.insert(chain_id, chain.try_into()?); + Ok(()) } } diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index eb0a22e586..effb869e0a 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -11,7 +11,9 @@ use crate::{ rpc::{RpcAuth, RPC}, }; use anyhow::*; +use e3_events::EvmEventConfigChain; use serde::{Deserialize, Serialize}; +use tracing::error; #[derive(Debug, Clone, PartialEq, Hash, Eq, Deserialize, Serialize)] pub struct ChainConfig { @@ -31,3 +33,30 @@ impl ChainConfig { .map_err(|e| anyhow!("Failed to parse RPC URL for chain {}: {}", self.name, e))?) } } + +impl TryFrom for EvmEventConfigChain { + type Error = anyhow::Error; + fn try_from(value: ChainConfig) -> std::result::Result { + let rpc = value.rpc_url()?; + let contracts = value.contracts.contracts(); + let mut lowest_block: Option = None; + for contract in contracts { + let deploy_block = contract.deploy_block(); + if deploy_block.unwrap_or(0) == 0 && !rpc.is_local() { + let rpc_url = rpc.url().to_string(); + let contract_address = contract.address(); + error!( + "Querying from block 0 on a non-local node ({}) without a specific deploy_block is not allowed.", + rpc_url + ); + bail!( + "Misconfiguration: Attempted to query historical events from genesis on a non-local node. \ + Please specify a `deploy_block` for contract address {contract_address} on rpc {rpc_url}" + ); + } + lowest_block = [lowest_block, deploy_block].into_iter().flatten().min(); + } + let start_block = lowest_block.unwrap_or(0); + Ok(EvmEventConfigChain::new(start_block)) + } +} diff --git a/crates/config/src/rpc.rs b/crates/config/src/rpc.rs index f814b55118..729137c939 100644 --- a/crates/config/src/rpc.rs +++ b/crates/config/src/rpc.rs @@ -12,60 +12,131 @@ use serde::Deserialize; use serde::Serialize; use url::Url; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RpcProtocol { + Http, + Https, + Ws, + Wss, +} + +impl RpcProtocol { + pub fn is_websocket(&self) -> bool { + matches!(self, RpcProtocol::Ws | RpcProtocol::Wss) + } + + pub fn is_secure(&self) -> bool { + matches!(self, RpcProtocol::Https | RpcProtocol::Wss) + } + + pub fn as_str(&self) -> &'static str { + match self { + RpcProtocol::Http => "http", + RpcProtocol::Https => "https", + RpcProtocol::Ws => "ws", + RpcProtocol::Wss => "wss", + } + } +} + #[derive(Clone)] -pub enum RPC { - Http(String), - Https(String), - Ws(String), - Wss(String), +pub struct RPC { + protocol: RpcProtocol, + url: Url, } impl RPC { pub fn from_url(url: &str) -> Result { let parsed = Url::parse(url).context("Invalid URL format")?; - match parsed.scheme() { - "http" => Ok(RPC::Http(url.to_string())), - "https" => Ok(RPC::Https(url.to_string())), - "ws" => Ok(RPC::Ws(url.to_string())), - "wss" => Ok(RPC::Wss(url.to_string())), + let protocol = match parsed.scheme() { + "http" => RpcProtocol::Http, + "https" => RpcProtocol::Https, + "ws" => RpcProtocol::Ws, + "wss" => RpcProtocol::Wss, _ => bail!("Invalid protocol. Expected: http://, https://, ws://, wss://"), + }; + + if parsed.host_str().is_none() { + bail!("URL must contain a host"); } + + Ok(RPC { + protocol, + url: parsed, + }) + } + + pub fn protocol(&self) -> RpcProtocol { + self.protocol + } + + pub fn url(&self) -> &Url { + &self.url + } + + pub fn hostname(&self) -> &str { + // Safe: validated in from_url() - http(s)/ws(s) schemes always require a host + self.url.host_str().expect("RPC URL always has a host") + } + + pub fn port(&self) -> u16 { + // Safe: http(s)/ws(s) always have known default ports + self.url + .port_or_known_default() + .expect("RPC URL always has a port") + } + + pub fn host_with_port(&self) -> String { + format!("{}:{}", self.hostname(), self.port()) } pub fn as_http_url(&self) -> Result { - match self { - RPC::Http(url) | RPC::Https(url) => Ok(url.clone()), - RPC::Ws(url) | RPC::Wss(url) => { - let mut parsed = - Url::parse(url).context(format!("Failed to parse URL: {}", url))?; - parsed - .set_scheme(if self.is_secure() { "https" } else { "http" }) - .map_err(|_| anyhow!("http(s) are valid schemes"))?; - Ok(parsed.to_string()) - } + if !self.protocol.is_websocket() { + Ok(self.url.to_string()) + } else { + let mut parsed = self.url.clone(); + let scheme = if self.protocol.is_secure() { + "https" + } else { + "http" + }; + parsed + .set_scheme(scheme) + .map_err(|_| anyhow!("http(s) are valid schemes"))?; + Ok(parsed.to_string()) } } pub fn as_ws_url(&self) -> Result { - match self { - RPC::Ws(url) | RPC::Wss(url) => Ok(url.clone()), - RPC::Http(url) | RPC::Https(url) => { - let mut parsed = - Url::parse(url).context(format!("Failed to parse URL: {}", url))?; - parsed - .set_scheme(if self.is_secure() { "wss" } else { "ws" }) - .map_err(|_| anyhow!("ws(s) are valid schemes"))?; - Ok(parsed.to_string()) - } + if self.protocol.is_websocket() { + Ok(self.url.to_string()) + } else { + let mut parsed = self.url.clone(); + let scheme = if self.protocol.is_secure() { + "wss" + } else { + "ws" + }; + parsed + .set_scheme(scheme) + .map_err(|_| anyhow!("ws(s) are valid schemes"))?; + Ok(parsed.to_string()) } } pub fn is_websocket(&self) -> bool { - matches!(self, RPC::Ws(_) | RPC::Wss(_)) + self.protocol.is_websocket() } pub fn is_secure(&self) -> bool { - matches!(self, RPC::Https(_) | RPC::Wss(_)) + self.protocol.is_secure() + } + + pub fn is_local(&self) -> bool { + match self.hostname() { + "localhost" | "127.0.0.1" | "::1" => true, + host => host.starts_with("127."), // 127.0.0.0/8 is all loopback + } } } diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 525a4868a7..fa93930c61 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -6,10 +6,13 @@ use super::EnclaveEventData; use crate::{CorrelationId, SyncEvmEvent}; +use crate::{EvmEventConfig, EvmEventConfigChain}; use actix::{Message, Recipient}; +use anyhow::Context; +use anyhow::Result; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, + collections::BTreeMap, fmt::{self, Display}, }; @@ -54,36 +57,32 @@ impl EvmEvent { #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct SyncStart { - /// The start block information for chains - pub evm_init_info: Vec<(u64, Option)>, // HashMap cannot derive Hash + /// The initial information for reading historical events from chains. This is generated from + /// from persisted information + pub evm_config: EvmEventConfig, + #[serde(skip)] + /// We include the sender here so that the evm can communicate directly with the sync actor pub sender: Option>, // Must be Option to allow serde deserialize on // EnclaveEvent as Default is required to be - // implemented + // implemented this is fine as this event is never + // shared } impl SyncStart { - pub fn new( - sender: impl Into>, - evm_init_info: HashMap>, - ) -> Self { + pub fn new(sender: impl Into>, evm_config: EvmEventConfig) -> Self { Self { sender: Some(sender.into()), - evm_init_info: evm_init_info.into_iter().collect(), + evm_config, } } - pub fn get_evm_init_for(&self, chain_id: u64) -> Option { - self.evm_init_info - .iter() - .find_map(|(ch_id, value)| { - if ch_id == &chain_id { - Some(value.clone()) - } else { - None - } - }) - .unwrap_or(None) + pub fn get_evm_config(&self, chain_id: u64) -> Result { + Ok(self + .evm_config + .get(&chain_id) + .context("No config found for chain")? + .clone()) } } diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index 02ead07af4..ae5d7e177a 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::collections::BTreeMap; + use crate::EvmEvent; use actix::Message; use serde::{Deserialize, Serialize}; @@ -22,3 +24,42 @@ impl From for SyncEvmEvent { SyncEvmEvent::Event(event) } } + +type ChainId = u64; +type DeployBlock = u64; + +/// Configuration value object for starting the evm reader for a specific chain +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmEventConfigChain { + deploy_block: DeployBlock, +} + +impl EvmEventConfigChain { + pub fn new(deploy_block: DeployBlock) -> Self { + Self { deploy_block } + } + pub fn deploy_block(&self) -> u64 { + self.deploy_block + } +} + +/// Configuration value object for starting the evm reader for all chains +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmEventConfig { + config: BTreeMap, // Need BTreeMap because of Hash +} + +impl EvmEventConfig { + pub fn new() -> Self { + Self { + config: BTreeMap::new(), + } + } + pub fn get(&self, chain_id: &ChainId) -> Option<&EvmEventConfigChain> { + self.config.get(&chain_id) + } + + pub fn insert(&mut self, key: ChainId, value: EvmEventConfigChain) { + self.config.insert(key, value); + } +} diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 16badbbdef..fb53e5d693 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -187,7 +187,7 @@ mod tests { use super::*; use e3_ciphernode_builder::EventSystem; - use e3_events::{CorrelationId, TestEvent}; + use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TestEvent}; use std::collections::HashMap; use tokio::sync::mpsc; @@ -219,7 +219,9 @@ mod tests { let chain_id = 1u64; // SyncStart: Init -> ForwardToSyncActor - bus.publish(SyncStart::new(collector.clone(), HashMap::new())) + let mut evm_config = EvmEventConfig::new(); + evm_config.insert(chain_id, EvmEventConfigChain::new(0)); + bus.publish(SyncStart::new(collector.clone(), evm_config)) .unwrap(); // Send EVM event while forwarding - should reach collector diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index 7ab371aa9d..faf514c76c 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -45,10 +45,10 @@ pub struct Filters { } impl Filters { - pub fn new(addresses: Vec

, start_block: Option) -> Self { + pub fn new(addresses: Vec
, start_block: u64) -> Self { let historical = Filter::new() .address(addresses.clone()) - .from_block(start_block.unwrap_or(0)); + .from_block(start_block); let current = Filter::new() .address(addresses) .from_block(BlockNumberOrTag::Latest); @@ -59,7 +59,7 @@ impl Filters { } } - pub fn from_routing_table(table: &HashMap, start_block: Option) -> Self { + pub fn from_routing_table(table: &HashMap, start_block: u64) -> Self { let addresses: Vec
= table.keys().cloned().collect(); Self::new(addresses, start_block) } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 8dc2ba27ea..7ce9dcec09 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -16,8 +16,8 @@ use alloy::{ use anyhow::Result; use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ - prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, GetEvents, - HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, + prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, EvmEventConfig, + EvmEventConfigChain, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, }; use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmReader}; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -161,8 +161,8 @@ async fn evm_reader() -> Result<()> { // SyncStart holds initialization information such as start block and earliest event // This should trigger all chains to start to sync - let mut evm_info = HashMap::new(); - evm_info.insert(chain_id, None); + let mut evm_info = EvmEventConfig::new(); + evm_info.insert(chain_id, EvmEventConfigChain::new(0)); bus.publish(SyncStart::new(sync, evm_info))?; sleep(Duration::from_secs(1)).await; @@ -242,8 +242,8 @@ async fn ensure_historical_events() -> Result<()> { ) }) .build(); - let mut evm_info = HashMap::new(); - evm_info.insert(chain_id, None); + let mut evm_info = EvmEventConfig::new(); + evm_info.insert(chain_id, EvmEventConfigChain::new(0)); bus.publish(SyncStart::new(sync, evm_info))?; for msg in live_events.clone() { diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index 40e03aeda8..f3679a9816 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -8,4 +8,5 @@ repository = "https://github.com/gnosisguild/enclave/crates/sync" [dependencies] actix.workspace = true +anyhow.workspace = true e3-events.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 3da65cc515..0bd2d0eb31 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -4,16 +4,27 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::HashMap; - -use actix::{Actor, AsyncContext, Handler, Message}; -use e3_events::{trap, BusHandle, EventPublisher, SyncEvmEvent, SyncStart}; - +use actix::{Actor, Addr, AsyncContext, Handler, Message}; +use anyhow::Context; +use e3_events::{trap, BusHandle, EType, EventPublisher, EvmEventConfig, SyncEvmEvent, SyncStart}; struct Sync { bus: BusHandle, + evm_config: Option, + // net_config: NetEventConfig, } -impl Sync {} +impl Sync { + pub fn new(bus: &BusHandle, evm_config: EvmEventConfig) -> Self { + Self { + evm_config: Some(evm_config), + bus: bus.clone(), + } + } + + pub fn setup(bus: &BusHandle, evm_config: EvmEventConfig) -> Addr { + Self::new(bus, evm_config).start() + } +} impl Actor for Sync { type Context = actix::Context; @@ -29,11 +40,14 @@ impl Handler for Sync { impl Handler for Sync { type Result = (); - fn handle(&mut self, msg: Bootstrap, ctx: &mut Self::Context) -> Self::Result { - trap(e3_events::EType::Sync, &self.bus.clone(), || { + fn handle(&mut self, _: Bootstrap, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Sync, &self.bus.clone(), || { + let evm_config = self.evm_config.take().context( + "EvmEventConfig was not set likely Bootstrap was called more than once.", + )?; + // Fetch snapshot state - self.bus - .publish(SyncStart::new(ctx.address(), HashMap::new())) + self.bus.publish(SyncStart::new(ctx.address(), evm_config)) }) } } From 67a3051be6b7afcd0f9c13d27958cd5dbd2a392d Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 07:50:59 +0000 Subject: [PATCH 083/102] refactor contract address handling to use typed Address instead of string --- .../src/ciphernode_builder.rs | 85 +++++++++---------- crates/ciphernode-builder/src/evm_system.rs | 25 ++++-- crates/cli/src/ciphernode/context.rs | 2 +- crates/cli/src/print_env.rs | 32 ++++--- crates/config/src/app_config.rs | 8 +- crates/config/src/chain_config.rs | 2 +- crates/config/src/contract.rs | 9 +- crates/evm/src/ciphernode_registry_sol.rs | 6 +- crates/evm/src/enclave_sol.rs | 3 +- crates/evm/src/enclave_sol_writer.rs | 4 +- crates/evm/tests/integration.rs | 14 +-- 11 files changed, 103 insertions(+), 87 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index f87208fe63..36886fbdaa 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -17,7 +17,9 @@ use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{InMemStore, RepositoriesFactory}; use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig}; -use e3_evm::{BondingRegistrySolReader, EnclaveSolWriter, EvmChainGateway}; +use e3_evm::{ + BondingRegistrySolReader, CiphernodeRegistrySolReader, EnclaveSolWriter, EvmChainGateway, +}; use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; @@ -400,69 +402,62 @@ impl CiphernodeBuilder { let mut system = EvmSystemChainBuilder::new(&bus, &provider); if self.contract_components.enclave { - let contract_address = chain.contracts.enclave.address(); let write_provider = provider_cache.ensure_write_provider(chain).await?; - - EnclaveSolWriter::attach(&bus, write_provider.clone(), &contract_address).await?; - let contract_address = contract_address.parse()?; - system = system.with_contract(move |next| { - (contract_address, EnclaveSolReader::setup(&next).recipient()) + let contract = &chain.contracts.enclave; + EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?).await?; + system.with_contract(contract.address()?, move |next| { + EnclaveSolReader::setup(&next).recipient() }); } if self.contract_components.enclave_reader { - let contract_address = chain.contracts.enclave.address().parse()?; + let contract = &chain.contracts.enclave; - system = system.with_contract(move |next| { - (contract_address, EnclaveSolReader::setup(&next).recipient()) + system.with_contract(contract.address()?, move |next| { + EnclaveSolReader::setup(&next).recipient() }); } if self.contract_components.bonding_registry { - let contract_address = chain.contracts.bonding_registry.address().parse()?; - system = system.with_contract(move |next| { - ( - contract_address, - BondingRegistrySolReader::setup(&next).recipient(), - ) + let contract = &chain.contracts.bonding_registry; + system.with_contract(contract.address()?, move |next| { + BondingRegistrySolReader::setup(&next).recipient() }); } if self.contract_components.ciphernode_registry { - let contract_address_str = chain.contracts.ciphernode_registry.address(); - let contract_address = contract_address_str.parse()?; - - system = system.with_contract(move |next| { - ( - contract_address, - CiphernodeRegistrySol::attach(&next).recipient(), - ) + let contract = &chain.contracts.ciphernode_registry; + + system.with_contract(contract.address()?, move |next| { + CiphernodeRegistrySolReader::setup(&next).recipient() }); + // TODO: Should we not let this pass and just use '?'? + // Above if we include enclave in the config and we don't have a wallet it will fail match provider_cache - .ensure_write_provider(chain) - .await - { - Ok(write_provider) => { - let _writer = CiphernodeRegistrySol::attach_writer( - &bus, - write_provider.clone(), - &contract_address_str, - self.pubkey_agg, - ) - .await?; - info!("CiphernodeRegistrySolWriter attached for publishing committees"); - - if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { - info!("Attaching CommitteeFinalizer for score sortition"); - e3_aggregator::CommitteeFinalizer::attach(&bus); - } + .ensure_write_provider(&chain) + .await + { + Ok(write_provider) => { + let _writer = CiphernodeRegistrySol::attach_writer( + &bus, + write_provider.clone(), + contract.address()?, + self.pubkey_agg, + ) + .await?; + info!("CiphernodeRegistrySolWriter attached for publishing committees"); + + if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { + info!("Attaching CommitteeFinalizer for score sortition"); + e3_aggregator::CommitteeFinalizer::attach(&bus); } - Err(e) => error!( - "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", - e - ), } + Err(e) => error!( + "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", + e + ), + } } system.build(); } diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 02c37286ff..22fd0dced7 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::mem::replace; + use actix::Actor; use alloy::{primitives::Address, providers::Provider}; use e3_events::{BusHandle, EventSubscriber, SyncStart}; @@ -12,8 +14,8 @@ use e3_evm::{ FixHistoricalOrder, OneShotRunner, SyncStartExtractor, }; -pub trait RouteFn: FnOnce(EvmEventProcessor) -> (Address, EvmEventProcessor) + Send {} -impl RouteFn for F where F: FnOnce(EvmEventProcessor) -> (Address, EvmEventProcessor) + Send {} +pub trait RouteFn: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} +impl RouteFn for F where F: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} type RouteFactory = Box; @@ -22,7 +24,7 @@ pub struct EvmSystemChainBuilder

{ provider: EthProvider

, bus: BusHandle, chain_id: u64, - route_factories: Vec, + route_factories: Vec<(Address, RouteFactory)>, } impl EvmSystemChainBuilder

{ @@ -36,26 +38,31 @@ impl EvmSystemChainBuilder

{ } } - pub fn with_contract(mut self, route_fn: F) -> Self { - self.route_factories.push(Box::new(route_fn)); + pub fn with_contract( + &mut self, + address: Address, + route_fn: F, + ) -> &mut Self { + self.route_factories.push((address, Box::new(route_fn))); self } - pub fn build(self) { + pub fn build(&mut self) { let gateway = FixHistoricalOrder::setup(EvmChainGateway::setup(&self.bus)); let runner = SyncStartExtractor::setup(OneShotRunner::setup({ let bus = self.bus.clone(); let provider = self.provider.clone(); let gateway = gateway.clone(); let chain_id = self.chain_id; - let route_factories = self.route_factories; + // Only gets consumed once so fine to do this + let route_factories = replace(&mut self.route_factories, Vec::new()); move |msg: SyncStart| { let config = msg.get_evm_config(chain_id)?; let gateway = gateway.recipient(); let mut router = EvmRouter::new(); - for route_fn in route_factories { - let (address, processor) = route_fn(gateway.clone()); + for (address, route_fn) in route_factories { + let processor = route_fn(gateway.clone()); router = router.add_route(address, &processor); } diff --git a/crates/cli/src/ciphernode/context.rs b/crates/cli/src/ciphernode/context.rs index 61843e8cd5..5d01af5bbb 100644 --- a/crates/cli/src/ciphernode/context.rs +++ b/crates/cli/src/ciphernode/context.rs @@ -66,7 +66,7 @@ pub(crate) struct ChainContext { impl ChainContext { pub(crate) async fn new(config: &AppConfig, selection: Option<&str>) -> Result { let chain = select_chain(config, selection)?; - let bonding_registry = parse_address(chain.contracts.bonding_registry.address())?; + let bonding_registry = parse_address(chain.contracts.bonding_registry.address_str())?; let rpc = chain.rpc_url()?; let cipher = Cipher::from_file(config.key_file()).await?; diff --git a/crates/cli/src/print_env.rs b/crates/cli/src/print_env.rs index 0a6ffa25f5..99ccba221d 100644 --- a/crates/cli/src/print_env.rs +++ b/crates/cli/src/print_env.rs @@ -15,18 +15,30 @@ pub fn extract_env_vars_vite(config: &AppConfig, chain: &str) -> String { let enclave_addr = &chain.contracts.enclave; let registry_addr = &chain.contracts.ciphernode_registry; let bonding_registry_addr = &chain.contracts.bonding_registry; - env_vars.push(format!("VITE_ENCLAVE_ADDRESS={}", enclave_addr.address())); - env_vars.push(format!("VITE_REGISTRY_ADDRESS={}", registry_addr.address())); + env_vars.push(format!( + "VITE_ENCLAVE_ADDRESS={}", + enclave_addr.address_str() + )); + env_vars.push(format!( + "VITE_REGISTRY_ADDRESS={}", + registry_addr.address_str() + )); env_vars.push(format!("VITE_RPC_URL={}", chain.rpc_url)); env_vars.push(format!( "VITE_BONDING_REGISTRY_ADDRESS={}", - bonding_registry_addr.address() + bonding_registry_addr.address_str() )); if let Some(e3_program) = &chain.contracts.e3_program { - env_vars.push(format!("VITE_E3_PROGRAM_ADDRESS={}", e3_program.address())); + env_vars.push(format!( + "VITE_E3_PROGRAM_ADDRESS={}", + e3_program.address_str() + )); } if let Some(fee_token) = &chain.contracts.fee_token { - env_vars.push(format!("VITE_FEE_TOKEN_ADDRESS={}", fee_token.address())); + env_vars.push(format!( + "VITE_FEE_TOKEN_ADDRESS={}", + fee_token.address_str() + )); } } @@ -41,18 +53,18 @@ pub fn extract_env_vars(config: &AppConfig, chain: &str) -> String { let enclave_addr = &chain.contracts.enclave; let registry_addr = &chain.contracts.ciphernode_registry; let bonding_registry_addr = &chain.contracts.bonding_registry; - env_vars.push(format!("ENCLAVE_ADDRESS={}", enclave_addr.address())); + env_vars.push(format!("ENCLAVE_ADDRESS={}", enclave_addr.address_str())); env_vars.push(format!("RPC_URL={}", chain.rpc_url)); - env_vars.push(format!("REGISTRY_ADDRESS={}", registry_addr.address())); + env_vars.push(format!("REGISTRY_ADDRESS={}", registry_addr.address_str())); env_vars.push(format!( "BONDING_REGISTRY_ADDRESS={}", - bonding_registry_addr.address() + bonding_registry_addr.address_str() )); if let Some(e3_program) = &chain.contracts.e3_program { - env_vars.push(format!("E3_PROGRAM_ADDRESS={}", e3_program.address())); + env_vars.push(format!("E3_PROGRAM_ADDRESS={}", e3_program.address_str())); } if let Some(fee_token) = &chain.contracts.fee_token { - env_vars.push(format!("FEE_TOKEN_ADDRESS={}", fee_token.address())); + env_vars.push(format!("FEE_TOKEN_ADDRESS={}", fee_token.address_str())); } } diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index 8bed2fe5d5..5074b23086 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -577,7 +577,7 @@ nodes: let chain = config.chains().first().unwrap(); assert_eq!(config.quic_port(), 1235); assert_eq!( - chain.contracts.ciphernode_registry.address(), + chain.contracts.ciphernode_registry.address_str(), "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" ); assert_eq!(config.peers(), vec!["one", "two"]); @@ -688,11 +688,11 @@ chains: assert_eq!(chain.name, "hardhat"); assert_eq!(chain.rpc_url, "ws://localhost:8545"); assert_eq!( - chain.contracts.enclave.address(), + chain.contracts.enclave.address_str(), "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" ); assert_eq!( - chain.contracts.ciphernode_registry.address(), + chain.contracts.ciphernode_registry.address_str(), "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" ); assert_eq!( @@ -803,7 +803,7 @@ chains: } ); assert_eq!( - chain.contracts.enclave.address(), + chain.contracts.enclave.address_str(), "0x1234567890123456789012345678901234567890" ); diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index effb869e0a..b570f7dd80 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -44,7 +44,7 @@ impl TryFrom for EvmEventConfigChain { let deploy_block = contract.deploy_block(); if deploy_block.unwrap_or(0) == 0 && !rpc.is_local() { let rpc_url = rpc.url().to_string(); - let contract_address = contract.address(); + let contract_address = contract.address_str(); error!( "Querying from block 0 on a non-local node ({}) without a specific deploy_block is not allowed.", rpc_url diff --git a/crates/config/src/contract.rs b/crates/config/src/contract.rs index c1e8dc0560..420d3c953c 100644 --- a/crates/config/src/contract.rs +++ b/crates/config/src/contract.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use alloy_primitives::Address; +use anyhow::Result; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Hash, Eq, Deserialize, Serialize, PartialEq)] @@ -17,7 +19,7 @@ pub enum Contract { } impl Contract { - pub fn address(&self) -> &String { + pub fn address_str(&self) -> &str { use Contract::*; match self { Full { address, .. } => address, @@ -25,6 +27,11 @@ impl Contract { } } + pub fn address(&self) -> Result

{ + let addr = self.address_str().parse()?; + Ok(addr) + } + pub fn deploy_block(&self) -> Option { use Contract::*; match self { diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index fba7022e5b..764d407146 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -245,10 +245,10 @@ impl CiphernodeRegistrySolWriter pub async fn attach( bus: &BusHandle, provider: EthProvider

, - contract_address: &str, + contract_address: Address, is_aggregator: bool, ) -> Result>> { - let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address.parse()?) + let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address) .await? .start(); @@ -533,7 +533,7 @@ impl CiphernodeRegistrySol { pub async fn attach_writer

( bus: &BusHandle, provider: EthProvider

, - contract_address: &str, + contract_address: Address, is_aggregator: bool, ) -> Result>> where diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 233ff16dd2..2bdc2a746a 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -10,6 +10,7 @@ use crate::{ }; use actix::Addr; use alloy::providers::{Provider, WalletProvider}; +use alloy_primitives::Address; use anyhow::Result; use e3_events::BusHandle; @@ -20,7 +21,7 @@ impl EnclaveSol { processor: &EvmEventProcessor, bus: &BusHandle, write_provider: EthProvider, - contract_address: &str, + contract_address: Address, ) -> Result> where W: Provider + WalletProvider + Clone + 'static, diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 0ac8204de0..b3e3df9516 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -55,9 +55,9 @@ impl EnclaveSolWriter

{ pub async fn attach( bus: &BusHandle, provider: EthProvider

, - contract_address: &str, + contract_address: Address, ) -> Result>> { - let addr = EnclaveSolWriter::new(bus, provider, contract_address.parse()?)?.start(); + let addr = EnclaveSolWriter::new(bus, provider, contract_address)?.start(); bus.subscribe_all(&["PlaintextAggregated", "Shutdown"], addr.clone().into()); Ok(addr) } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 7ce9dcec09..de10ec2e5f 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -151,11 +151,8 @@ async fn evm_reader() -> Result<()> { let contract_address = contract.address().clone(); let sync = FakeSyncActor::setup(&bus); EvmSystemChainBuilder::new(&bus, &provider) - .with_contract(move |upstream| { - ( - contract_address, - TestEventParser::setup(&upstream).recipient(), - ) + .with_contract(contract_address, move |upstream| { + TestEventParser::setup(&upstream).recipient() }) .build(); @@ -235,11 +232,8 @@ async fn ensure_historical_events() -> Result<()> { let sync = FakeSyncActor::setup(&bus); EvmSystemChainBuilder::new(&bus, &provider) - .with_contract(move |upstream| { - ( - contract_address, - TestEventParser::setup(&upstream).recipient(), - ) + .with_contract(contract_address, move |upstream| { + TestEventParser::setup(&upstream).recipient() }) .build(); let mut evm_info = EvmEventConfig::new(); From aae61bd85772534d4f8c4e87970fbb9c9bd834ff Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 09:55:33 +0000 Subject: [PATCH 084/102] refactor: integrate net package into ciphernode builder --- Cargo.lock | 2 + crates/ciphernode-builder/Cargo.toml | 2 + crates/ciphernode-builder/src/ciphernode.rs | 16 +- .../src/ciphernode_builder.rs | 216 ++++++++++++------ crates/ciphernode-builder/src/sync_builder.rs | 25 -- crates/cli/src/start.rs | 6 +- crates/config/src/chain_config.rs | 14 +- crates/entrypoint/src/helpers/shutdown.rs | 8 +- .../entrypoint/src/start/aggregator_start.rs | 35 +-- crates/entrypoint/src/start/start.rs | 28 +-- crates/net/Cargo.toml | 4 +- crates/sync/src/sync.rs | 15 +- crates/test-helpers/src/ciphernode_system.rs | 12 +- 13 files changed, 215 insertions(+), 168 deletions(-) delete mode 100644 crates/ciphernode-builder/src/sync_builder.rs diff --git a/Cargo.lock b/Cargo.lock index e5e37dc965..d46cac5244 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2882,8 +2882,10 @@ dependencies = [ "e3-fhe", "e3-keyshare", "e3-multithread", + "e3-net", "e3-request", "e3-sortition", + "e3-sync", "e3-trbfv", "e3-utils", "once_cell", diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 07b940c81e..495eda85f4 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -21,8 +21,10 @@ e3-evm.workspace = true e3-fhe.workspace = true e3-keyshare.workspace = true e3-multithread.workspace = true +e3-net.workspace = true e3-request.workspace = true e3-sortition.workspace = true +e3-sync.workspace = true e3-trbfv.workspace = true e3-utils.workspace = true rayon.workspace = true diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 2e67d36187..c4af8deaca 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -5,16 +5,22 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::Addr; +use anyhow::Result; use e3_data::{DataStore, InMemStore, StoreAddr}; use e3_events::{BusHandle, EnclaveEvent, HistoryCollector}; +use tokio::task::JoinHandle; -#[derive(Clone, Debug)] +/// A Sharable handle to a Ciphernode. NOTE: clones are available for use in the CiphernodeSystem +/// but they cannot await the task. +#[derive(Debug)] pub struct CiphernodeHandle { pub address: String, pub store: DataStore, pub bus: BusHandle, pub history: Option>>, pub errors: Option>>, + pub peer_id: String, + pub join_handle: JoinHandle>, } impl CiphernodeHandle { @@ -24,6 +30,8 @@ impl CiphernodeHandle { bus: BusHandle, history: Option>>, errors: Option>>, + peer_id: String, + join_handle: JoinHandle>, ) -> Self { Self { address, @@ -31,6 +39,8 @@ impl CiphernodeHandle { bus, history, errors, + peer_id, + join_handle, } } @@ -54,6 +64,10 @@ impl CiphernodeHandle { &self.store } + pub fn split(self) -> (BusHandle, JoinHandle>) { + (self.bus, self.join_handle) + } + pub fn in_mem_store(&self) -> Option<&Addr> { let addr = self.store.get_addr(); if let StoreAddr::InMem(ref store) = addr { diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 36886fbdaa..42372dd125 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::event_system::AggregateConfig; -use crate::{CiphernodeHandle, EventSystem, EvmSystemChainBuilder, ProviderCache}; +use crate::{CiphernodeHandle, EventSystem, EvmSystemChainBuilder, ProviderCache, WriteEnabled}; use actix::{Actor, Addr}; use anyhow::Result; use derivative::Derivative; @@ -13,22 +13,23 @@ use e3_aggregator::ext::{ PlaintextAggregatorExtension, PublicKeyAggregatorExtension, ThresholdPlaintextAggregatorExtension, }; +use e3_aggregator::CommitteeFinalizer; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{InMemStore, RepositoriesFactory}; -use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig}; -use e3_evm::{ - BondingRegistrySolReader, CiphernodeRegistrySolReader, EnclaveSolWriter, EvmChainGateway, -}; +use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EvmEventConfig}; +use e3_evm::{BondingRegistrySolReader, CiphernodeRegistrySolReader, EnclaveSolWriter}; use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; +use e3_net::{NetEventTranslator, NetRepositoryFactory}; use e3_request::E3Router; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; +use e3_sync::Synchronizer; use e3_utils::{rand_eth_addr, SharedRng}; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; @@ -69,6 +70,20 @@ pub struct CiphernodeBuilder { task_pool: Option, threads: Option, threshold_plaintext_agg: bool, + net_config: Option, +} + +// Simple Net Configuration +#[derive(Debug)] +struct NetConfig { + pub peers: Vec, + pub quic_port: u16, +} + +impl NetConfig { + pub fn new(peers: Vec, quic_port: u16) -> Self { + Self { peers, quic_port } + } } #[derive(Default, Debug)] @@ -121,6 +136,7 @@ impl CiphernodeBuilder { task_pool: None, threads: None, threshold_plaintext_agg: false, + net_config: None, } } @@ -283,6 +299,12 @@ impl CiphernodeBuilder { self } + /// Setup net package components. + pub fn with_net(mut self, peers: Vec, quic_port: u16) -> Self { + self.net_config = Some(NetConfig::new(peers, quic_port)); + self + } + fn create_local_bus() -> Addr> { EventBus::::new(EventBusConfig { deduplicate: true }).start() } @@ -390,77 +412,17 @@ impl CiphernodeBuilder { CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) .await?; - // TODO: gather an async handle from the event readers that closes when they shutdown and - // join it with the network manager joinhandle below - for chain in self - .chains - .iter() - .filter(|chain| chain.enabled.unwrap_or(true)) - { - let provider = provider_cache.ensure_read_provider(chain).await?; - - let mut system = EvmSystemChainBuilder::new(&bus, &provider); - - if self.contract_components.enclave { - let write_provider = provider_cache.ensure_write_provider(chain).await?; - let contract = &chain.contracts.enclave; - EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?).await?; - system.with_contract(contract.address()?, move |next| { - EnclaveSolReader::setup(&next).recipient() - }); - } - - if self.contract_components.enclave_reader { - let contract = &chain.contracts.enclave; - - system.with_contract(contract.address()?, move |next| { - EnclaveSolReader::setup(&next).recipient() - }); - } - - if self.contract_components.bonding_registry { - let contract = &chain.contracts.bonding_registry; - system.with_contract(contract.address()?, move |next| { - BondingRegistrySolReader::setup(&next).recipient() - }); - } - - if self.contract_components.ciphernode_registry { - let contract = &chain.contracts.ciphernode_registry; - - system.with_contract(contract.address()?, move |next| { - CiphernodeRegistrySolReader::setup(&next).recipient() - }); - - // TODO: Should we not let this pass and just use '?'? - // Above if we include enclave in the config and we don't have a wallet it will fail - match provider_cache - .ensure_write_provider(&chain) - .await - { - Ok(write_provider) => { - let _writer = CiphernodeRegistrySol::attach_writer( - &bus, - write_provider.clone(), - contract.address()?, - self.pubkey_agg, - ) - .await?; - info!("CiphernodeRegistrySolWriter attached for publishing committees"); - - if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { - info!("Attaching CommitteeFinalizer for score sortition"); - e3_aggregator::CommitteeFinalizer::attach(&bus); - } - } - Err(e) => error!( - "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", - e - ), - } - } - system.build(); - } + // Setup evm system + // TODO: gather an async handle from the event readers in thre following function + // that closes when they shutdown and join it with the network manager joinhandle externally + let evm_config = setup_evm_system( + &self.chains, + &mut provider_cache, + &bus, + &self.contract_components, + self.pubkey_agg, + ) + .await?; // E3 specific setup let mut e3_builder = E3Router::builder(&bus, store.clone()); @@ -507,15 +469,39 @@ impl CiphernodeBuilder { info!("Setting up KeyshareExtension (legacy)!"); e3_builder = e3_builder.with(KeyshareExtension::create(&bus, &addr, &self.cipher)) } + info!("building..."); + e3_builder.build().await?; + let (join_handle, peer_id) = if let Some(net_config) = self.net_config { + let repositories = store.repositories(); + let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( + bus.clone(), + net_config.peers, + &self.cipher, + net_config.quic_port, + repositories.libp2p_keypair(), + ) + .await?; + (join_handle, peer_id) + } else { + ( + tokio::spawn(std::future::ready(Ok(()))), + "-not set-".to_string(), + ) + }; + + Synchronizer::setup(&bus, evm_config); // TODO: add net config if required + Ok(CiphernodeHandle::new( addr.to_owned(), store, bus, history, errors, + peer_id, + join_handle, )) } @@ -590,3 +576,81 @@ fn create_aggregate_delays( Ok(delays) } + +async fn setup_evm_system( + chains: &Vec, + provider_cache: &mut ProviderCache, + bus: &BusHandle, + contract_components: &ContractComponents, + pubkey_agg: bool, +) -> Result { + let mut evm_config = EvmEventConfig::new(); + for chain in chains.iter().filter(|chain| chain.enabled.unwrap_or(true)) { + let provider = provider_cache.ensure_read_provider(chain).await?; + let chain_id = provider.chain_id(); + evm_config.insert(chain_id, chain.try_into()?); + let mut system = EvmSystemChainBuilder::new(&bus, &provider); + + if contract_components.enclave { + let write_provider = provider_cache.ensure_write_provider(chain).await?; + let contract = &chain.contracts.enclave; + EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?).await?; + system.with_contract(contract.address()?, move |next| { + EnclaveSolReader::setup(&next).recipient() + }); + } + + if contract_components.enclave_reader { + let contract = &chain.contracts.enclave; + + system.with_contract(contract.address()?, move |next| { + EnclaveSolReader::setup(&next).recipient() + }); + } + + if contract_components.bonding_registry { + let contract = &chain.contracts.bonding_registry; + system.with_contract(contract.address()?, move |next| { + BondingRegistrySolReader::setup(&next).recipient() + }); + } + + if contract_components.ciphernode_registry { + let contract = &chain.contracts.ciphernode_registry; + + system.with_contract(contract.address()?, move |next| { + CiphernodeRegistrySolReader::setup(&next).recipient() + }); + + // TODO: Should we not let this pass and just use '?'? + // Above if we include enclave in the config and we don't have a wallet it will fail + match provider_cache + .ensure_write_provider(&chain) + .await + { + Ok(write_provider) => { + CiphernodeRegistrySol::attach_writer( + &bus, + write_provider.clone(), + contract.address()?, + pubkey_agg, + ) + .await?; + info!("CiphernodeRegistrySolWriter attached for publishing committees"); + + if pubkey_agg { + info!("Attaching CommitteeFinalizer for score sortition"); + CommitteeFinalizer::attach(&bus); + } + } + Err(e) => error!( + "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", + e + ) + } + } + system.build(); + } + + Ok(evm_config) +} diff --git a/crates/ciphernode-builder/src/sync_builder.rs b/crates/ciphernode-builder/src/sync_builder.rs deleted file mode 100644 index 7d46bb7c78..0000000000 --- a/crates/ciphernode-builder/src/sync_builder.rs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use std::collections::HashMap; - -use alloy::primitives::Address; -use anyhow::Result; -use e3_config::chain_config::ChainConfig; -use e3_events::EvmEventConfig; -type ChainId = u64; -type DeployBlock = Option; - -pub struct SyncBuilder { - config: EvmEventConfig, -} - -impl SyncBuilder { - pub fn with_chain(&mut self, chain_id: u64, chain: ChainConfig) -> Result<()> { - self.config.insert(chain_id, chain.try_into()?); - Ok(()) - } -} diff --git a/crates/cli/src/start.rs b/crates/cli/src/start.rs index 8ab6b20fa1..2150cd4e2c 100644 --- a/crates/cli/src/start.rs +++ b/crates/cli/src/start.rs @@ -21,7 +21,7 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { // add cli peers to the config config.add_peers(peers); - let (bus, handle, peer_id) = match config.role() { + let node = match config.role() { // Launch in aggregator configuration NodeRole::Aggregator { pubkey_write_path, @@ -44,10 +44,10 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { "LAUNCHING CIPHERNODE: ({}/{}/{})", config.name(), address, - peer_id + node.peer_id ); - tokio::spawn(listen_for_shutdown(bus, handle)).await?; + tokio::spawn(listen_for_shutdown(node)).await?; Ok(()) } diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index b570f7dd80..63e241eee4 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -11,7 +11,7 @@ use crate::{ rpc::{RpcAuth, RPC}, }; use anyhow::*; -use e3_events::EvmEventConfigChain; +use e3_events::{EvmEventConfig, EvmEventConfigChain}; use serde::{Deserialize, Serialize}; use tracing::error; @@ -34,9 +34,9 @@ impl ChainConfig { } } -impl TryFrom for EvmEventConfigChain { +impl TryFrom<&ChainConfig> for EvmEventConfigChain { type Error = anyhow::Error; - fn try_from(value: ChainConfig) -> std::result::Result { + fn try_from(value: &ChainConfig) -> std::result::Result { let rpc = value.rpc_url()?; let contracts = value.contracts.contracts(); let mut lowest_block: Option = None; @@ -60,3 +60,11 @@ impl TryFrom for EvmEventConfigChain { Ok(EvmEventConfigChain::new(start_block)) } } + +impl TryFrom for EvmEventConfigChain { + type Error = anyhow::Error; + fn try_from(value: ChainConfig) -> std::result::Result { + let r = &value; + r.try_into() + } +} diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 1ff0d6748d..1b8f96e818 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -4,17 +4,17 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; -use e3_events::{prelude::*, BusHandle, Shutdown}; +use e3_ciphernode_builder::CiphernodeHandle; +use e3_events::{prelude::*, Shutdown}; use std::time::Duration; use tokio::{ select, signal::unix::{signal, SignalKind}, - task::JoinHandle, }; use tracing::{error, info}; -pub async fn listen_for_shutdown(bus: BusHandle, mut handle: JoinHandle>) { +pub async fn listen_for_shutdown(node: CiphernodeHandle) { + let (bus, mut handle) = node.split(); let mut sigterm = signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal stream"); select! { diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index c85e98cee9..3032e43d0c 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -6,12 +6,9 @@ use alloy::primitives::Address; use anyhow::Result; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{CiphernodeBuilder, CiphernodeHandle}; use e3_config::AppConfig; use e3_crypto::Cipher; -use e3_data::RepositoriesFactory; -use e3_events::BusHandle; -use e3_net::{NetEventTranslator, NetRepositoryFactory}; use e3_test_helpers::{PlaintextWriter, PublicKeyWriter}; use rand::SeedableRng; use rand_chacha::{rand_core::OsRng, ChaCha20Rng}; @@ -19,17 +16,16 @@ use std::{ path::PathBuf, sync::{Arc, Mutex}, }; -use tokio::task::JoinHandle; pub async fn execute( config: &AppConfig, address: Address, pubkey_write_path: Option, plaintext_write_path: Option, -) -> Result<(BusHandle, JoinHandle>, String)> { +) -> Result { let rng = Arc::new(Mutex::new(ChaCha20Rng::from_rng(OsRng)?)); let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); - let builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) + let node = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_address(&address.to_string()) .with_persistence(&config.log_file(), &config.db_file()) .with_chains(&config.chains()) @@ -39,30 +35,19 @@ pub async fn execute( .with_contract_ciphernode_registry() .with_max_threads() .with_pubkey_aggregation() - .with_threshold_plaintext_aggregation(); - - // TODO: put net package provisioning in the ciphernode-builder: - let node = builder.build().await?; - let store = node.store(); - let repositories = store.repositories(); - let bus = node.bus.clone(); - let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( - bus.clone(), - config.peers(), - &cipher, - config.quic_port(), - repositories.libp2p_keypair(), - ) - .await?; + .with_threshold_plaintext_aggregation() + .with_net(config.peers(), config.quic_port()) + .build() + .await?; // These are here purely for our integration test so leaving out of the builder if let Some(path) = pubkey_write_path { - PublicKeyWriter::attach(&path, bus.clone()); + PublicKeyWriter::attach(&path, node.bus().clone()); } if let Some(path) = plaintext_write_path { - PlaintextWriter::attach(&path, bus.clone()); + PlaintextWriter::attach(&path, node.bus().clone()); } - Ok((bus, join_handle, peer_id)) + Ok(node) } diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 2a4a42abef..9dae6086f7 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -6,7 +6,7 @@ use alloy::primitives::Address; use anyhow::Result; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{CiphernodeBuilder, CiphernodeHandle}; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; @@ -19,13 +19,10 @@ use tokio::task::JoinHandle; use tracing::instrument; #[instrument(name = "app", skip_all)] -pub async fn execute( - config: &AppConfig, - address: Address, -) -> Result<(BusHandle, JoinHandle>, String)> { +pub async fn execute(config: &AppConfig, address: Address) -> Result { let rng = Arc::new(Mutex::new(rand_chacha::ChaCha20Rng::from_rng(OsRng)?)); let cipher = Arc::new(Cipher::from_file(&config.key_file()).await?); - let builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) + let node = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_address(&address.to_string()) .with_persistence(&config.log_file(), &config.db_file()) .with_sortition_score() @@ -34,19 +31,10 @@ pub async fn execute( .with_contract_bonding_registry() .with_max_threads() .with_contract_ciphernode_registry() - .with_trbfv(); + .with_trbfv() + .with_net(config.peers(), config.quic_port()) + .build() + .await?; - let node = builder.build().await?; - let repositories = node.store().repositories(); - let bus = node.bus.clone(); - let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( - bus.clone(), - config.peers(), - &cipher, - config.quic_port(), - repositories.libp2p_keypair(), - ) - .await?; - - Ok((bus, join_handle, peer_id)) + Ok(node) } diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml index af340f2ead..46f00d41fc 100644 --- a/crates/net/Cargo.toml +++ b/crates/net/Cargo.toml @@ -27,7 +27,9 @@ tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } e3-events = { workspace = true } -e3-ciphernode-builder = { workspace = true } anyhow = { workspace = true } actix = { workspace = true } zeroize = { workspace = true } + +[dev-dependencies] +e3-ciphernode-builder = { workspace = true } diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 0bd2d0eb31..2a4a24398f 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -7,13 +7,13 @@ use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::Context; use e3_events::{trap, BusHandle, EType, EventPublisher, EvmEventConfig, SyncEvmEvent, SyncStart}; -struct Sync { +pub struct Synchronizer { bus: BusHandle, evm_config: Option, // net_config: NetEventConfig, } -impl Sync { +impl Synchronizer { pub fn new(bus: &BusHandle, evm_config: EvmEventConfig) -> Self { Self { evm_config: Some(evm_config), @@ -26,19 +26,22 @@ impl Sync { } } -impl Actor for Sync { +impl Actor for Synchronizer { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { ctx.notify(Bootstrap); } } -impl Handler for Sync { +impl Handler for Synchronizer { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result {} + fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + + // What do we do with these for now? + } } -impl Handler for Sync { +impl Handler for Synchronizer { type Result = (); fn handle(&mut self, _: Bootstrap, ctx: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index 61ef319129..8d577670d1 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -16,7 +16,7 @@ use tokio::time::timeout; type SetupFn<'a> = Box Pin> + 'a>> + 'a>; type ThenFn<'a> = - Box Pin> + 'a>> + 'a>; + Box Pin> + 'a>> + 'a>; /// This builds a ciphernode system using the actor model only. This helps us simulate the network /// in tests that we can run in the /crates/tests crate @@ -85,8 +85,8 @@ impl<'a> CiphernodeSystemBuilder<'a> { } for then_fn in self.thens { - for node in nodes.clone() { - then_fn(node).await?; + for node in nodes.iter() { + then_fn(&node).await?; } } @@ -140,7 +140,7 @@ impl CiphernodeSystem { Ok(CiphernodeHistory(history)) } pub async fn flush_all_history(&self, millis: u64) -> Result<()> { - let nodes = self.0.clone(); + let nodes = &self.0; for node in nodes.iter() { let Some(history) = node.history() else { break; @@ -199,6 +199,7 @@ mod tests { use e3_ciphernode_builder::EventSystem; use e3_data::InMemStore; use e3_events::{EventBus, EventBusConfig}; + use tokio::task::JoinHandle; async fn mock_setup_node(address: String) -> Result { // Create mock actors for the test @@ -207,6 +208,7 @@ mod tests { let history = EventBus::::history(&bus); let errors = EventBus::::error(&bus); let bus = EventSystem::new("test").with_event_bus(bus).handle()?; + let handle: JoinHandle> = tokio::spawn(async { Ok(()) }); Ok(CiphernodeHandle { address, @@ -214,6 +216,8 @@ mod tests { bus, history: Some(history), errors: Some(errors), + join_handle: handle, + peer_id: "-unknown peer id-".to_string(), }) } From f67d2677a0987e12d0bb24fd808daa67e40594cc Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 10:07:44 +0000 Subject: [PATCH 085/102] handle sync evm events in synchronizer --- crates/sync/src/sync.rs | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 2a4a24398f..d81f5e1f87 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -6,10 +6,18 @@ use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::Context; -use e3_events::{trap, BusHandle, EType, EventPublisher, EvmEventConfig, SyncEvmEvent, SyncStart}; +use e3_events::{ + trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, + SyncStart, +}; + +// NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p + +/// Manage the synchronization of events across. pub struct Synchronizer { bus: BusHandle, evm_config: Option, + evm_buffer: Vec, // net_config: NetEventConfig, } @@ -18,6 +26,7 @@ impl Synchronizer { Self { evm_config: Some(evm_config), bus: bus.clone(), + evm_buffer: Vec::new(), } } @@ -36,8 +45,21 @@ impl Actor for Synchronizer { impl Handler for Synchronizer { type Result = (); fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { - - // What do we do with these for now? + trap(EType::Sync, &self.bus.clone(), || { + match msg { + // Buffer events as the sync actor receives them + SyncEvmEvent::Event(event) => self.evm_buffer.push(event), + // When we hear that sync is complete send all events on chain then publish SyncEnd + SyncEvmEvent::HistoricalSyncComplete(_) => { + for evt in self.evm_buffer.drain(..) { + let (data, ts, _) = evt.split(); + self.bus.publish_from_remote(data, ts)?; + } + self.bus.publish(SyncEnd::new())?; + } + }; + Ok(()) + }) } } From ea5164098d7eced6af018457a81f415b6142840a Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 10:11:06 +0000 Subject: [PATCH 086/102] add better comment --- crates/sync/src/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index d81f5e1f87..0cc166adbb 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -71,7 +71,7 @@ impl Handler for Synchronizer { "EvmEventConfig was not set likely Bootstrap was called more than once.", )?; - // Fetch snapshot state + // TODO: Get information about what has and has not been synced then fire SyncStart self.bus.publish(SyncStart::new(ctx.address(), evm_config)) }) } From 34067b08666e8663385d8a896da6c9e362684b49 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 11:35:33 +0000 Subject: [PATCH 087/102] update timestamp catching --- crates/evm/src/events.rs | 10 ++++-- crates/evm/src/evm_read_interface.rs | 46 ++++++++++++++++++++++++---- crates/evm/src/evm_reader.rs | 11 +++++-- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index 09133e890f..bf02efb2fa 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -56,13 +56,19 @@ impl EnclaveEvmEvent { pub struct EvmLog { pub id: CorrelationId, pub log: Log, + pub timestamp: u64, pub chain_id: u64, } impl EvmLog { - pub fn new(log: Log, chain_id: u64) -> Self { + pub fn new(log: Log, chain_id: u64, timestamp: u64) -> Self { let id = CorrelationId::new(); - Self { log, chain_id, id } + Self { + log, + chain_id, + id, + timestamp, + } } pub fn get_id(&self) -> CorrelationId { diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index faf514c76c..24c7393da7 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -156,12 +156,14 @@ async fn stream_from_evm( let chain_id = provider.chain_id(); let provider_ref = provider.provider(); let mut last_id: Option = None; + let mut timestamp_tracker = TimestampTracker::new(); // Historical events match provider_ref.get_logs(&filters.historical).await { Ok(historical_logs) => { info!("Fetched {} historical events", historical_logs.len()); for log in historical_logs { - let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id)); + let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; + let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); last_id = Some(evt.get_id()); processor.do_send(evt) } @@ -188,7 +190,8 @@ async fn stream_from_evm( maybe_log = stream.next() => { match maybe_log { Some(log) => { - processor.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id))) + let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; + processor.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp))) } None => break, // Stream ended } @@ -212,10 +215,6 @@ async fn stream_from_evm( info!("Exiting stream loop"); } -fn is_local_node(rpc_url: &str) -> bool { - rpc_url.contains("localhost") || rpc_url.contains("127.0.0.1") -} - impl Handler for EvmReadInterface

{ type Result = (); @@ -227,3 +226,38 @@ impl Handler for EvmReadInterface

, // (block_number, timestamp) +} + +impl TimestampTracker { + fn new() -> Self { + Self { current: None } + } + + async fn get(&mut self, provider: &P, block_number: Option) -> u64 { + let Some(bn) = block_number else { + error!("BLOCK NUMBER NOT FOUND ON LOG!"); + return 0; + }; + + if let Some((cached_bn, ts)) = self.current { + if bn == cached_bn { + return ts; + } + } + + let ts = provider + .get_block_by_number(bn.into()) + .await + .ok() + .flatten() + .map(|b| b.header.timestamp) + .unwrap_or(0); + + self.current = Some((bn, ts)); + ts + } +} diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_reader.rs index ba4927fcb6..17bddf1d15 100644 --- a/crates/evm/src/evm_reader.rs +++ b/crates/evm/src/evm_reader.rs @@ -6,6 +6,7 @@ use actix::{Actor, Handler}; use e3_events::{hlc::HlcTimestamp, EnclaveEventData, EvmEvent}; +use tracing::info; use crate::{ events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, @@ -34,15 +35,19 @@ impl Handler for EvmReader { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { match msg.clone() { - EnclaveEvmEvent::Log(EvmLog { log, chain_id, id }) => { + EnclaveEvmEvent::Log(EvmLog { + log, + chain_id, + id, + timestamp, + }) => { let extractor = self.extractor; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { let err = "Log should always have metadata because we listen to non-pending blocks. If you are seeing this it is likely because there is an issue with how we are subscribing to blocks"; let block = log.block_number.expect(err); - let block_timestamp = log.block_timestamp.expect(err); let log_index = log.log_index.expect(err); - let ts = from_log_chain_id_to_ts(block_timestamp, log_index, chain_id); + let ts = from_log_chain_id_to_ts(timestamp, log_index, chain_id); self.next.do_send(EnclaveEvmEvent::Event(EvmEvent::new( // note we use the id from the log event above! id, event, block, ts, chain_id, From eacb2a4ef343973d0d16c3fe7c6aad24399ce87a Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 18:59:08 +0000 Subject: [PATCH 088/102] fix build errors --- crates/evm/src/events.rs | 3 ++- crates/evm/src/evm_hub.rs | 1 + crates/evm/src/evm_router.rs | 4 ++-- crates/evm/src/fix_historical_order.rs | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index bf02efb2fa..3d8913a698 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -81,7 +81,7 @@ use alloy_primitives::Address; #[cfg(test)] impl EvmLog { - pub fn test_log(address: Address, chain_id: u64) -> EvmLog { + pub fn test_log(address: Address, chain_id: u64, timestamp: u64) -> EvmLog { let id = CorrelationId::new(); EvmLog { log: Log { @@ -93,6 +93,7 @@ impl EvmLog { }, chain_id, id, + timestamp, } } } diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs index 9e98f4344f..3c22437424 100644 --- a/crates/evm/src/evm_hub.rs +++ b/crates/evm/src/evm_hub.rs @@ -74,6 +74,7 @@ mod tests { let log_event = EnclaveEvmEvent::Log(EvmLog::test_log( address!("0x1111111111111111111111111111111111111111"), 1, + 0, )); hub.send(log_event).await.unwrap(); diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 254949da27..068e5440c2 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -100,7 +100,7 @@ mod tests { let received_count = Arc::new(AtomicUsize::new(0)); let processor_addr = TestProcessor(received_count.clone()).start(); let addr = address!("0x1111111111111111111111111111111111111111"); - let test_log = EvmLog::test_log(addr, 1); + let test_log = EvmLog::test_log(addr, 1, 0); let test_address = test_log.log.address(); let router = EvmRouter::new() @@ -122,7 +122,7 @@ mod tests { let router_addr = address!("0x1111111111111111111111111111111111111111"); let log_addr = address!("0x2222222222222222222222222222222222222222"); - let test_log = EvmLog::test_log(log_addr, 1); + let test_log = EvmLog::test_log(log_addr, 1, 0); let router = EvmRouter::new() .add_route(router_addr, &processor_addr.recipient()) diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index 7416c122cc..c7ecaae70e 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -97,9 +97,9 @@ mod tests { let (tx, mut rx) = mpsc::unbounded_channel(); let fix = FixHistoricalOrder::setup(Collector(tx).start()); - let log_1 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 1)); - let log_2 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 2)); - let log_3 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 3)); + let log_1 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 1, 1)); + let log_2 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 2, 2)); + let log_3 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 3, 3)); let sync_complete = EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete::new( 1, From 2b63317ddedc474b6eaa4e73a4b9f3865c121993 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 21:34:41 +0000 Subject: [PATCH 089/102] fix event forwarding to sync --- Cargo.lock | 1 + crates/events/src/correlation_id.rs | 11 ++++- crates/events/src/enclave_event/sync_start.rs | 9 ++-- crates/events/src/sync.rs | 10 +++-- crates/evm/src/events.rs | 4 ++ crates/evm/src/evm_chain_gateway.rs | 25 +++++++++-- crates/evm/src/evm_read_interface.rs | 9 +++- crates/evm/src/evm_reader.rs | 3 ++ crates/evm/src/evm_router.rs | 7 ++- crates/evm/src/fix_historical_order.rs | 27 +++++++++++- crates/sync/Cargo.toml | 1 + crates/sync/src/sync.rs | 44 +++++++++++++++---- 12 files changed, 125 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 000eb00512..3589ddcdae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3403,6 +3403,7 @@ dependencies = [ "actix", "anyhow", "e3-events", + "tracing", ] [[package]] diff --git a/crates/events/src/correlation_id.rs b/crates/events/src/correlation_id.rs index b7f808deeb..ee3e3dadc5 100644 --- a/crates/events/src/correlation_id.rs +++ b/crates/events/src/correlation_id.rs @@ -5,7 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use std::{ + fmt, + fmt::Debug, fmt::Display, + fmt::Formatter, sync::atomic::{AtomicUsize, Ordering}, }; @@ -14,7 +17,7 @@ use serde::{Deserialize, Serialize}; static NEXT_CORRELATION_ID: AtomicUsize = AtomicUsize::new(1); /// CorrelationId provides a way to correlate commands and the events they create. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CorrelationId { id: usize, } @@ -31,3 +34,9 @@ impl Display for CorrelationId { write!(f, "{}", self.id) } } + +impl Debug for CorrelationId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.id) + } +} diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index fa93930c61..d4fe27d32c 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -11,10 +11,7 @@ use actix::{Message, Recipient}; use anyhow::Context; use anyhow::Result; use serde::{Deserialize, Serialize}; -use std::{ - collections::BTreeMap, - fmt::{self, Display}, -}; +use std::fmt::{self, Display}; /// This is a processed EvmEvent specifically typed for the Sync actor #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -51,6 +48,10 @@ impl EvmEvent { pub fn get_id(&self) -> CorrelationId { self.id } + + pub fn chain_id(&self) -> u64 { + self.chain_id + } } /// Dispatched by the Sync actor when initial data is read and the sync process needs to be started diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index ae5d7e177a..bc13534344 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -4,17 +4,17 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use crate::EvmEvent; use actix::Message; use serde::{Deserialize, Serialize}; - +type Chainid = u64; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub enum SyncEvmEvent { /// Signal that this reader has completed historical sync - HistoricalSyncComplete(u64), + HistoricalSyncComplete(ChainId), /// An actual event from the blockchain Event(EvmEvent), } @@ -62,4 +62,8 @@ impl EvmEventConfig { pub fn insert(&mut self, key: ChainId, value: EvmEventConfigChain) { self.config.insert(key, value); } + + pub fn chains(&self) -> HashSet { + self.config.keys().cloned().collect() + } } diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index 3d8913a698..4b59deaaeb 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -40,6 +40,9 @@ pub enum EnclaveEvmEvent { Event(EvmEvent), /// Raw log data from the provider Log(EvmLog), + /// Dummy event to report that an event was processed. This is required to ensure that the + /// appropriate events are ordered correctly + Processed(CorrelationId), } impl EnclaveEvmEvent { @@ -48,6 +51,7 @@ impl EnclaveEvmEvent { EnclaveEvmEvent::HistoricalSyncComplete(e) => e.get_id(), EnclaveEvmEvent::Log(e) => e.get_id(), EnclaveEvmEvent::Event(e) => e.get_id(), + EnclaveEvmEvent::Processed(id) => id.to_owned(), } } } diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index fb53e5d693..68c287311d 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -16,6 +16,7 @@ use e3_events::{ }; use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; +use tracing::info; /// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. /// It coordinates event flow between these components. @@ -98,6 +99,7 @@ impl EvmChainGateway { } fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { + info!("Processing SyncStart message"); // Received a SyncStart event from the event bus. Get the sender within that event and forward // all events to that actor let sender = msg.sender.context("No sender on SyncStart Message")?; @@ -110,6 +112,7 @@ impl EvmChainGateway { } fn handle_sync_end(&mut self, _: SyncEnd) -> Result<()> { + info!("Processing SyncEnd message"); let buffer = self.status.live()?; for evt in buffer { self.publish_evm_event(evt)?; @@ -138,19 +141,34 @@ impl EvmChainGateway { } fn handle_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { + info!( + "handling historical sync complete for chain_id({})", + event.chain_id + ); let sender = self.status.buffer_until_live()?; + info!("Sending historical sync complete event to sender."); sender.do_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id)); Ok(()) } fn handle_receive_evm_event(&mut self, msg: EvmEvent) -> Result<()> { match &mut self.status { - SyncStatus::BufferUntilLive(buffer) => buffer.push(msg), + SyncStatus::BufferUntilLive(buffer) => { + info!("saving evm event({}) to pre-live buffer", msg.get_id()); + buffer.push(msg) + } SyncStatus::ForwardToSyncActor(Some(sync_actor)) => { + info!("forwarding evm event({}) to SyncActor", msg.get_id()); sync_actor.do_send(msg.into()); } - SyncStatus::Live => self.publish_evm_event(msg)?, - SyncStatus::Init(buffer) => buffer.push(msg), + SyncStatus::Live => { + info!("publishing evm event({})", msg.get_id()); + self.publish_evm_event(msg)? + } + SyncStatus::Init(buffer) => { + info!("saving evm event({}) to pre-sync buffer", msg.get_id()); + buffer.push(msg) + } _ => (), }; Ok(()) @@ -188,7 +206,6 @@ mod tests { use e3_ciphernode_builder::EventSystem; use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TestEvent}; - use std::collections::HashMap; use tokio::sync::mpsc; struct SyncEventCollector { diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index 24c7393da7..14f6c9505b 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -165,6 +165,7 @@ async fn stream_from_evm( let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); last_id = Some(evt.get_id()); + info!("Sending event({})", evt.get_id()); processor.do_send(evt) } } @@ -174,9 +175,13 @@ async fn stream_from_evm( return; } } - + let historical_sync_event = HistoricalSyncComplete::new(chain_id, last_id); + info!( + "Historical Sync Complete event({})", + historical_sync_event.get_id() + ); processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete( - HistoricalSyncComplete::new(chain_id, last_id), + historical_sync_event, )); info!("Subscribing to live events"); diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_reader.rs index 17bddf1d15..acd80ef1fd 100644 --- a/crates/evm/src/evm_reader.rs +++ b/crates/evm/src/evm_reader.rs @@ -41,6 +41,7 @@ impl Handler for EvmReader { id, timestamp, }) => { + info!("processing event({})", msg.get_id()); let extractor = self.extractor; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { @@ -52,6 +53,8 @@ impl Handler for EvmReader { // note we use the id from the log event above! id, event, block, ts, chain_id, ))) + } else { + self.next.do_send(EnclaveEvmEvent::Processed(id)) } } hist @ EnclaveEvmEvent::HistoricalSyncComplete(..) => self.next.do_send(hist), diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 068e5440c2..790cb78487 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -8,7 +8,7 @@ use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; use actix::{Actor, Addr, Handler}; use alloy_primitives::Address; use std::collections::HashMap; -use tracing::error; +use tracing::{debug, error, info}; /// Directs EnclaveEvmEvent::Log events to the correct upstream processors. Drops all other event /// types @@ -50,7 +50,9 @@ impl Handler for EvmRouter { match msg.clone() { // Take all log events and route them EnclaveEvmEvent::Log(EvmLog { log, .. }) => { - if let Some(dest) = self.routing_table.get(&log.address()) { + let address = log.address(); + if let Some(dest) = self.routing_table.get(&address) { + debug!("Found address {address} in routing table forwarding to destination."); dest.do_send(msg); } else { error!( @@ -61,6 +63,7 @@ impl Handler for EvmRouter { } _ => { if let Some(fallback) = self.fallback.clone() { + info!("Sending event({}) to fallback", msg.get_id()); fallback.do_send(msg) } } diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index c7ecaae70e..be6d51dbcf 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -7,6 +7,8 @@ use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; use actix::{Actor, Addr, Handler}; use bloom::{BloomFilter, ASMS}; +use e3_events::CorrelationId; +use tracing::{info, warn}; pub struct FixHistoricalOrder { dest: EvmEventProcessor, @@ -34,11 +36,16 @@ impl FixHistoricalOrder { })) = self.pending_sync_complete { if self.seen_ids.contains(id) { + info!("Forwarding historical send complete event"); self.dest .do_send(self.pending_sync_complete.take().unwrap()); } } } + + fn track_id(&mut self, id: CorrelationId) { + self.seen_ids.insert(&id); + } } impl Actor for FixHistoricalOrder { @@ -49,18 +56,34 @@ impl Handler for FixHistoricalOrder { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + let id = msg.get_id(); + info!("Receiving EnclaveEvmEvent event({})", msg.get_id()); match msg { none_hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { prev_event: None, .. }) => { + info!( + "Historical order event({}) has no previous event. Forwarding...", + id + ); self.dest.do_send(none_hist); } - hist @ EnclaveEvmEvent::HistoricalSyncComplete(..) => { + hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { + prev_event: Some(prev), + .. + }) => { + info!( + "Historical order event({}) has previous event({}). Buffering...", + id, prev + ); + self.pending_sync_complete = Some(hist); } + EnclaveEvmEvent::Processed(id) => self.track_id(id), other => { - self.seen_ids.insert(&other.get_id()); + info!("Forwarding event({})", other.get_id()); + self.track_id(other.get_id()); self.dest.do_send(other); } } diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index f3679a9816..571cdf8b0e 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -10,3 +10,4 @@ repository = "https://github.com/gnosisguild/enclave/crates/sync" actix.workspace = true anyhow.workspace = true e3-events.workspace = true +tracing.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 0cc166adbb..9f4a90f712 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -4,35 +4,67 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::collections::{HashMap, HashSet}; + use actix::{Actor, Addr, AsyncContext, Handler, Message}; -use anyhow::Context; +use anyhow::{Context, Result}; use e3_events::{ trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, SyncStart, }; +use tracing::info; // NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p +type ChainId = u64; /// Manage the synchronization of events across. pub struct Synchronizer { bus: BusHandle, evm_config: Option, evm_buffer: Vec, + evm_to_sync: HashSet, // net_config: NetEventConfig, } impl Synchronizer { pub fn new(bus: &BusHandle, evm_config: EvmEventConfig) -> Self { + let evm_to_sync = evm_config.chains(); Self { evm_config: Some(evm_config), bus: bus.clone(), evm_buffer: Vec::new(), + evm_to_sync, } } pub fn setup(bus: &BusHandle, evm_config: EvmEventConfig) -> Addr { Self::new(bus, evm_config).start() } + + fn buffer_evm_event(&mut self, event: EvmEvent) { + info!("buffer evm event({})", event.get_id()); + self.evm_buffer.push(event); + } + + fn handle_sync_complete(&mut self, chain_id: u64) -> Result<()> { + info!("handle sync complete for chain({})", chain_id); + self.evm_to_sync.remove(&chain_id); + info!("{} chains left to sync...", self.evm_to_sync.len()); + if self.evm_to_sync.is_empty() { + self.handle_sync_end()?; + } + Ok(()) + } + + fn handle_sync_end(&mut self) -> Result<()> { + info!("all chains synced draining to bus and running sync end"); + for evt in self.evm_buffer.drain(..) { + let (data, ts, _) = evt.split(); + self.bus.publish_from_remote(data, ts)?; + } + self.bus.publish(SyncEnd::new())?; + Ok(()) + } } impl Actor for Synchronizer { @@ -48,14 +80,10 @@ impl Handler for Synchronizer { trap(EType::Sync, &self.bus.clone(), || { match msg { // Buffer events as the sync actor receives them - SyncEvmEvent::Event(event) => self.evm_buffer.push(event), + SyncEvmEvent::Event(event) => self.buffer_evm_event(event), // When we hear that sync is complete send all events on chain then publish SyncEnd - SyncEvmEvent::HistoricalSyncComplete(_) => { - for evt in self.evm_buffer.drain(..) { - let (data, ts, _) = evt.split(); - self.bus.publish_from_remote(data, ts)?; - } - self.bus.publish(SyncEnd::new())?; + SyncEvmEvent::HistoricalSyncComplete(chain_id) => { + self.handle_sync_complete(chain_id)? } }; Ok(()) From eb7c2d64614c2a791fa4c10fbd9bb402d8d51188 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 27 Jan 2026 21:44:33 +0000 Subject: [PATCH 090/102] evm_reader -> evm_parser --- crates/evm/src/bonding_registry_sol.rs | 6 +++--- crates/evm/src/ciphernode_registry_sol.rs | 8 ++++---- crates/evm/src/enclave_sol.rs | 4 ++-- crates/evm/src/enclave_sol_reader.rs | 6 +++--- crates/evm/src/{evm_reader.rs => evm_parser.rs} | 8 ++++---- crates/evm/src/fix_historical_order.rs | 2 +- crates/evm/src/lib.rs | 4 ++-- crates/evm/tests/integration.rs | 8 ++++---- 8 files changed, 23 insertions(+), 23 deletions(-) rename crates/evm/src/{evm_reader.rs => evm_parser.rs} (95%) diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index a54f948ca1..e9dca8572a 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{events::EvmEventProcessor, evm_reader::EvmReader}; +use crate::{events::EvmEventProcessor, evm_parser::EvmParser}; use actix::{Actor, Addr}; use alloy::{ primitives::{LogData, B256}, @@ -137,7 +137,7 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< /// Connects to BondingRegistry.sol converting EVM events to EnclaveEvents pub struct BondingRegistrySolReader; impl BondingRegistrySolReader { - pub fn setup(next: &EvmEventProcessor) -> Addr { - EvmReader::new(next, extractor).start() + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() } } diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 764d407146..45fc08a6e5 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -6,7 +6,7 @@ use crate::{ events::{EnclaveEvmEvent, EvmEventProcessor}, - evm_reader::EvmReader, + evm_parser::EvmParser, helpers::{send_tx_with_retry, EthProvider}, }; use actix::prelude::*; @@ -217,8 +217,8 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { - pub fn setup(next: &EvmEventProcessor) -> Addr { - EvmReader::new(next, extractor).start() + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() } } @@ -526,7 +526,7 @@ pub async fn publish_committee_to_registry) -> Addr { + pub fn attach(processor: &Recipient) -> Addr { CiphernodeRegistrySolReader::setup(processor) } diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index 2bdc2a746a..69df39b45c 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -6,7 +6,7 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - events::EvmEventProcessor, evm_reader::EvmReader, helpers::EthProvider, + events::EvmEventProcessor, evm_parser::EvmParser, helpers::EthProvider, }; use actix::Addr; use alloy::providers::{Provider, WalletProvider}; @@ -22,7 +22,7 @@ impl EnclaveSol { bus: &BusHandle, write_provider: EthProvider, contract_address: Address, - ) -> Result> + ) -> Result> where W: Provider + WalletProvider + Clone + 'static, { diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index 6cb717983b..8d89f9c8bb 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::events::EvmEventProcessor; -use crate::evm_reader::EvmReader; +use crate::evm_parser::EvmParser; use actix::{Actor, Addr}; use alloy::primitives::{LogData, B256}; use alloy::{sol, sol_types::SolEvent}; @@ -138,7 +138,7 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< pub struct EnclaveSolReader; impl EnclaveSolReader { - pub fn setup(next: &EvmEventProcessor) -> Addr { - EvmReader::new(next, extractor).start() + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() } } diff --git a/crates/evm/src/evm_reader.rs b/crates/evm/src/evm_parser.rs similarity index 95% rename from crates/evm/src/evm_reader.rs rename to crates/evm/src/evm_parser.rs index acd80ef1fd..2710fd10a9 100644 --- a/crates/evm/src/evm_reader.rs +++ b/crates/evm/src/evm_parser.rs @@ -13,16 +13,16 @@ use crate::{ ExtractorFn, }; -pub struct EvmReader { +pub struct EvmParser { next: EvmEventProcessor, extractor: ExtractorFn, } -impl Actor for EvmReader { +impl Actor for EvmParser { type Context = actix::Context; } -impl EvmReader { +impl EvmParser { pub fn new(next: &EvmEventProcessor, extractor: ExtractorFn) -> Self { Self { next: next.clone(), @@ -31,7 +31,7 @@ impl EvmReader { } } -impl Handler for EvmReader { +impl Handler for EvmParser { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { match msg.clone() { diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index be6d51dbcf..1faf833d21 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -8,7 +8,7 @@ use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; use actix::{Actor, Addr, Handler}; use bloom::{BloomFilter, ASMS}; use e3_events::CorrelationId; -use tracing::{info, warn}; +use tracing::info; pub struct FixHistoricalOrder { dest: EvmEventProcessor, diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index ad62c276a5..47f8b45f31 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -12,8 +12,8 @@ mod enclave_sol_writer; mod events; mod evm_chain_gateway; mod evm_hub; +mod evm_parser; mod evm_read_interface; -mod evm_reader; mod evm_router; mod fix_historical_order; pub mod helpers; @@ -31,8 +31,8 @@ pub use enclave_sol_writer::EnclaveSolWriter; pub use events::*; pub use evm_chain_gateway::*; pub use evm_hub::*; +pub use evm_parser::*; pub use evm_read_interface::*; -pub use evm_reader::*; pub use evm_router::*; pub use fix_historical_order::*; pub use helpers::*; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index de10ec2e5f..b87029282d 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -19,8 +19,8 @@ use e3_events::{ prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, EvmEventConfig, EvmEventConfigChain, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, }; -use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmReader}; -use std::{collections::HashMap, sync::Arc, time::Duration}; +use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; +use std::{sync::Arc, time::Duration}; use tokio::time::sleep; use tracing::subscriber::DefaultGuard; use tracing_subscriber::{fmt, EnvFilter}; @@ -56,8 +56,8 @@ fn test_event_extractor( struct TestEventParser; impl TestEventParser { - pub fn setup(next: &EvmEventProcessor) -> Addr { - EvmReader::new(next, test_event_extractor).start() + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, test_event_extractor).start() } } From 8cddbaf2f9dc24fbc88ebb92aa670db09c766420 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 28 Jan 2026 02:55:43 +0000 Subject: [PATCH 091/102] update timestamp max diff --- crates/events/src/enclave_event/sync_start.rs | 4 ++++ crates/events/src/hlc.rs | 3 +-- crates/evm/src/evm_read_interface.rs | 5 +++-- crates/sync/src/sync.rs | 13 +++++++++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index d4fe27d32c..3841785133 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -52,6 +52,10 @@ impl EvmEvent { pub fn chain_id(&self) -> u64 { self.chain_id } + + pub fn ts(&self) -> u128 { + self.ts + } } /// Dispatched by the Sync actor when initial data is read and the sync process needs to be started diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index 21b98e2655..c476fdca62 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -4,7 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use alloy::rpc::types::Log; use rand::Rng; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::{Arc, Mutex}; @@ -205,7 +204,7 @@ impl PartialEq for Hlc { } impl Hlc { - const DEFAULT_MAX_DRIFT: u64 = 60_000_000; // 60 sec + const DEFAULT_MAX_DRIFT: u64 = 5 * 60 * 1_000_000; // 5 min pub fn new(node: u32) -> Self { Self { diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index 14f6c9505b..e37596bf23 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -19,9 +19,10 @@ use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscribe use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::select; use tokio::sync::oneshot; -use tracing::{error, info, instrument}; +use tracing::{error, info, instrument, warn}; pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; @@ -176,7 +177,7 @@ async fn stream_from_evm( } } let historical_sync_event = HistoricalSyncComplete::new(chain_id, last_id); - info!( + warn!( "Historical Sync Complete event({})", historical_sync_event.get_id() ); diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 9f4a90f712..34de4c6ac3 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -9,8 +9,8 @@ use std::collections::{HashMap, HashSet}; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{ - trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, - SyncStart, + trap, BusHandle, EType, EnclaveEvent, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, + SyncEvmEvent, SyncStart, }; use tracing::info; @@ -58,9 +58,14 @@ impl Synchronizer { fn handle_sync_end(&mut self) -> Result<()> { info!("all chains synced draining to bus and running sync end"); + // Order all events (theoretically) + self.evm_buffer.sort_by_key(|i| i.ts()); + + // publish them in order for evt in self.evm_buffer.drain(..) { - let (data, ts, _) = evt.split(); - self.bus.publish_from_remote(data, ts)?; + let (data, _, _) = evt.split(); + self.bus.publish(data)?; // Use publish here as historical events will be correctly + // ordered as part of the preparatory process } self.bus.publish(SyncEnd::new())?; Ok(()) From ee7e9b6c045249c441a3127656cdc513067d9a36 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 28 Jan 2026 08:17:12 +0000 Subject: [PATCH 092/102] add test --- Cargo.lock | 2 + crates/sync/Cargo.toml | 4 ++ crates/sync/src/sync.rs | 140 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 143 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3589ddcdae..9c09850119 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3402,7 +3402,9 @@ version = "0.1.7" dependencies = [ "actix", "anyhow", + "e3-ciphernode-builder", "e3-events", + "tokio", "tracing", ] diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index 571cdf8b0e..97ba4b7fea 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -10,4 +10,8 @@ repository = "https://github.com/gnosisguild/enclave/crates/sync" actix.workspace = true anyhow.workspace = true e3-events.workspace = true +tokio.workspace = true tracing.workspace = true + +[dev-dependencies] +e3-ciphernode-builder.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 34de4c6ac3..27fc8efff0 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -4,13 +4,13 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{ - trap, BusHandle, EType, EnclaveEvent, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, - SyncEvmEvent, SyncStart, + trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, + SyncStart, }; use tracing::info; @@ -113,3 +113,137 @@ impl Handler for Synchronizer { #[derive(Message)] #[rtype("()")] pub struct Bootstrap; + +#[cfg(test)] +mod tests { + use super::*; + use e3_ciphernode_builder::EventSystem; + use e3_events::{ + CorrelationId, EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, + TestEvent, + }; + use e3_events::{EnclaveEvent, EventContextAccessors}; + use std::time::Duration; + use tokio::time::sleep; + + fn hlc_faucet(bus: &BusHandle, num: usize) -> Result> { + let mut queue = Vec::new(); + for _ in 0..num { + queue.push(bus.ts()?) + } + + Ok(queue.into_iter()) + } + + async fn settle() { + sleep(Duration::from_millis(100)).await; + } + + #[actix::test] + async fn test_synchronizer_full_flow() -> Result<()> { + // Setup event system and synchronizer + let system = EventSystem::new("test").with_fresh_bus(); + let bus = system.handle()?; + let history_collector = bus.history(); + + // Configure test chains + let mut evm_config = EvmEventConfig::new(); + evm_config.insert(1, EvmEventConfigChain::new(0)); + evm_config.insert(2, EvmEventConfigChain::new(0)); + + // Start synchronizer + let sync_addr = Synchronizer::setup(&bus, evm_config); + settle().await; + + // Verify SyncStart was published + let history = history_collector + .send(GetEvents::::new()) + .await?; + let sync_start_count = history + .into_iter() + .filter(|e| matches!(e.get_data(), EnclaveEventData::SyncStart(_))) + .count(); + assert!(sync_start_count > 0, "SyncStart should be dispatched"); + + // Create test events with timestamps + let mut timelord = hlc_faucet(&bus, 100)?; + let (chain_1, chain_2) = (1, 2); + let (block_1, block_2) = (1, 2); + + // Test events - timestamps generated in order + let h_2_1 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), + block_1, + timelord.next().unwrap(), + chain_2, + )); + + let h_1_1 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), + block_1, + timelord.next().unwrap(), + chain_1, + )); + + let h_1_2 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("1-second", 1)), + block_2, + timelord.next().unwrap(), + chain_1, + )); + + let h_2_2 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), + block_2, + timelord.next().unwrap(), + chain_2, + )); + + // Chain completion signals + let hc_1 = SyncEvmEvent::HistoricalSyncComplete(chain_1); + let hc_2 = SyncEvmEvent::HistoricalSyncComplete(chain_2); + + // Send events in mixed order to test sorting + sync_addr.send(h_2_2).await?; + sync_addr.send(h_2_1).await?; + sync_addr.send(hc_2).await?; + sync_addr.send(h_1_1).await?; + sync_addr.send(h_1_2).await?; + sync_addr.send(hc_1).await?; + + settle().await; + + // Get final event history and verify ordering + let history = history_collector + .send(GetEvents::::new()) + .await?; + + let events: Vec = history + .into_iter() + .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) + .collect(); + + let event_strings: Vec = events + .into_iter() + .filter_map(|e| { + if let EnclaveEventData::TestEvent(data) = e.into_data() { + Some(data.msg) + } else { + None + } + }) + .collect(); + + // Events should be published in timestamp order + assert_eq!( + event_strings, + vec!["2-first", "1-first", "1-second", "2-second"] + ); + + Ok(()) + } +} From ebe9faa8b2782bfa2a5c2d17d0d08940a9ea34b2 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 04:06:44 +0000 Subject: [PATCH 093/102] fix bad merge --- crates/ciphernode-builder/src/ciphernode_builder.rs | 5 ----- crates/ciphernode-builder/src/evm_system.rs | 4 ++-- crates/evm/src/enclave_sol_writer.rs | 2 +- crates/evm/src/evm_chain_gateway.rs | 9 ++++++--- crates/evm/src/evm_read_interface.rs | 4 ++-- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 7a64442338..0332fa1259 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -438,11 +438,6 @@ impl CiphernodeBuilder { )) } - if matches!(self.keyshare, Some(KeyshareKind::NonThreshold)) { - info!("Setting up KeyshareExtension (legacy)!"); - e3_builder = e3_builder.with(KeyshareExtension::create(&bus, &addr, &self.cipher)) - } - info!("building..."); e3_builder.build().await?; diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 22fd0dced7..0c44393463 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -8,7 +8,7 @@ use std::mem::replace; use actix::Actor; use alloy::{primitives::Address, providers::Provider}; -use e3_events::{BusHandle, EventSubscriber, SyncStart}; +use e3_events::{BusHandle, EventSubscriber, EventType, SyncStart}; use e3_evm::{ EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, FixHistoricalOrder, OneShotRunner, SyncStartExtractor, @@ -74,6 +74,6 @@ impl EvmSystemChainBuilder

{ Ok(()) } })); - self.bus.subscribe("SyncStart", runner.recipient()); + self.bus.subscribe(EventType::SyncStart, runner.recipient()); } } diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index d85e1f465d..7a19354cda 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -1,4 +1,4 @@ -/ SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: LGPL-3.0-only // // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 68c287311d..8474766e8d 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -11,8 +11,8 @@ use actix::{Addr, Recipient}; use anyhow::Result; use anyhow::{bail, Context}; use e3_events::{ - trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, SyncEnd, SyncEvmEvent, - SyncStart, + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, SyncEnd, + SyncEvmEvent, SyncStart, }; use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; @@ -94,7 +94,10 @@ impl EvmChainGateway { pub fn setup(bus: &BusHandle) -> Addr { let addr = Self::new(bus).start(); - bus.subscribe_all(&["SyncStart", "SyncEnd"], addr.clone().recipient()); + bus.subscribe_all( + &[EventType::SyncStart, EventType::SyncEnd], + addr.clone().recipient(), + ); addr } diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index e37596bf23..ece45967f2 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -15,7 +15,7 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy_primitives::Address; use anyhow::anyhow; -use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscriber}; +use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscriber, EventType}; use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; @@ -111,7 +111,7 @@ impl EvmReadInterface

{ let addr = EvmReadInterface::new(params).start(); - bus.subscribe("Shutdown", addr.clone().into()); + bus.subscribe(EventType::Shutdown, addr.clone().into()); addr } } From cf1af4db5811a548655d49e345cd0be6e82d0e14 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 04:38:21 +0000 Subject: [PATCH 094/102] fix bad test --- crates/tests/tests/integration.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 81a45446c6..8dd76a3513 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -12,11 +12,13 @@ use e3_ciphernode_builder::{CiphernodeBuilder, EventSystem}; use e3_crypto::Cipher; use e3_events::{ prelude::*, BusHandle, CiphertextOutputPublished, CommitteeFinalized, ConfigurationUpdated, - E3Requested, E3id, EnclaveEventData, OperatorActivationChanged, PlaintextAggregated, - TicketBalanceUpdated, + E3Requested, E3id, EnclaveEvent, EnclaveEventData, OperatorActivationChanged, + PlaintextAggregated, Seed, TakeEvents, TicketBalanceUpdated, }; use e3_fhe_params::{encode_bfv_params, BfvParamSet, BfvPreset}; use e3_multithread::{Multithread, MultithreadReport, ToReport}; +use e3_net::events::{GossipData, NetEvent}; +use e3_net::NetEventTranslator; use e3_test_helpers::ciphernode_system::CiphernodeSystemBuilder; use e3_test_helpers::{create_seed_from_u64, create_shared_rng_from_u64, AddToCommittee}; use e3_trbfv::helpers::calculate_error_size; @@ -25,8 +27,11 @@ use e3_utils::utility_types::ArcBytes; use fhe::bfv::PublicKey; use fhe_traits::{DeserializeParametrized, Serialize}; use num_bigint::BigUint; +use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; use std::time::{Duration, Instant}; use std::{fs, sync::Arc}; +use tokio::sync::{broadcast, mpsc}; pub fn save_snapshot(file_name: &str, bytes: &[u8]) { println!("### WRITING SNAPSHOT TO `{file_name}` ###"); @@ -540,16 +545,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { #[actix::test] async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { - use e3_events::{EnclaveEvent, TakeEvents}; - use e3_net::events::GossipData; - use e3_net::{events::NetEvent, NetEventTranslator}; - use rand::SeedableRng; - use rand_chacha::ChaCha20Rng; - use std::sync::Arc; - use tokio::sync::broadcast; - use tokio::sync::mpsc; - - let seed = e3_events::Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); + let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); // Setup elements in test let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network @@ -563,8 +559,8 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // Capture messages from output on msgs vec let event = E3Requested { e3_id: E3id::new("1235", 1), - threshold_m: 2, - threshold_n: 5, + threshold_m: 3, + threshold_n: 3, seed: seed.clone(), params: ArcBytes::from_bytes(&[1, 2, 3, 4]), ..E3Requested::default() @@ -572,7 +568,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // lets send an event from the network let _ = event_tx.send(NetEvent::GossipData(GossipData::GossipBytes( - bus.event_from(event.clone())?.to_bytes()?, + bus.event_from(event.clone(), None)?.to_bytes()?, ))); // check the history of the event bus @@ -625,7 +621,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { let mut builder = CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_trbfv() .with_address(addr) - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() @@ -832,7 +828,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { let mut builder = CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_trbfv() .with_address(addr) - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() From f730adb8fc55d30acbd49b22c729b4b50af95271 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 05:58:35 +0000 Subject: [PATCH 095/102] ensure all non-future ctx.notify() calls run synchronously --- crates/aggregator/src/committee_finalizer.rs | 5 ++-- crates/aggregator/src/publickey_aggregator.rs | 28 ++++++++++++------- .../src/threshold_plaintext_aggregator.rs | 5 ++-- crates/evm/src/ciphernode_registry_sol.rs | 9 +++--- crates/evm/src/enclave_sol_writer.rs | 5 ++-- crates/keyshare/src/threshold_keyshare.rs | 17 ++++++----- crates/multithread/src/multithread.rs | 1 + crates/net/src/document_publisher.rs | 11 ++++---- crates/net/src/net_sync_manager.rs | 2 +- crates/sortition/src/ciphernode_selector.rs | 7 +++-- crates/sortition/src/sortition.rs | 19 +++++++------ crates/utils/src/actix.rs | 23 ++++++++++++++- 12 files changed, 87 insertions(+), 45 deletions(-) diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index 9c2b531214..de319534d8 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -9,6 +9,7 @@ use e3_events::{ prelude::*, trap, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EType, EnclaveEvent, EnclaveEventData, EventType, Shutdown, }; +use e3_utils::NotifySync; use std::collections::HashMap; use std::time::Duration; use tracing::{error, info}; @@ -48,8 +49,8 @@ impl Handler for CommitteeFinalizer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::CommitteeRequested(data) => ctx.notify(data), - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::CommitteeRequested(data) => self.notify_sync(ctx, data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index a3ea9d3d04..bfdf3b0195 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -12,8 +12,10 @@ use e3_events::{ prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, }; +use e3_events::{trap, EType}; use e3_fhe::{Fhe, GetAggregatePublicKey}; use e3_utils::ArcBytes; +use e3_utils::NotifySync; use std::sync::Arc; use tracing::{error, info}; @@ -139,11 +141,14 @@ impl Actor for PublicKeyAggregator { impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::KeyshareCreated(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(_) => ctx.notify(Die), - _ => (), - } + trap(EType::KeyGeneration, &self.bus.clone(), || { + match msg.into_data() { + EnclaveEventData::KeyshareCreated(data) => self.notify_sync(ctx, data)?, + EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + _ => (), + }; + Ok(()) + }); } } @@ -163,10 +168,13 @@ impl Handler for PublicKeyAggregator { self.add_keyshare(pubkey, node)?; if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { - ctx.notify(ComputeAggregate { - keyshares: keyshares.clone(), - e3_id, - }) + self.notify_sync( + ctx, + ComputeAggregate { + keyshares: keyshares.clone(), + e3_id, + }, + )? } Ok(()) @@ -215,6 +223,6 @@ impl Handler for PublicKeyAggregator { impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { - ctx.stop() + ctx.stop(); } } diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index ca861ff876..5024eb8d95 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -20,6 +20,7 @@ use e3_trbfv::{ TrBFVResponse, }; use e3_utils::utility_types::ArcBytes; +use e3_utils::NotifySync; use tracing::{debug, error, info, trace}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -263,8 +264,8 @@ impl Handler for ThresholdPlaintextAggregator { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { EnclaveEventData::DecryptionshareCreated(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(_) => ctx.notify(Die), - EnclaveEventData::ComputeResponse(data) => ctx.notify(data), + EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + EnclaveEventData::ComputeResponse(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index ce83e1dfbd..c6fe4cae97 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -23,6 +23,7 @@ use e3_events::{ EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; +use e3_utils::NotifySync; use tracing::{error, info, trace}; sol!( @@ -290,21 +291,21 @@ impl Handler EnclaveEventData::PublicKeyAggregated(data) => { // Only publish if the src and destination chains match if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); + self.notify_sync(ctx, data); } } EnclaveEventData::CommitteeFinalizeRequested(data) => { if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); + self.notify_sync(ctx, data); } } EnclaveEventData::TicketGenerated(data) => { // Submit ticket if chain matches if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); + self.notify_sync(ctx, data); } } - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 7a19354cda..cc8153bb79 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -25,6 +25,7 @@ use e3_events::EnclaveEventData; use e3_events::EventType; use e3_events::Shutdown; use e3_events::{E3id, EType, PlaintextAggregated}; +use e3_utils::NotifySync; use tracing::info; sol!( @@ -79,10 +80,10 @@ impl Handler for E EnclaveEventData::PlaintextAggregated(data) => { // Only publish if the src and destination chains match if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); + self.notify_sync(ctx, data); } } - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 7156ff159c..92e4ba1e8d 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -27,6 +27,7 @@ use e3_trbfv::{ shares::{BfvEncryptedShares, EncryptableVec, Encrypted, ShamirShare, SharedSecret}, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; +use e3_utils::NotifySync; use e3_utils::{to_ordered_vec, utility_types::ArcBytes}; use fhe::bfv::BfvParameters; use fhe::bfv::{PublicKey, SecretKey}; @@ -407,8 +408,6 @@ impl ThresholdKeyshare { } pub fn handle_compute_response(&mut self, msg: TypedEvent) -> Result<()> { - self.bus.set_ctx(msg.get_ctx()); - self.state.set_ctx(msg.get_ctx()); match &msg.response { TrBFVResponse::GenEsiSss(_) => self.handle_gen_esi_sss_response(msg), TrBFVResponse::GenPkShareAndSkSss(_) => { @@ -943,16 +942,20 @@ impl Handler for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.clone().into_data() { - EnclaveEventData::CiphernodeSelected(data) => ctx.notify(msg.to_typed_event(data)), - EnclaveEventData::CiphertextOutputPublished(data) => ctx.notify(data), + EnclaveEventData::CiphernodeSelected(data) => { + self.notify_sync(ctx, msg.to_typed_event(data)) + } + EnclaveEventData::CiphertextOutputPublished(data) => self.notify_sync(ctx, data), EnclaveEventData::ThresholdShareCreated(data) => { let _ = self.handle_threshold_share_created(data, ctx.address()); } EnclaveEventData::EncryptionKeyCreated(data) => { let _ = self.handle_encryption_key_created(data, ctx.address()); } - EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), - EnclaveEventData::ComputeResponse(data) => ctx.notify(msg.to_typed_event(data)), + EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + EnclaveEventData::ComputeResponse(data) => { + self.notify_sync(ctx, msg.to_typed_event(data)) + } _ => (), } } @@ -1065,7 +1068,7 @@ impl Handler for ThresholdKeyshare { fn handle(&mut self, _: E3RequestComplete, ctx: &mut Self::Context) -> Self::Result { self.encryption_key_collector = None; self.decryption_key_collector = None; - ctx.notify(Die); + self.notify_sync(ctx, Die); } } diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index 7bf35affec..09bf9e8f29 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -33,6 +33,7 @@ use e3_trbfv::calculate_threshold_decryption::calculate_threshold_decryption; use e3_trbfv::gen_esi_sss::gen_esi_sss; use e3_trbfv::gen_pk_share_and_sk_sss::gen_pk_share_and_sk_sss; use e3_trbfv::{TrBFVError, TrBFVRequest, TrBFVResponse}; +use e3_utils::NotifySync; use e3_utils::SharedRng; use rand::Rng; use tracing::error; diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 785ec8eb58..7db3f39c29 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -21,6 +21,7 @@ use e3_events::{ }; use e3_utils::retry::{retry_with_backoff, to_retry}; use e3_utils::ArcBytes; +use e3_utils::NotifySync; use futures::TryFutureExt; use serde::{Deserialize, Serialize}; use std::{ @@ -137,8 +138,8 @@ impl Handler for DocumentPublisher { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { EnclaveEventData::PublishDocumentRequested(data) => ctx.notify(data), - EnclaveEventData::CiphernodeSelected(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), + EnclaveEventData::CiphernodeSelected(data) => self.notify_sync(ctx, data), + EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), _ => (), } } @@ -508,9 +509,9 @@ impl Handler for EventConverter { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::ThresholdShareCreated(data) => ctx.notify(data), - EnclaveEventData::EncryptionKeyCreated(data) => ctx.notify(data), - EnclaveEventData::DocumentReceived(data) => ctx.notify(data), + EnclaveEventData::ThresholdShareCreated(data) => self.notify_sync(ctx, data), + EnclaveEventData::EncryptionKeyCreated(data) => self.notify_sync(ctx, data), + EnclaveEventData::DocumentReceived(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index d876b1bfaa..7ef3b3a23f 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -11,7 +11,7 @@ use e3_events::{ EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, ReceiveEvents, Unsequenced, }; -use e3_utils::{retry_with_backoff, to_retry, OnceTake}; +use e3_utils::{retry_with_backoff, to_retry, NotifySync, OnceTake}; use futures::TryFutureExt; use libp2p::request_response::ResponseChannel; use std::{collections::HashMap, sync::Arc, time::Duration}; diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 22fef4d79b..f152e4fa27 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -15,6 +15,7 @@ use e3_events::{ EnclaveEvent, EnclaveEventData, EventType, Shutdown, TicketGenerated, TicketId, }; use e3_request::E3Meta; +use e3_utils::NotifySync; use std::collections::HashMap; use tracing::info; @@ -69,9 +70,9 @@ impl Handler for CiphernodeSelector { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { EnclaveEventData::E3Requested(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), - EnclaveEventData::CommitteeFinalized(data) => ctx.notify(data), - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 3c63a9d3fb..c61466bd36 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -15,6 +15,7 @@ use e3_events::{ PlaintextOutputPublished, Seed, TicketBalanceUpdated, }; use e3_events::{BusHandle, EnclaveEventData}; +use e3_utils::NotifySync; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tracing::info; @@ -232,14 +233,16 @@ impl Handler for Sortition { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::CiphernodeAdded(data) => ctx.notify(data.clone()), - EnclaveEventData::CiphernodeRemoved(data) => ctx.notify(data.clone()), - EnclaveEventData::TicketBalanceUpdated(data) => ctx.notify(data.clone()), - EnclaveEventData::OperatorActivationChanged(data) => ctx.notify(data.clone()), - EnclaveEventData::ConfigurationUpdated(data) => ctx.notify(data.clone()), - EnclaveEventData::CommitteePublished(data) => ctx.notify(data.clone()), - EnclaveEventData::PlaintextOutputPublished(data) => ctx.notify(data.clone()), - EnclaveEventData::CommitteeFinalized(data) => ctx.notify(data.clone()), + EnclaveEventData::CiphernodeAdded(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CiphernodeRemoved(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::TicketBalanceUpdated(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::OperatorActivationChanged(data) => { + self.notify_sync(ctx, data.clone()) + } + EnclaveEventData::ConfigurationUpdated(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CommitteePublished(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::PlaintextOutputPublished(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data.clone()), _ => (), } } diff --git a/crates/utils/src/actix.rs b/crates/utils/src/actix.rs index f4a18173b3..63cd1dbc02 100644 --- a/crates/utils/src/actix.rs +++ b/crates/utils/src/actix.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, ResponseActFuture, WrapFuture}; +use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; @@ -17,3 +17,24 @@ pub fn bail_result(a: &T, msg: impl Into) -> ResponseActFuture let m: String = msg.into(); Box::pin(async { Err(anyhow!(m)) }.into_actor(a)) } + +/// Extension trait for synchronous message handling +pub trait NotifySync +where + M: Message, + Self: Actor + Handler, +{ + /// Handles a message immediately without queuing. + /// Drop-in replacement for `ctx.notify(msg)` without interleaving other events. + fn notify_sync(&mut self, ctx: &mut Self::Context, msg: M) -> >::Result { + >::handle(self, msg, ctx) + } +} + +// Blanket implementation for all actors that handle the message +impl NotifySync for A +where + A: Actor + Handler, + M: Message, +{ +} From 7e10b0d21c939773d9e076e197d974f63d7194ee Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 07:20:32 +0000 Subject: [PATCH 096/102] refactor problematic handler causing issues with sync --- .../src/ciphernode_builder.rs | 8 +- crates/sortition/src/ciphernode_selector.rs | 92 ++++-------- crates/sortition/src/sortition.rs | 140 ++++++++++++------ 3 files changed, 132 insertions(+), 108 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 0332fa1259..775666a877 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -381,18 +381,20 @@ impl CiphernodeBuilder { // Use the configured backend directly let default_backend = self.sortition_backend.clone(); + let ciphernode_selector = + CiphernodeSelector::attach(&bus, repositories.ciphernode_selector(), &addr).await?; + let sortition = Sortition::attach( &bus, repositories.sortition(), repositories.node_state(), repositories.finalized_committees(), default_backend, + ciphernode_selector, + &addr, ) .await?; - CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) - .await?; - // Setup evm system // TODO: gather an async handle from the event readers in thre following function // that closes when they shutdown and join it with the network manager joinhandle externally diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index f152e4fa27..b2f75c674d 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -4,7 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::sortition::{GetNodeIndex, Sortition}; +use crate::sortition::Sortition; +use crate::WithSortitionPartyTicket; use actix::prelude::*; use anyhow::bail; use anyhow::Result; @@ -23,7 +24,6 @@ use tracing::info; /// emits a TicketGenerated event (score sortition) to the event bus pub struct CiphernodeSelector { bus: BusHandle, - sortition: Addr, address: String, e3_cache: Persistable>, } @@ -35,13 +35,11 @@ impl Actor for CiphernodeSelector { impl CiphernodeSelector { pub fn new( bus: &BusHandle, - sortition: &Addr, e3_cache: Persistable>, address: &str, ) -> Self { Self { bus: bus.clone(), - sortition: sortition.clone(), e3_cache, address: address.to_owned(), } @@ -49,14 +47,13 @@ impl CiphernodeSelector { pub async fn attach( bus: &BusHandle, - sortition: &Addr, selector_store: Repository>, address: &str, ) -> Result> { let e3_cache = selector_store.load_or_default(HashMap::new()).await?; - let addr = CiphernodeSelector::new(bus, sortition, e3_cache, address).start(); + let addr = CiphernodeSelector::new(bus, e3_cache, address).start(); - bus.subscribe(EventType::E3Requested, addr.clone().recipient()); + bus.subscribe(EventType::E3RequestComplete, addr.clone().recipient()); bus.subscribe(EventType::CommitteeFinalized, addr.clone().recipient()); bus.subscribe(EventType::Shutdown, addr.clone().recipient()); @@ -69,7 +66,6 @@ impl Handler for CiphernodeSelector { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::E3Requested(data) => ctx.notify(data), EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data), EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), @@ -78,14 +74,15 @@ impl Handler for CiphernodeSelector { } } -impl Handler for CiphernodeSelector { - type Result = ResponseFuture<()>; +impl Handler> for CiphernodeSelector { + type Result = (); - fn handle(&mut self, data: E3Requested, _ctx: &mut Self::Context) -> Self::Result { - let address = self.address.clone(); - let sortition = self.sortition.clone(); + fn handle( + &mut self, + data: WithSortitionPartyTicket, + _ctx: &mut Self::Context, + ) -> Self::Result { let bus = self.bus.clone(); - let chain_id = data.e3_id.chain_id(); trap(EType::Sortition, &bus.clone(), || { self.e3_cache.try_mutate(|mut cache| { @@ -99,58 +96,33 @@ impl Handler for CiphernodeSelector { seed: data.seed, threshold_n: data.threshold_n, threshold_m: data.threshold_m, - params: data.params, + params: data.params.clone(), esi_per_ct: data.esi_per_ct, - error_size: data.error_size, + error_size: data.error_size.clone(), }, ); Ok(cache) - }) - }); + })?; - Box::pin(async move { - let seed = data.seed; - let size = data.threshold_n; - info!( - "Calling GetNodeIndex address={} seed={} size={}", - address.clone(), - seed, - size - ); - // TODO: instead of this it would be better to pass the event theough sortition and - // then decorate it with this information WithIndex - if let Ok(found_result) = sortition - .send(GetNodeIndex { - chain_id, - seed, - address: address.clone(), - size, - }) - .await - { - let Some((_party_id, ticket_id)) = found_result else { - info!(node = address, "Ciphernode was not selected"); - return; - }; - - if let Some(tid) = ticket_id { - info!( - node = address, - ticket_id = tid, - "Ticket generated for score sortition" - ); - trap(EType::Sortition, &bus.clone(), || { - bus.publish(TicketGenerated { - e3_id: data.e3_id.clone(), - ticket_id: TicketId::Score(tid), - node: address.clone(), - })?; - Ok(()) - }) - } - } else { - info!("This node is not selected"); + if !data.is_selected() { + info!(node = &data.address(), "Ciphernode was not selected"); + return Ok(()); } + + if let Some(tid) = data.ticket_id() { + info!( + node = &data.address(), + ticket_id = tid, + "Ticket generated for score sortition" + ); + bus.publish(TicketGenerated { + e3_id: data.e3_id.clone(), + ticket_id: TicketId::Score(tid), + node: data.address().to_owned(), + })?; + } + + Ok(()) }) } } diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index c61466bd36..2720f5c5ef 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -5,19 +5,21 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::backends::{SortitionBackend, SortitionList}; +use crate::CiphernodeSelector; use actix::prelude::*; use alloy::primitives::U256; use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, - ConfigurationUpdated, EType, EnclaveEvent, EventType, OperatorActivationChanged, + ConfigurationUpdated, E3Requested, EType, EnclaveEvent, EventType, OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, }; use e3_events::{BusHandle, EnclaveEventData}; use e3_utils::NotifySync; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::ops::Deref; use tracing::info; use tracing::instrument; use tracing::warn; @@ -95,21 +97,6 @@ impl NodeStateStore { } } -/// Message: ask the `Sortition` whether `address` would be in the -/// committee of size `size` for randomness `seed` on `chain_id`. -#[derive(Message, Clone, Debug, PartialEq, Eq)] -#[rtype(result = "Option<(u64, Option)>")] -pub struct GetNodeIndex { - /// Round seed / randomness used by the sortition algorithm. - pub seed: Seed, - /// Hex-encoded node address (e.g., `"0x..."`). - pub address: String, - /// Committee size (top-N). - pub size: usize, - /// Target chain. - pub chain_id: u64, -} - /// Message: request the current set of registered node addresses for `chain_id`. #[derive(Message, Clone, Debug)] #[rtype(result = "Vec")] @@ -118,6 +105,47 @@ pub struct GetNodes { pub chain_id: u64, } +#[derive(Message, Clone, Debug, PartialEq, Eq)] +#[rtype(result = "()")] +pub struct WithSortitionPartyTicket { + inner: T, + party_ticket_id: Option<(u64, Option)>, + address: String, +} + +impl WithSortitionPartyTicket { + pub fn new(inner: T, party_ticket_id: Option<(u64, Option)>, address: &str) -> Self { + Self { + inner, + party_ticket_id, + address: address.to_owned(), + } + } + + pub fn is_selected(&self) -> bool { + self.party_ticket_id.is_some() + } + + pub fn address(&self) -> &str { + self.address.as_ref() + } + + pub fn ticket_id(&self) -> Option { + self.party_ticket_id.and_then(|(_, ticket_id)| ticket_id) + } + + pub fn party_id(&self) -> Option { + self.party_ticket_id.map(|(party_id, _)| party_id) + } +} + +impl Deref for WithSortitionPartyTicket { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + /// Message to get the finalized committee nodes for a specific E3. #[derive(Message, Clone, Debug)] #[rtype(result = "Vec")] @@ -143,6 +171,10 @@ pub struct Sortition { bus: BusHandle, /// Persistent map of finalized committees per E3 finalized_committees: Persistable>>, + /// Address for the CiphernodeSelector + ciphernode_selector: Addr, + /// Address for the current node + address: String, } /// Parameters for constructing a `Sortition` actor. @@ -156,6 +188,10 @@ pub struct SortitionParams { pub node_state: Persistable>, /// Persistent map of finalized committees per E3 pub finalized_committees: Persistable>>, + /// Address for the CiphernodeSelector + pub ciphernode_selector: Addr, + /// Address for the current node + pub address: String, } impl Sortition { @@ -165,6 +201,8 @@ impl Sortition { node_state: params.node_state, bus: params.bus, finalized_committees: params.finalized_committees, + ciphernode_selector: params.ciphernode_selector, + address: params.address, } } @@ -175,6 +213,8 @@ impl Sortition { node_state_store: Repository>, committees_store: Repository>>, default_backend: SortitionBackend, + ciphernode_selector: Addr, + address: &str, ) -> Result> { let mut backends = backends_store.load_or_default(HashMap::new()).await?; let node_state = node_state_store.load_or_default(HashMap::new()).await?; @@ -190,12 +230,15 @@ impl Sortition { backends, node_state, finalized_committees, + ciphernode_selector, + address: address.to_owned(), }) .start(); // Subscribe to all relevant events bus.subscribe_all( &[ + EventType::E3Requested, EventType::CiphernodeAdded, EventType::CiphernodeRemoved, EventType::TicketBalanceUpdated, @@ -222,6 +265,26 @@ impl Sortition { .ok_or_else(|| anyhow::anyhow!("No backend for chain_id {}", chain_id))?; Ok(backend.nodes()) } + + pub fn get_node_index( + &self, + seed: Seed, + size: usize, + chain_id: u64, + ) -> Option<(u64, Option)> { + let bus = self.bus.clone(); + let map = self.backends.get()?; + let state_map = self.node_state.get()?; + let backend = map.get(&chain_id)?; + let state = state_map.get(&chain_id)?; + + backend + .get_index(seed, size, self.address.clone(), chain_id, state) + .unwrap_or_else(|err| { + bus.err(EType::Sortition, err); + None + }) + } } impl Actor for Sortition { @@ -233,6 +296,7 @@ impl Handler for Sortition { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { + EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, data.clone()), EnclaveEventData::CiphernodeAdded(data) => self.notify_sync(ctx, data.clone()), EnclaveEventData::CiphernodeRemoved(data) => self.notify_sync(ctx, data.clone()), EnclaveEventData::TicketBalanceUpdated(data) => self.notify_sync(ctx, data.clone()), @@ -248,6 +312,21 @@ impl Handler for Sortition { } } +impl Handler for Sortition { + type Result = (); + fn handle(&mut self, msg: E3Requested, ctx: &mut Self::Context) -> Self::Result { + let chain_id = msg.e3_id.chain_id(); + let seed = msg.seed; + let threshold_n = msg.threshold_n; + self.ciphernode_selector + .do_send(WithSortitionPartyTicket::new( + msg, + self.get_node_index(seed, threshold_n, chain_id), + &self.address, + )) + } +} + impl Handler for Sortition { type Result = (); @@ -501,35 +580,6 @@ impl Handler for Sortition { } } -impl Handler for Sortition { - type Result = ResponseFuture)>>; - - fn handle(&mut self, msg: GetNodeIndex, _ctx: &mut Self::Context) -> Self::Result { - let backends_snapshot = self.backends.get(); - let node_state_snapshot = self.node_state.get(); - let bus = self.bus.clone(); - - Box::pin(async move { - if let (Some(map), Some(state_map)) = (backends_snapshot, node_state_snapshot) { - if let (Some(backend), Some(state)) = - (map.get(&msg.chain_id), state_map.get(&msg.chain_id)) - { - backend - .get_index(msg.seed, msg.size, msg.address.clone(), msg.chain_id, state) - .unwrap_or_else(|err| { - bus.err(EType::Sortition, err); - None - }) - } else { - None - } - } else { - None - } - }) - } -} - impl Handler for Sortition { type Result = Vec; From 730d47c36e2c449c9f4aca4475464c226637ca2b Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 07:33:42 +0000 Subject: [PATCH 097/102] wait for E3Requested to be received before dispatching CommitteFinalized --- crates/sortition/src/ciphernode_selector.rs | 1 - crates/tests/tests/integration.rs | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index b2f75c674d..7245c10b69 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -4,7 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::sortition::Sortition; use crate::WithSortitionPartyTicket; use actix::prelude::*; use anyhow::bail; diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 8dd76a3513..90f268185e 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -260,6 +260,11 @@ async fn test_trbfv_actor() -> Result<()> { println!("Emitting CommitteeFinalized with {} nodes", committee.len()); + let expected = vec!["E3Requested"]; + let _ = nodes + .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) + .await?; + bus.publish(CommitteeFinalized { e3_id: e3_id.clone(), committee, @@ -268,7 +273,7 @@ async fn test_trbfv_actor() -> Result<()> { let committee_finalized_timer = Instant::now(); - let expected = vec!["E3Requested", "CommitteeFinalized"]; + let expected = vec!["CommitteeFinalized"]; let _ = nodes .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; From 39662f597b7005581c80bc84a1aac3a8a25ad69e Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 07:45:03 +0000 Subject: [PATCH 098/102] remove notify --- .../src/threshold_plaintext_aggregator.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 5024eb8d95..2345773ea9 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -319,12 +319,15 @@ impl Handler for ThresholdPlaintextAggregator { .. })) = act.state.get() { - ctx.notify(ComputeAggregate { - shares: shares.clone(), - ciphertext_output: ciphertext_output.clone(), - threshold_m, - threshold_n, - }) + act.notify_sync( + ctx, + ComputeAggregate { + shares: shares.clone(), + ciphertext_output: ciphertext_output.clone(), + threshold_m, + threshold_n, + }, + ) } Ok(()) From 172ab1e6e2e86dbf62dd988a1348b88bc457c6ca Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 08:16:22 +0000 Subject: [PATCH 099/102] revert notify sync to just use notify for async handlers --- crates/evm/src/ciphernode_registry_sol.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index c6fe4cae97..3448590ce9 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -291,18 +291,18 @@ impl Handler EnclaveEventData::PublicKeyAggregated(data) => { // Only publish if the src and destination chains match if self.provider.chain_id() == data.e3_id.chain_id() { - self.notify_sync(ctx, data); + ctx.notify(data); } } EnclaveEventData::CommitteeFinalizeRequested(data) => { if self.provider.chain_id() == data.e3_id.chain_id() { - self.notify_sync(ctx, data); + ctx.notify(data); } } EnclaveEventData::TicketGenerated(data) => { // Submit ticket if chain matches if self.provider.chain_id() == data.e3_id.chain_id() { - self.notify_sync(ctx, data); + ctx.notify(data); } } EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), From 94819ccc490bc242c65ad1d620f50ced1c149e1e Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 08:22:35 +0000 Subject: [PATCH 100/102] rename functions --- crates/evm/src/evm_chain_gateway.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 8474766e8d..bdb0872c6c 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -109,7 +109,7 @@ impl EvmChainGateway { let mut buffer = self.status.forward_to_sync_actor(sender)?; // Drain any events that were buffered early for evt in buffer.drain(..) { - self.handle_receive_evm_event(evt)?; + self.process_evm_event(evt)?; } Ok(()) } @@ -132,18 +132,18 @@ impl EvmChainGateway { fn handle_evm_event(&mut self, msg: EnclaveEvmEvent) -> Result<()> { match msg { EnclaveEvmEvent::HistoricalSyncComplete(e) => { - self.handle_historical_sync_complete(e)?; + self.forward_historical_sync_complete(e)?; Ok(()) } EnclaveEvmEvent::Event(event) => { - self.handle_receive_evm_event(event)?; + self.process_evm_event(event)?; Ok(()) } _ => panic!("EvmChainGateway is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), } } - fn handle_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { + fn forward_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { info!( "handling historical sync complete for chain_id({})", event.chain_id @@ -154,7 +154,7 @@ impl EvmChainGateway { Ok(()) } - fn handle_receive_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + fn process_evm_event(&mut self, msg: EvmEvent) -> Result<()> { match &mut self.status { SyncStatus::BufferUntilLive(buffer) => { info!("saving evm event({}) to pre-live buffer", msg.get_id()); From 2bd71b78c9f3cf0c56f6812287cad188170bece9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 08:53:42 +0000 Subject: [PATCH 101/102] do_send -> try_send --- crates/evm/src/evm_chain_gateway.rs | 2 +- crates/net/src/net_sync_manager.rs | 17 +++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index bdb0872c6c..d730c1585f 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -150,7 +150,7 @@ impl EvmChainGateway { ); let sender = self.status.buffer_until_live()?; info!("Sending historical sync complete event to sender."); - sender.do_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id)); + sender.try_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id))?; Ok(()) } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 7ef3b3a23f..7d6c9a5a21 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -11,7 +11,7 @@ use e3_events::{ EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, ReceiveEvents, Unsequenced, }; -use e3_utils::{retry_with_backoff, to_retry, NotifySync, OnceTake}; +use e3_utils::{retry_with_backoff, to_retry, OnceTake}; use futures::TryFutureExt; use libp2p::request_response::ResponseChannel; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -99,11 +99,11 @@ impl Handler for NetSyncManager { /// SyncRequest is called on start up to fetch remote events impl Handler for NetSyncManager { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: OutgoingSyncRequested, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: OutgoingSyncRequested, ctx: &mut Self::Context) -> Self::Result { trap_fut( EType::Net, &self.bus.clone(), - handle_sync_request_event(self.tx.clone(), self.rx.clone(), self.bus.clone(), msg), + handle_sync_request_event(self.tx.clone(), self.rx.clone(), msg, ctx.address()), ) } } @@ -199,8 +199,8 @@ async fn sync_request( async fn handle_sync_request_event( net_cmds: mpsc::Sender, net_events: Arc>, - bus: BusHandle, event: OutgoingSyncRequested, + address: impl Into>, ) -> Result<()> { let value = retry_with_backoff( || { @@ -216,13 +216,6 @@ async fn handle_sync_request_event( ) .await?; - bus.publish(NetSyncEventsReceived::new( - value - .value - .events - .into_iter() - .map(|data| data.try_into()) - .collect::>>>()?, - ))?; + address.into().try_send(value)?; Ok(()) } From c1b7195bca8ef623115159b894d22ea5370713fd Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 29 Jan 2026 08:56:20 +0000 Subject: [PATCH 102/102] fix notify on future handler --- crates/evm/src/enclave_sol_writer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index cc8153bb79..68a1f6adc9 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -80,7 +80,7 @@ impl Handler for E EnclaveEventData::PlaintextAggregated(data) => { // Only publish if the src and destination chains match if self.provider.chain_id() == data.e3_id.chain_id() { - self.notify_sync(ctx, data); + ctx.notify(data); } } EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data),