From a366209a9ff1d0593fb0adfd1058e7c278f5a453 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 30 Jan 2026 09:47:24 +0000 Subject: [PATCH 01/63] refactor: make EventContextManager::set_ctx accept generic Into --- crates/data/src/persistable.rs | 7 +++++-- crates/events/src/bus_handle.rs | 7 +++++-- crates/events/src/enclave_event/mod.rs | 10 ++++++++++ crates/events/src/enclave_event/typed_event.rs | 7 +++++-- crates/events/src/traits.rs | 4 +++- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 7247a252b0..a47b0920d2 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -251,8 +251,11 @@ impl EventContextManager for Persistable { self.ctx.clone() } - fn set_ctx(&mut self, value: &EventContext) { - self.ctx = Some(value.clone()) + fn set_ctx(&mut self, value: C) + where + C: Into>, + { + self.ctx = Some(value.into().clone()) } } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 1863054021..61808d3607 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -175,8 +175,11 @@ impl EventSubscriber> for BusHandle { } impl EventContextManager for BusHandle { - fn set_ctx(&mut self, value: &EventContext) { - self.ctx = Some(value.clone()); + fn set_ctx(&mut self, value: C) + where + C: Into>, + { + self.ctx = Some(value.into().clone()); } fn get_ctx(&self) -> Option> { self.ctx.clone() diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 514ec33337..b9db88f62e 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -508,6 +508,16 @@ impl TryFrom> for EnclaveError { } } } +impl From> for EventContext { + fn from(value: EnclaveEvent) -> Self { + (&value).into() + } +} +impl From<&EnclaveEvent> for EventContext { + fn from(value: &EnclaveEvent) -> Self { + value.ctx.clone() + } +} // Add convenience method to EnclaveEvent impl EnclaveEvent { diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 7f3da4c0ee..f054474391 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -24,8 +24,11 @@ pub struct TypedEvent { } impl TypedEvent { - pub fn new(inner: T, ctx: EventContext) -> Self { - Self { inner, ctx } + pub fn new(inner: T, ctx: impl Into>) -> Self { + Self { + inner, + ctx: ctx.into(), + } } pub fn into_inner(self) -> T { diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 3feba8f434..364d057630 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -168,6 +168,8 @@ pub trait WithAggregateId { /// An EventContextManager hold the current event context for use in event publishing and /// persistence management pub trait EventContextManager { - fn set_ctx(&mut self, value: &EventContext); + fn set_ctx(&mut self, value: C) + where + C: Into>; fn get_ctx(&self) -> Option>; } From a7c560a06c3b10830470f2c94b7c449e57a5d9d9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 30 Jan 2026 11:13:08 +0000 Subject: [PATCH 02/63] remove redundant handlers --- .../src/threshold_plaintext_aggregator.rs | 127 ++++++------- .../enclave_event/decryptionshare_created.rs | 2 +- crates/sortition/src/ciphernode_selector.rs | 6 +- crates/sortition/src/sortition.rs | 172 +++++++++++------- 4 files changed, 179 insertions(+), 128 deletions(-) diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 2345773ea9..46855ff218 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -14,14 +14,14 @@ use e3_events::{ DecryptionshareCreated, Die, E3id, EType, EnclaveEvent, EnclaveEventData, PlaintextAggregated, Seed, }; -use e3_sortition::{GetNodesForE3, Sortition}; +use e3_sortition::{E3CommitteeContainsRequest, E3CommitteeContainsResponse, Sortition}; use e3_trbfv::{ calculate_threshold_decryption::CalculateThresholdDecryptionRequest, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; use e3_utils::utility_types::ArcBytes; use e3_utils::NotifySync; -use tracing::{debug, error, info, trace}; +use tracing::{debug, info, trace}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Collecting { @@ -272,67 +272,72 @@ impl Handler for ThresholdPlaintextAggregator { } impl Handler for ThresholdPlaintextAggregator { - type Result = ResponseActFuture>; - - fn handle(&mut self, event: DecryptionshareCreated, _: &mut Self::Context) -> Self::Result { - let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = - self.state.get() - else { - debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); - return Box::pin(fut::ready(Ok(()))); - }; - info!(event=?event, "Processing DecryptionShareCreated..."); - let address = event.node.clone(); - let party_id = event.party_id; - let e3_id = event.e3_id.clone(); - let decryption_share = event.decryption_share.clone(); - - Box::pin( - self.sortition - .send(GetNodesForE3 { - e3_id: e3_id.clone(), - chain_id: e3_id.chain_id(), - }) - .into_actor(self) - .map(move |res, act, ctx| { - let nodes = res?; - - if !nodes.contains(&address) { - trace!("Node {} not found in finalized committee", address); - return Ok(()); - } - - if e3_id != act.e3_id { - error!("Wrong e3_id sent to aggregator. This should not happen."); - return Ok(()); - } - - // Trust the party_id from the event - it's based on CommitteeFinalized order - // which is the authoritative source of truth for party IDs - act.add_share(party_id, decryption_share)?; - - if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { + type Result = (); + fn handle(&mut self, msg: DecryptionshareCreated, ctx: &mut Self::Context) -> Self::Result { + trap(EType::PublickeyAggregation, &self.bus.clone(), || { + let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = + self.state.get() + else { + debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); + return Ok(()); + }; + + let request = + E3CommitteeContainsRequest::new(&msg.e3_id, &msg.node, msg.clone(), ctx.address()); + self.sortition.try_send(request)?; + Ok(()) + }) + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + fn handle( + &mut self, + msg: E3CommitteeContainsResponse, + ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::PublickeyAggregation, &self.bus.clone(), || { + let e3_id = &msg.e3_id; + if *e3_id != self.e3_id { + bail!("Wrong e3_id sent to aggregator. This should not happen.") + }; + + if !msg.is_found_in_committee() { + trace!("Node {} not found in finalized committee", &msg.node); + return Ok(()); + }; + + // Trust the party_id from the event - it's based on CommitteeFinalized order + // which is the authoritative source of truth for party IDs + let DecryptionshareCreated { + party_id, + decryption_share, + .. + } = msg.into_inner(); + + self.add_share(party_id, decryption_share)?; + + if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { + threshold_m, + threshold_n, + shares, + ciphertext_output, + .. + })) = self.state.get() + { + self.notify_sync( + ctx, + ComputeAggregate { + shares: shares.clone(), + ciphertext_output: ciphertext_output.clone(), threshold_m, threshold_n, - shares, - ciphertext_output, - .. - })) = act.state.get() - { - act.notify_sync( - ctx, - ComputeAggregate { - shares: shares.clone(), - ciphertext_output: ciphertext_output.clone(), - threshold_m, - threshold_n, - }, - ) - } - - Ok(()) - }), - ) + }, + ) + } + Ok(()) + }) } } diff --git a/crates/events/src/enclave_event/decryptionshare_created.rs b/crates/events/src/enclave_event/decryptionshare_created.rs index 0df8177adb..1916db7068 100644 --- a/crates/events/src/enclave_event/decryptionshare_created.rs +++ b/crates/events/src/enclave_event/decryptionshare_created.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[rtype(result = "anyhow::Result<()>")] +#[rtype(result = "()")] pub struct DecryptionshareCreated { pub party_id: u64, pub decryption_share: Vec, // per index depending on what is required for the diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 7245c10b69..71f5820026 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::WithSortitionPartyTicket; +use crate::WithSortitionTicket; use actix::prelude::*; use anyhow::bail; use anyhow::Result; @@ -73,12 +73,12 @@ impl Handler for CiphernodeSelector { } } -impl Handler> for CiphernodeSelector { +impl Handler> for CiphernodeSelector { type Result = (); fn handle( &mut self, - data: WithSortitionPartyTicket, + data: WithSortitionTicket, _ctx: &mut Self::Context, ) -> Self::Result { let bus = self.bus.clone(); diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 2720f5c5ef..772261c7f5 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -11,9 +11,9 @@ use alloy::primitives::U256; use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ - prelude::*, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, - ConfigurationUpdated, E3Requested, EType, EnclaveEvent, EventType, OperatorActivationChanged, - PlaintextOutputPublished, Seed, TicketBalanceUpdated, + prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, + ConfigurationUpdated, E3Requested, E3id, EType, EnclaveEvent, EventType, + OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, }; use e3_events::{BusHandle, EnclaveEventData}; use e3_utils::NotifySync; @@ -97,23 +97,15 @@ impl NodeStateStore { } } -/// Message: request the current set of registered node addresses for `chain_id`. -#[derive(Message, Clone, Debug)] -#[rtype(result = "Vec")] -pub struct GetNodes { - /// Target chain. - pub chain_id: u64, -} - #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "()")] -pub struct WithSortitionPartyTicket { +pub struct WithSortitionTicket { inner: T, party_ticket_id: Option<(u64, Option)>, address: String, } -impl WithSortitionPartyTicket { +impl WithSortitionTicket { pub fn new(inner: T, party_ticket_id: Option<(u64, Option)>, address: &str) -> Self { Self { inner, @@ -139,21 +131,76 @@ impl WithSortitionPartyTicket { } } -impl Deref for WithSortitionPartyTicket { +impl Deref for WithSortitionTicket { type Target = T; fn deref(&self) -> &Self::Target { &self.inner } } -/// Message to get the finalized committee nodes for a specific E3. -#[derive(Message, Clone, Debug)] -#[rtype(result = "Vec")] -pub struct GetNodesForE3 { - /// E3 ID to get nodes for. - pub e3_id: e3_events::E3id, - /// Chain ID - pub chain_id: u64, +#[derive(Message, Clone, Debug, PartialEq, Eq)] +#[rtype(result = "()")] +pub struct E3CommitteeContainsRequest +where + T: Send + Sync, +{ + inner: T, + e3_id: E3id, + node: String, + sender: Recipient>, +} + +impl E3CommitteeContainsRequest +where + T: Send + Sync, +{ + pub fn new( + e3_id: &E3id, + node: &str, + inner: T, + sender: impl Into>>, + ) -> Self { + Self { + inner, + e3_id: e3_id.clone(), + node: node.to_owned(), + sender: sender.into(), + } + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq)] +#[rtype(result = "()")] +pub struct E3CommitteeContainsResponse { + inner: T, + is_found_in_committee: bool, +} + +impl E3CommitteeContainsResponse +where + T: Send + Sync, +{ + pub fn new(inner: T, is_found_in_committee: bool) -> Self { + Self { + inner, + is_found_in_committee, + } + } + + pub fn is_found_in_committee(&self) -> bool { + self.is_found_in_committee + } + + pub fn into_inner(self) -> T { + self.inner + } +} + +impl Deref for E3CommitteeContainsResponse { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.inner + } } /// Message to get the current node state. @@ -285,6 +332,18 @@ impl Sortition { None }) } + + fn get_committe(&self, e3_id: E3id) -> Vec { + self.finalized_committees + .get() + .and_then(|committees| committees.get(&e3_id).cloned()) + .unwrap_or_else(|| Vec::new()) + } + + fn committee_contains(&mut self, e3_id: E3id, node: String) -> bool { + let committee = self.get_committe(e3_id); + committee.contains(&node) + } } impl Actor for Sortition { @@ -314,16 +373,15 @@ impl Handler for Sortition { impl Handler for Sortition { type Result = (); - fn handle(&mut self, msg: E3Requested, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: E3Requested, _: &mut Self::Context) -> Self::Result { let chain_id = msg.e3_id.chain_id(); let seed = msg.seed; let threshold_n = msg.threshold_n; - self.ciphernode_selector - .do_send(WithSortitionPartyTicket::new( - msg, - self.get_node_index(seed, threshold_n, chain_id), - &self.address, - )) + self.ciphernode_selector.do_send(WithSortitionTicket::new( + msg, + self.get_node_index(seed, threshold_n, chain_id), + &self.address, + )); } } @@ -511,6 +569,28 @@ impl Handler for Sortition { } } } + +impl Handler> for Sortition +where + T: Clone + Send + Sync + 'static, +{ + type Result = (); + fn handle( + &mut self, + msg: E3CommitteeContainsRequest, + _: &mut Self::Context, + ) -> Self::Result { + trap(EType::Sortition, &self.bus.clone(), || { + let response = E3CommitteeContainsResponse::new( + msg.inner, + self.committee_contains(msg.e3_id, msg.node), + ); + msg.sender.try_send(response)?; + Ok(()) + }) + } +} + /// PlaintextOutputPublished is currently used as a signal to decrement the active jobs for the nodes in the committee /// But in reality, E3 Jobs might not emit that in case there are no votes or the job fails. /// We need to find a better way to handle the end of an E3, Reduce the jobs in case of of an Error @@ -580,40 +660,6 @@ impl Handler for Sortition { } } -impl Handler for Sortition { - type Result = Vec; - - fn handle(&mut self, msg: GetNodes, _ctx: &mut Self::Context) -> Self::Result { - self.get_nodes(msg.chain_id).unwrap_or_else(|err| { - tracing::warn!("Failed to get nodes for chain {}: {}", msg.chain_id, err); - Vec::new() - }) - } -} - -impl Handler for Sortition { - type Result = Vec; - - fn handle(&mut self, msg: GetNodesForE3, _ctx: &mut Self::Context) -> Self::Result { - if msg.e3_id.chain_id() != msg.chain_id { - tracing::warn!( - "Chain ID mismatch: e3_id has chain_id {}, but requested chain_id {}", - msg.e3_id.chain_id(), - msg.chain_id - ); - return Vec::new(); - } - - self.finalized_committees - .get() - .and_then(|committees| committees.get(&msg.e3_id).cloned()) - .unwrap_or_else(|| { - tracing::warn!("No finalized committee found for E3 {}", msg.e3_id); - Vec::new() - }) - } -} - impl Handler for Sortition { type Result = Option>; From f0c26dc3cea2b4d3193fe749129423da872c84ca Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 30 Jan 2026 11:23:47 +0000 Subject: [PATCH 03/63] tidy up --- crates/sortition/src/sortition.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 772261c7f5..d85593839d 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -8,7 +8,7 @@ use crate::backends::{SortitionBackend, SortitionList}; use crate::CiphernodeSelector; use actix::prelude::*; use alloy::primitives::U256; -use anyhow::Result; +use anyhow::{anyhow, Result}; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, @@ -306,10 +306,10 @@ impl Sortition { let map = self .backends .get() - .ok_or_else(|| anyhow::anyhow!("Could not get backends cache"))?; + .ok_or_else(|| anyhow!("Could not get backends cache"))?; let backend = map .get(&chain_id) - .ok_or_else(|| anyhow::anyhow!("No backend for chain_id {}", chain_id))?; + .ok_or_else(|| anyhow!("No backend for chain_id {}", chain_id))?; Ok(backend.nodes()) } @@ -333,15 +333,24 @@ impl Sortition { }) } - fn get_committe(&self, e3_id: E3id) -> Vec { + fn get_committe(&self, e3_id: &E3id) -> Vec { self.finalized_committees .get() - .and_then(|committees| committees.get(&e3_id).cloned()) + .and_then(|committees| committees.get(e3_id).cloned()) .unwrap_or_else(|| Vec::new()) } fn committee_contains(&mut self, e3_id: E3id, node: String) -> bool { - let committee = self.get_committe(e3_id); + let committee = self.get_committe(&e3_id); + + if committee.len() == 0 { + // Non blocking error + self.bus.err( + EType::Sortition, + anyhow!("No finalized committee found for E3 {}", e3_id), + ); + } + committee.contains(&node) } } From 8c2f68be30fb3c0e0fc20a0f86650f84ddd35cf5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 31 Jan 2026 06:29:03 +0000 Subject: [PATCH 04/63] tidy up memory management --- crates/aggregator/src/threshold_plaintext_aggregator.rs | 6 +++--- crates/sortition/src/sortition.rs | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 46855ff218..7e9cae7a66 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -281,9 +281,9 @@ impl Handler for ThresholdPlaintextAggregator { debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); return Ok(()); }; - - let request = - E3CommitteeContainsRequest::new(&msg.e3_id, &msg.node, msg.clone(), ctx.address()); + let node = msg.node.clone(); + let e3_id = msg.e3_id.clone(); + let request = E3CommitteeContainsRequest::new(e3_id, node, msg, ctx.address()); self.sortition.try_send(request)?; Ok(()) }) diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index d85593839d..1426d9e0bc 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -155,15 +155,15 @@ where T: Send + Sync, { pub fn new( - e3_id: &E3id, - node: &str, + e3_id: E3id, + node: String, inner: T, sender: impl Into>>, ) -> Self { Self { inner, - e3_id: e3_id.clone(), - node: node.to_owned(), + e3_id, + node, sender: sender.into(), } } From f97eef54eefab5bd6ee8fb66b602d548c277f775 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 31 Jan 2026 06:31:33 +0000 Subject: [PATCH 05/63] remove redundant handler --- crates/sortition/src/sortition.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 1426d9e0bc..54209aa5b9 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -203,11 +203,6 @@ impl Deref for E3CommitteeContainsResponse { } } -/// Message to get the current node state. -#[derive(Message, Clone, Debug)] -#[rtype(result = "Option>")] -pub struct GetNodeState; - /// Sortition actor that manages the sortition algorithm and the node state. pub struct Sortition { /// Persistent map of `chain_id -> SortitionBackend`. @@ -668,11 +663,3 @@ impl Handler for Sortition { } } } - -impl Handler for Sortition { - type Result = Option>; - - fn handle(&mut self, _msg: GetNodeState, _: &mut Self::Context) -> Self::Result { - self.node_state.get() - } -} From 6a13385aca53d6bc170a0d2586455ae5a19cd7bb Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 31 Jan 2026 12:10:59 +0000 Subject: [PATCH 06/63] refactor: add event context management to TypedEvent system - Update TypedEvent handlers to preserve event context across the system - Add publish_with_ctx method to maintain event causality chains - Refactor aggregator and keyshare components to use new context-aware publishing - Update bus handle to support both origin and context-aware event publishing - Ensure proper event correlation across distributed components --- crates/aggregator/src/committee_finalizer.rs | 2 +- crates/aggregator/src/publickey_aggregator.rs | 138 ++++++----- .../src/threshold_plaintext_aggregator.rs | 71 +++--- crates/ciphernode-builder/src/event_system.rs | 13 +- crates/entrypoint/src/helpers/shutdown.rs | 2 +- crates/events/src/bus_handle.rs | 65 ++++-- .../events/src/enclave_event/enclave_error.rs | 1 + crates/events/src/enclave_event/mod.rs | 4 + .../events/src/enclave_event/typed_event.rs | 4 + crates/events/src/sequencer.rs | 2 +- crates/events/src/traits.rs | 26 ++- crates/evm/src/evm_chain_gateway.rs | 4 +- crates/evm/tests/integration.rs | 6 +- .../keyshare/src/encryption_key_collector.rs | 16 +- crates/keyshare/src/threshold_keyshare.rs | 219 +++++++++++------- .../keyshare/src/threshold_share_collector.rs | 16 +- crates/multithread/src/multithread.rs | 30 +-- crates/net/src/document_publisher.rs | 202 +++++++++------- crates/net/src/events.rs | 6 +- crates/net/src/net_sync_manager.rs | 61 +++-- crates/request/src/router.rs | 6 +- crates/sortition/src/ciphernode_selector.rs | 64 ++--- crates/sortition/src/sortition.rs | 11 +- crates/sync/src/sync.rs | 15 +- crates/test-helpers/src/lib.rs | 2 +- crates/tests/tests/integration.rs | 38 ++- 26 files changed, 624 insertions(+), 400 deletions(-) diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index de319534d8..0d2d2b20b6 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -114,7 +114,7 @@ impl Handler for CommitteeFinalizer { info!(e3_id = %e3_id_clone, "Dispatching CommitteeFinalizeRequested event"); trap(EType::Sortition, &act.bus.clone(), || { - bus.publish(CommitteeFinalizeRequested { + bus.publish_origin(CommitteeFinalizeRequested { e3_id: e3_id_clone.clone(), })?; Ok(()) diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index bfdf3b0195..1ad56b4f05 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -10,7 +10,7 @@ use e3_bfv_client::client::compute_pk_commitment; use e3_data::Persistable; use e3_events::{ prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, KeyshareCreated, OrderedSet, - PublicKeyAggregated, Seed, + PublicKeyAggregated, Seed, TypedEvent, }; use e3_events::{trap, EType}; use e3_fhe::{Fhe, GetAggregatePublicKey}; @@ -142,8 +142,11 @@ impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { - match msg.into_data() { - EnclaveEventData::KeyshareCreated(data) => self.notify_sync(ctx, data)?, + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::KeyshareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), _ => (), }; @@ -152,71 +155,84 @@ impl Handler for PublicKeyAggregator { } } -impl Handler for PublicKeyAggregator { - type Result = Result<()>; - - fn handle(&mut self, event: KeyshareCreated, ctx: &mut Self::Context) -> Self::Result { - let e3_id = event.e3_id.clone(); - let pubkey = event.pubkey.clone(); - let node = event.node.clone(); - - if e3_id != self.e3_id { - error!("Wrong e3_id sent to aggregator. This should not happen."); - return Ok(()); - } +impl Handler> for PublicKeyAggregator { + type Result = (); - self.add_keyshare(pubkey, node)?; + fn handle( + &mut self, + event: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::PublickeyAggregation, &self.bus.clone(), || { + let (event, ec) = event.into_components(); + let e3_id = event.e3_id.clone(); + let pubkey = event.pubkey.clone(); + let node = event.node.clone(); + + if e3_id != self.e3_id { + error!("Wrong e3_id sent to aggregator. This should not happen."); + return Ok(()); + } - if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { - self.notify_sync( - ctx, - ComputeAggregate { - keyshares: keyshares.clone(), - e3_id, - }, - )? - } + self.add_keyshare(pubkey, node)?; + + if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { + self.notify_sync( + ctx, + TypedEvent::new( + ComputeAggregate { + keyshares: keyshares.clone(), + e3_id, + }, + ec, + ), + ) + } - Ok(()) + Ok(()) + }) } } -impl Handler for PublicKeyAggregator { - type Result = Result<()>; - - fn handle(&mut self, msg: ComputeAggregate, _: &mut Self::Context) -> Self::Result { - info!("Computing Aggregate PublicKey..."); - let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { - keyshares: msg.keyshares, - })?; - - let public_key_hash = compute_pk_commitment( - pubkey.clone(), - self.fhe.params.degree(), - self.fhe.params.plaintext(), - self.fhe.params.moduli().to_vec(), - )?; - - // Update the local state - self.set_pubkey(pubkey)?; - - if let Some(PublicKeyAggregatorState::Complete { - public_key: pubkey, - nodes, - .. - }) = self.state.get() - { - info!("Notifying network of PublicKey"); - info!("Sending PublicKeyAggregated..."); - let event = PublicKeyAggregated { - pubkey, - public_key_hash, - e3_id: msg.e3_id, +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::PublickeyAggregation, &self.bus.clone(), || { + let (msg, ec) = msg.into_components(); + info!("Computing Aggregate PublicKey..."); + let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { + keyshares: msg.keyshares, + })?; + + let public_key_hash = compute_pk_commitment( + pubkey.clone(), + self.fhe.params.degree(), + self.fhe.params.plaintext(), + self.fhe.params.moduli().to_vec(), + )?; + + // Update the local state + self.set_pubkey(pubkey)?; + + if let Some(PublicKeyAggregatorState::Complete { + public_key: pubkey, nodes, - }; - self.bus.publish(event)?; - } - Ok(()) + .. + }) = self.state.get() + { + info!("Notifying network of PublicKey"); + info!("Sending PublicKeyAggregated..."); + let event = PublicKeyAggregated { + pubkey, + public_key_hash, + e3_id: msg.e3_id, + nodes, + }; + self.bus.publish(event, ec)?; + } + Ok(()) + }) } } diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 7e9cae7a66..082226c342 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -12,7 +12,7 @@ use e3_data::Persistable; use e3_events::{ prelude::*, trap, BusHandle, ComputeRequest, ComputeResponse, CorrelationId, DecryptionshareCreated, Die, E3id, EType, EnclaveEvent, EnclaveEventData, PlaintextAggregated, - Seed, + Seed, TypedEvent, }; use e3_sortition::{E3CommitteeContainsRequest, E3CommitteeContainsResponse, Sortition}; use e3_trbfv::{ @@ -196,7 +196,8 @@ impl ThresholdPlaintextAggregator { }) } - pub fn handle_compute_aggregate(&mut self, msg: ComputeAggregate) -> Result<()> { + pub fn handle_compute_aggregate(&mut self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("create_calculate_threshold_decryption_event..."); let e3_id = self.e3_id.clone(); @@ -221,11 +222,12 @@ impl ThresholdPlaintextAggregator { CorrelationId::new(), e3_id, ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } - pub fn handle_compute_response(&mut self, msg: ComputeResponse) -> Result<()> { + pub fn handle_compute_response(&mut self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); ensure!( msg.e3_id == self.e3_id, "PlaintextAggregator should never receive incorrect e3_id msgs" @@ -250,7 +252,7 @@ impl ThresholdPlaintextAggregator { }; info!("Dispatching plaintext event {:?}", event); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } } @@ -262,18 +264,25 @@ impl Actor for ThresholdPlaintextAggregator { impl Handler for ThresholdPlaintextAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::DecryptionshareCreated(data) => ctx.notify(data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::DecryptionshareCreated(data) => ctx.notify(TypedEvent::new(data, ec)), EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), - EnclaveEventData::ComputeResponse(data) => self.notify_sync(ctx, data), + EnclaveEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for ThresholdPlaintextAggregator { +impl Handler> for ThresholdPlaintextAggregator { type Result = (); - fn handle(&mut self, msg: DecryptionshareCreated, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { trap(EType::PublickeyAggregation, &self.bus.clone(), || { let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = self.state.get() @@ -290,11 +299,13 @@ impl Handler for ThresholdPlaintextAggregator { } } -impl Handler> for ThresholdPlaintextAggregator { +impl Handler>> + for ThresholdPlaintextAggregator +{ type Result = (); fn handle( &mut self, - msg: E3CommitteeContainsResponse, + msg: E3CommitteeContainsResponse>, ctx: &mut Self::Context, ) -> Self::Result { trap(EType::PublickeyAggregation, &self.bus.clone(), || { @@ -310,11 +321,14 @@ impl Handler> for ThresholdP // Trust the party_id from the event - it's based on CommitteeFinalized order // which is the authoritative source of truth for party IDs - let DecryptionshareCreated { - party_id, - decryption_share, - .. - } = msg.into_inner(); + let ( + DecryptionshareCreated { + party_id, + decryption_share, + .. + }, + ec, + ) = msg.into_inner().into_components(); self.add_share(party_id, decryption_share)?; @@ -328,12 +342,15 @@ impl Handler> for ThresholdP { self.notify_sync( ctx, - ComputeAggregate { - shares: shares.clone(), - ciphertext_output: ciphertext_output.clone(), - threshold_m, - threshold_n, - }, + TypedEvent::new( + ComputeAggregate { + shares: shares.clone(), + ciphertext_output: ciphertext_output.clone(), + threshold_m, + threshold_n, + }, + ec, + ), ) } Ok(()) @@ -341,18 +358,18 @@ impl Handler> for ThresholdP } } -impl Handler for ThresholdPlaintextAggregator { +impl Handler> for ThresholdPlaintextAggregator { type Result = (); - fn handle(&mut self, msg: ComputeAggregate, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { trap(EType::PlaintextAggregation, &self.bus.clone(), || { self.handle_compute_aggregate(msg) }) } } -impl Handler for ThresholdPlaintextAggregator { +impl Handler> for ThresholdPlaintextAggregator { type Result = (); - fn handle(&mut self, msg: ComputeResponse, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { trap(EType::PlaintextAggregation, &self.bus.clone(), || { self.handle_compute_response(msg) }) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 921f845423..de86c1496f 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -13,8 +13,8 @@ use e3_data::{ }; use e3_events::hlc::Hlc; use e3_events::{ - BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, EventType, - Sequencer, StoreEventRequested, + BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, Sequencer, + StoreEventRequested, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -386,6 +386,7 @@ mod tests { use e3_events::CorrelationId; use e3_events::EnclaveEventData; + use e3_events::EventType; use e3_events::ReceiveEvents; use e3_events::TestEvent; use tempfile::TempDir; @@ -504,7 +505,7 @@ mod tests { ); // Push an event - handle.publish(TestEvent::new("pink", 1))?; + handle.publish_origin(TestEvent::new("pink", 1))?; sleep(Duration::from_millis(1)).await; // Now we have published an event all data should be written we can get the data from the store @@ -522,9 +523,9 @@ mod tests { let ts = handle.ts()?; // Push a few other events - handle.publish(TestEvent::new("yellow", 1))?; - handle.publish(TestEvent::new("red", 1))?; - handle.publish(TestEvent::new("white", 1))?; + handle.publish_origin(TestEvent::new("yellow", 1))?; + handle.publish_origin(TestEvent::new("red", 1))?; + handle.publish_origin(TestEvent::new("white", 1))?; sleep(Duration::from_millis(100)).await; // Get the event logs from the listener diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 1b8f96e818..97c2f59a33 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -22,7 +22,7 @@ pub async fn listen_for_shutdown(node: CiphernodeHandle) { info!("SIGTERM received, initiating graceful shutdown..."); // Stop the actor system - match bus.publish(Shutdown){ + match bus.publish_origin(Shutdown){ Ok(_) => (), Err(e) => error!("Shutdown failed to publish! {e}") } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 61808d3607..a2a17d4085 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -57,7 +57,7 @@ impl BusHandle { EventBus::>::history(&self.event_bus) } - /// Access the sequencer to internally dispatch am event to + /// Access the sequencer to internally dispatch an event to pub fn sequencer(&self) -> &Addr { &self.sequencer } @@ -84,16 +84,33 @@ impl BusHandle { } impl EventPublisher> for BusHandle { - fn publish(&self, data: impl Into) -> Result<()> { - let evt = self.event_from(data, self.get_ctx())?; - self.sequencer.do_send(evt); - Ok(()) + fn publish( + &self, + data: impl Into, + ctx: EventContext, + ) -> Result<()> { + self.publish_local(data, Some(ctx)) } - fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { - let evt = self.event_from_remote_source(data, self.get_ctx(), ts)?; - self.sequencer.do_send(evt); - Ok(()) + fn publish_origin(&self, data: impl Into) -> Result<()> { + self.publish_local(data, None) + } + + fn publish_from_remote( + &self, + data: impl Into, + remote_ts: u128, + ) -> Result<()> { + self.publish_from_remote_impl(data, remote_ts, None) + } + + fn publish_from_remote_as_response( + &self, + data: impl Into, + remote_ts: u128, + caused_by: EventContext, + ) -> Result<()> { + self.publish_from_remote_impl(data, remote_ts, Some(caused_by)) } fn naked_dispatch(&self, event: EnclaveEvent) { @@ -101,6 +118,28 @@ impl EventPublisher> for BusHandle { } } +impl BusHandle { + fn publish_from_remote_impl( + &self, + data: impl Into, + remote_ts: u128, + caused_by: Option>, + ) -> Result<()> { + let evt = self.event_from_remote_source(data, caused_by, remote_ts)?; + self.sequencer.do_send(evt); + Ok(()) + } + fn publish_local( + &self, + data: impl Into, + ctx: Option>, + ) -> Result<()> { + let evt = self.event_from(data, ctx)?; + self.sequencer.do_send(evt); + Ok(()) + } +} + impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { match self.event_from_error(err_type, error, self.get_ctx()) { @@ -279,13 +318,13 @@ mod tests { bus_c.subscribe(EventType::All, saver.clone().into()); // Publish events in causal order across buses - bus_a.publish(TestEvent::new("one", 1))?; + bus_a.publish_origin(TestEvent::new("one", 1))?; sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish(TestEvent::new("two", 2))?; + bus_b.publish_origin(TestEvent::new("two", 2))?; sleep(Duration::from_millis(5)).await; // next tick - bus_a.publish(TestEvent::new("three", 3))?; + bus_a.publish_origin(TestEvent::new("three", 3))?; sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish(TestEvent::new("four", 4))?; + bus_b.publish_origin(TestEvent::new("four", 4))?; sleep(Duration::from_millis(50)).await; // next tick // Get events diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 5f9100ded3..2c941fb514 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -45,6 +45,7 @@ pub enum EType { Data, Event, Computation, + DocumentPublishing, } impl EnclaveError { diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index b9db88f62e..483a07468f 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -280,6 +280,10 @@ where (self.payload, self.ctx.ts()) } + pub fn into_components(self) -> (EnclaveEventData, EventContext) { + (self.payload, self.ctx) + } + pub fn get_ctx(&self) -> &EventContext { &self.ctx } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index f054474391..3a76b9f550 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -38,6 +38,10 @@ impl TypedEvent { pub fn get_ctx(&self) -> &EventContext { &self.ctx } + + pub fn into_components(self) -> (T, EventContext) { + (self.inner, self.ctx) + } } impl Deref for TypedEvent { diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 552f502833..5cd4b694bc 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -73,7 +73,7 @@ mod tests { ]; for d in event_data.clone() { - bus.publish(d)?; + bus.publish_origin(d)?; } let expected = event_data diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 364d057630..a8bb3f4530 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -56,11 +56,11 @@ pub trait EventFactory { /// event ordering. /// /// This method should be used for events that originated from remote sources. + /// + /// The Option `caused_by` is for correlation when we send a remote request and receive a response. fn event_from_remote_source( &self, data: impl Into, - // NOTE: `caused_by` makes sense here as we could be sending out requests and receiving - // responses that relate to the request caused_by: Option>, ts: u128, ) -> Result; @@ -83,12 +83,30 @@ pub trait EventPublisher { /// to the event bus. /// /// This method should be used for events that have originated locally. - fn publish(&self, data: impl Into) -> Result<()>; + /// + /// The ctx parameter is to pass on the current context to the local event. + fn publish(&self, data: impl Into, ctx: EventContext) -> Result<()>; + /// This creates a context based on the given data. This should only be used when an event is + /// the origin event + fn publish_origin(&self, data: impl Into) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. /// /// This method should be used for events that originated from remote sources. - fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()>; + fn publish_from_remote(&self, data: impl Into, remote_ts: u128) -> Result<()>; + /// Create a new event from the given event data, apply the given remote HLC time to ensure correct + /// event ordering and publish it. + /// + /// This method should be used for events that originated from remote sources as a response to + /// a request we have sent + /// + /// The `caused_by` parameter is for correlation when we send a remote request and receive a response. + fn publish_from_remote_as_response( + &self, + data: impl Into, + remote_ts: u128, + caused_by: EventContext, + ) -> Result<()>; /// Dispatch the given event without applying any HLC transformation. fn naked_dispatch(&self, event: E); } diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index d730c1585f..0fca1d00b7 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -241,7 +241,7 @@ mod tests { // SyncStart: Init -> ForwardToSyncActor let mut evm_config = EvmEventConfig::new(); evm_config.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish(SyncStart::new(collector.clone(), evm_config)) + bus.publish_origin(SyncStart::new(collector.clone(), evm_config)) .unwrap(); // Send EVM event while forwarding - should reach collector @@ -285,7 +285,7 @@ mod tests { addr.do_send(EnclaveEvmEvent::Event(buffered_event)); // SyncEnd: BufferUntilLive -> Live (publishes buffered events to bus) - bus.publish(SyncEnd::new()).unwrap(); + bus.publish_origin(SyncEnd::new()).unwrap(); // Allow time for async message processing tokio::time::sleep(std::time::Duration::from_millis(50)).await; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index b87029282d..5e8e950720 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -108,7 +108,7 @@ impl Handler for FakeSyncActor { let (data, ts, _) = evt.split(); self.bus.publish_from_remote(data, ts)?; } - self.bus.publish(SyncEnd::new())?; + self.bus.publish_origin(SyncEnd::new())?; } }; Ok(()) @@ -160,7 +160,7 @@ async fn evm_reader() -> Result<()> { // This should trigger all chains to start to sync let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish(SyncStart::new(sync, evm_info))?; + bus.publish_origin(SyncStart::new(sync, evm_info))?; sleep(Duration::from_secs(1)).await; contract @@ -238,7 +238,7 @@ async fn ensure_historical_events() -> Result<()> { .build(); let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish(SyncStart::new(sync, evm_info))?; + bus.publish_origin(SyncStart::new(sync, evm_info))?; for msg in live_events.clone() { contract diff --git a/crates/keyshare/src/encryption_key_collector.rs b/crates/keyshare/src/encryption_key_collector.rs index b9bc2db99b..f98e6e896f 100644 --- a/crates/keyshare/src/encryption_key_collector.rs +++ b/crates/keyshare/src/encryption_key_collector.rs @@ -11,7 +11,9 @@ use std::{ }; use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, SpawnHandle}; -use e3_events::{E3id, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated}; +use e3_events::{ + E3id, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, TypedEvent, +}; use e3_trbfv::PartyId; use tracing::{info, warn}; @@ -96,9 +98,14 @@ impl Actor for EncryptionKeyCollector { } } -impl Handler for EncryptionKeyCollector { +impl Handler> for EncryptionKeyCollector { type Result = (); - fn handle(&mut self, msg: EncryptionKeyCreated, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); let start = Instant::now(); info!("EncryptionKeyCollector: EncryptionKeyCreated received"); @@ -141,7 +148,8 @@ impl Handler for EncryptionKeyCollector { ctx.cancel_future(handle); } - let event: AllEncryptionKeysCollected = self.keys.clone().into(); + let event: TypedEvent = + TypedEvent::new(self.keys.clone().into(), ec); self.parent.do_send(event); } diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 92e4ba1e8d..83a52d1038 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -12,8 +12,8 @@ use e3_events::{ prelude::*, trap, BusHandle, CiphernodeSelected, CiphertextOutputPublished, ComputeRequest, ComputeResponse, CorrelationId, DecryptionshareCreated, Die, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, EncryptionKey, EncryptionKeyCollectionFailed, - EncryptionKeyCreated, KeyshareCreated, PartyId, ThresholdShare, ThresholdShareCollectionFailed, - ThresholdShareCreated, TypedEvent, + EncryptionKeyCreated, EventContext, KeyshareCreated, PartyId, Sequenced, ThresholdShare, + ThresholdShareCollectionFailed, ThresholdShareCreated, TypedEvent, }; use e3_fhe::create_crp; use e3_trbfv::{ @@ -39,7 +39,7 @@ use std::{ mem, sync::{Arc, Mutex}, }; -use tracing::{error, info, warn}; +use tracing::{info, warn}; use crate::encryption_key_collector::{AllEncryptionKeysCollected, EncryptionKeyCollector}; use crate::threshold_share_collector::ThresholdShareCollector; @@ -375,7 +375,7 @@ impl ThresholdKeyshare { pub fn handle_threshold_share_created( &mut self, - msg: ThresholdShareCreated, + msg: TypedEvent, self_addr: Addr, ) -> Result<()> { let state = self.state.try_get()?; @@ -398,7 +398,7 @@ impl ThresholdKeyshare { pub fn handle_encryption_key_created( &mut self, - msg: EncryptionKeyCreated, + msg: TypedEvent, self_addr: Addr, ) -> Result<()> { info!("Received EncryptionKeyCreated forwarding to encryption key collector!"); @@ -429,6 +429,7 @@ impl ThresholdKeyshare { msg: TypedEvent, address: Addr, ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("CiphernodeSelected received."); // Ensure the collector is created let _ = self.ensure_collector(address.clone()); @@ -448,20 +449,23 @@ impl ThresholdKeyshare { CollectingEncryptionKeysData { sk_bfv: sk_bfv_encrypted.clone(), pk_bfv: pk_bfv_bytes.clone(), - ciphernode_selected: msg.into_inner(), + ciphernode_selected: msg, }, )) })?; let state = self.state.try_get()?; - self.bus.publish(EncryptionKeyCreated { - e3_id: state.e3_id.clone(), - key: Arc::new(EncryptionKey { - party_id: state.party_id, - pk_bfv: pk_bfv_bytes, - }), - external: false, - })?; + self.bus.publish( + EncryptionKeyCreated { + e3_id: state.e3_id.clone(), + key: Arc::new(EncryptionKey { + party_id: state.party_id, + pk_bfv: pk_bfv_bytes, + }), + external: false, + }, + ec, + )?; Ok(()) } @@ -469,8 +473,9 @@ impl ThresholdKeyshare { /// 1a. AllEncryptionKeysCollected - All BFV keys received, start share generation pub fn handle_all_encryption_keys_collected( &mut self, - msg: AllEncryptionKeysCollected, + msg: TypedEvent, ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!( "AllEncryptionKeysCollected - {} keys received", msg.keys.len() @@ -490,16 +495,21 @@ impl ThresholdKeyshare { }, )) })?; - self.handle_gen_esi_sss_requested(GenEsiSss(current.ciphernode_selected.clone()))?; - self.handle_gen_pk_share_and_sk_sss_requested(GenPkShareAndSkSss( - current.ciphernode_selected, + self.handle_gen_esi_sss_requested(TypedEvent::new( + GenEsiSss(current.ciphernode_selected.clone()), + ec.clone(), + ))?; + self.handle_gen_pk_share_and_sk_sss_requested(TypedEvent::new( + GenPkShareAndSkSss(current.ciphernode_selected), + ec, ))?; Ok(()) } /// 2. GenEsiSss - pub fn handle_gen_esi_sss_requested(&self, msg: GenEsiSss) -> Result<()> { + pub fn handle_gen_esi_sss_requested(&self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("GenEsiSss on ThresholdKeyshare"); let evt = msg.0; @@ -532,13 +542,14 @@ impl ThresholdKeyshare { e3_id, ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } /// 2a. GenEsiSss result pub fn handle_gen_esi_sss_response(&mut self, res: TypedEvent) -> Result<()> { - let output: GenEsiSssResponse = res.into_inner().try_into()?; + let (res, ec) = res.into_components(); + let output: GenEsiSssResponse = res.try_into()?; let esi_sss = output.esi_sss; @@ -580,13 +591,17 @@ impl ThresholdKeyshare { .. }) = self.state.get() { - self.handle_shares_generated()?; + self.handle_shares_generated(ec)?; } Ok(()) } /// 3. GenPkShareAndSkSss - pub fn handle_gen_pk_share_and_sk_sss_requested(&self, msg: GenPkShareAndSkSss) -> Result<()> { + pub fn handle_gen_pk_share_and_sk_sss_requested( + &self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("GenPkShareAndSkSss on ThresholdKeyshare"); let CiphernodeSelected { seed, e3_id, .. } = msg.0; let state = self @@ -611,7 +626,7 @@ impl ThresholdKeyshare { e3_id, ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } @@ -620,7 +635,8 @@ impl ThresholdKeyshare { &mut self, res: TypedEvent, ) -> Result<()> { - let TrBFVResponse::GenPkShareAndSkSss(output) = res.into_inner().response else { + let (res, ec) = res.into_components(); + let TrBFVResponse::GenPkShareAndSkSss(output) = res.response else { bail!("Error extracting data from compute process") }; @@ -658,13 +674,13 @@ impl ThresholdKeyshare { .. }) = self.state.get() { - self.handle_shares_generated()?; + self.handle_shares_generated(ec)?; } Ok(()) } /// 4. SharesGenerated - Encrypt shares with BFV and publish - pub fn handle_shares_generated(&mut self) -> Result<()> { + pub fn handle_shares_generated(&mut self, ec: EventContext) -> Result<()> { let Some(ThresholdKeyshareState { state: KeyshareState::AggregatingDecryptionKey(AggregatingDecryptionKey { @@ -735,12 +751,15 @@ impl ThresholdKeyshare { anyhow!("Failed to extract share for party {}", recipient_party_id) })?; - self.bus.publish(ThresholdShareCreated { - e3_id: e3_id.clone(), - share: Arc::new(party_share), - target_party_id: recipient_party_id as u64, - external: false, - })?; + self.bus.publish( + ThresholdShareCreated { + e3_id: e3_id.clone(), + share: Arc::new(party_share), + target_party_id: recipient_party_id as u64, + external: false, + }, + ec.clone(), + )?; } Ok(()) } @@ -750,8 +769,9 @@ impl ThresholdKeyshare { /// 5. AllThresholdSharesCollected - Decrypt received shares using BFV and aggregate pub fn handle_all_threshold_shares_collected( &self, - msg: AllThresholdSharesCollected, + msg: TypedEvent, ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("AllThresholdSharesCollected"); let cipher = self.cipher.clone(); let state = self.state.try_get()?; @@ -814,7 +834,7 @@ impl ThresholdKeyshare { e3_id.clone(), ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } @@ -823,7 +843,8 @@ impl ThresholdKeyshare { &mut self, res: TypedEvent, ) -> Result<()> { - let TrBFVResponse::CalculateDecryptionKey(output) = res.into_inner().response else { + let (res, ec) = res.into_components(); + let TrBFVResponse::CalculateDecryptionKey(output) = res.response else { bail!("Error extracting data from compute process") }; @@ -851,11 +872,14 @@ impl ThresholdKeyshare { let address = state.get_address().to_owned(); let current: ReadyForDecryption = state.clone().try_into()?; - self.bus.publish(KeyshareCreated { - pubkey: current.pk_share, - e3_id: e3_id.clone(), - node: address, - })?; + self.bus.publish( + KeyshareCreated { + pubkey: current.pk_share, + e3_id: e3_id.clone(), + node: address, + }, + ec, + )?; Ok(()) } @@ -863,8 +887,9 @@ impl ThresholdKeyshare { /// CiphertextOutputPublished pub fn handle_ciphertext_output_published( &mut self, - msg: CiphertextOutputPublished, + msg: TypedEvent, ) -> Result<()> { + let (msg, ec) = msg.into_components(); // Set state to decrypting self.state.try_mutate(|s| { use KeyshareState as K; @@ -899,7 +924,7 @@ impl ThresholdKeyshare { CorrelationId::new(), e3_id.clone(), ); - self.bus.publish(event)?; // CalculateDecryptionShareRequest + self.bus.publish(event, ec)?; // CalculateDecryptionShareRequest Ok(()) } @@ -908,7 +933,8 @@ impl ThresholdKeyshare { &mut self, res: TypedEvent, ) -> Result<()> { - let msg: CalculateDecryptionShareResponse = res.into_inner().try_into()?; + let (res, ec) = res.into_components(); + let msg: CalculateDecryptionShareResponse = res.try_into()?; let state = self.state.try_get()?; let party_id = state.party_id; let node = state.address; @@ -923,7 +949,7 @@ impl ThresholdKeyshare { }; // send the decryption share - self.bus.publish(event)?; + self.bus.publish(event, ec)?; // mark as complete self.state.try_mutate(|s| { @@ -941,20 +967,25 @@ impl ThresholdKeyshare { impl Handler for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.clone().into_data() { + let (msg, ec) = msg.into_components(); + match msg { EnclaveEventData::CiphernodeSelected(data) => { - self.notify_sync(ctx, msg.to_typed_event(data)) + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CiphertextOutputPublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) } - EnclaveEventData::CiphertextOutputPublished(data) => self.notify_sync(ctx, data), EnclaveEventData::ThresholdShareCreated(data) => { - let _ = self.handle_threshold_share_created(data, ctx.address()); + let _ = + self.handle_threshold_share_created(TypedEvent::new(data, ec), ctx.address()); } EnclaveEventData::EncryptionKeyCreated(data) => { - let _ = self.handle_encryption_key_created(data, ctx.address()); + let _ = + self.handle_encryption_key_created(TypedEvent::new(data, ec), ctx.address()); } EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), EnclaveEventData::ComputeResponse(data) => { - self.notify_sync(ctx, msg.to_typed_event(data)) + self.notify_sync(ctx, TypedEvent::new(data, ec)) } _ => (), } @@ -983,27 +1014,39 @@ impl Handler> for ThresholdKeyshare { } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: AllEncryptionKeysCollected, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_all_encryption_keys_collected(msg) }) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: AllThresholdSharesCollected, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_all_threshold_shares_collected(msg) }) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: CiphertextOutputPublished, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_ciphertext_output_published(msg) }) @@ -1017,23 +1060,24 @@ impl Handler for ThresholdKeyshare { msg: EncryptionKeyCollectionFailed, ctx: &mut Self::Context, ) -> Self::Result { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "Encryption key collection failed: {}", - msg.reason - ); - - // Clear the collector reference since it's stopped - self.encryption_key_collector = None; - - // Publish failure event to event bus for sync tracking - if let Err(e) = self.bus.publish(msg) { - error!("Failed to publish EncryptionKeyCollectionFailed: {}", e); - } - - // Stop this actor since we can't proceed without all encryption keys - ctx.stop(); + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "Encryption key collection failed: {}", + msg.reason + ); + + // Clear the collector reference since it's stopped + self.encryption_key_collector = None; + + // Publish failure event to event bus for sync tracking + self.bus.publish_origin(msg)?; + + // Stop this actor since we can't proceed without all encryption keys + ctx.stop(); + Ok(()) + }) } } @@ -1044,22 +1088,23 @@ impl Handler for ThresholdKeyshare { msg: ThresholdShareCollectionFailed, ctx: &mut Self::Context, ) -> Self::Result { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "Threshold share collection failed: {}", - msg.reason - ); + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "Threshold share collection failed: {}", + msg.reason + ); - // Clear the collector reference since it's stopped - self.decryption_key_collector = None; + // Clear the collector reference since it's stopped + self.decryption_key_collector = None; - // Publish failure event to event bus for sync tracking - if let Err(e) = self.bus.publish(msg) { - error!("Failed to publish ThresholdShareCollectionFailed: {}", e); - } + // Publish failure event to event bus for sync tracking + self.bus.publish_origin(msg)?; - ctx.stop(); + ctx.stop(); + Ok(()) + }) } } diff --git a/crates/keyshare/src/threshold_share_collector.rs b/crates/keyshare/src/threshold_share_collector.rs index d0393c34cf..34943d9f2e 100644 --- a/crates/keyshare/src/threshold_share_collector.rs +++ b/crates/keyshare/src/threshold_share_collector.rs @@ -11,7 +11,9 @@ use std::{ }; use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, SpawnHandle}; -use e3_events::{E3id, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated}; +use e3_events::{ + E3id, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated, TypedEvent, +}; use e3_trbfv::PartyId; use tracing::{info, warn}; @@ -68,9 +70,14 @@ impl Actor for ThresholdShareCollector { } } -impl Handler for ThresholdShareCollector { +impl Handler> for ThresholdShareCollector { type Result = (); - fn handle(&mut self, msg: ThresholdShareCreated, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); let start = Instant::now(); info!("ThresholdShareCollector: ThresholdShareCreated received by collector"); @@ -108,7 +115,8 @@ impl Handler for ThresholdShareCollector { ctx.cancel_future(handle); } - let event: AllThresholdSharesCollected = self.shares.clone().into(); + let event: TypedEvent = + TypedEvent::new(self.shares.clone().into(), ec); self.parent.do_send(event); } info!( diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index 09bf9e8f29..c5cbc7f90a 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -17,6 +17,7 @@ use actix::prelude::*; use actix::{Actor, Handler}; use anyhow::Result; use e3_crypto::Cipher; +use e3_events::trap_fut; use e3_events::BusHandle; use e3_events::ComputeRequestErrorKind; use e3_events::EType; @@ -26,6 +27,7 @@ use e3_events::ErrorDispatcher; use e3_events::Event; use e3_events::EventPublisher; use e3_events::EventSubscriber; +use e3_events::TypedEvent; use e3_events::{ComputeRequest, ComputeRequestError, ComputeResponse, EventType}; use e3_trbfv::calculate_decryption_key::calculate_decryption_key; use e3_trbfv::calculate_decryption_share::calculate_decryption_share; @@ -33,7 +35,6 @@ use e3_trbfv::calculate_threshold_decryption::calculate_threshold_decryption; use e3_trbfv::gen_esi_sss::gen_esi_sss; use e3_trbfv::gen_pk_share_and_sk_sss::gen_pk_share_and_sk_sss; use e3_trbfv::{TrBFVError, TrBFVRequest, TrBFVResponse}; -use e3_utils::NotifySync; use e3_utils::SharedRng; use rand::Rng; use tracing::error; @@ -99,33 +100,32 @@ impl Handler for Multithread { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { info!("Multithread received EnclaveEvent!"); - match msg.get_data() { - EnclaveEventData::ComputeRequest(data) => ctx.notify(data.clone()), + let (data, ec) = msg.into_components(); + match data { + EnclaveEventData::ComputeRequest(data) => ctx.notify(TypedEvent::new(data, ec)), _ => (), } } } -impl Handler for Multithread { +impl Handler> for Multithread { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: ComputeRequest, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { let cipher = self.cipher.clone(); let rng = self.rng.clone(); let bus = self.bus.clone(); let pool = self.task_pool.clone(); let report = self.report.clone(); - // TODO: replace with trap_fut - Box::pin(async move { - match handle_compute_request_event(msg, bus, cipher, rng, pool, report).await { - Ok(_) => (), - Err(e) => error!("{e}"), - } - }) + trap_fut( + EType::Computation, + &self.bus.clone(), + handle_compute_request_event(msg, bus, cipher, rng, pool, report), + ) } } async fn handle_compute_request_event( - msg: ComputeRequest, + msg: TypedEvent, bus: BusHandle, cipher: Arc, rng: SharedRng, @@ -134,7 +134,7 @@ async fn handle_compute_request_event( ) -> anyhow::Result<()> { let msg_string = msg.to_string(); let job_name = msg_string.clone(); - + let (msg, ctx) = msg.into_components(); // We spawn a thread on rayon moving to "sync"-land let (result, duration) = pool .spawn(job_name, TaskTimeouts::default(), move || { @@ -150,7 +150,7 @@ async fn handle_compute_request_event( }; match result { - Ok(val) => bus.publish(val)?, + Ok(val) => bus.publish(val, ctx)?, Err(e) => bus.err(EType::Computation, e), }; Ok(()) diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 7db3f39c29..56d73d84fc 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -14,10 +14,10 @@ use actix::prelude::*; use anyhow::Result; use chrono::{DateTime, Utc}; use e3_events::{ - prelude::*, BusHandle, CiphernodeSelected, CorrelationId, DocumentKind, DocumentMeta, - DocumentReceived, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, - EncryptionKeyCreated, Event, EventType, Filter, PartyId, PublishDocumentRequested, - ThresholdShareCreated, + prelude::*, trap, trap_fut, BusHandle, CiphernodeSelected, CorrelationId, DocumentKind, + DocumentMeta, DocumentReceived, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, + EncryptionKeyCreated, Event, EventContext, EventType, Filter, PartyId, + PublishDocumentRequested, Sequenced, ThresholdShareCreated, TypedEvent, }; use e3_utils::retry::{retry_with_backoff, to_retry}; use e3_utils::ArcBytes; @@ -55,7 +55,7 @@ pub struct DocumentPublisher { } impl DocumentPublisher { - /// Create a new NetEventTranslator actor + /// Create a new DocumentPublisher actor pub fn new( bus: &BusHandle, tx: &mpsc::Sender, @@ -153,42 +153,33 @@ impl Handler for DocumentPublisher { let rx = self.rx.clone(); let bus = self.bus.clone(); let topic = self.topic.clone(); - Box::pin(async move { - match handle_publish_document_requested(tx, rx, msg, topic).await { - Ok(_) => (), - Err(e) => { - error!(error=?e, "Could not handle publish document requested"); - bus.err(EType::IO, e) - } - } - }) + trap_fut( + EType::IO, + &bus.clone(), + handle_publish_document_requested(tx, rx, msg, topic, bus), + ) } } impl Handler for DocumentPublisher { type Result = (); fn handle(&mut self, msg: CiphernodeSelected, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_ciphernode_selected(msg) { - Ok(_) => (), - Err(e) => { - error!("{e}") - } - } + trap(EType::DocumentPublishing, &self.bus.clone(), || { + self.handle_ciphernode_selected(msg) + }) } } impl Handler for DocumentPublisher { type Result = (); fn handle(&mut self, msg: E3RequestComplete, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_e3_request_complete(msg) { - Ok(_) => (), - Err(e) => { - error!("{e}") - } - } + trap(EType::DocumentPublishing, &self.bus.clone(), || { + self.handle_e3_request_complete(msg) + }) } } +/// Receiving DocumentPublishedNotification from libp2p impl Handler for DocumentPublisher { type Result = ResponseFuture<()>; fn handle( @@ -199,18 +190,13 @@ impl Handler for DocumentPublisher { let ids = self.ids.clone(); let bus = self.bus.clone(); let tx = self.tx.clone(); - let msg = msg.clone(); let rx = self.rx.clone(); - - Box::pin(async move { - match handle_document_published_notification(tx, rx, bus.clone(), ids, msg).await { - Ok(_) => (), - Err(e) => { - error!(error=?e, "Could not handle document published notification"); - bus.err(EType::IO, e); - } - } - }) + let msg = msg.clone(); + trap_fut( + EType::IO, + &bus, + handle_document_published_notification(tx, rx, bus.clone(), ids, msg), + ) } } @@ -233,6 +219,7 @@ pub async fn handle_publish_document_requested( rx: Arc>, event: PublishDocumentRequested, topic: impl Into, + bus: BusHandle, ) -> Result<()> { let value = event.value; let key = Cid::from_content(&value); @@ -247,7 +234,7 @@ pub async fn handle_publish_document_requested( 1000, ) .await?; - let notification = DocumentPublishedNotification::new(event.meta, key); + let notification = DocumentPublishedNotification::new(event.meta, key, bus.ts()?); broadcast_document_published_notification(tx, rx, notification, topic).await?; Ok(()) } @@ -283,10 +270,13 @@ pub async fn handle_document_published_notification( .await?; debug!("Sending received event..."); - bus.publish(DocumentReceived { - meta: event.meta, - value, - })?; + bus.publish_from_remote( + DocumentReceived { + meta: event.meta, + value, + }, + event.ts, + )?; Ok(()) } @@ -421,6 +411,7 @@ impl EventConverter { receivable: ReceivableDocument, e3_id: &E3id, party_id: u64, + ctx: EventContext, ) -> Result<()> { let value = ArcBytes::from_bytes(&receivable.to_bytes()?); let meta = DocumentMeta::new( @@ -430,13 +421,17 @@ impl EventConverter { None, ); self.bus - .publish(PublishDocumentRequested::new(meta, value))?; + .publish(PublishDocumentRequested::new(meta, value), ctx)?; Ok(()) } /// Local node created a threshold share (already split per-party by ThresholdKeyshare). /// Publishes the single-party document with appropriate filter. - pub fn handle_threshold_share_created(&self, msg: ThresholdShareCreated) -> Result<()> { + pub fn handle_threshold_share_created( + &self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ctx) = msg.into_components(); if msg.external { return Ok(()); } @@ -447,29 +442,35 @@ impl EventConverter { msg.share.party_id, target_party_id, msg.e3_id ); + let e3_id = msg.e3_id.clone(); + self.publish_filtered( - ReceivableDocument::ThresholdShareCreated(msg.clone()), - &msg.e3_id, + ReceivableDocument::ThresholdShareCreated(msg), + &e3_id, target_party_id, + ctx, )?; Ok(()) } - fn handle_encryption_key_created(&self, msg: EncryptionKeyCreated) -> Result<()> { + fn handle_encryption_key_created(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); if msg.external { return Ok(()); } - let receivable = ReceivableDocument::EncryptionKeyCreated(msg.clone()); + + let meta = DocumentMeta::new(msg.e3_id.clone(), DocumentKind::TrBFV, vec![], None); + let receivable = ReceivableDocument::EncryptionKeyCreated(msg); let value = ArcBytes::from_bytes(&receivable.to_bytes()?); - let meta = DocumentMeta::new(msg.e3_id, DocumentKind::TrBFV, vec![], None); self.bus - .publish(PublishDocumentRequested::new(meta, value))?; + .publish(PublishDocumentRequested::new(meta, value), ctx)?; Ok(()) } /// Convert received document to internal events. /// Note: Filtering already happened in DocumentPublisher before DHT fetch. - fn handle_document_received(&self, msg: DocumentReceived) -> Result<()> { + fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); let receivable = ReceivableDocument::from_bytes(&msg.value.extract_bytes())?; match receivable { @@ -478,23 +479,29 @@ impl EventConverter { "Received ThresholdShareCreated from party {} for target party {}", evt.share.party_id, evt.target_party_id ); - self.bus.publish(ThresholdShareCreated { - external: true, - e3_id: evt.e3_id, - share: evt.share, - target_party_id: evt.target_party_id, - })?; + self.bus.publish( + ThresholdShareCreated { + external: true, + e3_id: evt.e3_id, + share: evt.share, + target_party_id: evt.target_party_id, + }, + ctx.clone(), + )?; } ReceivableDocument::EncryptionKeyCreated(evt) => { debug!( "Received EncryptionKeyCreated from party {}", evt.key.party_id ); - self.bus.publish(EncryptionKeyCreated { - external: true, - e3_id: evt.e3_id, - key: evt.key, - })?; + self.bus.publish( + EncryptionKeyCreated { + external: true, + e3_id: evt.e3_id, + key: evt.key, + }, + ctx, + )?; } } Ok(()) @@ -508,42 +515,58 @@ impl Actor for EventConverter { impl Handler for EventConverter { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::ThresholdShareCreated(data) => self.notify_sync(ctx, data), - EnclaveEventData::EncryptionKeyCreated(data) => self.notify_sync(ctx, data), - EnclaveEventData::DocumentReceived(data) => self.notify_sync(ctx, data), + let (data, ec) = msg.into_components(); + match data { + EnclaveEventData::ThresholdShareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::EncryptionKeyCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::DocumentReceived(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for EventConverter { +impl Handler> for EventConverter { type Result = (); - fn handle(&mut self, msg: ThresholdShareCreated, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_threshold_share_created(msg) { - Ok(_) => (), - Err(err) => error!("{err}"), - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::DocumentPublishing, &self.bus.clone(), || { + self.handle_threshold_share_created(msg) + }) } } -impl Handler for EventConverter { +impl Handler> for EventConverter { type Result = (); - fn handle(&mut self, msg: EncryptionKeyCreated, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_encryption_key_created(msg) { - Ok(_) => (), - Err(err) => error!("{err}"), - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::DocumentPublishing, &self.bus.clone(), || { + self.handle_encryption_key_created(msg) + }) } } -impl Handler for EventConverter { +impl Handler> for EventConverter { type Result = (); - fn handle(&mut self, msg: DocumentReceived, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_document_received(msg) { - Ok(_) => (), - Err(err) => error!("{err}"), - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::DocumentPublishing, &self.bus.clone(), || { + self.handle_document_received(msg) + }) } } @@ -612,7 +635,7 @@ mod tests { let e3_id = E3id::new("1243", 1); // 1. Send a request to publish - bus.publish(PublishDocumentRequested { + bus.publish_origin(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), })?; @@ -686,7 +709,7 @@ mod tests { let cid = Cid::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish(CiphernodeSelected { + bus.publish_origin(CiphernodeSelected { e3_id: e3_id.clone(), threshold_m: 3, threshold_n: 5, @@ -697,6 +720,7 @@ mod tests { GossipData::DocumentPublishedNotification(DocumentPublishedNotification { key: cid.clone(), meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), + ts: 123, }), ))?; @@ -748,7 +772,7 @@ mod tests { let e3_id = E3id::new("1243", 1); // Send a request to publish - bus.publish(PublishDocumentRequested { + bus.publish_origin(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), })?; @@ -798,7 +822,7 @@ mod tests { let cid = Cid::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish(CiphernodeSelected { + bus.publish_origin(CiphernodeSelected { e3_id: e3_id.clone(), threshold_m: 3, threshold_n: 5, @@ -815,6 +839,7 @@ mod tests { vec![], Some(expires_at), ), + ts: 123, }), ))?; @@ -828,6 +853,7 @@ mod tests { GossipData::DocumentPublishedNotification(DocumentPublishedNotification { key: cid.clone(), meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], Some(expires_at)), + ts: 123, }), ))?; diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index d394c80448..46feb6960b 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -71,6 +71,7 @@ pub struct SyncRequestValue { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct SyncResponseValue { pub events: Vec, + pub ts: u128, } #[derive(Message, Clone, Debug)] @@ -228,11 +229,12 @@ impl NetEvent { pub struct DocumentPublishedNotification { pub meta: DocumentMeta, pub key: Cid, + pub ts: u128, } impl DocumentPublishedNotification { - pub fn new(meta: DocumentMeta, key: Cid) -> Self { - Self { meta, key } + pub fn new(meta: DocumentMeta, key: Cid, ts: u128) -> Self { + Self { meta, key, ts } } pub fn to_bytes(&self) -> Result> { diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 7d6c9a5a21..49361fc40a 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -9,7 +9,7 @@ use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, - ReceiveEvents, Unsequenced, + ReceiveEvents, TypedEvent, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake}; use futures::TryFutureExt; @@ -69,7 +69,7 @@ impl NetSyncManager { while let Ok(event) = events.recv().await { debug!("Received event {:?}", event); match event { - NetEvent::OutgoingSyncRequestSucceeded(value) => addr.do_send(value), + // Someone is asking for our sync NetEvent::SyncRequestReceived(value) => addr.do_send(value), _ => (), } @@ -89,17 +89,23 @@ impl Actor for NetSyncManager { impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::OutgoingSyncRequested(data) => ctx.notify(data), + let (msg, ec) = msg.into_components(); + match msg { + // We are making a sync request of another node + EnclaveEventData::OutgoingSyncRequested(data) => ctx.notify(TypedEvent::new(data, ec)), _ => (), } } } /// SyncRequest is called on start up to fetch remote events -impl Handler for NetSyncManager { +impl Handler> for NetSyncManager { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: OutgoingSyncRequested, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { trap_fut( EType::Net, &self.bus.clone(), @@ -109,19 +115,28 @@ impl Handler for NetSyncManager { } /// We have received the sync response from the remote peer -impl Handler for NetSyncManager { +impl Handler> for NetSyncManager { type Result = (); - fn handle(&mut self, msg: OutgoingSyncRequestSucceeded, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { - self.bus.publish(NetSyncEventsReceived { - events: msg - .value - .events - .iter() - .cloned() - .map(|data| data.try_into()) - .collect::>>>()?, - })?; + let (msg, ctx) = msg.into_components(); + self.bus.publish_from_remote_as_response( + NetSyncEventsReceived { + events: msg + .value + .events + .iter() + .cloned() + .map(|data| data.try_into()) + .collect::>>>()?, + }, + msg.value.ts, + ctx, + )?; Ok(()) }); @@ -162,6 +177,7 @@ impl Handler for NetSyncManager { .cloned() .map(|ev| ev.try_into()) .collect::>()?, + ts: self.bus.ts()?, // NOTE: We are storing a local timestamp on this response }, channel: channel.to_owned(), })?; @@ -199,9 +215,13 @@ async fn sync_request( async fn handle_sync_request_event( net_cmds: mpsc::Sender, net_events: Arc>, - event: OutgoingSyncRequested, - address: impl Into>, + event: TypedEvent, + address: impl Into>>, ) -> Result<()> { + let (event, ctx) = event.into_components(); + + // Make the sync request + // value returned includes the timestamp from the remote peer let value = retry_with_backoff( || { sync_request( @@ -216,6 +236,7 @@ async fn handle_sync_request_event( ) .await?; - address.into().try_send(value)?; + // send the sync request succeeded to ourselves + address.into().try_send(TypedEvent::new(value, ctx))?; Ok(()) } diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index e52a316dba..d1a8d9de25 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -184,8 +184,8 @@ impl Handler for E3Router { } context.forward_message(&msg, &mut self.buffer); - - match msg.into_data() { + let (data, ctx) = msg.into_components(); + match data { EnclaveEventData::PlaintextAggregated(_) => { // Here we are detemining that by receiving the PlaintextAggregated event our request is // complete and we can notify everyone. This might change as we consider other factors @@ -195,7 +195,7 @@ impl Handler for E3Router { }; // Send to bus so all other actors can react to a request being complete. - self.bus.publish(event)?; + self.bus.publish(event, ctx)?; } EnclaveEventData::E3RequestComplete(_) => { // Note this will be sent above to the children who can kill themselves based on diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 71f5820026..8812fab129 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -10,6 +10,7 @@ use anyhow::bail; use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::E3RequestComplete; +use e3_events::TypedEvent; use e3_events::{ prelude::*, trap, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, E3id, EType, EnclaveEvent, EnclaveEventData, EventType, Shutdown, TicketGenerated, TicketId, @@ -64,26 +65,27 @@ impl CiphernodeSelector { impl Handler for CiphernodeSelector { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { + let (msg, ec) = msg.into_components(); + match msg { EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), - EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data), + EnclaveEventData::CommitteeFinalized(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } } -impl Handler> for CiphernodeSelector { +impl Handler>> for CiphernodeSelector { type Result = (); fn handle( &mut self, - data: WithSortitionTicket, - _ctx: &mut Self::Context, + data: WithSortitionTicket>, + _: &mut Self::Context, ) -> Self::Result { - let bus = self.bus.clone(); - - trap(EType::Sortition, &bus.clone(), || { + trap(EType::Sortition, &self.bus.clone(), || { self.e3_cache.try_mutate(|mut cache| { info!( "Mutating e3_cache: appending data: {:?}", @@ -107,18 +109,20 @@ impl Handler> for CiphernodeSelector { info!(node = &data.address(), "Ciphernode was not selected"); return Ok(()); } - if let Some(tid) = data.ticket_id() { info!( node = &data.address(), ticket_id = tid, "Ticket generated for score sortition" ); - bus.publish(TicketGenerated { - e3_id: data.e3_id.clone(), - ticket_id: TicketId::Score(tid), - node: data.address().to_owned(), - })?; + self.bus.publish( + TicketGenerated { + e3_id: data.e3_id.clone(), + ticket_id: TicketId::Score(tid), + node: data.address().to_owned(), + }, + data.get_ctx().clone(), + )?; } Ok(()) @@ -138,11 +142,16 @@ impl Handler for CiphernodeSelector { } } -impl Handler for CiphernodeSelector { +impl Handler> for CiphernodeSelector { type Result = (); - fn handle(&mut self, msg: CommitteeFinalized, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { trap(EType::Sortition, &self.bus.clone(), move || { + let (msg, ec) = msg.into_components(); info!("CiphernodeSelector received CommitteeFinalized."); let bus = self.bus.clone(); info!("Getting e3_cache..."); @@ -179,16 +188,19 @@ impl Handler for CiphernodeSelector { "Node is in finalized committee, emitting CiphernodeSelected" ); - bus.publish(CiphernodeSelected { - party_id: party_id as u64, - e3_id: msg.e3_id, - threshold_m: e3_meta.threshold_m, - threshold_n: e3_meta.threshold_n, - esi_per_ct: e3_meta.esi_per_ct, - error_size: e3_meta.error_size.clone(), - params: e3_meta.params.clone(), - seed: e3_meta.seed, - })?; + bus.publish( + CiphernodeSelected { + party_id: party_id as u64, + e3_id: msg.e3_id, + threshold_m: e3_meta.threshold_m, + threshold_n: e3_meta.threshold_n, + esi_per_ct: e3_meta.esi_per_ct, + error_size: e3_meta.error_size.clone(), + params: e3_meta.params.clone(), + seed: e3_meta.seed, + }, + ec, + )?; Ok(()) }) diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 54209aa5b9..9def6864ff 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -13,7 +13,7 @@ use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, ConfigurationUpdated, E3Requested, E3id, EType, EnclaveEvent, EventType, - OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, + OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, TypedEvent, }; use e3_events::{BusHandle, EnclaveEventData}; use e3_utils::NotifySync; @@ -358,8 +358,9 @@ impl Handler for Sortition { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, data.clone()), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), EnclaveEventData::CiphernodeAdded(data) => self.notify_sync(ctx, data.clone()), EnclaveEventData::CiphernodeRemoved(data) => self.notify_sync(ctx, data.clone()), EnclaveEventData::TicketBalanceUpdated(data) => self.notify_sync(ctx, data.clone()), @@ -375,9 +376,9 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: E3Requested, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { let chain_id = msg.e3_id.chain_id(); let seed = msg.seed; let threshold_n = msg.threshold_n; diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 27fc8efff0..f31396f19b 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -9,8 +9,8 @@ use std::collections::HashSet; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{ - trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, - SyncStart, + trap, BusHandle, EType, EventContext, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, + SyncEvmEvent, SyncStart, }; use tracing::info; @@ -57,6 +57,7 @@ impl Synchronizer { } fn handle_sync_end(&mut self) -> Result<()> { + // TODO: WORK OUT WHAT IS GOING ON HERE WITH PUBLISHING AND CONTEXT info!("all chains synced draining to bus and running sync end"); // Order all events (theoretically) self.evm_buffer.sort_by_key(|i| i.ts()); @@ -64,10 +65,11 @@ impl Synchronizer { // publish them in order for evt in self.evm_buffer.drain(..) { let (data, _, _) = evt.split(); - self.bus.publish(data)?; // Use publish here as historical events will be correctly - // ordered as part of the preparatory process + self.bus.publish_origin(data)?; // Use publish_origin here as historical events will be correctly + // ordered as part of the preparatory process and these + // events have no } - self.bus.publish(SyncEnd::new())?; + self.bus.publish_origin(SyncEnd::new())?; Ok(()) } } @@ -105,7 +107,8 @@ impl Handler for Synchronizer { )?; // TODO: Get information about what has and has not been synced then fire SyncStart - self.bus.publish(SyncStart::new(ctx.address(), evm_config)) + self.bus + .publish_origin(SyncStart::new(ctx.address(), evm_config)) }) } } diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index bd8c2c0068..25f9db0255 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -181,7 +181,7 @@ impl AddToCommittee { self.count += 1; - self.bus.publish(evt.clone())?; + self.bus.publish_origin(evt.clone())?; Ok(evt.into()) } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 90f268185e..286f8dbf3b 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -43,7 +43,7 @@ async fn setup_score_sortition_environment( eth_addrs: &Vec, chain_id: u64, ) -> Result<()> { - bus.publish(ConfigurationUpdated { + bus.publish_origin(ConfigurationUpdated { parameter: "ticketPrice".to_string(), old_value: U256::ZERO, new_value: U256::from(10_000_000u64), @@ -54,7 +54,7 @@ async fn setup_score_sortition_environment( for addr in eth_addrs { adder.add(addr).await?; - bus.publish(TicketBalanceUpdated { + bus.publish_origin(TicketBalanceUpdated { operator: addr.clone(), delta: I256::try_from(1_000_000_000u64).unwrap(), new_balance: U256::from(1_000_000_000u64), @@ -62,7 +62,7 @@ async fn setup_score_sortition_environment( chain_id, })?; - bus.publish(OperatorActivationChanged { + bus.publish_origin(OperatorActivationChanged { operator: addr.clone(), active: true, chain_id, @@ -238,7 +238,7 @@ async fn test_trbfv_actor() -> Result<()> { params, }; - bus.publish(e3_requested)?; + bus.publish_origin(e3_requested)?; // For score sortition, we need to wait for nodes to process E3Requested and run sortition // Since TicketGenerated is a local-only event (not shared across network), we can't collect it @@ -265,7 +265,7 @@ async fn test_trbfv_actor() -> Result<()> { .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; - bus.publish(CommitteeFinalized { + bus.publish_origin(CommitteeFinalized { e3_id: e3_id.clone(), committee, chain_id, @@ -383,7 +383,7 @@ async fn test_trbfv_actor() -> Result<()> { e3_id: e3_id.clone(), }; - bus.publish(ciphertext_published_event.clone())?; + bus.publish_origin(ciphertext_published_event.clone())?; println!("CiphertextOutputPublished event has been dispatched!"); @@ -520,9 +520,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { ..CiphernodeSelected::default() }; - bus.publish(evt_1.clone())?; - bus.publish(evt_2.clone())?; - bus.publish(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network + bus.publish_origin(evt_1.clone())?; + bus.publish_origin(evt_2.clone())?; + bus.publish_origin(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network // check the history of the event bus let history = history_collector @@ -606,15 +606,13 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { use e3_events::{EventBus, EventBusConfig, GetEvents, Shutdown, TakeEvents}; use e3_test_helpers::{create_random_eth_addrs, get_common_setup, simulate_libp2p_net}; use fhe::{ - bfv::{PublicKey, SecretKey}, + bfv::PublicKey, mbfv::{AggregateIter, PublicKeyShare}, }; use fhe_traits::Serialize; use std::time::Duration; use tokio::time::sleep; - type PkSkShareTuple = (PublicKeyShare, SecretKey, String); - async fn setup_local_ciphernode( bus: &BusHandle, rng: &e3_utils::SharedRng, @@ -676,7 +674,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { }; // Send e3request - bus.publish(E3Requested { + bus.publish_origin(E3Requested { e3_id: e3_id.clone(), threshold_m: 2, threshold_n: 2, @@ -685,7 +683,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { ..E3Requested::default() })?; - bus.publish(CommitteeFinalized { + bus.publish_origin(CommitteeFinalized { e3_id: e3_id.clone(), committee: eth_addrs.clone(), chain_id: 1, @@ -701,7 +699,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { assert_eq!(errors.len(), 0); // SEND SHUTDOWN! - bus.publish(Shutdown)?; + bus.publish_origin(Shutdown)?; // This is probably overkill but required to ensure that all the data is written sleep(Duration::from_secs(1)).await; @@ -766,7 +764,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { use e3_test_helpers::encrypt_ciphertext; let raw_plaintext = vec![vec![4, 5]]; let (ciphertext, expected) = encrypt_ciphertext(¶ms, pubkey, raw_plaintext)?; - bus.publish(CiphertextOutputPublished { + bus.publish_origin(CiphertextOutputPublished { ciphertext_output: ciphertext .iter() .map(|ct| ArcBytes::from_bytes(&ct.to_bytes())) @@ -913,7 +911,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { setup_score_sortition_environment(&bus, ð_addrs, 2).await?; // Send the computation requested event - bus.publish(E3Requested { + bus.publish_origin(E3Requested { e3_id: E3id::new("1234", 1), threshold_m: 2, threshold_n: 5, @@ -922,7 +920,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ..E3Requested::default() })?; - bus.publish(CommitteeFinalized { + bus.publish_origin(CommitteeFinalized { e3_id: E3id::new("1234", 1), committee: eth_addrs.clone(), chain_id: 1, @@ -957,7 +955,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ); // Send the computation requested event - bus.publish(E3Requested { + bus.publish_origin(E3Requested { e3_id: E3id::new("1234", 2), threshold_m: 2, threshold_n: 5, @@ -966,7 +964,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ..E3Requested::default() })?; - bus.publish(CommitteeFinalized { + bus.publish_origin(CommitteeFinalized { e3_id: E3id::new("1234", 2), committee: eth_addrs.clone(), chain_id: 2, From d0ab76adbe9fd9b049420b26f12c9e49d8a36c72 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 31 Jan 2026 12:56:57 +0000 Subject: [PATCH 07/63] refactor: add EventContext support to Persistable::try_mutate Update Persistable trait to support EventContext in try_mutate method for proper event synchronization. Add context tracking in persistable state and modify all callers to pass EventContext when mutating state. --- crates/aggregator/src/publickey_aggregator.rs | 21 +-- .../src/threshold_plaintext_aggregator.rs | 25 ++-- crates/data/src/persistable.rs | 20 ++- crates/keyshare/src/threshold_keyshare.rs | 16 +-- crates/sortition/src/ciphernode_selector.rs | 18 ++- crates/sortition/src/sortition.rs | 120 +++++++++++++----- 6 files changed, 153 insertions(+), 67 deletions(-) diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index 1ad56b4f05..6ba393280b 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -9,8 +9,8 @@ use anyhow::Result; use e3_bfv_client::client::compute_pk_commitment; use e3_data::Persistable; use e3_events::{ - prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, KeyshareCreated, OrderedSet, - PublicKeyAggregated, Seed, TypedEvent, + prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, EventContext, + KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, Sequenced, TypedEvent, }; use e3_events::{trap, EType}; use e3_fhe::{Fhe, GetAggregatePublicKey}; @@ -88,8 +88,13 @@ impl PublicKeyAggregator { } } - pub fn add_keyshare(&mut self, keyshare: ArcBytes, node: String) -> Result<()> { - self.state.try_mutate(|mut state| { + pub fn add_keyshare( + &mut self, + keyshare: ArcBytes, + node: String, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(&ec, |mut state| { let PublicKeyAggregatorState::Collecting { threshold_n, keyshares, @@ -119,8 +124,8 @@ impl PublicKeyAggregator { }) } - pub fn set_pubkey(&mut self, pubkey: Vec) -> Result<()> { - self.state.try_mutate(|mut state| { + pub fn set_pubkey(&mut self, pubkey: Vec, ec: &EventContext) -> Result<()> { + self.state.try_mutate(ec, |mut state| { let PublicKeyAggregatorState::Computing { keyshares, nodes } = &mut state else { return Ok(state); }; @@ -174,7 +179,7 @@ impl Handler> for PublicKeyAggregator { return Ok(()); } - self.add_keyshare(pubkey, node)?; + self.add_keyshare(pubkey, node, &ec)?; if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { self.notify_sync( @@ -213,7 +218,7 @@ impl Handler> for PublicKeyAggregator { )?; // Update the local state - self.set_pubkey(pubkey)?; + self.set_pubkey(pubkey, &ec)?; if let Some(PublicKeyAggregatorState::Complete { public_key: pubkey, diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 082226c342..54508cab42 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -11,8 +11,8 @@ use anyhow::{anyhow, bail, ensure, Result}; use e3_data::Persistable; use e3_events::{ prelude::*, trap, BusHandle, ComputeRequest, ComputeResponse, CorrelationId, - DecryptionshareCreated, Die, E3id, EType, EnclaveEvent, EnclaveEventData, PlaintextAggregated, - Seed, TypedEvent, + DecryptionshareCreated, Die, E3id, EType, EnclaveEvent, EnclaveEventData, EventContext, + PlaintextAggregated, Seed, Sequenced, TypedEvent, }; use e3_sortition::{E3CommitteeContainsRequest, E3CommitteeContainsResponse, Sortition}; use e3_trbfv::{ @@ -145,8 +145,13 @@ impl ThresholdPlaintextAggregator { } } - pub fn add_share(&mut self, party_id: u64, share: Vec) -> Result<()> { - self.state.try_mutate(|state| { + pub fn add_share( + &mut self, + party_id: u64, + share: Vec, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(ec, |state| { info!("Adding share for party_id={}", party_id); let current: Collecting = state.clone().try_into()?; let ciphertext_output = current.ciphertext_output; @@ -181,8 +186,12 @@ impl ThresholdPlaintextAggregator { }) } - pub fn set_decryption(&mut self, decrypted: Vec) -> Result<()> { - self.state.try_mutate(|mut state| { + pub fn set_decryption( + &mut self, + decrypted: Vec, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(ec, |mut state| { let ThresholdPlaintextAggregatorState::Computing(Computing { shares, .. }) = &mut state else { return Ok(state.clone()); @@ -243,7 +252,7 @@ impl ThresholdPlaintextAggregator { // Update the local state let plaintext = response.plaintext; - self.set_decryption(plaintext.clone())?; + self.set_decryption(plaintext.clone(), &ec)?; // Dispatch the PlaintextAggregated event let event = PlaintextAggregated { @@ -330,7 +339,7 @@ impl Handler>> ec, ) = msg.into_inner().into_components(); - self.add_share(party_id, decryption_share)?; + self.add_share(party_id, decryption_share, &ec)?; if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { threshold_m, diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index a47b0920d2..5b28d51cd5 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -13,7 +13,8 @@ use serde::{de::DeserializeOwned, Serialize}; pub trait PersistableData: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} impl PersistableData for T where T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} -/// AutoPersist enables a repository to generate a persistable container +/// AutoPersist enables a repository to generate a persistable container. This is not a database and +/// should not be thought of as a database. This is for creating actor snapshots. #[async_trait] pub trait AutoPersist where @@ -193,10 +194,25 @@ where } /// Mutate the content if available or return an error - pub fn try_mutate(&mut self, mutator: F) -> Result<()> + pub fn try_mutate_without_context(&mut self, mutator: F) -> Result<()> where F: FnOnce(T) -> Result, { + self.try_mutate_impl(mutator, None) + } + + pub fn try_mutate(&mut self, ctx: &EventContext, mutator: F) -> Result<()> + where + F: FnOnce(T) -> Result, + { + self.try_mutate_impl(mutator, Some(ctx.clone())) + } + + fn try_mutate_impl(&mut self, mutator: F, ctx: Option>) -> Result<()> + where + F: FnOnce(T) -> Result, + { + self.ctx = ctx; // Set the context let content = self.data.clone().ok_or(anyhow!("Data has not been set"))?; self.data = Some(mutator(content)?); self.write_to_store(); diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 83a52d1038..070f0e32b1 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -444,7 +444,7 @@ impl ThresholdKeyshare { let sk_bfv_encrypted = SensitiveBytes::new(sk_bytes, &self.cipher)?; let pk_bfv_bytes = ArcBytes::from_bytes(&pk_bfv.to_bytes()); - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { s.new_state(KeyshareState::CollectingEncryptionKeys( CollectingEncryptionKeysData { sk_bfv: sk_bfv_encrypted.clone(), @@ -483,7 +483,7 @@ impl ThresholdKeyshare { let current: CollectingEncryptionKeysData = self.state.try_get()?.try_into()?; - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { s.new_state(KeyshareState::GeneratingThresholdShare( GeneratingThresholdShareData { sk_sss: None, @@ -553,7 +553,7 @@ impl ThresholdKeyshare { let esi_sss = output.esi_sss; - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; info!("try_store_esi_sss"); @@ -642,7 +642,7 @@ impl ThresholdKeyshare { let (pk_share, sk_sss) = (output.pk_share, output.sk_sss); - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { info!("try_store_pk_share_and_sk_sss"); let current: GeneratingThresholdShareData = s.clone().try_into()?; let next = match current.esi_sss { @@ -850,7 +850,7 @@ impl ThresholdKeyshare { let (sk_poly_sum, es_poly_sum) = (output.sk_poly_sum, output.es_poly_sum); - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; info!("Try store decryption key"); @@ -891,7 +891,7 @@ impl ThresholdKeyshare { ) -> Result<()> { let (msg, ec) = msg.into_components(); // Set state to decrypting - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; let current: ReadyForDecryption = s.clone().try_into()?; @@ -949,10 +949,10 @@ impl ThresholdKeyshare { }; // send the decryption share - self.bus.publish(event, ec)?; + self.bus.publish(event, ec.clone())?; // mark as complete - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; info!("Decryption share sending process is complete"); diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 8812fab129..1a80597258 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -67,7 +67,9 @@ impl Handler for CiphernodeSelector { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { let (msg, ec) = msg.into_components(); match msg { - EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + EnclaveEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::CommitteeFinalized(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } @@ -86,7 +88,7 @@ impl Handler>> for CiphernodeSelecto _: &mut Self::Context, ) -> Self::Result { trap(EType::Sortition, &self.bus.clone(), || { - self.e3_cache.try_mutate(|mut cache| { + self.e3_cache.try_mutate(data.get_ctx(), |mut cache| { info!( "Mutating e3_cache: appending data: {:?}", data.e3_id.clone() @@ -121,7 +123,7 @@ impl Handler>> for CiphernodeSelecto ticket_id: TicketId::Score(tid), node: data.address().to_owned(), }, - data.get_ctx().clone(), + data.get_ctx().to_owned(), )?; } @@ -130,11 +132,15 @@ impl Handler>> for CiphernodeSelecto } } -impl Handler for CiphernodeSelector { +impl Handler> for CiphernodeSelector { type Result = (); - fn handle(&mut self, msg: E3RequestComplete, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::Sortition, &self.bus.clone(), move || { - self.e3_cache.try_mutate(|mut cache| { + self.e3_cache.try_mutate(msg.get_ctx(), |mut cache| { cache.remove(&msg.e3_id); Ok(cache) }) diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 9def6864ff..c1b969e3a2 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -262,7 +262,7 @@ impl Sortition { let node_state = node_state_store.load_or_default(HashMap::new()).await?; let finalized_committees = committees_store.load_or_default(HashMap::new()).await?; - backends.try_mutate(|mut list| { + backends.try_mutate_without_context(|mut list| { list.insert(u64::MAX, default_backend); Ok(list) })?; @@ -361,16 +361,30 @@ impl Handler for Sortition { let (msg, ec) = msg.into_components(); match msg { EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), - EnclaveEventData::CiphernodeAdded(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CiphernodeRemoved(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::TicketBalanceUpdated(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CiphernodeAdded(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CiphernodeRemoved(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::TicketBalanceUpdated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::OperatorActivationChanged(data) => { - self.notify_sync(ctx, data.clone()) + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::ConfigurationUpdated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CommitteePublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::PlaintextOutputPublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CommitteeFinalized(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) } - EnclaveEventData::ConfigurationUpdated(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CommitteePublished(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::PlaintextOutputPublished(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data.clone()), _ => (), } } @@ -390,14 +404,15 @@ impl Handler> for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CiphernodeAdded, _ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); let chain_id = msg.chain_id; let addr = msg.address.clone(); - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + if let Err(err) = self.node_state.try_mutate(&ec, |mut state_map| { let chain_state = state_map .entry(chain_id) .or_insert_with(NodeStateStore::default); @@ -410,7 +425,7 @@ impl Handler for Sortition { self.bus.err(EType::Sortition, err); } - if let Err(err) = self.backends.try_mutate(move |mut list_map| { + if let Err(err) = self.backends.try_mutate(&ec, move |mut list_map| { let default_backend = list_map .get(&u64::MAX) .cloned() @@ -429,14 +444,19 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CiphernodeRemoved, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); let chain_id = msg.chain_id; let addr = msg.address.clone(); - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + if let Err(err) = self.node_state.try_mutate(&ec, |mut state_map| { if let Some(chain_state) = state_map.get_mut(&chain_id) { chain_state.nodes.remove(&addr); } @@ -445,7 +465,7 @@ impl Handler for Sortition { self.bus.err(EType::Sortition, err); } - if let Err(err) = self.backends.try_mutate(move |mut list_map| { + if let Err(err) = self.backends.try_mutate(&ec, move |mut list_map| { if let Some(backend) = list_map.get_mut(&chain_id) { backend.remove(addr); } @@ -458,11 +478,16 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: TicketBalanceUpdated, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + if let Err(err) = self.node_state.try_mutate(&ec, |mut state_map| { let chain_state = state_map .entry(msg.chain_id) .or_insert_with(NodeStateStore::default); @@ -486,11 +511,16 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: OperatorActivationChanged, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + if let Err(err) = self.node_state.try_mutate(&ec, |mut state_map| { // Update all entries for this operator across all chains for (_, chain_state) in state_map.iter_mut() { let node = chain_state @@ -513,12 +543,17 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: ConfigurationUpdated, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); if msg.parameter == "ticketPrice" { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + if let Err(err) = self.node_state.try_mutate(&ec, |mut state_map| { let chain_state = state_map .entry(msg.chain_id) .or_insert_with(NodeStateStore::default); @@ -537,11 +572,16 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CommitteePublished, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + if let Err(err) = self.node_state.try_mutate(&ec, |mut state_map| { let chain_id = msg.e3_id.chain_id(); let e3_id_str = format!("{}:{}", chain_id, msg.e3_id.e3_id()); let chain_state = state_map @@ -600,11 +640,16 @@ where /// But in reality, E3 Jobs might not emit that in case there are no votes or the job fails. /// We need to find a better way to handle the end of an E3, Reduce the jobs in case of of an Error /// so the tickets do not get locked up. -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: PlaintextOutputPublished, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + if let Err(err) = self.node_state.try_mutate(&ec,|mut state_map| { let chain_id = msg.e3_id.chain_id(); let e3_id_str = format!("{}:{}", chain_id, msg.e3_id.e3_id()); @@ -646,17 +691,22 @@ impl Handler for Sortition { } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CommitteeFinalized, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); info!( e3_id = %msg.e3_id, committee_size = msg.committee.len(), "Storing finalized committee" ); - if let Err(err) = self.finalized_committees.try_mutate(|mut committees| { + if let Err(err) = self.finalized_committees.try_mutate(&ec, |mut committees| { committees.insert(msg.e3_id.clone(), msg.committee.clone()); Ok(committees) }) { From bc8dffd857cc0e83833b6db8e682142714a8d163 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 04:34:37 +0000 Subject: [PATCH 08/63] fixed better error handling was causing failing test --- Cargo.lock | 2 + crates/events/src/bus_handle.rs | 6 ++ crates/events/src/enclave_event/mod.rs | 10 ++- crates/events/src/eventbus.rs | 3 +- crates/net/Cargo.toml | 1 + crates/net/src/cid.rs | 9 +-- crates/net/src/document_publisher.rs | 90 +++++++++++++++++--------- crates/utils/Cargo.toml | 1 + crates/utils/src/formatters.rs | 36 +++++++++++ 9 files changed, 122 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c542b28c4e..56fe12b6e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3274,6 +3274,7 @@ dependencies = [ "async-trait", "bincode", "chrono", + "derivative", "e3-ciphernode-builder", "e3-config", "e3-crypto", @@ -3560,6 +3561,7 @@ dependencies = [ "derivative", "rand 0.8.5", "rand_chacha 0.3.1", + "regex", "serde", "tokio", "tracing", diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index a2a17d4085..2105ca42fb 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -81,6 +81,12 @@ impl BusHandle { let pipe = BusHandlePipe::new(other.to_owned(), predicate).start(); self.subscribe(EventType::All, pipe.into()); } + + pub fn with_ec(&self, ec: &EventContext) -> Self { + let mut bus = self.clone(); + bus.set_ctx(ec.clone()); + bus + } } impl EventPublisher> for BusHandle { diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 483a07468f..34e30881de 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -56,7 +56,7 @@ pub use decryptionshare_created::*; pub use die::*; pub use e3_request_complete::*; pub use e3_requested::*; -use e3_utils::{colorize, Color}; +use e3_utils::{colorize, colorize_event_ids, Color}; pub use enclave_error::*; pub use encryption_key_collection_failed::*; pub use encryption_key_created::*; @@ -534,7 +534,13 @@ impl EnclaveEvent { impl fmt::Display for EnclaveEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let t = self.event_type(); - f.write_str(&format!("{} {:?}", colorize(t, Color::Cyan), self)) + let colorized_debug = colorize_event_ids(self); + + let s = match t.as_str() { + "EnclaveError" => format!("{} {}", colorize(t, Color::Red), colorized_debug), + _ => format!("{} {}", colorize(t, Color::Cyan), colorized_debug), + }; + f.write_str(&s) } } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index f899f5b1aa..cf3be8cc63 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -8,6 +8,7 @@ use crate::traits::{ErrorEvent, Event}; use crate::EventType; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; +use e3_utils::{colorize, Color}; use std::collections::{HashMap, VecDeque}; use std::marker::PhantomData; use tracing::info; @@ -131,7 +132,7 @@ impl Handler for EventBus { } // TODO: workshop to work out best display format - tracing::info!(">>> {}", event); + tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); self.track(event); } } diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml index 46f00d41fc..37c2540de3 100644 --- a/crates/net/Cargo.toml +++ b/crates/net/Cargo.toml @@ -13,6 +13,7 @@ async-std = { workspace = true } async-trait = { workspace = true } bincode = { workspace = true } chrono = { workspace = true } +derivative = { workspace = true } futures = { workspace = true } e3-crypto = { workspace = true } e3-config = { workspace = true } diff --git a/crates/net/src/cid.rs b/crates/net/src/cid.rs index e9ee58df05..86678b39ca 100644 --- a/crates/net/src/cid.rs +++ b/crates/net/src/cid.rs @@ -4,13 +4,14 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::ops::{Deref, DerefMut}; - +use derivative::Derivative; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use std::ops::{Deref, DerefMut}; -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Cid(pub Vec); +#[derive(Derivative, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derivative(Debug)] +pub struct Cid(#[derivative(Debug(format_with = "e3_utils::formatters::hexf"))] pub Vec); impl Cid { pub fn from_content(content: &[u8]) -> Self { diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 56d73d84fc..b74cf77b95 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -11,6 +11,7 @@ use crate::{ Cid, }; use actix::prelude::*; +use anyhow::Context; use anyhow::Result; use chrono::{DateTime, Utc}; use e3_events::{ @@ -136,44 +137,65 @@ impl Actor for DocumentPublisher { impl Handler for DocumentPublisher { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::PublishDocumentRequested(data) => ctx.notify(data), - EnclaveEventData::CiphernodeSelected(data) => self.notify_sync(ctx, data), - EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::PublishDocumentRequested(data) => { + ctx.notify(TypedEvent::new(data, ec)) + } + EnclaveEventData::CiphernodeSelected(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for DocumentPublisher { +impl Handler> for DocumentPublisher { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: PublishDocumentRequested, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { let tx = self.tx.clone(); - let msg = msg.clone(); + let (msg, ec) = msg.into_components(); let rx = self.rx.clone(); let bus = self.bus.clone(); let topic = self.topic.clone(); trap_fut( EType::IO, - &bus.clone(), + &bus.with_ec(&ec), handle_publish_document_requested(tx, rx, msg, topic, bus), ) } } -impl Handler for DocumentPublisher { +impl Handler> for DocumentPublisher { type Result = (); - fn handle(&mut self, msg: CiphernodeSelected, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::DocumentPublishing, &self.bus.clone(), || { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { self.handle_ciphernode_selected(msg) }) } } -impl Handler for DocumentPublisher { +impl Handler> for DocumentPublisher { type Result = (); - fn handle(&mut self, msg: E3RequestComplete, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::DocumentPublishing, &self.bus.clone(), || { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { self.handle_e3_request_complete(msg) }) } @@ -471,8 +493,8 @@ impl EventConverter { /// Note: Filtering already happened in DocumentPublisher before DHT fetch. fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { let (msg, ctx) = msg.into_components(); - let receivable = ReceivableDocument::from_bytes(&msg.value.extract_bytes())?; - + let receivable = ReceivableDocument::from_bytes(&msg.value.extract_bytes()) + .context("Could not deserialize document bytes")?; match receivable { ReceivableDocument::ThresholdShareCreated(evt) => { debug!( @@ -538,9 +560,11 @@ impl Handler> for EventConverter { msg: TypedEvent, _ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::DocumentPublishing, &self.bus.clone(), || { - self.handle_threshold_share_created(msg) - }) + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_threshold_share_created(msg), + ) } } @@ -551,9 +575,11 @@ impl Handler> for EventConverter { msg: TypedEvent, _ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::DocumentPublishing, &self.bus.clone(), || { - self.handle_encryption_key_created(msg) - }) + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_encryption_key_created(msg), + ) } } @@ -564,9 +590,11 @@ impl Handler> for EventConverter { msg: TypedEvent, _ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::DocumentPublishing, &self.bus.clone(), || { - self.handle_document_received(msg) - }) + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_document_received(msg), + ) } } @@ -853,7 +881,7 @@ mod tests { GossipData::DocumentPublishedNotification(DocumentPublishedNotification { key: cid.clone(), meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], Some(expires_at)), - ts: 123, + ts: 100, }), ))?; @@ -874,7 +902,7 @@ mod tests { net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { key: cid, correlation_id, - value, + value, // This will error because this is not a ReceivableDocument which is fine })?; // wait for events to settle @@ -882,8 +910,12 @@ mod tests { // Check event was dispatched let events = history.send(GetEvents::new()).await?; - let Some(EnclaveEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = - events.last().map(|e| e.get_data()) + let Some(EnclaveEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = events + .iter() + // Filter out the error + .filter(|e| e.event_type() != "EnclaveError") + .last() + .map(|e| e.get_data()) else { bail!("No event sent"); }; diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index bc2563d2eb..b031a29e86 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -12,6 +12,7 @@ alloy.workspace = true anyhow.workspace = true derivative.workspace = true rand.workspace = true +regex.workspace = true rand_chacha.workspace = true serde.workspace = true tokio.workspace = true diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index b8094d900b..d1711d74aa 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -5,6 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use core::fmt; +use std::sync::LazyLock; + +use regex::Regex; // Custom formatter function for hex display pub fn hexf(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result { @@ -36,6 +39,7 @@ pub fn truncate(s: String) -> String { } } +#[derive(Clone, Copy)] pub enum Color { Black = 30, Red = 31, @@ -58,3 +62,35 @@ pub enum Color { pub fn colorize(s: T, color: Color) -> String { format!("\x1b[{}m{}\x1b[0m", color as u8, s) } + +// Pre-compile the regex for efficiency +static EVENT_ID_RE: LazyLock = + LazyLock::new(|| Regex::new(r"EventId\(0x([a-fA-F0-9]+)\)").unwrap()); + +/// Colorizes `EventId(0x...)` patterns in the debug output of a value. +fn hash_to_color(hex_str: &str) -> u8 { + let hash: u32 = hex_str + .bytes() + .fold(0u32, |acc, b| acc.wrapping_mul(31).wrapping_add(b as u32)); + 22 + (hash as u8 % 200) // 200 colors, skipping dark ones +} + +pub fn colorize_event_ids(value: &T) -> String { + let s = format!("{:?}", value); + let mut result = String::with_capacity(s.len() + 100); + let mut last_end = 0; + for cap in EVENT_ID_RE.captures_iter(&s) { + let full_match = cap.get(0).unwrap(); + let hex_str = cap.get(1).unwrap().as_str(); + result.push_str(&s[last_end..full_match.start()]); + let color = hash_to_color(hex_str); + result.push_str(&format!( + "\x1b[38;5;{}m{}\x1b[0m", + color, + full_match.as_str() + )); + last_end = full_match.end(); + } + result.push_str(&s[last_end..]); + result +} From 6010772622777f8a648dbaac493daccbd238bc53 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 05:00:33 +0000 Subject: [PATCH 09/63] update cargo lock with correct regex version --- examples/CRISP/Cargo.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/CRISP/Cargo.lock b/examples/CRISP/Cargo.lock index b96e8ea803..d86251d7ca 100644 --- a/examples/CRISP/Cargo.lock +++ b/examples/CRISP/Cargo.lock @@ -2540,6 +2540,7 @@ dependencies = [ "derivative", "rand 0.8.5", "rand_chacha 0.3.1", + "regex", "serde", "tokio", "tracing", @@ -4814,9 +4815,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", From 01337e22b37a4f48e84c80cd8ec2b9c77c7e973a Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 05:13:33 +0000 Subject: [PATCH 10/63] publish_origin -> publish_without_context --- crates/aggregator/src/committee_finalizer.rs | 2 +- crates/ciphernode-builder/src/event_system.rs | 8 ++--- crates/entrypoint/src/helpers/shutdown.rs | 2 +- crates/events/src/bus_handle.rs | 10 +++--- crates/events/src/sequencer.rs | 2 +- crates/events/src/traits.rs | 2 +- crates/evm/src/evm_chain_gateway.rs | 4 +-- crates/evm/tests/integration.rs | 6 ++-- crates/keyshare/src/threshold_keyshare.rs | 4 +-- crates/net/src/document_publisher.rs | 8 ++--- crates/sync/src/sync.rs | 10 +++--- crates/test-helpers/src/lib.rs | 2 +- crates/tests/tests/integration.rs | 34 +++++++++---------- 13 files changed, 47 insertions(+), 47 deletions(-) diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index 0d2d2b20b6..bba43955d1 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -114,7 +114,7 @@ impl Handler for CommitteeFinalizer { info!(e3_id = %e3_id_clone, "Dispatching CommitteeFinalizeRequested event"); trap(EType::Sortition, &act.bus.clone(), || { - bus.publish_origin(CommitteeFinalizeRequested { + bus.publish_without_context(CommitteeFinalizeRequested { e3_id: e3_id_clone.clone(), })?; Ok(()) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index de86c1496f..ed875d86a9 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -505,7 +505,7 @@ mod tests { ); // Push an event - handle.publish_origin(TestEvent::new("pink", 1))?; + handle.publish_without_context(TestEvent::new("pink", 1))?; sleep(Duration::from_millis(1)).await; // Now we have published an event all data should be written we can get the data from the store @@ -523,9 +523,9 @@ mod tests { let ts = handle.ts()?; // Push a few other events - handle.publish_origin(TestEvent::new("yellow", 1))?; - handle.publish_origin(TestEvent::new("red", 1))?; - handle.publish_origin(TestEvent::new("white", 1))?; + handle.publish_without_context(TestEvent::new("yellow", 1))?; + handle.publish_without_context(TestEvent::new("red", 1))?; + handle.publish_without_context(TestEvent::new("white", 1))?; sleep(Duration::from_millis(100)).await; // Get the event logs from the listener diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 97c2f59a33..353ced3c75 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -22,7 +22,7 @@ pub async fn listen_for_shutdown(node: CiphernodeHandle) { info!("SIGTERM received, initiating graceful shutdown..."); // Stop the actor system - match bus.publish_origin(Shutdown){ + match bus.publish_without_context(Shutdown){ Ok(_) => (), Err(e) => error!("Shutdown failed to publish! {e}") } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 2105ca42fb..1c8aa8510b 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -98,7 +98,7 @@ impl EventPublisher> for BusHandle { self.publish_local(data, Some(ctx)) } - fn publish_origin(&self, data: impl Into) -> Result<()> { + fn publish_without_context(&self, data: impl Into) -> Result<()> { self.publish_local(data, None) } @@ -324,13 +324,13 @@ mod tests { bus_c.subscribe(EventType::All, saver.clone().into()); // Publish events in causal order across buses - bus_a.publish_origin(TestEvent::new("one", 1))?; + bus_a.publish_without_context(TestEvent::new("one", 1))?; sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish_origin(TestEvent::new("two", 2))?; + bus_b.publish_without_context(TestEvent::new("two", 2))?; sleep(Duration::from_millis(5)).await; // next tick - bus_a.publish_origin(TestEvent::new("three", 3))?; + bus_a.publish_without_context(TestEvent::new("three", 3))?; sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish_origin(TestEvent::new("four", 4))?; + bus_b.publish_without_context(TestEvent::new("four", 4))?; sleep(Duration::from_millis(50)).await; // next tick // Get events diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 5cd4b694bc..b98db473f7 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -73,7 +73,7 @@ mod tests { ]; for d in event_data.clone() { - bus.publish_origin(d)?; + bus.publish_without_context(d)?; } let expected = event_data diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index a8bb3f4530..64dfa0fd2d 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -88,7 +88,7 @@ pub trait EventPublisher { fn publish(&self, data: impl Into, ctx: EventContext) -> Result<()>; /// This creates a context based on the given data. This should only be used when an event is /// the origin event - fn publish_origin(&self, data: impl Into) -> Result<()>; + fn publish_without_context(&self, data: impl Into) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. /// diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 0fca1d00b7..4c08bd2b95 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -241,7 +241,7 @@ mod tests { // SyncStart: Init -> ForwardToSyncActor let mut evm_config = EvmEventConfig::new(); evm_config.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_origin(SyncStart::new(collector.clone(), evm_config)) + bus.publish_without_context(SyncStart::new(collector.clone(), evm_config)) .unwrap(); // Send EVM event while forwarding - should reach collector @@ -285,7 +285,7 @@ mod tests { addr.do_send(EnclaveEvmEvent::Event(buffered_event)); // SyncEnd: BufferUntilLive -> Live (publishes buffered events to bus) - bus.publish_origin(SyncEnd::new()).unwrap(); + bus.publish_without_context(SyncEnd::new()).unwrap(); // Allow time for async message processing tokio::time::sleep(std::time::Duration::from_millis(50)).await; diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 5e8e950720..2b377efc53 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -108,7 +108,7 @@ impl Handler for FakeSyncActor { let (data, ts, _) = evt.split(); self.bus.publish_from_remote(data, ts)?; } - self.bus.publish_origin(SyncEnd::new())?; + self.bus.publish_without_context(SyncEnd::new())?; } }; Ok(()) @@ -160,7 +160,7 @@ async fn evm_reader() -> Result<()> { // This should trigger all chains to start to sync let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_origin(SyncStart::new(sync, evm_info))?; + bus.publish_without_context(SyncStart::new(sync, evm_info))?; sleep(Duration::from_secs(1)).await; contract @@ -238,7 +238,7 @@ async fn ensure_historical_events() -> Result<()> { .build(); let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_origin(SyncStart::new(sync, evm_info))?; + bus.publish_without_context(SyncStart::new(sync, evm_info))?; for msg in live_events.clone() { contract diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 070f0e32b1..1b0ec6760f 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -1072,7 +1072,7 @@ impl Handler for ThresholdKeyshare { self.encryption_key_collector = None; // Publish failure event to event bus for sync tracking - self.bus.publish_origin(msg)?; + self.bus.publish_without_context(msg)?; // Stop this actor since we can't proceed without all encryption keys ctx.stop(); @@ -1100,7 +1100,7 @@ impl Handler for ThresholdKeyshare { self.decryption_key_collector = None; // Publish failure event to event bus for sync tracking - self.bus.publish_origin(msg)?; + self.bus.publish_without_context(msg)?; ctx.stop(); Ok(()) diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index b74cf77b95..4ba72afa6d 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -663,7 +663,7 @@ mod tests { let e3_id = E3id::new("1243", 1); // 1. Send a request to publish - bus.publish_origin(PublishDocumentRequested { + bus.publish_without_context(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), })?; @@ -737,7 +737,7 @@ mod tests { let cid = Cid::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish_origin(CiphernodeSelected { + bus.publish_without_context(CiphernodeSelected { e3_id: e3_id.clone(), threshold_m: 3, threshold_n: 5, @@ -800,7 +800,7 @@ mod tests { let e3_id = E3id::new("1243", 1); // Send a request to publish - bus.publish_origin(PublishDocumentRequested { + bus.publish_without_context(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), })?; @@ -850,7 +850,7 @@ mod tests { let cid = Cid::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish_origin(CiphernodeSelected { + bus.publish_without_context(CiphernodeSelected { e3_id: e3_id.clone(), threshold_m: 3, threshold_n: 5, diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index f31396f19b..61e989e1b9 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -65,11 +65,11 @@ impl Synchronizer { // publish them in order for evt in self.evm_buffer.drain(..) { let (data, _, _) = evt.split(); - self.bus.publish_origin(data)?; // Use publish_origin here as historical events will be correctly - // ordered as part of the preparatory process and these - // events have no + self.bus.publish_without_context(data)?; // Use publish_without_context here as historical events will be correctly + // ordered as part of the preparatory process and these + // events have no } - self.bus.publish_origin(SyncEnd::new())?; + self.bus.publish_without_context(SyncEnd::new())?; Ok(()) } } @@ -108,7 +108,7 @@ impl Handler for Synchronizer { // TODO: Get information about what has and has not been synced then fire SyncStart self.bus - .publish_origin(SyncStart::new(ctx.address(), evm_config)) + .publish_without_context(SyncStart::new(ctx.address(), evm_config)) }) } } diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 25f9db0255..58cf2ce48b 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -181,7 +181,7 @@ impl AddToCommittee { self.count += 1; - self.bus.publish_origin(evt.clone())?; + self.bus.publish_without_context(evt.clone())?; Ok(evt.into()) } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 286f8dbf3b..cf8523221f 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -43,7 +43,7 @@ async fn setup_score_sortition_environment( eth_addrs: &Vec, chain_id: u64, ) -> Result<()> { - bus.publish_origin(ConfigurationUpdated { + bus.publish_without_context(ConfigurationUpdated { parameter: "ticketPrice".to_string(), old_value: U256::ZERO, new_value: U256::from(10_000_000u64), @@ -54,7 +54,7 @@ async fn setup_score_sortition_environment( for addr in eth_addrs { adder.add(addr).await?; - bus.publish_origin(TicketBalanceUpdated { + bus.publish_without_context(TicketBalanceUpdated { operator: addr.clone(), delta: I256::try_from(1_000_000_000u64).unwrap(), new_balance: U256::from(1_000_000_000u64), @@ -62,7 +62,7 @@ async fn setup_score_sortition_environment( chain_id, })?; - bus.publish_origin(OperatorActivationChanged { + bus.publish_without_context(OperatorActivationChanged { operator: addr.clone(), active: true, chain_id, @@ -238,7 +238,7 @@ async fn test_trbfv_actor() -> Result<()> { params, }; - bus.publish_origin(e3_requested)?; + bus.publish_without_context(e3_requested)?; // For score sortition, we need to wait for nodes to process E3Requested and run sortition // Since TicketGenerated is a local-only event (not shared across network), we can't collect it @@ -265,7 +265,7 @@ async fn test_trbfv_actor() -> Result<()> { .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; - bus.publish_origin(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: e3_id.clone(), committee, chain_id, @@ -383,7 +383,7 @@ async fn test_trbfv_actor() -> Result<()> { e3_id: e3_id.clone(), }; - bus.publish_origin(ciphertext_published_event.clone())?; + bus.publish_without_context(ciphertext_published_event.clone())?; println!("CiphertextOutputPublished event has been dispatched!"); @@ -520,9 +520,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { ..CiphernodeSelected::default() }; - bus.publish_origin(evt_1.clone())?; - bus.publish_origin(evt_2.clone())?; - bus.publish_origin(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network + bus.publish_without_context(evt_1.clone())?; + bus.publish_without_context(evt_2.clone())?; + bus.publish_without_context(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network // check the history of the event bus let history = history_collector @@ -674,7 +674,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { }; // Send e3request - bus.publish_origin(E3Requested { + bus.publish_without_context(E3Requested { e3_id: e3_id.clone(), threshold_m: 2, threshold_n: 2, @@ -683,7 +683,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { ..E3Requested::default() })?; - bus.publish_origin(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: e3_id.clone(), committee: eth_addrs.clone(), chain_id: 1, @@ -699,7 +699,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { assert_eq!(errors.len(), 0); // SEND SHUTDOWN! - bus.publish_origin(Shutdown)?; + bus.publish_without_context(Shutdown)?; // This is probably overkill but required to ensure that all the data is written sleep(Duration::from_secs(1)).await; @@ -764,7 +764,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { use e3_test_helpers::encrypt_ciphertext; let raw_plaintext = vec![vec![4, 5]]; let (ciphertext, expected) = encrypt_ciphertext(¶ms, pubkey, raw_plaintext)?; - bus.publish_origin(CiphertextOutputPublished { + bus.publish_without_context(CiphertextOutputPublished { ciphertext_output: ciphertext .iter() .map(|ct| ArcBytes::from_bytes(&ct.to_bytes())) @@ -911,7 +911,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { setup_score_sortition_environment(&bus, ð_addrs, 2).await?; // Send the computation requested event - bus.publish_origin(E3Requested { + bus.publish_without_context(E3Requested { e3_id: E3id::new("1234", 1), threshold_m: 2, threshold_n: 5, @@ -920,7 +920,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ..E3Requested::default() })?; - bus.publish_origin(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: E3id::new("1234", 1), committee: eth_addrs.clone(), chain_id: 1, @@ -955,7 +955,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ); // Send the computation requested event - bus.publish_origin(E3Requested { + bus.publish_without_context(E3Requested { e3_id: E3id::new("1234", 2), threshold_m: 2, threshold_n: 5, @@ -964,7 +964,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ..E3Requested::default() })?; - bus.publish_origin(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: E3id::new("1234", 2), committee: eth_addrs.clone(), chain_id: 2, From d91dbef4f6d984902e28609eed596b7e3ab55c44 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 06:31:53 +0000 Subject: [PATCH 11/63] pass context to errors and fix up colorizing to avoid looking like errors --- crates/aggregator/src/committee_finalizer.rs | 26 ++-- crates/aggregator/src/publickey_aggregator.rs | 27 ++-- .../src/threshold_plaintext_aggregator.rs | 142 ++++++++++-------- crates/evm/src/evm_chain_gateway.rs | 4 +- crates/evm/tests/integration.rs | 2 +- crates/keyshare/src/threshold_keyshare.rs | 40 +++-- crates/net/src/net_event_translator.rs | 2 +- crates/net/src/net_sync_manager.rs | 4 +- crates/request/src/router.rs | 2 +- crates/sync/src/sync.rs | 8 +- crates/utils/src/formatters.rs | 49 +++++- 11 files changed, 188 insertions(+), 118 deletions(-) diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index bba43955d1..bf682d9b7b 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -7,7 +7,7 @@ use actix::prelude::*; use e3_events::{ prelude::*, trap, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EType, - EnclaveEvent, EnclaveEventData, EventType, Shutdown, + EnclaveEvent, EnclaveEventData, EventType, Shutdown, TypedEvent, }; use e3_utils::NotifySync; use std::collections::HashMap; @@ -48,23 +48,31 @@ impl Actor for CommitteeFinalizer { impl Handler for CommitteeFinalizer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::CommitteeRequested(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::CommitteeRequested(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } } -impl Handler for CommitteeFinalizer { +impl Handler> for CommitteeFinalizer { type Result = (); - fn handle(&mut self, msg: CommitteeRequested, ctx: &mut Self::Context) -> Self::Result { + // TODO: Remove all async from this function. Remove reliance on e3_evm package. Add unit test. + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let ec = msg.get_ctx().clone(); let e3_id = msg.e3_id.clone(); let submission_deadline = msg.submission_deadline; const FINALIZATION_BUFFER_SECONDS: u64 = 1; - let e3_id_for_log = e3_id.clone(); let fut = async move { match e3_evm::helpers::get_current_timestamp().await { @@ -113,10 +121,10 @@ impl Handler for CommitteeFinalizer { move |act, _ctx| { info!(e3_id = %e3_id_clone, "Dispatching CommitteeFinalizeRequested event"); - trap(EType::Sortition, &act.bus.clone(), || { - bus.publish_without_context(CommitteeFinalizeRequested { + trap(EType::Sortition, &act.bus.with_ec(&ec), || { + bus.publish(CommitteeFinalizeRequested { e3_id: e3_id_clone.clone(), - })?; + },ec)?; Ok(()) }); diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index 6ba393280b..4e99a213b6 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -146,17 +146,14 @@ impl Actor for PublicKeyAggregator { impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - let (msg, ec) = msg.into_components(); - match msg { - EnclaveEventData::KeyshareCreated(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), - _ => (), - }; - Ok(()) - }); + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::KeyshareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + _ => (), + }; } } @@ -168,8 +165,8 @@ impl Handler> for PublicKeyAggregator { event: TypedEvent, ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::PublickeyAggregation, &self.bus.clone(), || { - let (event, ec) = event.into_components(); + let (event, ec) = event.into_components(); + trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { let e3_id = event.e3_id.clone(); let pubkey = event.pubkey.clone(); let node = event.node.clone(); @@ -203,8 +200,8 @@ impl Handler> for PublicKeyAggregator { type Result = (); fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::PublickeyAggregation, &self.bus.clone(), || { - let (msg, ec) = msg.into_components(); + let (msg, ec) = msg.into_components(); + trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { info!("Computing Aggregate PublicKey..."); let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { keyshares: msg.keyshares, diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 54508cab42..ba0060e63b 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -292,19 +292,23 @@ impl Handler> for ThresholdPlaintextAggregato msg: TypedEvent, ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::PublickeyAggregation, &self.bus.clone(), || { - let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = - self.state.get() - else { - debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); - return Ok(()); - }; - let node = msg.node.clone(); - let e3_id = msg.e3_id.clone(); - let request = E3CommitteeContainsRequest::new(e3_id, node, msg, ctx.address()); - self.sortition.try_send(request)?; - Ok(()) - }) + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = + self.state.get() + else { + debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); + return Ok(()); + }; + let node = msg.node.clone(); + let e3_id = msg.e3_id.clone(); + let request = E3CommitteeContainsRequest::new(e3_id, node, msg, ctx.address()); + self.sortition.try_send(request)?; + Ok(()) + }, + ) } } @@ -317,71 +321,79 @@ impl Handler>> msg: E3CommitteeContainsResponse>, ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::PublickeyAggregation, &self.bus.clone(), || { - let e3_id = &msg.e3_id; - if *e3_id != self.e3_id { - bail!("Wrong e3_id sent to aggregator. This should not happen.") - }; - - if !msg.is_found_in_committee() { - trace!("Node {} not found in finalized committee", &msg.node); - return Ok(()); - }; - - // Trust the party_id from the event - it's based on CommitteeFinalized order - // which is the authoritative source of truth for party IDs - let ( - DecryptionshareCreated { - party_id, - decryption_share, + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let e3_id = &msg.e3_id; + if *e3_id != self.e3_id { + bail!("Wrong e3_id sent to aggregator. This should not happen.") + }; + + if !msg.is_found_in_committee() { + trace!("Node {} not found in finalized committee", &msg.node); + return Ok(()); + }; + + // Trust the party_id from the event - it's based on CommitteeFinalized order + // which is the authoritative source of truth for party IDs + let ( + DecryptionshareCreated { + party_id, + decryption_share, + .. + }, + ec, + ) = msg.into_inner().into_components(); + + self.add_share(party_id, decryption_share, &ec)?; + + if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { + threshold_m, + threshold_n, + shares, + ciphertext_output, .. - }, - ec, - ) = msg.into_inner().into_components(); - - self.add_share(party_id, decryption_share, &ec)?; - - if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { - threshold_m, - threshold_n, - shares, - ciphertext_output, - .. - })) = self.state.get() - { - self.notify_sync( - ctx, - TypedEvent::new( - ComputeAggregate { - shares: shares.clone(), - ciphertext_output: ciphertext_output.clone(), - threshold_m, - threshold_n, - }, - ec, - ), - ) - } - Ok(()) - }) + })) = self.state.get() + { + self.notify_sync( + ctx, + TypedEvent::new( + ComputeAggregate { + shares: shares.clone(), + ciphertext_output: ciphertext_output.clone(), + threshold_m, + threshold_n, + }, + ec, + ), + ) + } + Ok(()) + }, + ) } } impl Handler> for ThresholdPlaintextAggregator { type Result = (); fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::PlaintextAggregation, &self.bus.clone(), || { - self.handle_compute_aggregate(msg) - }) + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_aggregate(msg), + ) } } impl Handler> for ThresholdPlaintextAggregator { type Result = (); fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::PlaintextAggregation, &self.bus.clone(), || { - self.handle_compute_response(msg) - }) + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg), + ) } } diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 4c08bd2b95..928960c087 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -185,7 +185,7 @@ impl Actor for EvmChainGateway { impl Handler for EvmChainGateway { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.clone(), || { + trap(EType::Evm, &self.bus.with_ec(msg.get_ctx()), || { match msg.into_data() { EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e)?, @@ -198,7 +198,7 @@ impl Handler for EvmChainGateway { impl Handler for EvmChainGateway { type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) } } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 2b377efc53..2504a32e2f 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -97,7 +97,7 @@ impl FakeSyncActor { impl Handler for FakeSyncActor { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { match msg { // Buffer events as the sync actor receives them diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 1b0ec6760f..16b6b57eb3 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -995,9 +995,11 @@ impl Handler for ThresholdKeyshare { impl Handler> for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_compute_response(msg) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg), + ) } } @@ -1008,9 +1010,11 @@ impl Handler> for ThresholdKeyshare { msg: TypedEvent, ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_ciphernode_selected(msg, ctx.address()) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_ciphernode_selected(msg, ctx.address()), + ) } } @@ -1021,9 +1025,11 @@ impl Handler> for ThresholdKeyshare { msg: TypedEvent, _: &mut Self::Context, ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_all_encryption_keys_collected(msg) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_all_encryption_keys_collected(msg), + ) } } @@ -1034,9 +1040,11 @@ impl Handler> for ThresholdKeyshare { msg: TypedEvent, _: &mut Self::Context, ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_all_threshold_shares_collected(msg) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_all_threshold_shares_collected(msg), + ) } } @@ -1047,9 +1055,11 @@ impl Handler> for ThresholdKeyshare { msg: TypedEvent, _: &mut Self::Context, ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_ciphertext_output_published(msg) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_ciphertext_output_published(msg), + ) } } diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 962827d5bf..4ff06270e4 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -171,7 +171,7 @@ impl Handler for NetEventTranslator { impl Handler for NetEventTranslator { type Result = (); fn handle(&mut self, event: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Net, &self.bus.clone(), || { + trap(EType::Net, &self.bus.with_ec(event.get_ctx()), || { let sent_events = self.sent_events.clone(); let tx = self.tx.clone(); let evt = event.clone(); diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 49361fc40a..2256e9c3a6 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -108,7 +108,7 @@ impl Handler> for NetSyncManager { ) -> Self::Result { trap_fut( EType::Net, - &self.bus.clone(), + &self.bus.with_ec(msg.get_ctx()), handle_sync_request_event(self.tx.clone(), self.rx.clone(), msg, ctx.address()), ) } @@ -122,7 +122,7 @@ impl Handler> for NetSyncManager { msg: TypedEvent, _: &mut Self::Context, ) -> Self::Result { - trap(EType::Net, &self.bus.clone(), || { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { let (msg, ctx) = msg.into_components(); self.bus.publish_from_remote_as_response( NetSyncEventsReceived { diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index d1a8d9de25..5d3b7410e4 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -150,7 +150,7 @@ impl Actor for E3Router { impl Handler for E3Router { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Event, &self.bus.clone(), || { + trap(EType::Event, &self.bus.with_ec(msg.get_ctx()), || { // If we are shutting down then bail on anything else if let EnclaveEventData::Shutdown(_) = msg.get_data() { for (_, ctx) in self.contexts.iter() { diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 61e989e1b9..3df3fad412 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -67,7 +67,8 @@ impl Synchronizer { let (data, _, _) = evt.split(); self.bus.publish_without_context(data)?; // Use publish_without_context here as historical events will be correctly // ordered as part of the preparatory process and these - // events have no + // events have no prior or causal events as + // they come from the blockchain } self.bus.publish_without_context(SyncEnd::new())?; Ok(()) @@ -81,9 +82,12 @@ impl Actor for Synchronizer { } } +// TODO: Make SyncEvmevent carry EnclaveEvent as the evm package should be in the +// business of creating EnclaveEvents for the rest of the application. It should not be +// Synchronizers responsability to do this otherwise it knows too much about the evm package impl Handler for Synchronizer { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { match msg { // Buffer events as the sync actor receives them diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index d1711d74aa..6f7892ce22 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -67,12 +67,49 @@ pub fn colorize(s: T, color: Color) -> String { static EVENT_ID_RE: LazyLock = LazyLock::new(|| Regex::new(r"EventId\(0x([a-fA-F0-9]+)\)").unwrap()); -/// Colorizes `EventId(0x...)` patterns in the debug output of a value. -fn hash_to_color(hex_str: &str) -> u8 { - let hash: u32 = hex_str +/// Hashes a string to an ANSI 256 color within `[hue_min, hue_max)` degrees. +/// +/// # Examples +/// ``` +/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 300.0) // orange to purple +/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 330.0) // full spectrum, no red +/// hash_str_to_ansi_color_in_hue_range(s, 0.0, 360.0) // full spectrum +/// ``` +fn hash_str_to_ansi_color_in_hue_range(s: &str, hue_min: f32, hue_max: f32) -> u8 { + let hash: u32 = s .bytes() .fold(0u32, |acc, b| acc.wrapping_mul(31).wrapping_add(b as u32)); - 22 + (hash as u8 % 200) // 200 colors, skipping dark ones + + let hue = hue_min + (hash as f32 % (hue_max - hue_min)); + + let (r, g, b) = hsv_to_rgb(hue, 1.0, 1.0); + + rgb_to_ansi256(r, g, b) +} + +fn hsv_to_rgb(h: f32, s: f32, v: f32) -> (f32, f32, f32) { + let c = v * s; + let x = c * (1.0 - ((h / 60.0) % 2.0 - 1.0).abs()); + let m = v - c; + + let (r, g, b) = match (h / 60.0) as u32 { + 0 => (c, x, 0.0), + 1 => (x, c, 0.0), + 2 => (0.0, c, x), + 3 => (0.0, x, c), + 4 => (x, 0.0, c), + _ => (c, 0.0, x), + }; + + (r + m, g + m, b + m) +} + +fn rgb_to_ansi256(r: f32, g: f32, b: f32) -> u8 { + let r6 = (r * 5.0).round() as u8; + let g6 = (g * 5.0).round() as u8; + let b6 = (b * 5.0).round() as u8; + + 16 + 36 * r6 + 6 * g6 + b6 } pub fn colorize_event_ids(value: &T) -> String { @@ -83,7 +120,9 @@ pub fn colorize_event_ids(value: &T) -> String { let full_match = cap.get(0).unwrap(); let hex_str = cap.get(1).unwrap().as_str(); result.push_str(&s[last_end..full_match.start()]); - let color = hash_to_color(hex_str); + let color = hash_str_to_ansi_color_in_hue_range(hex_str, 30.0, 330.0); // Avoiding red so + // it does not look + // like errors result.push_str(&format!( "\x1b[38;5;{}m{}\x1b[0m", color, From 9261e60f207f206292fea6b0eb50aa961e641bba Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 06:45:39 +0000 Subject: [PATCH 12/63] updates --- .../keyshare/src/threshold_share_collector.rs | 6 + crates/sortition/src/ciphernode_selector.rs | 121 ++++++++++-------- crates/sync/src/sync.rs | 1 + 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/crates/keyshare/src/threshold_share_collector.rs b/crates/keyshare/src/threshold_share_collector.rs index 34943d9f2e..11a416a1be 100644 --- a/crates/keyshare/src/threshold_share_collector.rs +++ b/crates/keyshare/src/threshold_share_collector.rs @@ -33,11 +33,17 @@ pub(crate) enum CollectorState { pub struct ThresholdShareCollectionTimeout; pub struct ThresholdShareCollector { + /// The E3id for the round e3_id: E3id, + /// The partys the collector expects to receive from todo: HashSet, + /// The parent actor that has requested collection parent: Addr, + /// The current state of the collector state: CollectorState, + /// The shares collected shares: HashMap>, + /// A timeout handle for when this collector will report failure timeout_handle: Option, } diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 1a80597258..a82d72fffa 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -87,7 +87,7 @@ impl Handler>> for CiphernodeSelecto data: WithSortitionTicket>, _: &mut Self::Context, ) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), || { + trap(EType::Sortition, &self.bus.with_ec(data.get_ctx()), || { self.e3_cache.try_mutate(data.get_ctx(), |mut cache| { info!( "Mutating e3_cache: appending data: {:?}", @@ -139,12 +139,16 @@ impl Handler> for CiphernodeSelector { msg: TypedEvent, _: &mut Self::Context, ) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), move || { - self.e3_cache.try_mutate(msg.get_ctx(), |mut cache| { - cache.remove(&msg.e3_id); - Ok(cache) - }) - }) + trap( + EType::Sortition, + &self.bus.with_ec(msg.get_ctx()), + move || { + self.e3_cache.try_mutate(msg.get_ctx(), |mut cache| { + cache.remove(&msg.e3_id); + Ok(cache) + }) + }, + ) } } @@ -156,60 +160,65 @@ impl Handler> for CiphernodeSelector { msg: TypedEvent, _ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), move || { - let (msg, ec) = msg.into_components(); - info!("CiphernodeSelector received CommitteeFinalized."); - let bus = self.bus.clone(); - info!("Getting e3_cache..."); - let Some(e3_cache) = self.e3_cache.get() else { - bail!("Could not get cache"); - }; - - info!("Getting e3_meta..."); - let Some(e3_meta) = e3_cache.get(&msg.e3_id) else { - bail!( - "Could not find E3Meta on CiphernodeSelector for {}", - msg.e3_id - ); - }; - - // Check if this node is in the finalized committee - if !msg.committee.contains(&self.address) { - info!(node = self.address, "Node not in finalized committee"); - return Ok(()); - } + trap( + EType::Sortition, + &self.bus.with_ec(msg.get_ctx()), + move || { + let (msg, ec) = msg.into_components(); + info!("CiphernodeSelector received CommitteeFinalized."); + let bus = self.bus.clone(); + info!("Getting e3_cache..."); + let Some(e3_cache) = self.e3_cache.get() else { + bail!("Could not get cache"); + }; + + info!("Getting e3_meta..."); + let Some(e3_meta) = e3_cache.get(&msg.e3_id) else { + bail!( + "Could not find E3Meta on CiphernodeSelector for {}", + msg.e3_id + ); + }; + + // Check if this node is in the finalized committee + if !msg.committee.contains(&self.address) { + info!(node = self.address, "Node not in finalized committee"); + return Ok(()); + } + + // Retrieve E3 metadata from repository + let Some(party_id) = msg.committee.iter().position(|addr| addr == &self.address) + else { + info!( + node = self.address, + "Node address not found in committee list (should not happen)" + ); + return Ok(()); + }; - // Retrieve E3 metadata from repository - let Some(party_id) = msg.committee.iter().position(|addr| addr == &self.address) else { info!( node = self.address, - "Node address not found in committee list (should not happen)" + party_id = party_id, + "Node is in finalized committee, emitting CiphernodeSelected" ); - return Ok(()); - }; - - info!( - node = self.address, - party_id = party_id, - "Node is in finalized committee, emitting CiphernodeSelected" - ); - - bus.publish( - CiphernodeSelected { - party_id: party_id as u64, - e3_id: msg.e3_id, - threshold_m: e3_meta.threshold_m, - threshold_n: e3_meta.threshold_n, - esi_per_ct: e3_meta.esi_per_ct, - error_size: e3_meta.error_size.clone(), - params: e3_meta.params.clone(), - seed: e3_meta.seed, - }, - ec, - )?; - Ok(()) - }) + bus.publish( + CiphernodeSelected { + party_id: party_id as u64, + e3_id: msg.e3_id, + threshold_m: e3_meta.threshold_m, + threshold_n: e3_meta.threshold_n, + esi_per_ct: e3_meta.esi_per_ct, + error_size: e3_meta.error_size.clone(), + params: e3_meta.params.clone(), + seed: e3_meta.seed, + }, + ec, + )?; + + Ok(()) + }, + ) } } diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 3df3fad412..631347434c 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -88,6 +88,7 @@ impl Actor for Synchronizer { impl Handler for Synchronizer { type Result = (); fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) -> Self::Result { + // TODO: extract contect from enclave event trap(EType::Sync, &self.bus.clone(), || { match msg { // Buffer events as the sync actor receives them From 2fd50db2f84dbb6c107a60ad49188f118dbb6eaf Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 06:46:19 +0000 Subject: [PATCH 13/63] update doc tests --- crates/utils/src/formatters.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index 6f7892ce22..3a8d8a7b1e 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -71,9 +71,9 @@ static EVENT_ID_RE: LazyLock = /// /// # Examples /// ``` -/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 300.0) // orange to purple -/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 330.0) // full spectrum, no red -/// hash_str_to_ansi_color_in_hue_range(s, 0.0, 360.0) // full spectrum +/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 300.0); // orange to purple +/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 330.0); // full spectrum, no red +/// hash_str_to_ansi_color_in_hue_range(s, 0.0, 360.0); // full spectrum /// ``` fn hash_str_to_ansi_color_in_hue_range(s: &str, hue_min: f32, hue_max: f32) -> u8 { let hash: u32 = s From 5c65e67251277794c35f2a16c3d1874b57581f81 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 06:48:31 +0000 Subject: [PATCH 14/63] ensure doctests are ignored --- crates/utils/src/formatters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index 3a8d8a7b1e..ab7d6cc0a7 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -70,7 +70,7 @@ static EVENT_ID_RE: LazyLock = /// Hashes a string to an ANSI 256 color within `[hue_min, hue_max)` degrees. /// /// # Examples -/// ``` +/// ```ignore /// hash_str_to_ansi_color_in_hue_range(s, 30.0, 300.0); // orange to purple /// hash_str_to_ansi_color_in_hue_range(s, 30.0, 330.0); // full spectrum, no red /// hash_str_to_ansi_color_in_hue_range(s, 0.0, 360.0); // full spectrum From 98e0bdd3b5e3da22775021e6162d63b6be312374 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 06:49:27 +0000 Subject: [PATCH 15/63] typo --- crates/utils/src/formatters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index ab7d6cc0a7..77eaa3c9c6 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -67,7 +67,7 @@ pub fn colorize(s: T, color: Color) -> String { static EVENT_ID_RE: LazyLock = LazyLock::new(|| Regex::new(r"EventId\(0x([a-fA-F0-9]+)\)").unwrap()); -/// Hashes a string to an ANSI 256 color within `[hue_min, hue_max)` degrees. +/// Hashes a string to an ANSI 256 color within `[hue_min, hue_max]` degrees. /// /// # Examples /// ```ignore From d332516bcf6470bf90ef51fb27e7d9039eee3483 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 07:33:10 +0000 Subject: [PATCH 16/63] refactor EVM system architecture and routing configuration --- crates/ciphernode-builder/src/evm_system.rs | 72 +++++++++++++++------ crates/evm/src/evm_read_interface.rs | 24 +++---- crates/evm/src/evm_router.rs | 4 +- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 0c44393463..5edc3c8363 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -48,32 +48,64 @@ impl EvmSystemChainBuilder

{ } pub fn build(&mut self) { - let gateway = FixHistoricalOrder::setup(EvmChainGateway::setup(&self.bus)); - let runner = SyncStartExtractor::setup(OneShotRunner::setup({ + // Think about the following in reverse order + + // Gateway is the final step before connecting to the bus + let next = EvmChainGateway::setup(&self.bus); + + // Fix the historical order to avoid missing historical events + let next = FixHistoricalOrder::setup(next); + + // This will run once when the SyncStart event is received + let next = OneShotRunner::setup({ + // Clone self refs for closure let bus = self.bus.clone(); let provider = self.provider.clone(); - let gateway = gateway.clone(); let chain_id = self.chain_id; - // Only gets consumed once so fine to do this + + // Only gets consumed once so fine to use replace to clean out route_factories let route_factories = replace(&mut self.route_factories, Vec::new()); + + // The event is defined here move |msg: SyncStart| { - let config = msg.get_evm_config(chain_id)?; - let gateway = gateway.recipient(); - let mut router = EvmRouter::new(); - - for (address, route_fn) in route_factories { - let processor = route_fn(gateway.clone()); - router = router.add_route(address, &processor); - } - - router = router.add_fallback(&gateway); - let filters = - Filters::from_routing_table(router.get_routing_table(), config.deploy_block()); - let router = router.start(); - EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); + // Extract config + let deploy_block = msg.get_evm_config(chain_id)?.deploy_block(); + + // Pass next to the router + let router = configure_router(next, route_factories); + + // Extract filters from the router + let filters = filters_from_router(&router, deploy_block); + + // Setup and start the read interface and the router + EvmReadInterface::setup(&provider, router.start(), &bus, filters); Ok(()) } - })); - self.bus.subscribe(EventType::SyncStart, runner.recipient()); + }); + + // We get a SyncStart event and sent to oneShotRunner + let next = SyncStartExtractor::setup(next); + + // Finaly subscribe to the bus and wait for SyncStart + self.bus.subscribe(EventType::SyncStart, next.recipient()); } } + +/// Setup a router with a fallback and route factories all forwarding to next +fn configure_router( + next: impl Into, + route_factories: Vec<(Address, Box)>, +) -> EvmRouter { + let next = next.into(); + let mut router = EvmRouter::new(); + router.add_fallback(&next); + for (address, route_fn) in route_factories { + let processor = route_fn(next.clone()); + router.add_route(address, &processor); + } + router +} + +fn filters_from_router(router: &EvmRouter, deploy_block: u64) -> Filters { + Filters::from_routing_table(router.get_routing_table(), deploy_block) +} diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index ece45967f2..e6313014a8 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -28,7 +28,7 @@ pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; pub struct EvmReadInterfaceParams

{ provider: EthProvider

, - processor: Recipient, + next: Recipient, bus: BusHandle, filters: Filters, } @@ -75,8 +75,8 @@ pub struct EvmReadInterface

{ shutdown_rx: Option>, /// The sender for the shutdown signal this is only used internally shutdown_tx: Option>, - /// Processor to forward events an actor - processor: EvmEventProcessor, + /// Processor to forward events + next: EvmEventProcessor, /// Event bus for error propagation only bus: BusHandle, /// Filters to configure when to seek from @@ -90,7 +90,7 @@ impl EvmReadInterface

{ provider: Some(params.provider), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), - processor: params.processor, + next: params.next, bus: params.bus, filters: params.filters, } @@ -98,13 +98,13 @@ impl EvmReadInterface

{ pub fn setup( provider: &EthProvider

, - next: &EvmEventProcessor, + next: impl Into, bus: &BusHandle, filters: Filters, ) -> Addr { let params = EvmReadInterfaceParams { provider: provider.clone(), - processor: next.clone(), + next: next.into(), bus: bus.clone(), filters, }; @@ -122,7 +122,7 @@ impl Actor for EvmReadInterface

{ fn started(&mut self, ctx: &mut Self::Context) { // let reader_addr = ctx.address(); let bus = self.bus.clone(); - let processor = self.processor.clone(); + let next = self.next.clone(); let filters = self.filters.clone(); let Some(provider) = self.provider.take() else { @@ -137,7 +137,7 @@ impl Actor for EvmReadInterface

{ }; ctx.spawn( - async move { stream_from_evm(provider, processor, shutdown, &bus, filters).await } + async move { stream_from_evm(provider, next, shutdown, &bus, filters).await } .into_actor(self), ); } @@ -149,7 +149,7 @@ impl Actor for EvmReadInterface

{ #[instrument(name = "evm_interface", skip_all)] async fn stream_from_evm( provider: EthProvider

, - processor: EvmEventProcessor, + next: EvmEventProcessor, mut shutdown: oneshot::Receiver<()>, bus: &BusHandle, filters: Filters, @@ -167,7 +167,7 @@ async fn stream_from_evm( let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); last_id = Some(evt.get_id()); info!("Sending event({})", evt.get_id()); - processor.do_send(evt) + next.do_send(evt) } } Err(e) => { @@ -181,7 +181,7 @@ async fn stream_from_evm( "Historical Sync Complete event({})", historical_sync_event.get_id() ); - processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + next.do_send(EnclaveEvmEvent::HistoricalSyncComplete( historical_sync_event, )); @@ -197,7 +197,7 @@ async fn stream_from_evm( match maybe_log { Some(log) => { let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; - processor.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp))) + next.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp))) } None => break, // Stream ended } diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 790cb78487..a5d55a0972 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -25,12 +25,12 @@ impl EvmRouter { } } - pub fn add_route(mut self, address: Address, dest: &EvmEventProcessor) -> Self { + pub fn add_route(&mut self, address: Address, dest: &EvmEventProcessor) -> &mut Self { self.routing_table.insert(address, dest.clone()); self } - pub fn add_fallback(mut self, fallback: &EvmEventProcessor) -> Self { + pub fn add_fallback(&mut self, fallback: &EvmEventProcessor) -> &mut Self { self.fallback = Some(fallback.clone()); self } From 61399e78a6a33b724a99174a8c0a2d2948df47e7 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 11:04:21 +0000 Subject: [PATCH 17/63] update test --- crates/ciphernode-builder/src/evm_system.rs | 5 +- .../enclave_event/evm_sync_events_received.rs | 5 +- crates/events/src/enclave_event/sync_start.rs | 17 +- crates/evm/src/evm_chain_gateway.rs | 189 ++++++++++++------ crates/evm/src/evm_router.rs | 8 +- crates/evm/tests/integration.rs | 46 +---- crates/sync/src/sync.rs | 46 ++--- 7 files changed, 174 insertions(+), 142 deletions(-) diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 5edc3c8363..991af572cb 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -97,11 +97,10 @@ fn configure_router( route_factories: Vec<(Address, Box)>, ) -> EvmRouter { let next = next.into(); - let mut router = EvmRouter::new(); - router.add_fallback(&next); + let mut router = EvmRouter::new().add_fallback(&next); for (address, route_fn) in route_factories { let processor = route_fn(next.clone()); - router.add_route(address, &processor); + router = router.add_route(address, &processor); } router } diff --git a/crates/events/src/enclave_event/evm_sync_events_received.rs b/crates/events/src/enclave_event/evm_sync_events_received.rs index 0fcf1e3347..f6d9325bf8 100644 --- a/crates/events/src/enclave_event/evm_sync_events_received.rs +++ b/crates/events/src/enclave_event/evm_sync_events_received.rs @@ -14,11 +14,12 @@ use super::{EnclaveEvent, Unsequenced}; #[rtype(result = "()")] pub struct EvmSyncEventsReceived { pub events: Vec>, + pub chain_id: u64, } impl EvmSyncEventsReceived { - pub fn new(events: Vec>) -> Self { - Self { events } + pub fn new(events: Vec>, chain_id: u64) -> Self { + Self { events, chain_id } } } diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 3841785133..038e4d90a5 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -4,8 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use super::EnclaveEventData; -use crate::{CorrelationId, SyncEvmEvent}; +use super::{EnclaveEventData, EvmSyncEventsReceived}; +use crate::CorrelationId; use crate::{EvmEventConfig, EvmEventConfigChain}; use actix::{Message, Recipient}; use anyhow::Context; @@ -68,14 +68,17 @@ pub struct SyncStart { #[serde(skip)] /// We include the sender here so that the evm can communicate directly with the sync actor - pub sender: Option>, // Must be Option to allow serde deserialize on - // EnclaveEvent as Default is required to be - // implemented this is fine as this event is never - // shared + pub sender: Option>, // Must be Option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented this is fine as this event is never + // shared } impl SyncStart { - pub fn new(sender: impl Into>, evm_config: EvmEventConfig) -> Self { + pub fn new( + sender: impl Into>, + evm_config: EvmEventConfig, + ) -> Self { Self { sender: Some(sender.into()), evm_config, diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 928960c087..f9c37eb5bb 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -4,17 +4,19 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::mem::take; + use crate::events::EnclaveEvmEvent; use crate::HistoricalSyncComplete; use actix::{Actor, Handler}; use actix::{Addr, Recipient}; use anyhow::Result; use anyhow::{bail, Context}; +use e3_events::EType; use e3_events::{ - trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, SyncEnd, - SyncEvmEvent, SyncStart, + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventFactory, EventSubscriber, EventType, + EvmSyncEventsReceived, SyncEnd, SyncStart, Unsequenced, }; -use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; use tracing::info; @@ -25,16 +27,28 @@ pub struct EvmChainGateway { status: SyncStatus, } +#[derive(Clone, Default, Debug)] +struct ForwardToSyncActorData { + pub sender: Option>, + pub buffer: Vec>, +} + +impl ForwardToSyncActorData { + pub fn add_event(&mut self, event: EnclaveEvent) { + self.buffer.push(event); + } +} + /// This state machine coordinates the function of the EvmChainGateway #[derive(Clone, Debug)] enum SyncStatus { /// Intial State - Init(Vec), // Include a buffer to hold events that arrive too early + Init(Vec>), // Include a buffer to hold events that arrive too early /// After SyncStart we forward all events to SyncActor - ForwardToSyncActor(Option>), + ForwardToSyncActor(ForwardToSyncActorData), /// Once the chain has completed historical sync then we buffer all "live" events until sync is /// complete - BufferUntilLive(Vec), + BufferUntilLive(Vec>), /// Forward all events directly to the bus Live, } @@ -48,8 +62,8 @@ impl Default for SyncStatus { impl SyncStatus { pub fn forward_to_sync_actor( &mut self, - sender: Recipient, - ) -> Result> { + sender: Recipient, + ) -> Result>> { let Self::Init(buffer) = self else { bail!( "Cannot change state to ForwardToSyncActor when state is {:?}", @@ -58,23 +72,27 @@ impl SyncStatus { }; let buffer = std::mem::take(buffer); - *self = SyncStatus::ForwardToSyncActor(Some(sender)); + *self = SyncStatus::ForwardToSyncActor(ForwardToSyncActorData { + sender: Some(sender), + buffer: Vec::new(), + }); Ok(buffer) } - pub fn buffer_until_live(&mut self) -> Result> { + pub fn buffer_until_live(&mut self) -> Result { let Self::ForwardToSyncActor(sender) = self else { bail!( "Cannot change state to BufferUntilLive when state is {:?}", self ); }; - let sender = std::mem::take(sender).context("Cannot call buffer_until_live twice")?; + + let state_data = take(sender); *self = SyncStatus::BufferUntilLive(vec![]); - Ok(sender) + Ok(state_data) } - pub fn live(&mut self) -> Result> { + pub fn live(&mut self) -> Result>> { let Self::BufferUntilLive(buffer) = self else { bail!("Cannot change state to Live when state is {:?}", self); }; @@ -123,9 +141,8 @@ impl EvmChainGateway { Ok(()) } - fn publish_evm_event(&mut self, msg: EvmEvent) -> Result<()> { - let (data, ts, _) = msg.split(); - self.bus.publish_from_remote(data, ts)?; + fn publish_evm_event(&mut self, msg: EnclaveEvent) -> Result<()> { + self.bus.naked_dispatch(msg); Ok(()) } @@ -136,7 +153,10 @@ impl EvmChainGateway { Ok(()) } EnclaveEvmEvent::Event(event) => { - self.process_evm_event(event)?; + info!("Received event!"); + let (data,ts,_) = event.split(); + let enclave_event = self.bus.event_from_remote_source(data,None,ts)?; + self.process_evm_event(enclave_event)?; Ok(()) } _ => panic!("EvmChainGateway is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), @@ -148,31 +168,28 @@ impl EvmChainGateway { "handling historical sync complete for chain_id({})", event.chain_id ); - let sender = self.status.buffer_until_live()?; + let state = self.status.buffer_until_live()?; info!("Sending historical sync complete event to sender."); - sender.try_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id))?; + let sender = state + .sender + .context("ForwardToSyncActor state must hold a sender")?; + let event = EvmSyncEventsReceived::new(state.buffer, event.chain_id); + sender.try_send(event)?; Ok(()) } - fn process_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + fn process_evm_event(&mut self, msg: EnclaveEvent) -> Result<()> { match &mut self.status { - SyncStatus::BufferUntilLive(buffer) => { - info!("saving evm event({}) to pre-live buffer", msg.get_id()); - buffer.push(msg) - } - SyncStatus::ForwardToSyncActor(Some(sync_actor)) => { - info!("forwarding evm event({}) to SyncActor", msg.get_id()); - sync_actor.do_send(msg.into()); - } - SyncStatus::Live => { - info!("publishing evm event({})", msg.get_id()); - self.publish_evm_event(msg)? - } SyncStatus::Init(buffer) => { - info!("saving evm event({}) to pre-sync buffer", msg.get_id()); - buffer.push(msg) + info!("Buffering until Forwarding... {:?}", msg); + buffer.push(msg); } - _ => (), + SyncStatus::BufferUntilLive(buffer) => { + info!("Buffering until live... {:?}", msg); + buffer.push(msg); + } + SyncStatus::ForwardToSyncActor(state) => state.add_event(msg), + SyncStatus::Live => self.publish_evm_event(msg)?, }; Ok(()) } @@ -199,6 +216,7 @@ impl Handler for EvmChainGateway { impl Handler for EvmChainGateway { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { + info!("Handler"); trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) } } @@ -208,28 +226,41 @@ mod tests { use super::*; use e3_ciphernode_builder::EventSystem; - use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TestEvent}; + use e3_events::{ + CorrelationId, EvmEvent, EvmEventConfig, EvmEventConfigChain, GetEvents, TakeEvents, + TestEvent, + }; use tokio::sync::mpsc; + use tracing_subscriber::{fmt, EnvFilter}; struct SyncEventCollector { - tx: mpsc::UnboundedSender, + tx: mpsc::UnboundedSender, } impl Actor for SyncEventCollector { type Context = actix::Context; } - impl Handler for SyncEventCollector { + impl Handler for SyncEventCollector { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) { + fn handle(&mut self, msg: EvmSyncEventsReceived, _: &mut Self::Context) { let _ = self.tx.send(msg); } } #[actix::test] - async fn test_evm_chain_gateway() { + async fn test_evm_chain_gateway() -> Result<()> { + let _foo = tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new("info")) + .with_test_writer() + .finish(), + ); + let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle().unwrap(); + let bus: BusHandle = system.handle()?; + + let history_collector = bus.history(); let (tx, mut rx) = mpsc::unbounded_channel(); let collector = SyncEventCollector { tx }.start(); @@ -248,7 +279,7 @@ mod tests { let evm_event = EvmEvent::new( CorrelationId::new(), TestEvent { - msg: "test".to_string(), + msg: "Before Complete".to_string(), entropy: 1, } .into(), @@ -256,25 +287,45 @@ mod tests { 12345, chain_id, ); - // This will actually arrive earlier than SyncStart but aught to be buffered - addr.do_send(EnclaveEvmEvent::Event(evm_event)); - let received = rx.recv().await.unwrap(); - assert!(matches!(received, SyncEvmEvent::Event(_))); + // This will actually arrive earlier than SyncStart but aught to be buffered + addr.send(EnclaveEvmEvent::Event(evm_event)).await?; // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive - addr.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + addr.send(EnclaveEvmEvent::HistoricalSyncComplete( HistoricalSyncComplete::new(chain_id, None), - )); + )) + .await?; + // Normal Synchronizer will take this and wait for other events before flushing events to + // the bus here we simulate it let received = rx.recv().await.unwrap(); - assert!(matches!(received, SyncEvmEvent::HistoricalSyncComplete(_))); + for event in received.events { + bus.naked_dispatch(event); + } // Send EVM event while buffering - should be buffered (not received) let buffered_event = EvmEvent::new( CorrelationId::new(), TestEvent { - msg: "buffered".to_string(), + msg: "Before SyncEnd".to_string(), + entropy: 2, + } + .into(), + 101, + 12346, + chain_id, + ); + addr.send(EnclaveEvmEvent::Event(buffered_event)).await?; + + // The Synchronizer will publish the SyncEnd event when it has all the information it needs + // and has published everything to the bus + bus.publish_without_context(SyncEnd::new())?; + + let after_event = EvmEvent::new( + CorrelationId::new(), + TestEvent { + msg: "After SyncEnd".to_string(), entropy: 2, } .into(), @@ -282,15 +333,39 @@ mod tests { 12346, chain_id, ); - addr.do_send(EnclaveEvmEvent::Event(buffered_event)); - // SyncEnd: BufferUntilLive -> Live (publishes buffered events to bus) - bus.publish_without_context(SyncEnd::new()).unwrap(); + addr.send(EnclaveEvmEvent::Event(after_event)).await?; + + let full = history_collector.send(TakeEvents::new(5)).await?; - // Allow time for async message processing - tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let test_events: Vec = full + .iter() + .filter_map(|e| { + if let EnclaveEventData::TestEvent(TestEvent { msg, .. }) = e.get_data() { + Some(msg.to_string()) + } else { + None + } + }) + .collect(); - // Verify no more messages were sent to collector (buffered events go to bus, not collector) - assert!(rx.try_recv().is_err()); + assert_eq!( + test_events, + vec!["Before Complete", "Before SyncEnd", "After SyncEnd"] + ); + + let event_types: Vec = full.iter().map(|e| e.event_type()).collect(); + + assert_eq!( + event_types, + vec![ + "SyncStart", + "TestEvent", + "SyncEnd", + "TestEvent", + "TestEvent" + ] + ); + Ok(()) } } diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index a5d55a0972..9a4251b867 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; -use actix::{Actor, Addr, Handler}; +use actix::{Actor, Handler}; use alloy_primitives::Address; use std::collections::HashMap; use tracing::{debug, error, info}; @@ -25,12 +25,12 @@ impl EvmRouter { } } - pub fn add_route(&mut self, address: Address, dest: &EvmEventProcessor) -> &mut Self { + pub fn add_route(mut self, address: Address, dest: &EvmEventProcessor) -> Self { self.routing_table.insert(address, dest.clone()); self } - pub fn add_fallback(&mut self, fallback: &EvmEventProcessor) -> &mut Self { + pub fn add_fallback(mut self, fallback: &EvmEventProcessor) -> Self { self.fallback = Some(fallback.clone()); self } @@ -46,7 +46,7 @@ impl Actor for EvmRouter { impl Handler for EvmRouter { type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { match msg.clone() { // Take all log events and route them EnclaveEvmEvent::Log(EvmLog { log, .. }) => { diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 2504a32e2f..66d34084e1 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -16,8 +16,8 @@ use alloy::{ use anyhow::Result; use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ - prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, EvmEventConfig, - EvmEventConfigChain, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, + prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEventConfig, + EvmEventConfigChain, EvmSyncEventsReceived, GetEvents, SyncEnd, SyncStart, TestEvent, }; use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; use std::{sync::Arc, time::Duration}; @@ -61,24 +61,8 @@ impl TestEventParser { } } -async fn get_msgs(history_collector: &Addr>) -> Result> { - let history = history_collector - .send(GetEvents::::new()) - .await?; - let msgs: Vec = history - .into_iter() - .filter_map(|evt| match evt.into_data() { - EnclaveEventData::TestEvent(data) => Some(data.msg), - _ => None, - }) - .collect(); - - Ok(msgs) -} - struct FakeSyncActor { bus: BusHandle, - buffer: Vec, } impl Actor for FakeSyncActor { @@ -87,30 +71,18 @@ impl Actor for FakeSyncActor { impl FakeSyncActor { pub fn setup(bus: &BusHandle) -> Addr { - Self { - bus: bus.clone(), - buffer: Vec::new(), - } - .start() + Self { bus: bus.clone() }.start() } } -impl Handler for FakeSyncActor { +impl Handler for FakeSyncActor { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, mut msg: EvmSyncEventsReceived, _: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { - match msg { - // Buffer events as the sync actor receives them - SyncEvmEvent::Event(event) => self.buffer.push(event), - // When we hear that sync is complete send all events on chain then publish SyncEnd - SyncEvmEvent::HistoricalSyncComplete(_) => { - for evt in self.buffer.drain(..) { - let (data, ts, _) = evt.split(); - self.bus.publish_from_remote(data, ts)?; - } - self.bus.publish_without_context(SyncEnd::new())?; - } - }; + for evt in msg.events.drain(..) { + self.bus.naked_dispatch(evt); + } + self.bus.publish_without_context(SyncEnd::new())?; Ok(()) }) } diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 631347434c..5fd55024a9 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -9,8 +9,9 @@ use std::collections::HashSet; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{ - trap, BusHandle, EType, EventContext, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, - SyncEvmEvent, SyncStart, + trap, BusHandle, EType, EnclaveEvent, EventContext, EventContextAccessors, EventPublisher, + EvmEvent, EvmEventConfig, EvmSyncEventsReceived, SyncEnd, SyncEvmEvent, SyncStart, TypedEvent, + Unsequenced, }; use tracing::info; @@ -21,7 +22,7 @@ type ChainId = u64; pub struct Synchronizer { bus: BusHandle, evm_config: Option, - evm_buffer: Vec, + evm_events: Vec>, evm_to_sync: HashSet, // net_config: NetEventConfig, } @@ -32,8 +33,8 @@ impl Synchronizer { Self { evm_config: Some(evm_config), bus: bus.clone(), - evm_buffer: Vec::new(), evm_to_sync, + evm_events: Vec::new(), } } @@ -41,14 +42,11 @@ impl Synchronizer { Self::new(bus, evm_config).start() } - fn buffer_evm_event(&mut self, event: EvmEvent) { - info!("buffer evm event({})", event.get_id()); - self.evm_buffer.push(event); - } - - fn handle_sync_complete(&mut self, chain_id: u64) -> Result<()> { + fn handle_sync_complete(&mut self, mut msg: EvmSyncEventsReceived) -> Result<()> { + let chain_id = msg.chain_id; info!("handle sync complete for chain({})", chain_id); self.evm_to_sync.remove(&chain_id); + self.evm_events.append(&mut msg.events); info!("{} chains left to sync...", self.evm_to_sync.len()); if self.evm_to_sync.is_empty() { self.handle_sync_end()?; @@ -57,18 +55,13 @@ impl Synchronizer { } fn handle_sync_end(&mut self) -> Result<()> { - // TODO: WORK OUT WHAT IS GOING ON HERE WITH PUBLISHING AND CONTEXT info!("all chains synced draining to bus and running sync end"); // Order all events (theoretically) - self.evm_buffer.sort_by_key(|i| i.ts()); + self.evm_events.sort_by_key(|i| i.ts()); // publish them in order - for evt in self.evm_buffer.drain(..) { - let (data, _, _) = evt.split(); - self.bus.publish_without_context(data)?; // Use publish_without_context here as historical events will be correctly - // ordered as part of the preparatory process and these - // events have no prior or causal events as - // they come from the blockchain + for evt in self.evm_events.drain(..) { + self.bus.naked_dispatch(evt); } self.bus.publish_without_context(SyncEnd::new())?; Ok(()) @@ -82,22 +75,11 @@ impl Actor for Synchronizer { } } -// TODO: Make SyncEvmevent carry EnclaveEvent as the evm package should be in the -// business of creating EnclaveEvents for the rest of the application. It should not be -// Synchronizers responsability to do this otherwise it knows too much about the evm package -impl Handler for Synchronizer { +impl Handler for Synchronizer { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) -> Self::Result { - // TODO: extract contect from enclave event + fn handle(&mut self, msg: EvmSyncEventsReceived, _: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { - match msg { - // Buffer events as the sync actor receives them - SyncEvmEvent::Event(event) => self.buffer_evm_event(event), - // When we hear that sync is complete send all events on chain then publish SyncEnd - SyncEvmEvent::HistoricalSyncComplete(chain_id) => { - self.handle_sync_complete(chain_id)? - } - }; + self.handle_sync_complete(msg)?; Ok(()) }) } From b014b6ac90589a1446bc550f922534683fe94b87 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 11:59:00 +0000 Subject: [PATCH 18/63] refactor: move EvmEvent from events to evm crate and simplify sync flow - Move EvmEvent struct definition from events crate to evm crate - Remove unused SyncEvmEvent enum and simplify sync message handling - Add tracing utility to test-helpers for cleaner test setup - Update imports and dependencies across affected crates --- Cargo.lock | 3 + crates/events/src/enclave_event/sync_start.rs | 48 +----------- crates/events/src/sync.rs | 20 +---- crates/evm/Cargo.toml | 1 + crates/evm/src/events.rs | 47 ++++++++++- crates/evm/src/evm_chain_gateway.rs | 7 +- crates/evm/src/evm_parser.rs | 4 +- crates/evm/tests/integration.rs | 15 +--- crates/sync/Cargo.toml | 1 + crates/sync/src/sync.rs | 77 ++++++++----------- crates/test-helpers/Cargo.toml | 1 + crates/test-helpers/src/utils.rs | 13 +++- 12 files changed, 104 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96bfaf914c..3be47ce1c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3076,6 +3076,7 @@ dependencies = [ "e3-evm", "e3-fhe-params", "e3-sortition", + "e3-test-helpers", "e3-trbfv", "e3-utils", "futures-util", @@ -3440,6 +3441,7 @@ dependencies = [ "anyhow", "e3-ciphernode-builder", "e3-events", + "e3-test-helpers", "tokio", "tracing", ] @@ -3476,6 +3478,7 @@ dependencies = [ "rand_chacha 0.3.1", "tokio", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 038e4d90a5..388db36859 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -4,8 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use super::{EnclaveEventData, EvmSyncEventsReceived}; -use crate::CorrelationId; +use super::EvmSyncEventsReceived; use crate::{EvmEventConfig, EvmEventConfigChain}; use actix::{Message, Recipient}; use anyhow::Context; @@ -13,51 +12,6 @@ use anyhow::Result; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; -/// This is a processed EvmEvent specifically typed for the Sync actor -#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[rtype(result = "()")] -pub struct EvmEvent { - data: EnclaveEventData, - block: u64, - chain_id: u64, - ts: u128, - id: CorrelationId, -} - -impl EvmEvent { - pub fn new( - id: CorrelationId, - data: EnclaveEventData, - block: u64, - ts: u128, - chain_id: u64, - ) -> Self { - Self { - id, - data, - block, - ts, - chain_id, - } - } - - pub fn split(self) -> (EnclaveEventData, u128, u64) { - (self.data, self.ts, self.block) - } - - pub fn get_id(&self) -> CorrelationId { - self.id - } - - pub fn chain_id(&self) -> u64 { - self.chain_id - } - - pub fn ts(&self) -> u128 { - self.ts - } -} - /// Dispatched by the Sync actor when initial data is read and the sync process needs to be started #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index bc13534344..13d75a9cf1 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -4,26 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::{BTreeMap, HashSet}; - -use crate::EvmEvent; -use actix::Message; use serde::{Deserialize, Serialize}; -type Chainid = u64; -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[rtype(result = "()")] -pub enum SyncEvmEvent { - /// Signal that this reader has completed historical sync - HistoricalSyncComplete(ChainId), - /// An actual event from the blockchain - Event(EvmEvent), -} - -impl From for SyncEvmEvent { - fn from(event: EvmEvent) -> SyncEvmEvent { - SyncEvmEvent::Event(event) - } -} +use std::collections::{BTreeMap, HashSet}; type ChainId = u64; type DeployBlock = u64; diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index b3e77c6bda..f41e5e035f 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -35,5 +35,6 @@ zeroize = { workspace = true } e3-evm = { workspace = true } e3-entrypoint = { workspace = true } e3-ciphernode-builder = { workspace = true } +e3-test-helpers = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } tracing-subscriber = { workspace = true } diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index 4b59deaaeb..e4688c544a 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -6,7 +6,7 @@ use actix::{Message, Recipient}; use alloy::rpc::types::Log; -use e3_events::{CorrelationId, EvmEvent}; +use e3_events::{CorrelationId, EnclaveEventData}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -31,6 +31,51 @@ impl HistoricalSyncComplete { } } +/// This is a processed EvmEvent specifically typed for the Sync actor +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct EvmEvent { + data: EnclaveEventData, + block: u64, + chain_id: u64, + ts: u128, + id: CorrelationId, +} + +impl EvmEvent { + pub fn new( + id: CorrelationId, + data: EnclaveEventData, + block: u64, + ts: u128, + chain_id: u64, + ) -> Self { + Self { + id, + data, + block, + ts, + chain_id, + } + } + + pub fn split(self) -> (EnclaveEventData, u128, u64) { + (self.data, self.ts, self.block) + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } + + pub fn chain_id(&self) -> u64 { + self.chain_id + } + + pub fn ts(&self) -> u128 { + self.ts + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub enum EnclaveEvmEvent { diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index f9c37eb5bb..538dc6ccf4 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -223,13 +223,12 @@ impl Handler for EvmChainGateway { #[cfg(test)] mod tests { + use crate::EvmEvent; + use super::*; use e3_ciphernode_builder::EventSystem; - use e3_events::{ - CorrelationId, EvmEvent, EvmEventConfig, EvmEventConfigChain, GetEvents, TakeEvents, - TestEvent, - }; + use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TakeEvents, TestEvent}; use tokio::sync::mpsc; use tracing_subscriber::{fmt, EnvFilter}; diff --git a/crates/evm/src/evm_parser.rs b/crates/evm/src/evm_parser.rs index 2710fd10a9..da31fa9161 100644 --- a/crates/evm/src/evm_parser.rs +++ b/crates/evm/src/evm_parser.rs @@ -5,12 +5,12 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler}; -use e3_events::{hlc::HlcTimestamp, EnclaveEventData, EvmEvent}; +use e3_events::{hlc::HlcTimestamp, EnclaveEventData}; use tracing::info; use crate::{ events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, - ExtractorFn, + EvmEvent, ExtractorFn, }; pub struct EvmParser { diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 66d34084e1..6ce4503fb1 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -22,8 +22,6 @@ use e3_events::{ use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; use std::{sync::Arc, time::Duration}; use tokio::time::sleep; -use tracing::subscriber::DefaultGuard; -use tracing_subscriber::{fmt, EnvFilter}; sol!( #[sol(rpc)] @@ -88,18 +86,9 @@ impl Handler for FakeSyncActor { } } -fn add_tracing() -> DefaultGuard { - tracing::subscriber::set_default( - fmt() - .with_env_filter(EnvFilter::new("info")) - .with_test_writer() - .finish(), - ) -} - #[actix::test] async fn evm_reader() -> Result<()> { - let _guard = add_tracing(); + let _guard = e3_test_helpers::with_tracing("info"); // Create a WS provider // NOTE: Anvil must be available on $PATH @@ -169,7 +158,7 @@ async fn evm_reader() -> Result<()> { } #[actix::test] async fn ensure_historical_events() -> Result<()> { - let _guard = add_tracing(); + let _guard = e3_test_helpers::with_tracing("info"); // Create a WS provider // NOTE: Anvil must be available on $PATH diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index 97ba4b7fea..a9821ed402 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -15,3 +15,4 @@ tracing.workspace = true [dev-dependencies] e3-ciphernode-builder.workspace = true +e3-test-helpers.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 5fd55024a9..c424cf8b95 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -9,9 +9,8 @@ use std::collections::HashSet; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{ - trap, BusHandle, EType, EnclaveEvent, EventContext, EventContextAccessors, EventPublisher, - EvmEvent, EvmEventConfig, EvmSyncEventsReceived, SyncEnd, SyncEvmEvent, SyncStart, TypedEvent, - Unsequenced, + trap, BusHandle, EType, EnclaveEvent, EventContextAccessors, EventPublisher, EvmEventConfig, + EvmSyncEventsReceived, SyncEnd, SyncStart, Unsequenced, }; use tracing::info; @@ -42,19 +41,19 @@ impl Synchronizer { Self::new(bus, evm_config).start() } - fn handle_sync_complete(&mut self, mut msg: EvmSyncEventsReceived) -> Result<()> { + fn handle_evm_sync_events_received(&mut self, mut msg: EvmSyncEventsReceived) -> Result<()> { let chain_id = msg.chain_id; info!("handle sync complete for chain({})", chain_id); self.evm_to_sync.remove(&chain_id); self.evm_events.append(&mut msg.events); info!("{} chains left to sync...", self.evm_to_sync.len()); if self.evm_to_sync.is_empty() { - self.handle_sync_end()?; + self.sort_and_finalize()?; } Ok(()) } - fn handle_sync_end(&mut self) -> Result<()> { + fn sort_and_finalize(&mut self) -> Result<()> { info!("all chains synced draining to bus and running sync end"); // Order all events (theoretically) self.evm_events.sort_by_key(|i| i.ts()); @@ -79,7 +78,7 @@ impl Handler for Synchronizer { type Result = (); fn handle(&mut self, msg: EvmSyncEventsReceived, _: &mut Self::Context) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { - self.handle_sync_complete(msg)?; + self.handle_evm_sync_events_received(msg)?; Ok(()) }) } @@ -108,11 +107,10 @@ pub struct Bootstrap; mod tests { use super::*; use e3_ciphernode_builder::EventSystem; + use e3_events::{EnclaveEvent, EventFactory}; use e3_events::{ - CorrelationId, EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, - TestEvent, + EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, TestEvent, }; - use e3_events::{EnclaveEvent, EventContextAccessors}; use std::time::Duration; use tokio::time::sleep; @@ -131,9 +129,10 @@ mod tests { #[actix::test] async fn test_synchronizer_full_flow() -> Result<()> { + let _guard = e3_test_helpers::with_tracing("info"); // Setup event system and synchronizer let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; + let bus: BusHandle = system.handle()?; let history_collector = bus.history(); // Configure test chains @@ -157,62 +156,48 @@ mod tests { // Create test events with timestamps let mut timelord = hlc_faucet(&bus, 100)?; - let (chain_1, chain_2) = (1, 2); - let (block_1, block_2) = (1, 2); // Test events - timestamps generated in order - let h_2_1 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), + let h_2_1 = bus.event_from_remote_source( EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), - block_1, + None, timelord.next().unwrap(), - chain_2, - )); + )?; - let h_1_1 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), + let h_1_1 = bus.event_from_remote_source( EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), - block_1, + None, timelord.next().unwrap(), - chain_1, - )); + )?; - let h_1_2 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), + let h_1_2 = bus.event_from_remote_source( EnclaveEventData::TestEvent(TestEvent::new("1-second", 1)), - block_2, + None, timelord.next().unwrap(), - chain_1, - )); + )?; - let h_2_2 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), + let h_2_2 = bus.event_from_remote_source( EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), - block_2, + None, timelord.next().unwrap(), - chain_2, - )); - - // Chain completion signals - let hc_1 = SyncEvmEvent::HistoricalSyncComplete(chain_1); - let hc_2 = SyncEvmEvent::HistoricalSyncComplete(chain_2); + )?; // Send events in mixed order to test sorting - sync_addr.send(h_2_2).await?; - sync_addr.send(h_2_1).await?; - sync_addr.send(hc_2).await?; - sync_addr.send(h_1_1).await?; - sync_addr.send(h_1_2).await?; - sync_addr.send(hc_1).await?; + sync_addr + .send(EvmSyncEventsReceived::new(vec![h_2_2, h_2_1], 2)) + .await?; + sync_addr + .send(EvmSyncEventsReceived::new(vec![h_1_1, h_1_2], 1)) + .await?; settle().await; // Get final event history and verify ordering - let history = history_collector + let full = history_collector .send(GetEvents::::new()) .await?; - - let events: Vec = history + println!("full = {}", full.len()); + let events: Vec = full .into_iter() .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) .collect(); diff --git a/crates/test-helpers/Cargo.toml b/crates/test-helpers/Cargo.toml index 10c882f257..6cf492e796 100644 --- a/crates/test-helpers/Cargo.toml +++ b/crates/test-helpers/Cargo.toml @@ -35,3 +35,4 @@ rand = { workspace = true } rand_chacha = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/crates/test-helpers/src/utils.rs b/crates/test-helpers/src/utils.rs index ffa6a5a132..2f1f532b8b 100644 --- a/crates/test-helpers/src/utils.rs +++ b/crates/test-helpers/src/utils.rs @@ -6,7 +6,18 @@ use std::{fmt::Debug, fs, io::Write, path::PathBuf}; -use tracing::{error, trace}; +use tracing::{error, subscriber::DefaultGuard, trace}; +use tracing_subscriber::{fmt, EnvFilter}; + +/// Use this at the top of a test to include tracing +pub fn with_tracing(level: &str) -> DefaultGuard { + tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new(level)) + .with_test_writer() + .finish(), + ) +} pub fn write_file_with_dirs(path: &PathBuf, content: &[u8]) -> std::io::Result<()> { let abs_path = if path.is_absolute() { From f3a211eb8e12728a95220955caf70d173ed3a2ba Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 12:11:19 +0000 Subject: [PATCH 19/63] tidy up api --- crates/evm/src/events.rs | 11 ++++++++++- crates/evm/src/evm_chain_gateway.rs | 22 +++------------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index e4688c544a..12e2434918 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -6,7 +6,10 @@ use actix::{Message, Recipient}; use alloy::rpc::types::Log; -use e3_events::{CorrelationId, EnclaveEventData}; +use anyhow::Result; +use e3_events::{ + BusHandle, CorrelationId, EnclaveEvent, EnclaveEventData, EventFactory, Unsequenced, +}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -74,6 +77,12 @@ impl EvmEvent { pub fn ts(&self) -> u128 { self.ts } + + pub fn into_enclave_event(self, bus: &BusHandle) -> Result> { + let data = self.data; + let ts = self.ts; + bus.event_from_remote_source(data, None, ts) + } } #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 538dc6ccf4..ee0a22a982 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -120,7 +120,6 @@ impl EvmChainGateway { } fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { - info!("Processing SyncStart message"); // Received a SyncStart event from the event bus. Get the sender within that event and forward // all events to that actor let sender = msg.sender.context("No sender on SyncStart Message")?; @@ -133,7 +132,6 @@ impl EvmChainGateway { } fn handle_sync_end(&mut self, _: SyncEnd) -> Result<()> { - info!("Processing SyncEnd message"); let buffer = self.status.live()?; for evt in buffer { self.publish_evm_event(evt)?; @@ -153,10 +151,7 @@ impl EvmChainGateway { Ok(()) } EnclaveEvmEvent::Event(event) => { - info!("Received event!"); - let (data,ts,_) = event.split(); - let enclave_event = self.bus.event_from_remote_source(data,None,ts)?; - self.process_evm_event(enclave_event)?; + self.process_evm_event(event.into_enclave_event(&self.bus)?)?; Ok(()) } _ => panic!("EvmChainGateway is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), @@ -164,12 +159,7 @@ impl EvmChainGateway { } fn forward_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { - info!( - "handling historical sync complete for chain_id({})", - event.chain_id - ); let state = self.status.buffer_until_live()?; - info!("Sending historical sync complete event to sender."); let sender = state .sender .context("ForwardToSyncActor state must hold a sender")?; @@ -180,14 +170,8 @@ impl EvmChainGateway { fn process_evm_event(&mut self, msg: EnclaveEvent) -> Result<()> { match &mut self.status { - SyncStatus::Init(buffer) => { - info!("Buffering until Forwarding... {:?}", msg); - buffer.push(msg); - } - SyncStatus::BufferUntilLive(buffer) => { - info!("Buffering until live... {:?}", msg); - buffer.push(msg); - } + SyncStatus::Init(buffer) => buffer.push(msg), + SyncStatus::BufferUntilLive(buffer) => buffer.push(msg), SyncStatus::ForwardToSyncActor(state) => state.add_event(msg), SyncStatus::Live => self.publish_evm_event(msg)?, }; From fd77f2b332cd1027cf6d34f2cdc1c0942b57d133 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 12:56:42 +0000 Subject: [PATCH 20/63] refactor: add block to event context --- crates/data/src/commit_log_event_log.rs | 2 +- crates/data/src/in_mem_event_log.rs | 2 +- crates/data/src/write_buffer.rs | 2 + crates/events/src/bus_handle.rs | 19 ++++++--- crates/events/src/enclave_event/mod.rs | 17 +++++--- .../events/src/enclave_event/typed_event.rs | 4 ++ crates/events/src/event_context.rs | 39 +++++++++++++++---- crates/events/src/traits.rs | 15 ++++++- crates/evm/src/events.rs | 2 +- crates/net/src/document_publisher.rs | 1 + crates/net/src/events.rs | 2 +- crates/net/src/net_event_translator.rs | 7 ++-- crates/net/src/net_sync_manager.rs | 1 + crates/sync/src/sync.rs | 7 +++- 14 files changed, 92 insertions(+), 28 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 46dc04dea9..059fdca595 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -82,7 +82,7 @@ mod tests { use tempfile::tempdir; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) + EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123, None) } #[test] diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index a283ded98d..d8b227f200 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -49,7 +49,7 @@ mod tests { use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) + EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123, None) } #[test] diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index c144b33403..8e82aeceb2 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -201,6 +201,7 @@ mod tests { EventId::hash(1), old_hlc.into(), aggregate_id.clone(), + None, ) .sequence(1); @@ -210,6 +211,7 @@ mod tests { EventId::hash(2), new_hlc.into(), aggregate_id.clone(), + None, ) .sequence(2); diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 1c8aa8510b..17fd7b63a8 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -106,8 +106,9 @@ impl EventPublisher> for BusHandle { &self, data: impl Into, remote_ts: u128, + block: Option, ) -> Result<()> { - self.publish_from_remote_impl(data, remote_ts, None) + self.publish_from_remote_impl(data, remote_ts, None, block) } fn publish_from_remote_as_response( @@ -115,8 +116,9 @@ impl EventPublisher> for BusHandle { data: impl Into, remote_ts: u128, caused_by: EventContext, + block: Option, ) -> Result<()> { - self.publish_from_remote_impl(data, remote_ts, Some(caused_by)) + self.publish_from_remote_impl(data, remote_ts, Some(caused_by), block) } fn naked_dispatch(&self, event: EnclaveEvent) { @@ -130,8 +132,9 @@ impl BusHandle { data: impl Into, remote_ts: u128, caused_by: Option>, + block: Option, ) -> Result<()> { - let evt = self.event_from_remote_source(data, caused_by, remote_ts)?; + let evt = self.event_from_remote_source(data, caused_by, remote_ts, block)?; self.sequencer.do_send(evt); Ok(()) } @@ -166,6 +169,7 @@ impl EventFactory> for BusHandle { data.into(), caused_by, ts.into(), + None, )) } @@ -174,12 +178,14 @@ impl EventFactory> for BusHandle { data: impl Into, caused_by: Option>, ts: u128, + block: Option, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), caused_by, ts.into(), + block, )) } } @@ -268,7 +274,9 @@ mod tests { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { let ts = msg.ts(); - self.dest.publish_from_remote(msg.into_data(), ts).unwrap() + self.dest + .publish_from_remote(msg.into_data(), ts, None) + .unwrap() } } @@ -414,7 +422,8 @@ where fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { if (self.predicate)(&msg) { let (data, ts) = msg.split(); - let _ = self.handle.publish_from_remote(data, ts); + let _ = self.handle.publish_from_remote(data, ts, None); // TODO: check if this is fine + // to erase block data } } } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 34e30881de..0a6ae8f906 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -305,6 +305,9 @@ impl EventContextAccessors for EnclaveEvent { fn aggregate_id(&self) -> AggregateId { self.ctx.aggregate_id() } + fn block(&self) -> Option { + self.ctx.block() + } } impl EventContextSeq for EnclaveEvent { @@ -316,8 +319,9 @@ impl EventContextSeq for EnclaveEvent { impl EnclaveEvent { pub fn clone_unsequenced(&self) -> EnclaveEvent { let ts = self.ts(); + let block = self.block(); let data = self.clone().into_data(); - EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts) + EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts, block) } pub fn to_typed_event(&self, data: T) -> TypedEvent { @@ -339,7 +343,7 @@ impl EnclaveEvent { impl EnclaveEvent { /// test-helpers only utility function to create a new unsequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, None, time).into_sequenced(seq) + EnclaveEvent::::new_with_timestamp(data, None, time, None).into_sequenced(seq) } /// test-helpers only utility function to remove time information from an event @@ -384,8 +388,8 @@ impl ErrorEvent for EnclaveEvent { let aggregate_id = AggregateId::new(0); // Error events use default aggregate_id let ctx = caused_by - .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) - .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)); + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id, None)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id, None)); Ok(EnclaveEvent { payload: payload.into(), @@ -549,6 +553,7 @@ impl EventConstructorWithTimestamp for EnclaveEvent { data: Self::Data, caused_by: Option>, ts: u128, + block: Option, ) -> Self { let payload: EnclaveEventData = data.into(); let id = EventId::hash(&payload); @@ -556,8 +561,8 @@ impl EventConstructorWithTimestamp for EnclaveEvent { EnclaveEvent { payload, ctx: caused_by - .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) - .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)), + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id, block)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id, block)), } } } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 3a76b9f550..ed5a546791 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -71,6 +71,10 @@ impl EventContextAccessors for TypedEvent { fn aggregate_id(&self) -> AggregateId { self.ctx.aggregate_id() } + + fn block(&self) -> Option { + self.ctx.block() + } } impl EventContextSeq for TypedEvent { diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 6cb5b9bafa..7218ce7c48 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -57,6 +57,7 @@ pub struct EventContext { seq: S::Seq, ts: u128, aggregate_id: AggregateId, + block: Option, } impl EventContext { @@ -66,6 +67,7 @@ impl EventContext { origin_id: EventId, ts: u128, aggregate_id: AggregateId, + block: Option, ) -> Self { Self { id, @@ -74,11 +76,17 @@ impl EventContext { seq: (), ts, aggregate_id, + block, } } - pub fn new_origin(id: EventId, ts: u128, aggregate_id: AggregateId) -> Self { - Self::new(id, id, id, ts, aggregate_id) + pub fn new_origin( + id: EventId, + ts: u128, + aggregate_id: AggregateId, + block: Option, + ) -> Self { + Self::new(id, id, id, ts, aggregate_id, block) } pub fn from_cause( @@ -86,8 +94,16 @@ impl EventContext { cause: EventContext, ts: u128, aggregate_id: AggregateId, + block: Option, ) -> Self { - EventContext::new(id, cause.id(), cause.origin_id(), ts, aggregate_id) + EventContext::new( + id, + cause.id(), + cause.origin_id(), + ts, + aggregate_id, + cause.block.max(block), // block watermark + ) } pub fn sequence(self, value: u64) -> EventContext { @@ -98,6 +114,7 @@ impl EventContext { origin_id: self.origin_id, ts: self.ts, aggregate_id: self.aggregate_id, + block: self.block, } } } @@ -122,6 +139,10 @@ impl EventContextAccessors for EventContext { fn aggregate_id(&self) -> AggregateId { self.aggregate_id } + + fn block(&self) -> Option { + self.block + } } impl EventContextSeq for EventContext { @@ -147,16 +168,17 @@ mod tests { EventId::hash(1), 1, AggregateId::new(1), + None, ) .sequence(1); events.push(one.clone()); - let two = - EventContext::from_cause(EventId::hash(2), one, 2, AggregateId::new(1)).sequence(2); + let two = EventContext::from_cause(EventId::hash(2), one, 2, AggregateId::new(1), None) + .sequence(2); events.push(two.clone()); - let three = - EventContext::from_cause(EventId::hash(3), two, 3, AggregateId::new(1)).sequence(3); + let three = EventContext::from_cause(EventId::hash(3), two, 3, AggregateId::new(1), None) + .sequence(3); events.push(three.clone()); assert_eq!( @@ -169,6 +191,7 @@ mod tests { causation_id: EventId::hash(1), ts: 1, aggregate_id: AggregateId::new(1), + block: None }, EventContext { seq: 2, @@ -177,6 +200,7 @@ mod tests { causation_id: EventId::hash(1), ts: 2, aggregate_id: AggregateId::new(1), + block: None }, EventContext { seq: 3, @@ -185,6 +209,7 @@ mod tests { causation_id: EventId::hash(2), ts: 3, aggregate_id: AggregateId::new(1), + block: None }, ] ) diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 64dfa0fd2d..ecdd0d56b7 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -58,11 +58,13 @@ pub trait EventFactory { /// This method should be used for events that originated from remote sources. /// /// The Option `caused_by` is for correlation when we send a remote request and receive a response. + /// Block should be provided when the event is from the blockchain fn event_from_remote_source( &self, data: impl Into, caused_by: Option>, ts: u128, + block: Option, ) -> Result; } @@ -87,13 +89,18 @@ pub trait EventPublisher { /// The ctx parameter is to pass on the current context to the local event. fn publish(&self, data: impl Into, ctx: EventContext) -> Result<()>; /// This creates a context based on the given data. This should only be used when an event is - /// the origin event + /// the origin event and does not originate remotely. This is also useful in tests. fn publish_without_context(&self, data: impl Into) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. /// /// This method should be used for events that originated from remote sources. - fn publish_from_remote(&self, data: impl Into, remote_ts: u128) -> Result<()>; + fn publish_from_remote( + &self, + data: impl Into, + remote_ts: u128, + block: Option, + ) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. /// @@ -106,6 +113,7 @@ pub trait EventPublisher { data: impl Into, remote_ts: u128, caused_by: EventContext, + block: Option, ) -> Result<()>; /// Dispatch the given event without applying any HLC transformation. fn naked_dispatch(&self, event: E); @@ -134,6 +142,7 @@ pub trait EventConstructorWithTimestamp: Event + Sized { data: Self::Data, caused_by: Option>, ts: u128, + block: Option, ) -> Self; } @@ -171,6 +180,8 @@ pub trait EventContextAccessors { fn ts(&self) -> u128; /// The aggregate id for this event fn aggregate_id(&self) -> AggregateId; + /// The highest block watermark we have seen + fn block(&self) -> Option; } pub trait EventContextSeq { diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index 12e2434918..366b91e639 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -81,7 +81,7 @@ impl EvmEvent { pub fn into_enclave_event(self, bus: &BusHandle) -> Result> { let data = self.data; let ts = self.ts; - bus.event_from_remote_source(data, None, ts) + bus.event_from_remote_source(data, None, ts, Some(self.block)) } } diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 4ba72afa6d..27f9da2622 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -298,6 +298,7 @@ pub async fn handle_document_published_notification( value, }, event.ts, + None, )?; Ok(()) diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 46feb6960b..89bf9ba2ff 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -307,7 +307,7 @@ mod tests { fn test_enclave_event_gossip_lifecycle() -> anyhow::Result<()> { // event is created locally let event: EnclaveEvent = - EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), None, 31415); + EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), None, 31415, None); // event is sequenced after bus.publish() adds a sequence number let event: EnclaveEvent = event.into_sequenced(90210); diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 4ff06270e4..65276c288f 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -161,9 +161,10 @@ impl Handler for NetEventTranslator { fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { let LibP2pEvent(data) = msg; let event: EnclaveEvent = data.try_into()?; - self.sent_events.insert(event.id()); - let (data, ts) = event.split(); - self.bus.publish_from_remote(data, ts)?; + let id = event.id(); + let (data, ec) = event.into_components(); + self.bus.publish_from_remote(data, ec.ts(), None)?; + self.sent_events.insert(id); Ok(()) } } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 2256e9c3a6..df79a958a8 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -136,6 +136,7 @@ impl Handler> for NetSyncManager { }, msg.value.ts, ctx, + None, )?; Ok(()) diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index c424cf8b95..2cd7b9bf0a 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -92,6 +92,7 @@ impl Handler for Synchronizer { "EvmEventConfig was not set likely Bootstrap was called more than once.", )?; + // What was the last block we processed for each aggregate? // TODO: Get information about what has and has not been synced then fire SyncStart self.bus .publish_without_context(SyncStart::new(ctx.address(), evm_config)) @@ -162,24 +163,28 @@ mod tests { EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), None, timelord.next().unwrap(), + Some(1), )?; let h_1_1 = bus.event_from_remote_source( EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), None, timelord.next().unwrap(), + Some(1), )?; let h_1_2 = bus.event_from_remote_source( - EnclaveEventData::TestEvent(TestEvent::new("1-second", 1)), + EnclaveEventData::TestEvent(TestEvent::new("1-second", 2)), None, timelord.next().unwrap(), + Some(2), )?; let h_2_2 = bus.event_from_remote_source( EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), None, timelord.next().unwrap(), + Some(2), )?; // Send events in mixed order to test sorting From 336712efee190cdb8d0d7bb7ca8cb2f5b1bc6bd1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 13:35:55 +0000 Subject: [PATCH 21/63] refactor: add block tracking to write buffer --- crates/data/src/write_buffer.rs | 127 ++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 31 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 8e82aeceb2..54ba89ce53 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -36,7 +36,7 @@ impl AggregateConfig { } } -#[derive(Debug)] +#[derive(Debug, Default)] struct AggregateBuffer { buffer: Vec, } @@ -52,6 +52,8 @@ pub struct WriteBuffer { dest: Option>, /// Per-aggregate buffers for organizing inserts aggregate_buffers: HashMap, + /// Per-aggregate blockheight highwatermarks + block_height_seen: HashMap, /// Per-aggregate wait time configuration config: AggregateConfig, } @@ -65,6 +67,7 @@ impl WriteBuffer { Self { dest: None, aggregate_buffers: HashMap::new(), + block_height_seen: HashMap::new(), config: AggregateConfig::new(HashMap::new()), } } @@ -73,22 +76,17 @@ impl WriteBuffer { Self { dest: None, aggregate_buffers: HashMap::new(), + block_height_seen: HashMap::new(), config, } } fn handle_insert(&mut self, msg: Insert) { - let aggregate_id = if let Some(event_ctx) = msg.ctx() { - event_ctx.aggregate_id().clone() - } else { - AggregateId::new(0) - }; - - let agg_buffer = self - .aggregate_buffers - .entry(aggregate_id) - .or_insert_with(|| AggregateBuffer::new()); - agg_buffer.buffer.push(msg.clone()); + handle_insert( + msg, + &mut self.aggregate_buffers, + &mut self.block_height_seen, + ) } fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { @@ -172,6 +170,31 @@ fn process_expired_inserts( (updated_buffers, all_expired_inserts) } +fn handle_insert( + msg: Insert, + aggregate_buffers: &mut HashMap, + block_height_seen: &mut HashMap, +) { + let aggregate_id = msg + .ctx() + .map(|ec| { + if let Some(block) = ec.block() { + block_height_seen + .entry(ec.aggregate_id().clone()) + .and_modify(|e| *e = (*e).max(block)) + .or_insert(block); + } + ec.aggregate_id().clone() + }) + .unwrap_or_else(|| AggregateId::new(0)); + + aggregate_buffers + .entry(aggregate_id) + .or_default() + .buffer + .push(msg); +} + impl Handler for WriteBuffer { type Result = (); @@ -186,6 +209,60 @@ mod tests { use crate::events::Insert; use e3_events::{hlc::HlcTimestamp, EventContext, EventId}; + fn insert_with_block(agg: AggregateId, block: u64) -> Insert { + Insert::new_with_context( + "key", + b"value".to_vec(), + EventContext::new_origin(EventId::hash(1), 123, agg, Some(block)).sequence(1), + ) + } + + fn insert_without_block(agg: AggregateId) -> Insert { + Insert::new_with_context( + "key", + b"value".to_vec(), + EventContext::new_origin(EventId::hash(1), 123, agg, None).sequence(1), + ) + } + + fn insert_no_context() -> Insert { + Insert::new("key", b"value".to_vec()) + } + + #[test] + fn tracks_highwater_mark() { + let mut buffers = HashMap::new(); + let mut heights = HashMap::new(); + let agg_1 = AggregateId::new(1); + let agg_42 = AggregateId::new(42); + + // Initial inserts for both aggregates + handle_insert(insert_with_block(agg_42, 24), &mut buffers, &mut heights); + handle_insert(insert_with_block(agg_1, 25), &mut buffers, &mut heights); + + // Higher block for agg_42 - should update + handle_insert(insert_with_block(agg_42, 100), &mut buffers, &mut heights); + + // Lower block for agg_42 - should NOT update + handle_insert(insert_with_block(agg_42, 50), &mut buffers, &mut heights); + + // No block context - should not affect heights + handle_insert(insert_without_block(agg_1), &mut buffers, &mut heights); + + // No context at all - uses default aggregate + handle_insert(insert_no_context(), &mut buffers, &mut heights); + + // Verify highwater marks + assert_eq!(heights.get(&agg_42), Some(&100)); + assert_eq!(heights.get(&agg_1), Some(&25)); + assert_eq!(heights.get(&AggregateId::new(0)), None); + + // Verify buffer counts + assert_eq!(buffers.get(&agg_42).unwrap().buffer.len(), 3); + assert_eq!(buffers.get(&agg_1).unwrap().buffer.len(), 2); + assert_eq!(buffers.get(&AggregateId::new(0)).unwrap().buffer.len(), 1); + } + #[test] fn test_process_expired_inserts() { let aggregate_id = AggregateId::new(1); @@ -195,25 +272,13 @@ mod tests { let old_hlc = HlcTimestamp::new(500_000, 0, 1); // 0.5 seconds ago let new_hlc = HlcTimestamp::new(3_000_000, 0, 2); // 3 seconds from epoch - let old_ctx = EventContext::new( - EventId::hash(1), - EventId::hash(1), - EventId::hash(1), - old_hlc.into(), - aggregate_id.clone(), - None, - ) - .sequence(1); - - let new_ctx = EventContext::new( - EventId::hash(2), - EventId::hash(2), - EventId::hash(2), - new_hlc.into(), - aggregate_id.clone(), - None, - ) - .sequence(2); + let old_ctx = + EventContext::new_origin(EventId::hash(1), old_hlc.into(), aggregate_id.clone(), None) + .sequence(1); + + let new_ctx = + EventContext::new_origin(EventId::hash(2), new_hlc.into(), aggregate_id.clone(), None) + .sequence(2); let old_insert = Insert::new_with_context("old_key", b"old_value".to_vec(), old_ctx); let new_insert = Insert::new_with_context("new_key", b"new_value".to_vec(), new_ctx); From 8d5b204e9468cb4df12d62bc5ffe05af0854e415 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 13:57:30 +0000 Subject: [PATCH 22/63] save aggregate block to rolling snapshot --- crates/data/src/write_buffer.rs | 21 +++++++++++++++++++-- crates/events/src/events.rs | 13 +++++++++++-- crates/events/src/sequencer.rs | 7 +++++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 54ba89ce53..12fcae74c3 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -89,12 +89,24 @@ impl WriteBuffer { ) } + fn get_highest_block(&self, agg: AggregateId, block: Option) -> u64 { + block + .max(self.block_height_seen.get(&agg).map(ToOwned::to_owned)) + .unwrap_or(0) + } + fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { // Store the sequence number as an Insert message so snapshots hold the most recent event // they were created against + let aggregate_id = msg.aggregate_id(); self.handle_insert(Insert::new( - &format!("//aggregate_seq/{}", msg.aggregate_id()), - msg.seq().to_le_bytes().to_vec(), // Same as bincode avoiding result + &format!("//aggregate_seq/{}", aggregate_id), + encode_u64(msg.seq()), // Same as bincode avoiding result + )); + + self.handle_insert(Insert::new( + &format!("//aggregate_block/{}", aggregate_id), + encode_u64(self.get_highest_block(aggregate_id, msg.block())), )); let now = SystemTime::now() @@ -130,6 +142,11 @@ impl Handler for WriteBuffer { } } +/// Encode the same as bincode without using a result +fn encode_u64(value: u64) -> Vec { + value.to_le_bytes().to_vec() +} + fn process_expired_inserts( aggregate_buffers: &HashMap, config: &HashMap, diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 5fd9ae190d..69a78fcf6a 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -14,11 +14,16 @@ use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; pub struct CommitSnapshot { seq: u64, aggregate_id: AggregateId, + block: Option, } impl CommitSnapshot { - pub fn new(seq: u64, aggregate_id: AggregateId) -> Self { - Self { seq, aggregate_id } + pub fn new(seq: u64, aggregate_id: AggregateId, block: Option) -> Self { + Self { + seq, + aggregate_id, + block, + } } pub fn seq(&self) -> u64 { @@ -28,6 +33,10 @@ impl CommitSnapshot { pub fn aggregate_id(&self) -> AggregateId { self.aggregate_id } + + pub fn block(&self) -> Option { + self.block + } } /// Direct event received by the EventStore to store an event diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index b98db473f7..bc92a5610a 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -49,8 +49,11 @@ impl Handler for Sequencer { fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { let event = msg.into_event(); let seq = event.seq(); - self.buffer - .do_send(CommitSnapshot::new(seq, event.aggregate_id())); + self.buffer.do_send(CommitSnapshot::new( + seq, + event.aggregate_id(), + event.block(), + )); self.bus.do_send(event) } } From 5852eb2d70c3fd65b6c8f8aaf6947a1660e62d90 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 21:12:33 +0000 Subject: [PATCH 23/63] remove redundant import --- crates/evm/src/evm_read_interface.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index e6313014a8..38c87c3104 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -19,7 +19,6 @@ use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscribe use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; -use std::time::{SystemTime, UNIX_EPOCH}; use tokio::select; use tokio::sync::oneshot; use tracing::{error, info, instrument, warn}; From b1f341de6810577220936d566cf4a025d44d15ce Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 1 Feb 2026 23:46:17 +0000 Subject: [PATCH 24/63] feat: add sync bootstrap and sanpshot infrastructure (wip) --- Cargo.lock | 3 + .../src/ciphernode_builder.rs | 6 +- crates/ciphernode-builder/src/event_system.rs | 4 +- crates/config/src/store_keys.rs | 14 +- crates/data/Cargo.toml | 1 + crates/data/src/data_store.rs | 32 ++- crates/data/src/repositories.rs | 9 +- crates/data/src/write_buffer.rs | 54 ++++- .../events/src/enclave_event/enclave_error.rs | 6 +- crates/events/src/enclave_event/mod.rs | 8 +- crates/events/src/event_context.rs | 26 ++- crates/events/src/events.rs | 25 +-- crates/events/src/eventstore.rs | 35 +++- crates/events/src/eventstore_router.rs | 3 +- crates/events/src/sequencer.rs | 36 ++-- crates/events/src/sync.rs | 9 + crates/sync/Cargo.toml | 2 + crates/sync/src/lib.rs | 2 + crates/sync/src/repo.rs | 23 +++ crates/sync/src/sync.rs | 184 ++++++++++++++++-- crates/utils/src/error.rs | 7 + crates/utils/src/lib.rs | 2 + .../enclave-contracts/deployed_contracts.json | 108 ++++++++++ 23 files changed, 517 insertions(+), 82 deletions(-) create mode 100644 crates/sync/src/repo.rs create mode 100644 crates/utils/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index 3be47ce1c8..1d10187872 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,6 +2975,7 @@ dependencies = [ "async-trait", "bincode", "commitlog", + "e3-config", "e3-events", "e3-utils", "once_cell", @@ -3440,6 +3441,8 @@ dependencies = [ "actix", "anyhow", "e3-ciphernode-builder", + "e3-config", + "e3-data", "e3-events", "e3-test-helpers", "tokio", diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 775666a877..9af2c52ec9 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -356,7 +356,7 @@ impl CiphernodeBuilder { if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { EventSystem::persisted(&addr, log_path, kv_path) .with_event_bus(local_bus) - .with_aggregate_config(aggregate_config) + .with_aggregate_config(aggregate_config.clone()) } else { if let Some(ref store) = self.in_mem_store { EventSystem::in_mem_from_store(&addr, store) @@ -462,7 +462,7 @@ impl CiphernodeBuilder { ) }; - Synchronizer::setup(&bus, evm_config); // TODO: add net config if required + Synchronizer::setup(&bus, &evm_config, &repositories, &aggregate_config); Ok(CiphernodeHandle::new( addr.to_owned(), @@ -523,7 +523,7 @@ fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { /// Build delay configuration for a specific chain fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, u64) { - let aggregate_id = AggregateId::new(actual_chain_id as usize); + let aggregate_id = AggregateId::from_chain_id(Some(actual_chain_id)); let finalization_ms = chain.finalization_ms.unwrap_or(0); let delay_us = finalization_ms * 1000; // ms → microseconds (aggregate_id, delay_us) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index ed875d86a9..0bde9637f2 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -326,7 +326,7 @@ impl EventSystem { .store .get_or_init(|| InMemStore::new(true).start()) .clone(); - DataStore::from_in_mem(&addr, &self.buffer()) + DataStore::from_in_mem_with_buffer(&addr, &self.buffer()) } EventSystemBackend::Persisted(b) => { let addr = b @@ -336,7 +336,7 @@ impl EventSystem { SledStore::new(&handle, &b.sled_path) })? .clone(); - DataStore::from_sled_store(&addr, &self.buffer()) + DataStore::from_sled_store_with_buffer(&addr, &self.buffer()) } }; self.wire_if_ready(); diff --git a/crates/config/src/store_keys.rs b/crates/config/src/store_keys.rs index 6b0eae21ae..b9797242fb 100644 --- a/crates/config/src/store_keys.rs +++ b/crates/config/src/store_keys.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_events::E3id; +use e3_events::{AggregateId, E3id}; pub struct StoreKeys; @@ -76,4 +76,16 @@ impl StoreKeys { pub fn ciphernode_selector() -> String { String::from("//ciphernode_selector") } + + pub fn aggregate_seq(aggregate_id: AggregateId) -> String { + format!("//aggregate_seq/{}", aggregate_id) + } + + pub fn aggregate_block(aggregate_id: AggregateId) -> String { + format!("//aggregate_block/{}", aggregate_id) + } + + pub fn aggregate_ts(aggregate_id: AggregateId) -> String { + format!("//aggregate_ts/{}", aggregate_id) + } } diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index 59c4aa6987..895bfd7bb8 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/gnosisguild/enclave/crates/data" actix = { workspace = true } e3-events = { workspace = true } e3-utils = { workspace = true } +e3-config = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } sled = { workspace = true } diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index d4516f858f..ceeede2bb5 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -171,7 +171,10 @@ impl DataStore { } } - pub fn from_sled_store(addr: &Addr, write_buffer: &Addr) -> Self { + pub fn from_sled_store_with_buffer( + addr: &Addr, + write_buffer: &Addr, + ) -> Self { Self { addr: StoreAddr::Sled(addr.clone()), get: addr.clone().recipient(), @@ -182,7 +185,10 @@ impl DataStore { } } - pub fn from_in_mem(addr: &Addr, write_buffer: &Addr) -> Self { + pub fn from_in_mem_with_buffer( + addr: &Addr, + write_buffer: &Addr, + ) -> Self { Self { addr: StoreAddr::InMem(addr.clone()), get: addr.clone().recipient(), @@ -192,6 +198,28 @@ impl DataStore { scope: vec![], } } + + pub fn from_in_mem(addr: &Addr) -> Self { + Self { + addr: StoreAddr::InMem(addr.clone()), + get: addr.clone().recipient(), + insert: addr.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + } + } + + pub fn from_sled_store(addr: &Addr) -> Self { + Self { + addr: StoreAddr::Sled(addr.clone()), + get: addr.clone().recipient(), + insert: addr.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + } + } } impl From<&Addr> for DataStore { diff --git a/crates/data/src/repositories.rs b/crates/data/src/repositories.rs index 4b89ef6b8c..63cce29663 100644 --- a/crates/data/src/repositories.rs +++ b/crates/data/src/repositories.rs @@ -3,9 +3,10 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::{DataStore, InMemStore, Repository}; +use actix::Actor; -use crate::{DataStore, Repository}; - +#[derive(Clone)] pub struct Repositories { pub store: DataStore, } @@ -27,6 +28,10 @@ impl Repositories { pub fn new(store: DataStore) -> Self { Repositories { store } } + + pub fn in_mem() -> Self { + DataStore::from_in_mem(&InMemStore::new(false).start()).into() + } } impl From> for Repositories { diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 12fcae74c3..933166b228 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -5,8 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler, Message, Recipient}; +use e3_config::StoreKeys; use e3_events::hlc::HlcTimestamp; -use e3_events::{AggregateId, CommitSnapshot, EventContextAccessors}; +use e3_events::{AggregateId, CommitSnapshot, EventContext, EventContextAccessors}; use std::{ collections::HashMap, time::{SystemTime, UNIX_EPOCH}, @@ -14,6 +15,10 @@ use std::{ use crate::{Insert, InsertBatch}; +#[derive(Message)] +#[rtype(result = "()")] +pub struct Start; + /// Central configuration for aggregates in the WriteBuffer #[derive(Debug, Clone)] pub struct AggregateConfig { @@ -34,6 +39,10 @@ impl AggregateConfig { pub fn indexed_ids(&self) -> Vec { self.delays.keys().map(|id| id.to_usize()).collect() } + + pub fn aggregates(&self) -> Vec { + self.delays.keys().cloned().collect() + } } #[derive(Debug, Default)] @@ -47,7 +56,14 @@ impl AggregateBuffer { } } +pub enum WriteBufferStatus { + Paused, + Runnning, +} + pub struct WriteBuffer { + /// The status of the write buffer + status: WriteBufferStatus, /// Destination recipient for batched inserts dest: Option>, /// Per-aggregate buffers for organizing inserts @@ -65,6 +81,7 @@ impl Actor for WriteBuffer { impl WriteBuffer { pub fn new() -> Self { Self { + status: WriteBufferStatus::Paused, dest: None, aggregate_buffers: HashMap::new(), block_height_seen: HashMap::new(), @@ -74,6 +91,7 @@ impl WriteBuffer { pub fn with_config(config: AggregateConfig) -> Self { Self { + status: WriteBufferStatus::Paused, dest: None, aggregate_buffers: HashMap::new(), block_height_seen: HashMap::new(), @@ -82,6 +100,10 @@ impl WriteBuffer { } fn handle_insert(&mut self, msg: Insert) { + if let WriteBufferStatus::Paused = self.status { + return; + } + handle_insert( msg, &mut self.aggregate_buffers, @@ -96,19 +118,31 @@ impl WriteBuffer { } fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { + if let WriteBufferStatus::Paused = self.status { + return; + } + // Store the sequence number as an Insert message so snapshots hold the most recent event // they were created against let aggregate_id = msg.aggregate_id(); + + // NOTE: can not use repository here as we need to insert operations in stream self.handle_insert(Insert::new( - &format!("//aggregate_seq/{}", aggregate_id), - encode_u64(msg.seq()), // Same as bincode avoiding result + &StoreKeys::aggregate_seq(aggregate_id), + encode_u64(msg.seq()), )); self.handle_insert(Insert::new( - &format!("//aggregate_block/{}", aggregate_id), + &StoreKeys::aggregate_block(aggregate_id), encode_u64(self.get_highest_block(aggregate_id, msg.block())), )); + self.handle_insert(Insert::new( + // NOT NEW NEED CTX + &StoreKeys::aggregate_ts(aggregate_id), + encode_u128(msg.ts()), + )); + let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_else(|_| std::time::Duration::from_secs(0)) @@ -147,6 +181,11 @@ fn encode_u64(value: u64) -> Vec { value.to_le_bytes().to_vec() } +/// Encode the same as bincode without using a result +fn encode_u128(value: u128) -> Vec { + value.to_le_bytes().to_vec() +} + fn process_expired_inserts( aggregate_buffers: &HashMap, config: &HashMap, @@ -220,6 +259,13 @@ impl Handler for WriteBuffer { } } +impl Handler for WriteBuffer { + type Result = (); + fn handle(&mut self, msg: Start, ctx: &mut Self::Context) -> Self::Result { + self.status = WriteBufferStatus::Runnning; + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 2c941fb514..93881c076d 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::Message; +use actix::{Addr, Message}; use serde::{Deserialize, Serialize}; use std::{ fmt::{self, Display}, @@ -12,7 +12,9 @@ use std::{ pin::Pin, }; -use crate::{BusHandle, ErrorDispatcher}; +use crate::{BusHandle, ErrorDispatcher, EventBus}; + +use super::EnclaveEvent; pub trait FromError { fn from_error(err_type: EType, error: impl Into) -> Self; diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 0a6ae8f906..0e35fd2834 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -437,12 +437,8 @@ impl EnclaveEventData { impl WithAggregateId for EnclaveEventData { fn get_aggregate_id(&self) -> AggregateId { - let maybe_e3_id = self.get_e3_id(); - if let Some(e3_id) = maybe_e3_id { - AggregateId::new(e3_id.chain_id() as usize) - } else { - AggregateId::new(0) - } + let chain_id = self.get_e3_id().map(|e3_id| e3_id.chain_id()); + AggregateId::from_chain_id(chain_id) } } diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 7218ce7c48..55e1ee7acd 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -23,15 +23,31 @@ impl AggregateId { pub fn to_usize(&self) -> usize { self.0 } + + /// Create AggregateId from Option + /// None → AggregateId(0), Some(chain_id) → AggregateId(chain_id) + pub fn from_chain_id(chain_id: Option) -> Self { + match chain_id { + None => Self::new(0), + Some(id) => Self::new(id.try_into().unwrap_or(0)), + } + } + + /// Convert back to Option + /// AggregateId(0) → None, otherwise → Some(chain_id) + pub fn to_chain_id(&self) -> Option { + if self.0 == 0 { + None + } else { + Some(self.0 as u64) + } + } } impl From> for AggregateId { fn from(value: Option) -> Self { - if let Some(e3_id) = value { - Self::new(e3_id.chain_id() as usize) - } else { - Self::new(0) - } + let chain_id = value.map(|e3_id| e3_id.chain_id()); + Self::from_chain_id(chain_id) } } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 69a78fcf6a..49a0cf197f 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -6,36 +6,39 @@ use actix::{Message, Recipient}; -use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; +use crate::{ + AggregateId, CorrelationId, EnclaveEvent, EventContext, EventContextAccessors, EventContextSeq, + Sequenced, Unsequenced, +}; /// Direct event received by the snapshot buffer in order to save snapshot to disk #[derive(Message, Debug)] #[rtype("()")] pub struct CommitSnapshot { - seq: u64, - aggregate_id: AggregateId, - block: Option, + ec: EventContext, } impl CommitSnapshot { - pub fn new(seq: u64, aggregate_id: AggregateId, block: Option) -> Self { + pub fn new(event: &EnclaveEvent) -> Self { Self { - seq, - aggregate_id, - block, + ec: event.get_ctx().clone(), } } pub fn seq(&self) -> u64 { - self.seq + self.ec.seq() } pub fn aggregate_id(&self) -> AggregateId { - self.aggregate_id + self.ec.aggregate_id() } pub fn block(&self) -> Option { - self.block + self.ec.block() + } + + pub fn ts(&self) -> u128 { + self.ec.ts() } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 239bd68cc0..fce0ec5da3 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -10,11 +10,15 @@ use crate::{ }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; -use tracing::error; +use e3_utils::major_issue; +use tracing::{error, warn}; + +const MAX_STORAGE_ERRORS: u64 = 10; pub struct EventStore { index: I, log: L, + storage_errors: u64, } impl EventStore { @@ -23,7 +27,15 @@ impl EventStore { let sender = msg.sender; let ts = event.ts(); if let Some(_) = self.index.get(ts)? { - bail!("Event already stored at timestamp {ts}!"); + warn!("Event already stored at timestamp {ts}! This might happen when recovering from a snapshot. Skipping storage"); + self.storage_errors += 1; + if self.storage_errors > MAX_STORAGE_ERRORS { + bail!( + "The eventstore had too many storage errors! {}", + self.storage_errors + ); + } + return Ok(()); } let seq = self.log.append(&event)?; self.index.insert(ts, seq)?; @@ -52,7 +64,11 @@ impl EventStore { impl EventStore { pub fn new(index: I, log: L) -> Self { - Self { index, log } + Self { + index, + log, + storage_errors: 0, + } } } @@ -63,10 +79,10 @@ impl Actor for EventStore { impl Handler for EventStore { type Result = (); fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { - match self.handle_store_event_requested(msg) { - Ok(_) => (), - Err(e) => panic!("{e}"), // panic here because when event storage fails we really need - // to just give up + if let Err(e) = self.handle_store_event_requested(msg) { + panic!("{}", major_issue("Could not store event in eventstore.", e)) + // panic here because when event storage fails we really need + // to just give up } } } @@ -74,9 +90,8 @@ impl Handler for EventStore< impl Handler for EventStore { type Result = (); fn handle(&mut self, msg: GetEventsAfter, _: &mut Self::Context) -> Self::Result { - match self.handle_get_events_after(msg) { - Ok(_) => (), - Err(e) => error!("{e}"), + if let Err(e) = self.handle_get_events_after(msg) { + error!("{e}"); } } } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 4d9a6b3321..cb52ec29ac 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -12,6 +12,7 @@ use crate::{ use crate::{CorrelationId, ReceiveEvents}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; +use e3_utils::major_issue; use std::collections::HashMap; use tracing::error; @@ -66,7 +67,7 @@ impl Handler for EventStoreR fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { if let Err(e) = self.handle_store_event_requested(msg) { - error!("Failed to route store event request: {}", e); + panic!("{}", major_issue("Could not store event in eventstore.", e)) } } } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index bc92a5610a..5e788f7fbd 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -4,12 +4,13 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; - use crate::{ events::{CommitSnapshot, EventStored, StoreEventRequested}, - EnclaveEvent, EventBus, EventContextAccessors, EventContextSeq, Sequenced, Unsequenced, + EnclaveEvent, EventBus, Sequenced, Unsequenced, }; +use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; +use anyhow::Result; +use e3_utils::major_issue; /// Component to sequence the storage of events pub struct Sequencer { @@ -30,6 +31,13 @@ impl Sequencer { buffer: buffer.into(), } } + + fn handle_event_stored(&self, msg: EventStored) -> Result<()> { + let event = msg.into_event(); + self.buffer.try_send(CommitSnapshot::new(&event))?; + self.bus.try_send(event)?; + Ok(()) + } } impl Actor for Sequencer { @@ -39,22 +47,24 @@ impl Actor for Sequencer { impl Handler> for Sequencer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - self.eventstore - .do_send(StoreEventRequested::new(msg, ctx.address())) + if let Err(e) = self + .eventstore + .try_send(StoreEventRequested::new(msg, ctx.address())) + { + panic!("{}", major_issue("Could not store event in eventstore.", e)) + } } } impl Handler for Sequencer { type Result = (); fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { - let event = msg.into_event(); - let seq = event.seq(); - self.buffer.do_send(CommitSnapshot::new( - seq, - event.aggregate_id(), - event.block(), - )); - self.bus.do_send(event) + if let Err(e) = self.handle_event_stored(msg) { + panic!( + "{}", + major_issue("Could not send event to buffer or bus.", e) + ) + } } } diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index 13d75a9cf1..cac5157f00 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -37,6 +37,11 @@ impl EvmEventConfig { config: BTreeMap::new(), } } + pub fn from_config(config: impl Into>) -> Self { + EvmEventConfig { + config: config.into(), + } + } pub fn get(&self, chain_id: &ChainId) -> Option<&EvmEventConfigChain> { self.config.get(&chain_id) } @@ -48,4 +53,8 @@ impl EvmEventConfig { pub fn chains(&self) -> HashSet { self.config.keys().cloned().collect() } + + pub fn deploy_block(&self, chain_id: u64) -> Option { + self.get(&chain_id).map(|c| c.deploy_block()) + } } diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index a9821ed402..fd3289b711 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -10,6 +10,8 @@ repository = "https://github.com/gnosisguild/enclave/crates/sync" actix.workspace = true anyhow.workspace = true e3-events.workspace = true +e3-data.workspace = true +e3-config.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index 07b21af2e0..6d0d970c05 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +mod repo; mod sync; +pub use repo::*; pub use sync::*; diff --git a/crates/sync/src/repo.rs b/crates/sync/src/repo.rs new file mode 100644 index 0000000000..54ba0fdb13 --- /dev/null +++ b/crates/sync/src/repo.rs @@ -0,0 +1,23 @@ +use e3_config::StoreKeys; +use e3_data::{Repositories, Repository}; +use e3_events::AggregateId; + +pub trait SyncRepositoryFactory { + fn aggregate_seq(&self, aggregate_id: AggregateId) -> Repository; + fn aggregate_block(&self, aggregate_id: AggregateId) -> Repository; + fn aggregate_ts(&self, aggregate_id: AggregateId) -> Repository; +} + +impl SyncRepositoryFactory for Repositories { + fn aggregate_seq(&self, aggregate_id: AggregateId) -> Repository { + Repository::new(self.store.scope(StoreKeys::aggregate_seq(aggregate_id))) + } + + fn aggregate_block(&self, aggregate_id: AggregateId) -> Repository { + Repository::new(self.store.scope(StoreKeys::aggregate_block(aggregate_id))) + } + + fn aggregate_ts(&self, aggregate_id: AggregateId) -> Repository { + Repository::new(self.store.scope(StoreKeys::aggregate_ts(aggregate_id))) + } +} diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 2cd7b9bf0a..6f508092ea 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -4,14 +4,16 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::HashSet; - +use crate::SyncRepositoryFactory; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; +use e3_data::{AggregateConfig, Repositories, WriteBuffer}; use e3_events::{ - trap, BusHandle, EType, EnclaveEvent, EventContextAccessors, EventPublisher, EvmEventConfig, - EvmSyncEventsReceived, SyncEnd, SyncStart, Unsequenced, + trap, trap_fut, AggregateId, BusHandle, EType, EnclaveEvent, EventContextAccessors, + EventPublisher, EvmEventConfig, EvmEventConfigChain, EvmSyncEventsReceived, SyncEnd, SyncStart, + Unsequenced, }; +use std::collections::{BTreeMap, HashMap, HashSet}; use tracing::info; // NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p @@ -23,22 +25,47 @@ pub struct Synchronizer { evm_config: Option, evm_events: Vec>, evm_to_sync: HashSet, + repositories: Repositories, + write_buffer: Addr, // net_config: NetEventConfig, + aggregate_config: AggregateConfig, } impl Synchronizer { - pub fn new(bus: &BusHandle, evm_config: EvmEventConfig) -> Self { + pub fn new( + bus: BusHandle, + evm_config: EvmEventConfig, + repositories: Repositories, + aggregate_config: AggregateConfig, + write_buffer: Addr, + ) -> Self { let evm_to_sync = evm_config.chains(); Self { evm_config: Some(evm_config), - bus: bus.clone(), + bus, evm_to_sync, evm_events: Vec::new(), + repositories, + aggregate_config, + write_buffer, } } - pub fn setup(bus: &BusHandle, evm_config: EvmEventConfig) -> Addr { - Self::new(bus, evm_config).start() + pub fn setup( + bus: &BusHandle, + evm_config: &EvmEventConfig, + repositories: &Repositories, + aggregate_config: &AggregateConfig, + write_buffer: &Addr, + ) -> Addr { + Self::new( + bus.clone(), + evm_config.clone(), + repositories.clone(), + aggregate_config.clone(), + write_buffer.clone(), + ) + .start() } fn handle_evm_sync_events_received(&mut self, mut msg: EvmSyncEventsReceived) -> Result<()> { @@ -85,18 +112,134 @@ impl Handler for Synchronizer { } impl Handler for Synchronizer { - type Result = (); + type Result = actix::ResponseFuture<()>; fn handle(&mut self, _: Bootstrap, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Sync, &self.bus.clone(), || { - let evm_config = self.evm_config.take().context( - "EvmEventConfig was not set likely Bootstrap was called more than once.", - )?; - - // What was the last block we processed for each aggregate? - // TODO: Get information about what has and has not been synced then fire SyncStart - self.bus - .publish_without_context(SyncStart::new(ctx.address(), evm_config)) - }) + let address = ctx.address(); + let repositories = self.repositories.clone(); + let evm_config = self.evm_config.take(); + let aggregates = self.aggregate_config.aggregates(); + let bus = self.bus.clone(); + trap_fut( + EType::Sync, + &self.bus.clone(), + handle_bootstrap(bus, address, repositories, evm_config, aggregates), + ) + } +} + +async fn handle_bootstrap( + bus: BusHandle, + address: Addr, + repositories: Repositories, + evm_config: Option, + aggregates: Vec, +) -> Result<()> { + let evm_config = evm_config + .context("EvmEventConfig was not set likely Bootstrap was called more than once.")?; + // ============================================================ + // Phase 1: Load Snapshot + // ============================================================ + + // 1.1 Read snapshot from disk (may not exist on first boot) + let snapshot = SnapshotMeta::read_from_disk(aggregates, evm_config, repositories).await?; + + // 1.2 Extract state, last_applied_hlc, last_block_number (use defaults if no snapshot) + + // 1.4 Pause WriteBuffer streaming (don't write replayed mutations to disk) + + // ============================================================ + // Phase 2: Replay Missed Events + // ============================================================ + + // 2.1 Query EventStore for all events WHERE hlc > last_applied_hlc ORDER BY hlc + + // 2.2 For each event: + // - Route to appropriate actor by aggregate_id + // - Apply event mutation (in-memory only) + // - Track highest block_number seen (if event has one) + + // ============================================================ + // Phase 3: Determine Blockchain Resume Point + // ============================================================ + + // 3.1 Get highest block_number from replayed events (if any had block numbers) + + // 3.2 Fall back to snapshot.last_block_number if no blockchain events replayed + + // 3.3 Calculate resume_block = max(config.deploy_block, highest_block + 1) + + // ============================================================ + // Phase 4: Resume Normal Operation + // ============================================================ + + // 4.1 Update WriteBuffer watermark to current HLC + + // 4.2 Resume WriteBuffer streaming (mutations now flow to disk) + + // 4.3 Subscribe to blockchain from resume_block + + // 4.4 Return Ok / start event loop + // Get the sequences for each aggregate + // bus.publish_without_context(SyncStart::new(address, aggregate_states.as_evm_config()))?; + Ok(()) +} + +/// Latest event information in store +pub struct AggregateState { + ts: u128, + aggregate_id: AggregateId, + seq: u64, + block: u64, +} + +struct SnapshotMeta { + aggregate_state: Vec, +} + +impl SnapshotMeta { + pub async fn read_from_disk( + ids: Vec, + initial_evm_config: EvmEventConfig, + repositories: Repositories, + ) -> Result { + let mut aggregate_state = Vec::new(); + for aggregate_id in ids { + let deploy_block = aggregate_id + .to_chain_id() + .and_then(|chain_id| initial_evm_config.deploy_block(chain_id)) + .unwrap_or(0); + let seq_repo = repositories.aggregate_seq(aggregate_id); + let block_repo = repositories.aggregate_block(aggregate_id); + let ts_repo = repositories.aggregate_ts(aggregate_id); + let seq = seq_repo.read().await?.unwrap_or(0); + let block = block_repo.read().await?.unwrap_or(deploy_block); + let ts = ts_repo.read().await?.unwrap_or(0); + let agg_state = AggregateState { + aggregate_id, + seq, + block, + ts, + }; + aggregate_state.push(agg_state); + } + + Ok(Self { aggregate_state }) + } + + pub fn as_evm_config(&self) -> EvmEventConfig { + let map: BTreeMap = self + .aggregate_state + .iter() + .map(|s| (s.aggregate_id.to_chain_id(), s.block)) + .filter_map(|s| { + if let Some(chain) = s.0 { + Some((chain, EvmEventConfigChain::new(s.1))) + } else { + None + } + }) + .collect(); + EvmEventConfig::from_config(map) } } @@ -140,9 +283,10 @@ mod tests { let mut evm_config = EvmEventConfig::new(); evm_config.insert(1, EvmEventConfigChain::new(0)); evm_config.insert(2, EvmEventConfigChain::new(0)); + let repositories = Repositories::in_mem(); // Start synchronizer - let sync_addr = Synchronizer::setup(&bus, evm_config); + let sync_addr = Synchronizer::setup(&bus, evm_config, &repositories); settle().await; // Verify SyncStart was published diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs new file mode 100644 index 0000000000..ef2f5f752e --- /dev/null +++ b/crates/utils/src/error.rs @@ -0,0 +1,7 @@ +use tracing::error; + +/// Formats panic errors so they are seen in logs clearly +pub fn major_issue(msg: &str, e: impl Into) -> String { + error!("\n\n\nMAJOR ISSUE: {msg}.\n\nThe error supplied was: {:?}\n\n As a precaution we are crashing the system.\n\n\n", e.into()); + format!("System has crashed. Nothing personal. Goodbye.") +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index ec17ce7a9c..454c5b4791 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -6,6 +6,7 @@ pub mod actix; pub mod alloy; +pub mod error; pub mod formatters; pub mod helpers; pub mod path; @@ -13,6 +14,7 @@ pub mod retry; pub mod utility_types; pub use actix::*; pub use alloy::*; +pub use error::*; pub use formatters::*; pub use helpers::*; pub use path::*; diff --git a/packages/enclave-contracts/deployed_contracts.json b/packages/enclave-contracts/deployed_contracts.json index bb782c5a49..9429c1f1b5 100644 --- a/packages/enclave-contracts/deployed_contracts.json +++ b/packages/enclave-contracts/deployed_contracts.json @@ -106,5 +106,113 @@ "blockNumber": 10043257, "address": "0xC39b101f2FB4ea677c1EA18f92C15CDD54Af40c2" } + }, + "localhost": { + "PoseidonT3": { + "blockNumber": 3, + "address": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93" + }, + "MockUSDC": { + "constructorArgs": { + "initialSupply": "1000000" + }, + "blockNumber": 4, + "address": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" + }, + "EnclaveToken": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "blockNumber": 5, + "address": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + }, + "EnclaveTicketToken": { + "constructorArgs": { + "baseToken": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0", + "registry": "0x0000000000000000000000000000000000000001", + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "blockNumber": 7, + "address": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707" + }, + "SlashingManager": { + "constructorArgs": { + "admin": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "bondingRegistry": "0x0000000000000000000000000000000000000001" + }, + "blockNumber": 8, + "address": "0x0165878A594ca255338adfa4d48449f69242Eb8F" + }, + "BondingRegistry": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "ticketToken": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707", + "licenseToken": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9", + "registry": "0x0000000000000000000000000000000000000001", + "slashedFundsTreasury": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "ticketPrice": "10000000", + "licenseRequiredBond": "100000000000000000000", + "minTicketBalance": "1", + "exitDelay": "604800" + }, + "proxyRecords": { + "initData": "0x7333fa82000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660000000000000000000000005fc8d32690cc91d4c39d9d3abcbd16989f875707000000000000000000000000cf7ed3acca5a467e9e704c703e8d87f634fb0fc90000000000000000000000000000000000000000000000000000000000000001000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000000000000000000000000000000000000009896800000000000000000000000000000000000000000000000056bc75e2d6310000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000093a80", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6", + "proxyAdminAddress": "0x94099942864EA81cCF197E9D71ac53310b1468D8", + "implementationAddress": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" + }, + "blockNumber": 8, + "address": "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6" + }, + "CiphernodeRegistryOwnable": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "enclaveAddress": "0x0000000000000000000000000000000000000001", + "submissionWindow": "10" + }, + "proxyRecords": { + "initData": "0x1794bb3c000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000a", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", + "proxyAdminAddress": "0x6F1216D1BFe15c98520CA1434FC1d9D57AC95321", + "implementationAddress": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318" + }, + "blockNumber": 11, + "address": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788" + }, + "Enclave": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "registry": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", + "bondingRegistry": "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6", + "feeToken": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0", + "maxDuration": "2592000", + "params": [ + "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000ffffee0010000000000000000000000000000000000000000000000000000000ffffc400100000000000000000000000000000000000000000000000000000000000000013300000000000000000000000000000000000000000000000000000000000000" + ] + }, + "proxyRecords": { + "initData": "0xefe0308b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266000000000000000000000000610178da211fef7d417bc0e6fed39f05609ad7880000000000000000000000002279b7a0a67db372996a5fab50d91eaa73d2ebe60000000000000000000000009fe46736679d2d9a65f0992f2272de9f3c7fa6e00000000000000000000000000000000000000000000000000000000000278d0000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000ffffee0010000000000000000000000000000000000000000000000000000000ffffc400100000000000000000000000000000000000000000000000000000000000000013300000000000000000000000000000000000000000000000000000000000000", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0", + "proxyAdminAddress": "0x1F708C24a0D3A740cD47cC0444E9480899f3dA7D", + "implementationAddress": "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e" + }, + "blockNumber": 13, + "address": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0" + }, + "MockComputeProvider": { + "blockNumber": 23, + "address": "0x59b670e9fA9D0A427751Af201D676719a970857b" + }, + "MockDecryptionVerifier": { + "blockNumber": 24, + "address": "0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1" + }, + "MockE3Program": { + "blockNumber": 25, + "address": "0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44" + } } } \ No newline at end of file From 5c4657f8789d2c43bf56938b57ec51fc04a93010 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 3 Feb 2026 08:15:50 +0000 Subject: [PATCH 25/63] add headers --- crates/sync/src/repo.rs | 6 ++++++ crates/utils/src/error.rs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/crates/sync/src/repo.rs b/crates/sync/src/repo.rs index 54ba0fdb13..d839beb770 100644 --- a/crates/sync/src/repo.rs +++ b/crates/sync/src/repo.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; use e3_events::AggregateId; diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs index ef2f5f752e..8ffd8c62b1 100644 --- a/crates/utils/src/error.rs +++ b/crates/utils/src/error.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use tracing::error; /// Formats panic errors so they are seen in logs clearly From 7fe66b1947547a442b3971a7b35203df2d9dc340 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 08:55:35 +0000 Subject: [PATCH 26/63] convert write buffer into snapshot buffer --- crates/aggregator/src/repo.rs | 3 +- .../src/ciphernode_builder.rs | 9 +- crates/ciphernode-builder/src/event_system.rs | 104 ++--- crates/config/src/lib.rs | 2 - crates/data/src/data_store.rs | 13 +- crates/data/src/in_mem.rs | 2 +- crates/data/src/lib.rs | 6 - crates/data/src/persistable.rs | 6 +- crates/data/src/sled_db.rs | 7 +- crates/data/src/sled_store.rs | 3 +- crates/data/src/write_buffer.rs | 396 ------------------ .../events.rs => events/src/data_events.rs} | 3 +- .../events/src/enclave_event/enclave_error.rs | 55 ++- crates/events/src/events.rs | 36 +- crates/{data => events}/src/into_key.rs | 0 crates/events/src/lib.rs | 8 + crates/events/src/sequencer.rs | 9 +- .../src/snapshot_buffer/aggregate_config.rs | 34 ++ crates/events/src/snapshot_buffer/batch.rs | 49 +++ .../src/snapshot_buffer/batch_router.rs | 169 ++++++++ crates/events/src/snapshot_buffer/mod.rs | 8 + .../src/snapshot_buffer/snapshot_buffer.rs | 98 +++++ .../src/snapshot_buffer/timelock_queue.rs | 365 ++++++++++++++++ crates/{config => events}/src/store_keys.rs | 2 +- crates/evm/src/repo.rs | 2 +- crates/fhe/src/repo.rs | 3 +- crates/keyshare/src/repo.rs | 3 +- crates/net/src/repo.rs | 2 +- crates/request/src/repo.rs | 2 +- crates/sortition/src/repo.rs | 3 +- crates/sortition/src/sortition.rs | 6 +- crates/sync/src/repo.rs | 2 +- crates/sync/src/sync.rs | 32 +- 33 files changed, 883 insertions(+), 559 deletions(-) delete mode 100644 crates/data/src/write_buffer.rs rename crates/{data/src/events.rs => events/src/data_events.rs} (97%) rename crates/{data => events}/src/into_key.rs (100%) create mode 100644 crates/events/src/snapshot_buffer/aggregate_config.rs create mode 100644 crates/events/src/snapshot_buffer/batch.rs create mode 100644 crates/events/src/snapshot_buffer/batch_router.rs create mode 100644 crates/events/src/snapshot_buffer/mod.rs create mode 100644 crates/events/src/snapshot_buffer/snapshot_buffer.rs create mode 100644 crates/events/src/snapshot_buffer/timelock_queue.rs rename crates/{config => events}/src/store_keys.rs (98%) diff --git a/crates/aggregator/src/repo.rs b/crates/aggregator/src/repo.rs index 769c7da7d2..303bd38e93 100644 --- a/crates/aggregator/src/repo.rs +++ b/crates/aggregator/src/repo.rs @@ -4,9 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use crate::{PublicKeyAggregatorState, ThresholdPlaintextAggregatorState}; diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 9af2c52ec9..c82664eb4e 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -4,7 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::event_system::AggregateConfig; use crate::{CiphernodeHandle, EventSystem, EvmSystemChainBuilder, ProviderCache, WriteEnabled}; use actix::{Actor, Addr}; use anyhow::Result; @@ -14,7 +13,9 @@ use e3_aggregator::CommitteeFinalizer; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{InMemStore, RepositoriesFactory}; -use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EvmEventConfig}; +use e3_events::{ + AggregateConfig, AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EvmEventConfig, +}; use e3_evm::{BondingRegistrySolReader, CiphernodeRegistrySolReader, EnclaveSolWriter}; use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; @@ -462,7 +463,9 @@ impl CiphernodeBuilder { ) }; - Synchronizer::setup(&bus, &evm_config, &repositories, &aggregate_config); + let buffer = event_system.buffer()?; + + Synchronizer::setup(&bus, &evm_config, &repositories, &aggregate_config, &buffer); Ok(CiphernodeHandle::new( addr.to_owned(), diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 0bde9637f2..6a5942d713 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -8,13 +8,13 @@ use crate::get_enclave_event_bus; use actix::{Actor, Addr, Recipient}; use anyhow::{anyhow, Result}; use e3_data::{ - CommitLogEventLog, DataStore, ForwardTo, InMemEventLog, InMemSequenceIndex, InMemStore, - InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, + CommitLogEventLog, DataStore, InMemEventLog, InMemSequenceIndex, InMemStore, SledSequenceIndex, + SledStore, }; use e3_events::hlc::Hlc; use e3_events::{ - BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, Sequencer, - StoreEventRequested, + AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, + EventStoreRouter, InsertBatch, Sequencer, SnapshotBuffer, StoreEventRequested, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -22,13 +22,19 @@ use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; -pub use e3_data::AggregateConfig; - struct InMemBackend { eventstores: OnceCell>>>, store: OnceCell>, } +impl InMemBackend { + fn get_or_init_store(&self) -> Addr { + self.store + .get_or_init(|| InMemStore::new(true).start()) + .clone() + } +} + /// Hold the Persistent EventStore instance and SledStore struct PersistedBackend { log_path: PathBuf, @@ -37,6 +43,14 @@ struct PersistedBackend { store: OnceCell>, } +impl PersistedBackend { + fn get_or_init_store(&self, handle: &BusHandle) -> Result> { + self.store + .get_or_try_init(|| SledStore::new(handle, &self.sled_path)) + .cloned() + } +} + /// An EventSystemBackend is holding the potentially persistent structures for the system enum EventSystemBackend { InMem(InMemBackend), @@ -64,7 +78,7 @@ pub struct EventSystem { /// EventSystem backend either persisted or in memory backend: EventSystemBackend, /// WriteBuffer for batching inserts from actors into a snapshot - buffer: OnceCell>, + buffer: OnceCell>, /// EventSystem Sequencer sequencer: OnceCell>, /// EventSystem eventbus @@ -185,16 +199,18 @@ impl EventSystem { } /// Get the buffer address - pub fn buffer(&self) -> Addr { - let buffer = self - .buffer - .get_or_init(|| { - let config = self.aggregate_config(); - WriteBuffer::with_config(config).start() + pub fn buffer(&self) -> Result> { + self.buffer + .get_or_try_init(|| { + let store: Recipient = match &self.backend { + EventSystemBackend::InMem(b) => b.get_or_init_store().into(), + EventSystemBackend::Persisted(b) => { + b.get_or_init_store(&self.handle()?)?.into() + } + }; + SnapshotBuffer::spawn(&self.aggregate_config(), store) }) - .clone(); - self.wire_if_ready(); - buffer + .cloned() } /// Get the sequencer address @@ -202,7 +218,7 @@ impl EventSystem { self.sequencer .get_or_try_init(|| { let router = self.eventstore_router()?; - Ok(Sequencer::new(&self.eventbus(), router, self.buffer()).start()) + Ok(Sequencer::new(&self.eventbus(), router).start()) }) .cloned() } @@ -320,50 +336,16 @@ impl EventSystem { /// Get the DataStore pub fn store(&self) -> Result { - let store = match &self.backend { - EventSystemBackend::InMem(b) => { - let addr = b - .store - .get_or_init(|| InMemStore::new(true).start()) - .clone(); - DataStore::from_in_mem_with_buffer(&addr, &self.buffer()) - } - EventSystemBackend::Persisted(b) => { - let addr = b - .store - .get_or_try_init(|| { - let handle = self.handle()?; - SledStore::new(&handle, &b.sled_path) - })? - .clone(); - DataStore::from_sled_store_with_buffer(&addr, &self.buffer()) - } - }; - self.wire_if_ready(); - Ok(store) - } - - // We need to ensure that once the buffer and store are created they are connected so that - // inserts are sent between the two actors. This internal function ensures this happens. - fn wire_if_ready(&self) { - let buffer = match self.buffer.get() { - Some(b) => b, - None => return, - }; - - let store: Option> = match &self.backend { - EventSystemBackend::InMem(b) => b.store.get().cloned().map(Into::into), - EventSystemBackend::Persisted(b) => b.store.get().cloned().map(Into::into), - }; - - let Some(store) = store else { - return; - }; - - // Now we know both are ready, so initialization will succeed - self.wired.get_or_init(|| { - buffer.do_send(ForwardTo::new(store)); - }); + match &self.backend { + EventSystemBackend::InMem(b) => Ok(DataStore::from_in_mem_with_buffer( + &b.get_or_init_store(), + self.buffer()?, + )), + EventSystemBackend::Persisted(b) => Ok(DataStore::from_sled_store_with_buffer( + &b.get_or_init_store(&self.handle()?)?, + self.buffer()?, + )), + } } fn node_id(name: &str) -> u32 { diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 5c8b7199f9..2e18c6f92d 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -10,11 +10,9 @@ pub mod contract; pub mod load_config; pub mod paths_engine; pub mod rpc; -mod store_keys; pub mod validation; mod yaml; pub use app_config::*; pub use contract::*; pub use rpc::*; -pub use store_keys::*; diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index ceeede2bb5..4e00f7ddc8 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -6,12 +6,13 @@ use std::borrow::Cow; -use crate::{Get, Insert, InsertSync, Remove, WriteBuffer}; -use crate::{InMemStore, IntoKey, SledStore}; +use crate::{InMemStore, SledStore}; use actix::{Addr, Recipient}; use anyhow::anyhow; use anyhow::Context; use anyhow::Result; +use e3_events::IntoKey; +use e3_events::{Get, Insert, InsertSync, Remove}; use serde::{Deserialize, Serialize}; use tracing::error; @@ -173,12 +174,12 @@ impl DataStore { pub fn from_sled_store_with_buffer( addr: &Addr, - write_buffer: &Addr, + write_buffer: impl Into>, ) -> Self { Self { addr: StoreAddr::Sled(addr.clone()), get: addr.clone().recipient(), - insert: write_buffer.clone().recipient(), + insert: write_buffer.into(), insert_sync: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], @@ -187,12 +188,12 @@ impl DataStore { pub fn from_in_mem_with_buffer( addr: &Addr, - write_buffer: &Addr, + write_buffer: impl Into>, ) -> Self { Self { addr: StoreAddr::InMem(addr.clone()), get: addr.clone().recipient(), - insert: write_buffer.clone().recipient(), + insert: write_buffer.into(), insert_sync: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index ed0f52f60f..75cb789869 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertBatch, InsertSync, Remove}; use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; +use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; use std::collections::BTreeMap; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 83827a3aa4..31f8b63bff 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -6,11 +6,9 @@ mod commit_log_event_log; mod data_store; -mod events; mod in_mem; mod in_mem_event_log; mod in_mem_sequence_index; -mod into_key; mod persistable; mod repositories; mod repository; @@ -19,15 +17,12 @@ mod sled_sequence_index; mod sled_store; mod sled_utils; mod snapshot; -mod write_buffer; pub use commit_log_event_log::*; pub use data_store::*; -pub use events::*; pub use in_mem::*; pub use in_mem_event_log::*; pub use in_mem_sequence_index::*; -pub use into_key::IntoKey; pub use persistable::*; pub use repositories::*; pub use repository::*; @@ -35,4 +30,3 @@ pub use sled_db::*; pub use sled_sequence_index::*; pub use sled_store::*; pub use snapshot::*; -pub use write_buffer::*; diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 5b28d51cd5..3472a27473 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -3,11 +3,11 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, Remove, Repository}; +use crate::Repository; use actix::Recipient; use anyhow::*; use async_trait::async_trait; -use e3_events::{EventContext, EventContextManager, Sequenced}; +use e3_events::{EventContext, EventContextManager, Get, Insert, Remove, Sequenced}; use serde::{de::DeserializeOwned, Serialize}; pub trait PersistableData: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} @@ -279,7 +279,7 @@ impl EventContextManager for Persistable { mod tests { use actix::{Actor, Addr, Handler, Message}; - use crate::{Get, Insert, Remove}; + use e3_events::{Get, Insert, Remove}; use super::{Persistable, StoreConnector}; diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 641ea39797..72e508d425 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -4,15 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::sled_utils::{clear_all_caches, get_or_open_db_tree}; use anyhow::{Context, Result}; +use e3_events::{Get, Insert, Remove}; use sled::{transaction::ConflictableTransactionError, Tree}; use std::path::PathBuf; -use crate::{ - sled_utils::{clear_all_caches, get_or_open_db_tree}, - Get, Insert, Remove, -}; - pub struct SledDb { db: Tree, } diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 6bf35bea46..660ddb20b3 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,10 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertBatch, InsertSync, Remove, SledDb}; +use crate::SledDb; use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::Result; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData, EventType}; +use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; use std::path::PathBuf; use tracing::{error, info}; diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs deleted file mode 100644 index 933166b228..0000000000 --- a/crates/data/src/write_buffer.rs +++ /dev/null @@ -1,396 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::{Actor, Handler, Message, Recipient}; -use e3_config::StoreKeys; -use e3_events::hlc::HlcTimestamp; -use e3_events::{AggregateId, CommitSnapshot, EventContext, EventContextAccessors}; -use std::{ - collections::HashMap, - time::{SystemTime, UNIX_EPOCH}, -}; - -use crate::{Insert, InsertBatch}; - -#[derive(Message)] -#[rtype(result = "()")] -pub struct Start; - -/// Central configuration for aggregates in the WriteBuffer -#[derive(Debug, Clone)] -pub struct AggregateConfig { - pub delays: HashMap, -} - -impl AggregateConfig { - /// Create a new AggregateConfig with the specified delays - pub fn new(mut delays: HashMap) -> Self { - // Always handle AggregatId of 0 with a delay of 0 - if let None = delays.get(&AggregateId::new(0)) { - delays.insert(AggregateId::new(0), 0); - } - Self { delays } - } - - /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured - pub fn indexed_ids(&self) -> Vec { - self.delays.keys().map(|id| id.to_usize()).collect() - } - - pub fn aggregates(&self) -> Vec { - self.delays.keys().cloned().collect() - } -} - -#[derive(Debug, Default)] -struct AggregateBuffer { - buffer: Vec, -} - -impl AggregateBuffer { - fn new() -> Self { - Self { buffer: Vec::new() } - } -} - -pub enum WriteBufferStatus { - Paused, - Runnning, -} - -pub struct WriteBuffer { - /// The status of the write buffer - status: WriteBufferStatus, - /// Destination recipient for batched inserts - dest: Option>, - /// Per-aggregate buffers for organizing inserts - aggregate_buffers: HashMap, - /// Per-aggregate blockheight highwatermarks - block_height_seen: HashMap, - /// Per-aggregate wait time configuration - config: AggregateConfig, -} - -impl Actor for WriteBuffer { - type Context = actix::Context; -} - -impl WriteBuffer { - pub fn new() -> Self { - Self { - status: WriteBufferStatus::Paused, - dest: None, - aggregate_buffers: HashMap::new(), - block_height_seen: HashMap::new(), - config: AggregateConfig::new(HashMap::new()), - } - } - - pub fn with_config(config: AggregateConfig) -> Self { - Self { - status: WriteBufferStatus::Paused, - dest: None, - aggregate_buffers: HashMap::new(), - block_height_seen: HashMap::new(), - config, - } - } - - fn handle_insert(&mut self, msg: Insert) { - if let WriteBufferStatus::Paused = self.status { - return; - } - - handle_insert( - msg, - &mut self.aggregate_buffers, - &mut self.block_height_seen, - ) - } - - fn get_highest_block(&self, agg: AggregateId, block: Option) -> u64 { - block - .max(self.block_height_seen.get(&agg).map(ToOwned::to_owned)) - .unwrap_or(0) - } - - fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { - if let WriteBufferStatus::Paused = self.status { - return; - } - - // Store the sequence number as an Insert message so snapshots hold the most recent event - // they were created against - let aggregate_id = msg.aggregate_id(); - - // NOTE: can not use repository here as we need to insert operations in stream - self.handle_insert(Insert::new( - &StoreKeys::aggregate_seq(aggregate_id), - encode_u64(msg.seq()), - )); - - self.handle_insert(Insert::new( - &StoreKeys::aggregate_block(aggregate_id), - encode_u64(self.get_highest_block(aggregate_id, msg.block())), - )); - - self.handle_insert(Insert::new( - // NOT NEW NEED CTX - &StoreKeys::aggregate_ts(aggregate_id), - encode_u128(msg.ts()), - )); - - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|_| std::time::Duration::from_secs(0)) - .as_micros() as u64; - - if let Some(ref dest) = self.dest { - let (updated_buffers, expired_inserts) = - process_expired_inserts(&self.aggregate_buffers, &self.config.delays, now); - if !expired_inserts.is_empty() { - let batch = InsertBatch::new(expired_inserts); - dest.do_send(batch); - } - - self.aggregate_buffers = updated_buffers; - } - } -} - -impl Handler for WriteBuffer { - type Result = (); - fn handle(&mut self, msg: ForwardTo, _: &mut Self::Context) -> Self::Result { - self.dest = Some(msg.dest()) - } -} - -impl Handler for WriteBuffer { - type Result = (); - - fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - self.handle_insert(msg) - } -} - -/// Encode the same as bincode without using a result -fn encode_u64(value: u64) -> Vec { - value.to_le_bytes().to_vec() -} - -/// Encode the same as bincode without using a result -fn encode_u128(value: u128) -> Vec { - value.to_le_bytes().to_vec() -} - -fn process_expired_inserts( - aggregate_buffers: &HashMap, - config: &HashMap, - now: u64, -) -> (HashMap, Vec) { - let mut updated_buffers = HashMap::new(); - let mut all_expired_inserts = Vec::new(); - - for (aggregate_id, agg_buffer) in aggregate_buffers { - let delay_micros = config.get(aggregate_id).copied().unwrap_or(0); - let cutoff_time = now.saturating_sub(delay_micros); - let mut expired_inserts = Vec::new(); - let mut remaining_inserts = Vec::new(); - - for insert in &agg_buffer.buffer { - if let Some(ctx) = insert.ctx() { - let event_wall_time = HlcTimestamp::wall_time(ctx.ts()); - if event_wall_time < cutoff_time { - expired_inserts.push(insert.clone()); - } else { - remaining_inserts.push(insert.clone()); - } - } else { - // If there is no context just flush it - expired_inserts.push(insert.clone()); - } - } - - all_expired_inserts.extend(expired_inserts); - - if !remaining_inserts.is_empty() { - let mut new_agg_buffer = AggregateBuffer::new(); - new_agg_buffer.buffer = remaining_inserts; - updated_buffers.insert(aggregate_id.clone(), new_agg_buffer); - } - } - - (updated_buffers, all_expired_inserts) -} - -fn handle_insert( - msg: Insert, - aggregate_buffers: &mut HashMap, - block_height_seen: &mut HashMap, -) { - let aggregate_id = msg - .ctx() - .map(|ec| { - if let Some(block) = ec.block() { - block_height_seen - .entry(ec.aggregate_id().clone()) - .and_modify(|e| *e = (*e).max(block)) - .or_insert(block); - } - ec.aggregate_id().clone() - }) - .unwrap_or_else(|| AggregateId::new(0)); - - aggregate_buffers - .entry(aggregate_id) - .or_default() - .buffer - .push(msg); -} - -impl Handler for WriteBuffer { - type Result = (); - - fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { - self.handle_commit_snapshot(msg) - } -} - -impl Handler for WriteBuffer { - type Result = (); - fn handle(&mut self, msg: Start, ctx: &mut Self::Context) -> Self::Result { - self.status = WriteBufferStatus::Runnning; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::events::Insert; - use e3_events::{hlc::HlcTimestamp, EventContext, EventId}; - - fn insert_with_block(agg: AggregateId, block: u64) -> Insert { - Insert::new_with_context( - "key", - b"value".to_vec(), - EventContext::new_origin(EventId::hash(1), 123, agg, Some(block)).sequence(1), - ) - } - - fn insert_without_block(agg: AggregateId) -> Insert { - Insert::new_with_context( - "key", - b"value".to_vec(), - EventContext::new_origin(EventId::hash(1), 123, agg, None).sequence(1), - ) - } - - fn insert_no_context() -> Insert { - Insert::new("key", b"value".to_vec()) - } - - #[test] - fn tracks_highwater_mark() { - let mut buffers = HashMap::new(); - let mut heights = HashMap::new(); - let agg_1 = AggregateId::new(1); - let agg_42 = AggregateId::new(42); - - // Initial inserts for both aggregates - handle_insert(insert_with_block(agg_42, 24), &mut buffers, &mut heights); - handle_insert(insert_with_block(agg_1, 25), &mut buffers, &mut heights); - - // Higher block for agg_42 - should update - handle_insert(insert_with_block(agg_42, 100), &mut buffers, &mut heights); - - // Lower block for agg_42 - should NOT update - handle_insert(insert_with_block(agg_42, 50), &mut buffers, &mut heights); - - // No block context - should not affect heights - handle_insert(insert_without_block(agg_1), &mut buffers, &mut heights); - - // No context at all - uses default aggregate - handle_insert(insert_no_context(), &mut buffers, &mut heights); - - // Verify highwater marks - assert_eq!(heights.get(&agg_42), Some(&100)); - assert_eq!(heights.get(&agg_1), Some(&25)); - assert_eq!(heights.get(&AggregateId::new(0)), None); - - // Verify buffer counts - assert_eq!(buffers.get(&agg_42).unwrap().buffer.len(), 3); - assert_eq!(buffers.get(&agg_1).unwrap().buffer.len(), 2); - assert_eq!(buffers.get(&AggregateId::new(0)).unwrap().buffer.len(), 1); - } - - #[test] - fn test_process_expired_inserts() { - let aggregate_id = AggregateId::new(1); - - // Create test inserts with different timestamps (in microseconds) - // Create proper HlcTimestamps and encode them to u128 - let old_hlc = HlcTimestamp::new(500_000, 0, 1); // 0.5 seconds ago - let new_hlc = HlcTimestamp::new(3_000_000, 0, 2); // 3 seconds from epoch - - let old_ctx = - EventContext::new_origin(EventId::hash(1), old_hlc.into(), aggregate_id.clone(), None) - .sequence(1); - - let new_ctx = - EventContext::new_origin(EventId::hash(2), new_hlc.into(), aggregate_id.clone(), None) - .sequence(2); - - let old_insert = Insert::new_with_context("old_key", b"old_value".to_vec(), old_ctx); - let new_insert = Insert::new_with_context("new_key", b"new_value".to_vec(), new_ctx); - let insert_no_ctx = Insert::new("no_ctx_key", b"no_ctx_value".to_vec()); - - // Set up aggregate buffer with mixed inserts - let mut agg_buffer = AggregateBuffer::new(); - agg_buffer.buffer.push(old_insert.clone()); - agg_buffer.buffer.push(new_insert.clone()); - agg_buffer.buffer.push(insert_no_ctx.clone()); - - let mut aggregate_buffers = HashMap::new(); - aggregate_buffers.insert(aggregate_id.clone(), agg_buffer); - - // Set config with 1 second delay - let mut delays = HashMap::new(); - delays.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds - let config = AggregateConfig::new(delays); - - // Use current time of 2 seconds, so old insert (0.5s) and insert without context should expire, - // new insert (3s) should remain - let now = 2_000_000; // 2 seconds in microseconds - - let (updated_buffers, expired_inserts) = - process_expired_inserts(&aggregate_buffers, &config.delays, now); - - // Verify expired inserts (old insert and insert without context) - assert_eq!(expired_inserts.len(), 2); - assert!(expired_inserts.contains(&old_insert)); - assert!(expired_inserts.contains(&insert_no_ctx)); - - // Verify remaining inserts in buffer - assert_eq!(updated_buffers.len(), 1); - let remaining_buffer = updated_buffers.get(&aggregate_id).unwrap(); - assert_eq!(remaining_buffer.buffer.len(), 1); - assert!(remaining_buffer.buffer.contains(&new_insert)); - } -} - -#[derive(Message)] -#[rtype("()")] -pub struct ForwardTo(Recipient); - -impl ForwardTo { - pub fn new(dest: impl Into>) -> Self { - Self(dest.into()) - } - - pub fn dest(self) -> Recipient { - self.0 - } -} diff --git a/crates/data/src/events.rs b/crates/events/src/data_events.rs similarity index 97% rename from crates/data/src/events.rs rename to crates/events/src/data_events.rs index 6f36b8c53c..5964049eaa 100644 --- a/crates/data/src/events.rs +++ b/crates/events/src/data_events.rs @@ -4,10 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::IntoKey; +use crate::{EventContext, IntoKey, Sequenced}; use actix::Message; use anyhow::Result; -use e3_events::{EventContext, Sequenced}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 93881c076d..d456a42eaa 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -4,17 +4,19 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Addr, Message}; +use actix::Message; +use e3_utils::major_issue; use serde::{Deserialize, Serialize}; use std::{ fmt::{self, Display}, future::Future, pin::Pin, }; +use tracing::warn; -use crate::{BusHandle, ErrorDispatcher, EventBus}; +use crate::{BusHandle, ErrorDispatcher}; -use super::EnclaveEvent; +use super::{EnclaveEvent, Unsequenced}; pub trait FromError { fn from_error(err_type: EType, error: impl Into) -> Self; @@ -70,7 +72,7 @@ impl FromError for EnclaveError { /// Function to run a closure that returns a result. If result is an Err variant it is trapped and /// sent to the bus as an ErrorEvent -pub fn trap(err_type: EType, bus: &BusHandle, runner: F) +pub fn trap(err_type: EType, bus: &impl ErrorDispatcher>, runner: F) where F: FnOnce() -> anyhow::Result<()>, { @@ -97,3 +99,48 @@ where } }) } + +// The following dispatchers should be used where you don't have the BusHandle available. + +// A struct that panics on errors +pub struct PanicDispatcher; + +impl PanicDispatcher { + pub fn new() -> Self { + Self {} + } +} + +impl ErrorDispatcher> for PanicDispatcher { + fn err(&self, _: EType, error: impl Into) { + panic!("{}", major_issue("Failure!", error)); + } +} + +// A struct that warns on errors +pub struct WarningDispatcher; +impl WarningDispatcher { + pub fn new() -> Self { + Self {} + } +} +impl ErrorDispatcher> for WarningDispatcher { + fn err(&self, err_type: EType, error: impl Into) { + tracing::warn!("{:?} Failure! {}", err_type, error.into()); + } +} + +// A struct that logs errors on errors +// Avoid using this over BusHandle +pub struct LogErrorDispatcher; +impl LogErrorDispatcher { + pub fn new() -> Self { + Self {} + } +} + +impl ErrorDispatcher> for LogErrorDispatcher { + fn err(&self, err_type: EType, error: impl Into) { + tracing::error!("{:?} Failure! {}", err_type, error.into()); + } +} diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 49a0cf197f..660881e14e 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -6,41 +6,7 @@ use actix::{Message, Recipient}; -use crate::{ - AggregateId, CorrelationId, EnclaveEvent, EventContext, EventContextAccessors, EventContextSeq, - Sequenced, Unsequenced, -}; - -/// Direct event received by the snapshot buffer in order to save snapshot to disk -#[derive(Message, Debug)] -#[rtype("()")] -pub struct CommitSnapshot { - ec: EventContext, -} - -impl CommitSnapshot { - pub fn new(event: &EnclaveEvent) -> Self { - Self { - ec: event.get_ctx().clone(), - } - } - - pub fn seq(&self) -> u64 { - self.ec.seq() - } - - pub fn aggregate_id(&self) -> AggregateId { - self.ec.aggregate_id() - } - - pub fn block(&self) -> Option { - self.ec.block() - } - - pub fn ts(&self) -> u128 { - self.ec.ts() - } -} +use crate::{CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; /// Direct event received by the EventStore to store an event #[derive(Message, Debug)] diff --git a/crates/data/src/into_key.rs b/crates/events/src/into_key.rs similarity index 100% rename from crates/data/src/into_key.rs rename to crates/events/src/into_key.rs diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index b9221447d7..b956799785 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -6,6 +6,7 @@ mod bus_handle; mod correlation_id; +mod data_events; mod e3id; mod enclave_event; mod event_context; @@ -15,15 +16,19 @@ mod events; mod eventstore; mod eventstore_router; pub mod hlc; +mod into_key; mod ordered_set; pub mod prelude; mod seed; mod sequencer; +mod snapshot_buffer; +mod store_keys; mod sync; mod traits; pub use bus_handle::*; pub use correlation_id::*; +pub use data_events::*; pub use e3id::*; pub use enclave_event::*; pub use event_context::*; @@ -32,8 +37,11 @@ pub use eventbus::*; pub use events::*; pub use eventstore::*; pub use eventstore_router::*; +pub use into_key::*; pub use ordered_set::*; pub use seed::*; pub use sequencer::*; +pub use snapshot_buffer::*; +pub use store_keys::*; pub use sync::*; pub use traits::*; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 5e788f7fbd..f9e0ee6d8a 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - events::{CommitSnapshot, EventStored, StoreEventRequested}, + events::{EventStored, StoreEventRequested}, EnclaveEvent, EventBus, Sequenced, Unsequenced, }; use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; @@ -16,25 +16,22 @@ use e3_utils::major_issue; pub struct Sequencer { bus: Addr>>, eventstore: Recipient, - buffer: Recipient, } impl Sequencer { pub fn new( bus: &Addr>>, eventstore: impl Into>, - buffer: impl Into>, ) -> Self { Self { bus: bus.clone(), eventstore: eventstore.into(), - buffer: buffer.into(), } } fn handle_event_stored(&self, msg: EventStored) -> Result<()> { let event = msg.into_event(); - self.buffer.try_send(CommitSnapshot::new(&event))?; + self.bus.try_send(event)?; Ok(()) } @@ -62,7 +59,7 @@ impl Handler for Sequencer { if let Err(e) = self.handle_event_stored(msg) { panic!( "{}", - major_issue("Could not send event to buffer or bus.", e) + major_issue("Could not send event to snapshot_buffer or bus.", e) ) } } diff --git a/crates/events/src/snapshot_buffer/aggregate_config.rs b/crates/events/src/snapshot_buffer/aggregate_config.rs new file mode 100644 index 0000000000..75bfef3506 --- /dev/null +++ b/crates/events/src/snapshot_buffer/aggregate_config.rs @@ -0,0 +1,34 @@ +use crate::AggregateId; +use std::collections::HashMap; + +/// Central configuration for aggregates in the WriteBuffer +#[derive(Debug, Clone)] +pub struct AggregateConfig { + pub delays: HashMap, +} + +impl AggregateConfig { + pub fn get_delay(&self, id: &AggregateId) -> u64 { + self.delays.get(id).cloned().unwrap_or(0) + } +} + +impl AggregateConfig { + /// Create a new AggregateConfig with the specified delays + pub fn new(mut delays: HashMap) -> Self { + // Always handle AggregatId of 0 with a delay of 0 + if let None = delays.get(&AggregateId::new(0)) { + delays.insert(AggregateId::new(0), 0); + } + Self { delays } + } + + /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured + pub fn indexed_ids(&self) -> Vec { + self.delays.keys().map(|id| id.to_usize()).collect() + } + + pub fn aggregates(&self) -> Vec { + self.delays.keys().cloned().collect() + } +} diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs new file mode 100644 index 0000000000..b493a5e643 --- /dev/null +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -0,0 +1,49 @@ +use std::mem::replace; + +use actix::{Actor, ActorContext, Addr, Handler, Message, Recipient}; + +use crate::{trap, EType, Insert, InsertBatch, PanicDispatcher}; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct Flush; + +pub struct Batch { + inserts: Vec, + db: Recipient, +} + +impl Batch { + pub fn new(db: impl Into>, inserts: Vec) -> Self { + Self { + inserts, + db: db.into(), + } + } + pub fn spawn(db: impl Into>, inserts: Vec) -> Addr { + Self::new(db, inserts).start() + } +} + +impl Actor for Batch { + type Context = actix::Context; +} + +impl Handler for Batch { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + self.inserts.push(msg) + } +} + +impl Handler for Batch { + type Result = (); + fn handle(&mut self, _: Flush, ctx: &mut Self::Context) -> Self::Result { + let inserts = replace(&mut self.inserts, Vec::new()); + trap(EType::IO, &PanicDispatcher::new(), || { + self.db.try_send(InsertBatch::new(inserts))?; + ctx.stop(); + Ok(()) + }) + } +} diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs new file mode 100644 index 0000000000..feb1fa2d35 --- /dev/null +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -0,0 +1,169 @@ +use super::{ + batch::{Batch, Flush}, + timelock_queue::StartTimelock, + AggregateConfig, +}; +use crate::{ + trap, AggregateId, EType, EnclaveEvent, EventContextAccessors, EventContextSeq, Insert, + InsertBatch, PanicDispatcher, Sequenced, StoreKeys, +}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use anyhow::Context; +use std::{ + collections::HashMap, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tracing::warn; + +type Seq = u64; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct FlushSeq(pub Seq); + +impl FlushSeq { + pub fn seq(&self) -> u64 { + self.0 + } +} + +#[derive(Debug)] +pub struct BatchRouter { + config: AggregateConfig, + aggregates: HashMap, + batches: HashMap>, + block_height_seen: HashMap, + timelock_queue: Recipient, + db: Recipient, +} + +impl Actor for BatchRouter { + type Context = actix::Context; +} + +impl BatchRouter { + pub fn new( + config: &AggregateConfig, + timelock_queue: impl Into>, + db: impl Into>, + ) -> Self { + Self { + batches: HashMap::new(), + aggregates: HashMap::new(), + config: config.clone(), + timelock_queue: timelock_queue.into(), + block_height_seen: HashMap::new(), + db: db.into(), + } + } + + fn get_highest_block(&mut self, agg: AggregateId, block: Option) -> u64 { + let highest = block + .into_iter() + .chain(self.block_height_seen.get(&agg).copied()) + .max() + .unwrap_or(0); + + self.block_height_seen.insert(agg, highest); + highest + } +} + +impl Handler for BatchRouter { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + let ctx = msg.ctx().context( + "Insert events sent to the BatchRouter MUST have a context but one was not found.", + )?; + let seq = ctx.seq(); + if self.batches.contains_key(&seq) { + if let Some(batch) = self.batches.get(&seq) { + batch.try_send(msg)?; + } + } else { + warn!("Insert received for flushed seq {}", seq); + } + Ok(()) + }) + } +} + +impl Handler> for BatchRouter { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + let ec = msg.get_ctx(); + trap(EType::IO, &PanicDispatcher::new(), || { + let prev_seq = ec.seq() - 1; + if self.batches.contains_key(&prev_seq) { + let prev_agg = self + .aggregates + .get(&prev_seq) + .context("invariant: prev_agg MUST exist if batches has a batch")?; + + let delay = self.config.get_delay(prev_agg); + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_micros() as u64; + + self.timelock_queue + .try_send(StartTimelock::new(prev_seq, now, delay))?; + } + + let agg_id = ec.aggregate_id(); + let highest_block = self.get_highest_block(agg_id, ec.block()); + + let batch = Batch::spawn( + self.db.clone(), + vec![ + Insert::new_with_context( + &StoreKeys::aggregate_seq(agg_id), + encode_u64(ec.seq()), + ec.clone(), + ), + Insert::new_with_context( + &StoreKeys::aggregate_block(agg_id), + encode_u64(highest_block), + ec.clone(), + ), + Insert::new_with_context( + &StoreKeys::aggregate_ts(agg_id), + encode_u128(ec.ts()), + ec.clone(), + ), + ], + ); + + self.batches.insert(ec.seq(), batch); + self.aggregates.insert(ec.seq(), ec.aggregate_id()); + + Ok(()) + }) + } +} + +impl Handler for BatchRouter { + type Result = (); + fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(batch) = self.batches.get(&msg.seq()) { + batch.try_send(Flush)?; + self.batches.remove(&msg.seq()); + self.aggregates.remove(&msg.seq()); + } + Ok(()) + }) + } +} + +/// Encode the same as bincode without using a result +fn encode_u64(value: u64) -> Vec { + value.to_le_bytes().to_vec() +} + +/// Encode the same as bincode without using a result +fn encode_u128(value: u128) -> Vec { + value.to_le_bytes().to_vec() +} diff --git a/crates/events/src/snapshot_buffer/mod.rs b/crates/events/src/snapshot_buffer/mod.rs new file mode 100644 index 0000000000..8a178f7b83 --- /dev/null +++ b/crates/events/src/snapshot_buffer/mod.rs @@ -0,0 +1,8 @@ +mod aggregate_config; +mod batch; +mod batch_router; +mod snapshot_buffer; +mod timelock_queue; + +pub use aggregate_config::*; +pub use snapshot_buffer::*; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs new file mode 100644 index 0000000000..c6f406d119 --- /dev/null +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -0,0 +1,98 @@ +use super::{ + batch_router::{BatchRouter, FlushSeq}, + timelock_queue::{StartTimelock, TimelockQueue}, + AggregateConfig, +}; +use crate::{trap, EType, Insert, InsertBatch, PanicDispatcher}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use anyhow::Result; + +#[derive(Message)] +#[rtype(result = "()")] +struct SetDependencies { + router: Addr, + timelock: Recipient, +} + +impl SetDependencies { + pub fn new(router: Addr, timelock: impl Into>) -> Self { + Self { + router: router.into(), + timelock: timelock.into(), + } + } +} + +pub struct SnapshotBuffer { + router: Option>, + timelock: Option>, +} + +impl SnapshotBuffer { + pub fn new() -> Self { + SnapshotBuffer { + router: None, + timelock: None, + } + } + + pub fn spawn( + config: &AggregateConfig, + store: impl Into>, + ) -> Result> { + let me = Self::new().start(); + let store = store.into(); + let router = BatchRouter::new(config, me.clone(), store.clone()).start(); + let timelock = TimelockQueue::new(me.clone()).start(); + me.try_send(SetDependencies::new(router, timelock))?; + Ok(me) + } +} +impl Actor for SnapshotBuffer { + type Context = actix::Context; +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref timelock) = self.timelock { + timelock.try_send(msg)?; + } + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: SetDependencies, _: &mut Self::Context) -> Self::Result { + let SetDependencies { timelock, router } = msg; + self.timelock = Some(timelock); + self.router = Some(router); + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs new file mode 100644 index 0000000000..e54eb8ea01 --- /dev/null +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -0,0 +1,365 @@ +use crate::{trap, EType, PanicDispatcher}; +use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient}; +use std::{ + cmp::{Ordering, Reverse}, + collections::BinaryHeap, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use super::batch_router::FlushSeq; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct StartTimelock { + seq: u64, + now: u64, + delay: u64, +} + +impl StartTimelock { + pub fn new(seq: u64, now: u64, delay: u64) -> Self { + Self { seq, now, delay } + } +} + +pub trait Clock: Send + Sync + 'static { + fn now_micros(&self) -> u64; +} + +// Production implementation +#[derive(Clone, Default)] +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_micros(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_micros() as u64 + } +} + +#[derive(Message)] +#[rtype(result = "()")] +pub struct Tick; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct Timelock { + expiry: u64, + seq: u64, +} + +impl Timelock { + pub fn new(expiry: u64, seq: u64) -> Self { + Self { expiry, seq } + } +} + +impl Ord for Timelock { + fn cmp(&self, other: &Self) -> Ordering { + self.expiry.cmp(&other.expiry) + } +} + +impl PartialOrd for Timelock { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +pub struct TimelockQueue { + timelocks: BinaryHeap>, + batch_router: Recipient, + clock: Arc, +} + +impl TimelockQueue { + pub fn new(batch_router: impl Into>) -> Self { + Self::with_clock(batch_router, Arc::new(SystemClock)) + } + + pub fn spawn(batch_router: impl Into>) -> Addr { + Self::new(batch_router).start() + } + + pub fn with_clock(batch_router: impl Into>, clock: Arc) -> Self { + Self { + batch_router: batch_router.into(), + timelocks: BinaryHeap::new(), + clock, + } + } + + fn next_timelock_lt(&mut self, now: u64) -> bool { + if let Some(peek) = self.timelocks.peek() { + peek.0.expiry <= now + } else { + false + } + } +} + +impl Actor for TimelockQueue { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + // Send Tick to self every second + ctx.run_interval(Duration::from_secs(1), |_, ctx| { + ctx.address().do_send(Tick); + }); + } +} + +impl Handler for TimelockQueue { + type Result = (); + fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { + let expiry = msg.now + msg.delay; + self.timelocks.push(Reverse(Timelock::new(expiry, msg.seq))); + } +} + +impl Handler for TimelockQueue { + type Result = (); + fn handle(&mut self, _: Tick, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + let now_time = self.clock.now_micros(); + + while self.timelocks.len() > 0 && self.next_timelock_lt(now_time) { + if let Some(tl) = self.timelocks.pop() { + let seq = tl.0.seq; + self.batch_router.try_send(FlushSeq(seq))?; + } + } + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use actix::prelude::*; + use std::sync::{ + atomic::{AtomicU64, Ordering as AtomicOrdering}, + Arc, Mutex, + }; + + // ==================== Mock Clock ==================== + + #[derive(Clone)] + pub struct MockClock { + current_time: Arc, + } + + impl MockClock { + pub fn new(initial_time: u64) -> Self { + Self { + current_time: Arc::new(AtomicU64::new(initial_time)), + } + } + + pub fn set(&self, time: u64) { + self.current_time.store(time, AtomicOrdering::SeqCst); + } + + pub fn advance(&self, micros: u64) { + self.current_time.fetch_add(micros, AtomicOrdering::SeqCst); + } + } + + impl Clock for MockClock { + fn now_micros(&self) -> u64 { + self.current_time.load(AtomicOrdering::SeqCst) + } + } + + // ==================== Mock Router ==================== + + struct MockBatchRouter { + received_seqs: Arc>>, + } + + impl MockBatchRouter { + fn new(received_seqs: Arc>>) -> Self { + Self { received_seqs } + } + } + + impl Actor for MockBatchRouter { + type Context = Context; + } + + impl Handler for MockBatchRouter { + type Result = (); + + fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { + self.received_seqs.lock().unwrap().push(msg.0); + } + } + + // ==================== Tests ==================== + + #[actix_rt::test] + async fn test_tick_with_mock_clock_no_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); // Start at t=1000 + let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + + // Add timelock expiring at t=2000 + queue + .send(StartTimelock::new(42, 1000, 1000)) + .await + .unwrap(); + + // Tick at t=1000 - nothing should expire + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + assert!(received_seqs.lock().unwrap().is_empty()); + } + + #[actix_rt::test] + async fn test_tick_with_mock_clock_after_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + + // Add timelock expiring at t=2000 + queue + .send(StartTimelock::new(42, 1000, 1000)) + .await + .unwrap(); + + // Advance clock past expiry + clock.set(2500); + + // Now tick should flush + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 1); + assert_eq!(seqs[0], 42); + } + + #[actix_rt::test] + async fn test_tick_exact_expiry_boundary() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + + // Add timelock expiring at exactly t=2000 + queue + .send(StartTimelock::new(42, 1000, 1000)) + .await + .unwrap(); + + // Set clock to exact expiry time + clock.set(2000); + + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + // Should flush at exact expiry (<=) + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 1); + assert_eq!(seqs[0], 42); + } + + #[actix_rt::test] + async fn test_tick_one_microsecond_before_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + + // Add timelock expiring at t=2000 + queue + .send(StartTimelock::new(42, 1000, 1000)) + .await + .unwrap(); + + // Set clock to 1 microsecond before expiry + clock.set(1999); + + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + // Should NOT flush + assert!(received_seqs.lock().unwrap().is_empty()); + } + + #[actix_rt::test] + async fn test_multiple_timelocks_partial_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(0); + let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + + // Add timelocks with different expiries + queue.send(StartTimelock::new(1, 0, 1000)).await.unwrap(); // expires at 1000 + queue.send(StartTimelock::new(2, 0, 2000)).await.unwrap(); // expires at 2000 + queue.send(StartTimelock::new(3, 0, 3000)).await.unwrap(); // expires at 3000 + + // Advance to 1500 - only first should expire + clock.set(1500); + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + { + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 1); + assert_eq!(seqs[0], 1); + } + + // Advance to 2500 - second should expire + clock.set(2500); + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + { + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 2); + assert_eq!(seqs[1], 2); + } + + // Advance to 5000 - third should expire + clock.set(5000); + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + { + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 3); + assert_eq!(seqs[2], 3); + } + } + + #[actix_rt::test] + async fn test_clock_advance_helper() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + + queue.send(StartTimelock::new(42, 1000, 500)).await.unwrap(); + + // Use advance instead of set + clock.advance(600); // Now at 1600, expiry is 1500 + + queue.send(Tick).await.unwrap(); + actix_rt::time::sleep(Duration::from_millis(10)).await; + + assert_eq!(received_seqs.lock().unwrap().len(), 1); + } +} diff --git a/crates/config/src/store_keys.rs b/crates/events/src/store_keys.rs similarity index 98% rename from crates/config/src/store_keys.rs rename to crates/events/src/store_keys.rs index b9797242fb..d0b89d22c6 100644 --- a/crates/config/src/store_keys.rs +++ b/crates/events/src/store_keys.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_events::{AggregateId, E3id}; +use crate::{AggregateId, E3id}; pub struct StoreKeys; diff --git a/crates/evm/src/repo.rs b/crates/evm/src/repo.rs index 1fc79ae011..c9b37b92a9 100644 --- a/crates/evm/src/repo.rs +++ b/crates/evm/src/repo.rs @@ -4,8 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; +use e3_events::StoreKeys; use crate::EvmReadInterfaceState; diff --git a/crates/fhe/src/repo.rs b/crates/fhe/src/repo.rs index 3c17fbeec6..289d272fad 100644 --- a/crates/fhe/src/repo.rs +++ b/crates/fhe/src/repo.rs @@ -4,9 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use crate::FheSnapshot; diff --git a/crates/keyshare/src/repo.rs b/crates/keyshare/src/repo.rs index e7d17eb618..90fe4864cb 100644 --- a/crates/keyshare/src/repo.rs +++ b/crates/keyshare/src/repo.rs @@ -4,9 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use crate::ThresholdKeyshareState; diff --git a/crates/net/src/repo.rs b/crates/net/src/repo.rs index a4f2c518a4..43840164da 100644 --- a/crates/net/src/repo.rs +++ b/crates/net/src/repo.rs @@ -4,8 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; +use e3_events::StoreKeys; pub trait NetRepositoryFactory { fn libp2p_keypair(&self) -> Repository>; diff --git a/crates/request/src/repo.rs b/crates/request/src/repo.rs index 455c6de9b0..e076348895 100644 --- a/crates/request/src/repo.rs +++ b/crates/request/src/repo.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; use e3_events::E3id; +use e3_events::StoreKeys; use crate::{E3ContextSnapshot, E3Meta, E3RouterSnapshot}; diff --git a/crates/sortition/src/repo.rs b/crates/sortition/src/repo.rs index 75a92d0ccb..ffa3b84cc9 100644 --- a/crates/sortition/src/repo.rs +++ b/crates/sortition/src/repo.rs @@ -6,9 +6,8 @@ use crate::backends::SortitionBackend; use crate::sortition::NodeStateStore; -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use e3_request::E3Meta; use std::collections::HashMap; diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 5befd49a84..87aa1eb628 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -13,8 +13,8 @@ use anyhow::{anyhow, Result}; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, - ConfigurationUpdated, E3Requested, E3id, EType, EnclaveEvent, EventType, - OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, TypedEvent, + ConfigurationUpdated, E3Requested, EType, EnclaveEvent, EventType, OperatorActivationChanged, + PlaintextOutputPublished, Seed, TicketBalanceUpdated, TypedEvent, }; use e3_events::{BusHandle, E3id, EnclaveEventData}; use e3_utils::NotifySync; @@ -370,7 +370,7 @@ impl Handler for Sortition { EnclaveEventData::TicketBalanceUpdated(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } - EnclaveEventData::OpertorActivationChanged(data) => { + EnclaveEventData::OperatorActivationChanged(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } EnclaveEventData::ConfigurationUpdated(data) => { diff --git a/crates/sync/src/repo.rs b/crates/sync/src/repo.rs index d839beb770..a8f763a122 100644 --- a/crates/sync/src/repo.rs +++ b/crates/sync/src/repo.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; use e3_events::AggregateId; +use e3_events::StoreKeys; pub trait SyncRepositoryFactory { fn aggregate_seq(&self, aggregate_id: AggregateId) -> Repository; diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 6f508092ea..56c088c6d7 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -7,13 +7,13 @@ use crate::SyncRepositoryFactory; use actix::{Actor, Addr, AsyncContext, Handler, Message}; use anyhow::{Context, Result}; -use e3_data::{AggregateConfig, Repositories, WriteBuffer}; +use e3_data::Repositories; use e3_events::{ - trap, trap_fut, AggregateId, BusHandle, EType, EnclaveEvent, EventContextAccessors, - EventPublisher, EvmEventConfig, EvmEventConfigChain, EvmSyncEventsReceived, SyncEnd, SyncStart, - Unsequenced, + trap, trap_fut, AggregateConfig, AggregateId, BusHandle, EType, EnclaveEvent, + EventContextAccessors, EventPublisher, EvmEventConfig, EvmEventConfigChain, + EvmSyncEventsReceived, SnapshotBuffer, SyncEnd, Unsequenced, }; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use tracing::info; // NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p @@ -26,7 +26,7 @@ pub struct Synchronizer { evm_events: Vec>, evm_to_sync: HashSet, repositories: Repositories, - write_buffer: Addr, + snapshot_buffer: Addr, // net_config: NetEventConfig, aggregate_config: AggregateConfig, } @@ -37,7 +37,7 @@ impl Synchronizer { evm_config: EvmEventConfig, repositories: Repositories, aggregate_config: AggregateConfig, - write_buffer: Addr, + snapshot_buffer: Addr, ) -> Self { let evm_to_sync = evm_config.chains(); Self { @@ -47,7 +47,7 @@ impl Synchronizer { evm_events: Vec::new(), repositories, aggregate_config, - write_buffer, + snapshot_buffer, } } @@ -56,14 +56,14 @@ impl Synchronizer { evm_config: &EvmEventConfig, repositories: &Repositories, aggregate_config: &AggregateConfig, - write_buffer: &Addr, + snapshot_buffer: &Addr, ) -> Addr { Self::new( bus.clone(), evm_config.clone(), repositories.clone(), aggregate_config.clone(), - write_buffer.clone(), + snapshot_buffer.clone(), ) .start() } @@ -250,11 +250,13 @@ pub struct Bootstrap; #[cfg(test)] mod tests { use super::*; + use actix::io::WriteHandler; use e3_ciphernode_builder::EventSystem; use e3_events::{EnclaveEvent, EventFactory}; use e3_events::{ EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, TestEvent, }; + use std::collections::HashMap; use std::time::Duration; use tokio::time::sleep; @@ -284,9 +286,15 @@ mod tests { evm_config.insert(1, EvmEventConfigChain::new(0)); evm_config.insert(2, EvmEventConfigChain::new(0)); let repositories = Repositories::in_mem(); - + let snapshot_buffer = WriteBuffer::new().start(); // Start synchronizer - let sync_addr = Synchronizer::setup(&bus, evm_config, &repositories); + let sync_addr = Synchronizer::setup( + &bus, + &evm_config, + &repositories, + &AggregateConfig::new(HashMap::new()), + &snapshot_buffer, + ); settle().await; // Verify SyncStart was published From b1fcf68bf46e54ac57dff15950fc9dd30000fff8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 08:57:06 +0000 Subject: [PATCH 27/63] headers --- crates/events/src/snapshot_buffer/aggregate_config.rs | 5 +++++ crates/events/src/snapshot_buffer/batch.rs | 5 +++++ crates/events/src/snapshot_buffer/batch_router.rs | 5 +++++ crates/events/src/snapshot_buffer/mod.rs | 5 +++++ crates/events/src/snapshot_buffer/snapshot_buffer.rs | 5 +++++ crates/events/src/snapshot_buffer/timelock_queue.rs | 5 +++++ 6 files changed, 30 insertions(+) diff --git a/crates/events/src/snapshot_buffer/aggregate_config.rs b/crates/events/src/snapshot_buffer/aggregate_config.rs index 75bfef3506..3829fc54e1 100644 --- a/crates/events/src/snapshot_buffer/aggregate_config.rs +++ b/crates/events/src/snapshot_buffer/aggregate_config.rs @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. use crate::AggregateId; use std::collections::HashMap; diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index b493a5e643..4ca59cb23c 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. use std::mem::replace; use actix::{Actor, ActorContext, Addr, Handler, Message, Recipient}; diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index feb1fa2d35..9a77f7dbf8 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. use super::{ batch::{Batch, Flush}, timelock_queue::StartTimelock, diff --git a/crates/events/src/snapshot_buffer/mod.rs b/crates/events/src/snapshot_buffer/mod.rs index 8a178f7b83..1be85bd75b 100644 --- a/crates/events/src/snapshot_buffer/mod.rs +++ b/crates/events/src/snapshot_buffer/mod.rs @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. mod aggregate_config; mod batch; mod batch_router; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index c6f406d119..4342bf1f4d 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. use super::{ batch_router::{BatchRouter, FlushSeq}, timelock_queue::{StartTimelock, TimelockQueue}, diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index e54eb8ea01..3770ea9ab4 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -1,3 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. use crate::{trap, EType, PanicDispatcher}; use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient}; use std::{ From 646392245bfcf9f7f3f50662dedc94a7fad32a24 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 09:20:19 +0000 Subject: [PATCH 28/63] compiling --- .../src/snapshot_buffer/timelock_queue.rs | 47 +++++++++++-------- crates/sync/src/sync.rs | 3 +- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index 3770ea9ab4..ef4bc7a240 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -149,6 +149,7 @@ mod tests { atomic::{AtomicU64, Ordering as AtomicOrdering}, Arc, Mutex, }; + use tokio::time::sleep; // ==================== Mock Clock ==================== @@ -205,13 +206,14 @@ mod tests { // ==================== Tests ==================== - #[actix_rt::test] + #[actix::test] async fn test_tick_with_mock_clock_no_expiry() { let received_seqs = Arc::new(Mutex::new(Vec::new())); let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); let clock = MockClock::new(1000); // Start at t=1000 - let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); // Add timelock expiring at t=2000 queue @@ -221,18 +223,19 @@ mod tests { // Tick at t=1000 - nothing should expire queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; assert!(received_seqs.lock().unwrap().is_empty()); } - #[actix_rt::test] + #[actix::test] async fn test_tick_with_mock_clock_after_expiry() { let received_seqs = Arc::new(Mutex::new(Vec::new())); let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); let clock = MockClock::new(1000); - let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); // Add timelock expiring at t=2000 queue @@ -245,20 +248,21 @@ mod tests { // Now tick should flush queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; let seqs = received_seqs.lock().unwrap(); assert_eq!(seqs.len(), 1); assert_eq!(seqs[0], 42); } - #[actix_rt::test] + #[actix::test] async fn test_tick_exact_expiry_boundary() { let received_seqs = Arc::new(Mutex::new(Vec::new())); let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); let clock = MockClock::new(1000); - let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); // Add timelock expiring at exactly t=2000 queue @@ -270,7 +274,7 @@ mod tests { clock.set(2000); queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; // Should flush at exact expiry (<=) let seqs = received_seqs.lock().unwrap(); @@ -278,13 +282,14 @@ mod tests { assert_eq!(seqs[0], 42); } - #[actix_rt::test] + #[actix::test] async fn test_tick_one_microsecond_before_expiry() { let received_seqs = Arc::new(Mutex::new(Vec::new())); let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); let clock = MockClock::new(1000); - let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); // Add timelock expiring at t=2000 queue @@ -296,19 +301,20 @@ mod tests { clock.set(1999); queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; // Should NOT flush assert!(received_seqs.lock().unwrap().is_empty()); } - #[actix_rt::test] + #[actix::test] async fn test_multiple_timelocks_partial_expiry() { let received_seqs = Arc::new(Mutex::new(Vec::new())); let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); let clock = MockClock::new(0); - let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); // Add timelocks with different expiries queue.send(StartTimelock::new(1, 0, 1000)).await.unwrap(); // expires at 1000 @@ -318,7 +324,7 @@ mod tests { // Advance to 1500 - only first should expire clock.set(1500); queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; { let seqs = received_seqs.lock().unwrap(); @@ -329,7 +335,7 @@ mod tests { // Advance to 2500 - second should expire clock.set(2500); queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; { let seqs = received_seqs.lock().unwrap(); @@ -340,7 +346,7 @@ mod tests { // Advance to 5000 - third should expire clock.set(5000); queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; { let seqs = received_seqs.lock().unwrap(); @@ -349,13 +355,14 @@ mod tests { } } - #[actix_rt::test] + #[actix::test] async fn test_clock_advance_helper() { let received_seqs = Arc::new(Mutex::new(Vec::new())); let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); let clock = MockClock::new(1000); - let queue = TimelockQueue::with_clock(mock_router.recipient(), clock.clone()).start(); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); queue.send(StartTimelock::new(42, 1000, 500)).await.unwrap(); @@ -363,7 +370,7 @@ mod tests { clock.advance(600); // Now at 1600, expiry is 1500 queue.send(Tick).await.unwrap(); - actix_rt::time::sleep(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; assert_eq!(received_seqs.lock().unwrap().len(), 1); } diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 56c088c6d7..809834c241 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -250,7 +250,6 @@ pub struct Bootstrap; #[cfg(test)] mod tests { use super::*; - use actix::io::WriteHandler; use e3_ciphernode_builder::EventSystem; use e3_events::{EnclaveEvent, EventFactory}; use e3_events::{ @@ -286,7 +285,7 @@ mod tests { evm_config.insert(1, EvmEventConfigChain::new(0)); evm_config.insert(2, EvmEventConfigChain::new(0)); let repositories = Repositories::in_mem(); - let snapshot_buffer = WriteBuffer::new().start(); + let snapshot_buffer = system.buffer()?; // Start synchronizer let sync_addr = Synchronizer::setup( &bus, From 1b153699a25f83f87bdf53a2c6a6face0ac704fa Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 09:21:21 +0000 Subject: [PATCH 29/63] remove warnings --- crates/config/src/chain_config.rs | 2 +- crates/entrypoint/src/start/start.rs | 4 ---- crates/events/src/enclave_event/enclave_error.rs | 1 - crates/evm/src/evm_chain_gateway.rs | 2 +- crates/indexer/tests/integration.rs | 2 +- crates/multithread/src/multithread.rs | 1 - crates/net/src/document_publisher.rs | 2 +- crates/net/src/net_sync_manager.rs | 2 +- crates/trbfv/tests/integration.rs | 2 +- 9 files changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index 63e241eee4..69a203932b 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -11,7 +11,7 @@ use crate::{ rpc::{RpcAuth, RPC}, }; use anyhow::*; -use e3_events::{EvmEventConfig, EvmEventConfigChain}; +use e3_events::EvmEventConfigChain; use serde::{Deserialize, Serialize}; use tracing::error; diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 9dae6086f7..4129bb6167 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -9,13 +9,9 @@ use anyhow::Result; use e3_ciphernode_builder::{CiphernodeBuilder, CiphernodeHandle}; use e3_config::AppConfig; use e3_crypto::Cipher; -use e3_data::RepositoriesFactory; -use e3_events::BusHandle; -use e3_net::{NetEventTranslator, NetRepositoryFactory}; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; use std::sync::{Arc, Mutex}; -use tokio::task::JoinHandle; use tracing::instrument; #[instrument(name = "app", skip_all)] diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index d456a42eaa..547aca60b5 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -12,7 +12,6 @@ use std::{ future::Future, pin::Pin, }; -use tracing::warn; use crate::{BusHandle, ErrorDispatcher}; diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index ee0a22a982..f5b198408d 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -14,7 +14,7 @@ use anyhow::Result; use anyhow::{bail, Context}; use e3_events::EType; use e3_events::{ - trap, BusHandle, EnclaveEvent, EnclaveEventData, EventFactory, EventSubscriber, EventType, + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, EvmSyncEventsReceived, SyncEnd, SyncStart, Unsequenced, }; use e3_events::{Event, EventPublisher}; diff --git a/crates/indexer/tests/integration.rs b/crates/indexer/tests/integration.rs index bd6d1be158..b7abb70697 100644 --- a/crates/indexer/tests/integration.rs +++ b/crates/indexer/tests/integration.rs @@ -12,7 +12,7 @@ use alloy::{ use e3_bfv_client::compute_pk_commitment; use e3_evm_helpers::contracts::ReadOnly; use e3_fhe_params::DEFAULT_BFV_PRESET; -use e3_fhe_params::{build_bfv_params_from_set_arc, BfvPreset}; +use e3_fhe_params::build_bfv_params_from_set_arc; use e3_indexer::{DataStore, EnclaveIndexer, InMemoryStore}; use eyre::Result; use fhe::bfv::{PublicKey, SecretKey}; diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index c5cbc7f90a..e0b40fe459 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -24,7 +24,6 @@ use e3_events::EType; use e3_events::EnclaveEvent; use e3_events::EnclaveEventData; use e3_events::ErrorDispatcher; -use e3_events::Event; use e3_events::EventPublisher; use e3_events::EventSubscriber; use e3_events::TypedEvent; diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 27f9da2622..d906b1af1b 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -31,7 +31,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::sync::{broadcast, mpsc}; -use tracing::{debug, error, info}; +use tracing::{debug, info}; const KADEMLIA_PUT_TIMEOUT: Duration = Duration::from_secs(30); const KADEMLIA_GET_TIMEOUT: Duration = Duration::from_secs(30); diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index df79a958a8..6b34ef57ba 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -8,7 +8,7 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, + EnclaveEventData, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, ReceiveEvents, TypedEvent, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake}; diff --git a/crates/trbfv/tests/integration.rs b/crates/trbfv/tests/integration.rs index 45d73261f8..72fb33d71f 100644 --- a/crates/trbfv/tests/integration.rs +++ b/crates/trbfv/tests/integration.rs @@ -13,7 +13,7 @@ use e3_bfv_client::decode_bytes_to_vec_u64; use e3_crypto::Cipher; use e3_fhe::create_crp; use e3_fhe_params::DEFAULT_BFV_PRESET; -use e3_fhe_params::{encode_bfv_params, BfvParamSet, BfvPreset}; +use e3_fhe_params::{encode_bfv_params, BfvParamSet}; use e3_test_helpers::{create_seed_from_u64, create_shared_rng_from_u64, usecase_helpers}; use e3_trbfv::{ calculate_decryption_share::{ From 00b2c9ceacc1cd019d9b9e144e0190c8819085a9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 09:26:02 +0000 Subject: [PATCH 30/63] remove warnings --- crates/ciphernode-builder/src/event_system.rs | 2 -- crates/evm/src/evm_hub.rs | 2 +- crates/evm/src/sync_start_extractor.rs | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 6a5942d713..c539efb405 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -85,8 +85,6 @@ pub struct EventSystem { eventbus: OnceCell>>, /// EventSystem BusHandle handle: OnceCell, - /// A OnceLock that is used to indicate whether the system is wired to write snapshots - wired: OnceCell<()>, /// Hlc override hlc: OnceCell, /// Central configuration for aggregates, including delays and other settings diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs index 3c22437424..5dcf92bd9f 100644 --- a/crates/evm/src/evm_hub.rs +++ b/crates/evm/src/evm_hub.rs @@ -29,7 +29,7 @@ impl Actor for EvmHub { impl Handler for EvmHub { type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { let EnclaveEvmEvent::Log { .. } = msg.clone() else { return; }; diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs index 5cd2a52ee3..5c98db97d3 100644 --- a/crates/evm/src/sync_start_extractor.rs +++ b/crates/evm/src/sync_start_extractor.rs @@ -26,7 +26,7 @@ impl Actor for SyncStartExtractor { impl Handler for SyncStartExtractor { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { if let EnclaveEventData::SyncStart(evt) = msg.into_data() { self.dest.do_send(evt) } From 1a04c310ab85d065f47f6345b31bc5fd8e6e4c25 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 10:16:44 +0000 Subject: [PATCH 31/63] remove wired from system --- crates/ciphernode-builder/src/event_system.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index c539efb405..cb95d2cdef 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -111,7 +111,6 @@ impl EventSystem { sequencer: OnceCell::new(), eventbus: OnceCell::new(), handle: OnceCell::new(), - wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), @@ -130,7 +129,6 @@ impl EventSystem { sequencer: OnceCell::new(), eventbus: OnceCell::new(), handle: OnceCell::new(), - wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), @@ -151,7 +149,6 @@ impl EventSystem { sequencer: OnceCell::new(), eventbus: OnceCell::new(), handle: OnceCell::new(), - wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), @@ -435,9 +432,6 @@ mod tests { let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); - - // Wiring happened automatically - assert!(system.wired.get().is_some()); } #[actix::test] @@ -447,9 +441,6 @@ mod tests { let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); - - // Wiring happened automatically - assert!(system.wired.get().is_some()); } #[actix::test] From 6f1ed6419365cb388c97700696359c46dfd20ca3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 10:18:41 +0000 Subject: [PATCH 32/63] apply clock to snapshot router --- .../src/snapshot_buffer/batch_router.rs | 25 ++++++++++++++----- .../src/snapshot_buffer/snapshot_buffer.rs | 17 ++++++++++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 9a77f7dbf8..5c2ad16146 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use super::{ batch::{Batch, Flush}, - timelock_queue::StartTimelock, + timelock_queue::{Clock, StartTimelock}, AggregateConfig, }; use crate::{ @@ -16,6 +16,7 @@ use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Context; use std::{ collections::HashMap, + sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; use tracing::warn; @@ -32,7 +33,6 @@ impl FlushSeq { } } -#[derive(Debug)] pub struct BatchRouter { config: AggregateConfig, aggregates: HashMap, @@ -40,6 +40,7 @@ pub struct BatchRouter { block_height_seen: HashMap, timelock_queue: Recipient, db: Recipient, + clock: Arc, } impl Actor for BatchRouter { @@ -51,6 +52,20 @@ impl BatchRouter { config: &AggregateConfig, timelock_queue: impl Into>, db: impl Into>, + ) -> Self { + Self::with_clock( + config, + timelock_queue, + db, + Arc::new(super::timelock_queue::SystemClock), + ) + } + + pub fn with_clock( + config: &AggregateConfig, + timelock_queue: impl Into>, + db: impl Into>, + clock: Arc, ) -> Self { Self { batches: HashMap::new(), @@ -59,6 +74,7 @@ impl BatchRouter { timelock_queue: timelock_queue.into(), block_height_seen: HashMap::new(), db: db.into(), + clock, } } @@ -108,10 +124,7 @@ impl Handler> for BatchRouter { let delay = self.config.get_delay(prev_agg); - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|_| Duration::from_secs(0)) - .as_micros() as u64; + let now = self.clock.now_micros(); self.timelock_queue .try_send(StartTimelock::new(prev_seq, now, delay))?; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 4342bf1f4d..6ef207b6c9 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -5,12 +5,13 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use super::{ batch_router::{BatchRouter, FlushSeq}, - timelock_queue::{StartTimelock, TimelockQueue}, + timelock_queue::{Clock, StartTimelock, TimelockQueue}, AggregateConfig, }; use crate::{trap, EType, Insert, InsertBatch, PanicDispatcher}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; +use std::sync::Arc; #[derive(Message)] #[rtype(result = "()")] @@ -44,15 +45,25 @@ impl SnapshotBuffer { pub fn spawn( config: &AggregateConfig, store: impl Into>, + ) -> Result> { + Self::with_clock(config, store, Arc::new(super::timelock_queue::SystemClock)) + } + + pub fn with_clock( + config: &AggregateConfig, + store: impl Into>, + clock: Arc, ) -> Result> { let me = Self::new().start(); let store = store.into(); - let router = BatchRouter::new(config, me.clone(), store.clone()).start(); - let timelock = TimelockQueue::new(me.clone()).start(); + let router = + BatchRouter::with_clock(config, me.clone(), store.clone(), clock.clone()).start(); + let timelock = TimelockQueue::with_clock(me.clone(), clock).start(); me.try_send(SetDependencies::new(router, timelock))?; Ok(me) } } + impl Actor for SnapshotBuffer { type Context = actix::Context; } From aa0b000a6b6a5ffdbc00d19040d988cb2db740b5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 12:24:05 +0000 Subject: [PATCH 33/63] test snapshotbuffer --- crates/events/src/enclave_event/mod.rs | 9 +- crates/events/src/enclave_event/test_event.rs | 12 ++ .../src/snapshot_buffer/batch_router.rs | 7 +- .../src/snapshot_buffer/snapshot_buffer.rs | 164 +++++++++++++++++- .../src/snapshot_buffer/timelock_queue.rs | 53 ++++-- crates/indexer/tests/integration.rs | 2 +- 6 files changed, 216 insertions(+), 31 deletions(-) diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 0e35fd2834..3999493248 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -341,11 +341,17 @@ impl EnclaveEvent { #[cfg(feature = "test-helpers")] impl EnclaveEvent { - /// test-helpers only utility function to create a new unsequenced event + /// test-helpers only utility function to create a new sequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { EnclaveEvent::::new_with_timestamp(data, None, time, None).into_sequenced(seq) } + /// test-helpers only utility function to create a new sequenced event + pub fn from_data_ec(data: EnclaveEventData, ec: EventContext) -> Self { + EnclaveEvent::::new_with_timestamp(data, Some(ec.clone()), ec.ts(), ec.block()) + .into_sequenced(ec.seq()) + } + /// test-helpers only utility function to remove time information from an event pub fn strip_ts(&self) -> EnclaveEvent { EnclaveEvent::new_stored_event(self.get_data().clone(), 0, self.seq()) @@ -430,6 +436,7 @@ impl EnclaveEventData { EnclaveEventData::TicketSubmitted(ref data) => Some(data.e3_id.clone()), EnclaveEventData::EncryptionKeyCreated(ref data) => Some(data.e3_id.clone()), EnclaveEventData::ComputeResponse(ref data) => Some(data.e3_id.clone()), + EnclaveEventData::TestEvent(ref data) => data.e3_id.clone(), _ => None, } } diff --git a/crates/events/src/enclave_event/test_event.rs b/crates/events/src/enclave_event/test_event.rs index ab26068958..e38603589d 100644 --- a/crates/events/src/enclave_event/test_event.rs +++ b/crates/events/src/enclave_event/test_event.rs @@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize}; pub struct TestEvent { pub msg: String, pub entropy: u64, + pub e3_id: Option, } impl TestEvent { @@ -19,6 +20,15 @@ impl TestEvent { Self { msg: msg.to_owned(), entropy, + e3_id: None, + } + } + + pub fn with_e3_id(self, id: E3id) -> Self { + Self { + msg: self.msg, + entropy: self.entropy, + e3_id: Some(id), } } } @@ -26,6 +36,8 @@ impl TestEvent { #[cfg(test)] use std::fmt::{self, Display}; +use crate::{AggregateId, E3id}; + #[cfg(test)] impl Display for TestEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 5c2ad16146..b3ae3ab28a 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -14,11 +14,7 @@ use crate::{ }; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Context; -use std::{ - collections::HashMap, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; +use std::{collections::HashMap, sync::Arc}; use tracing::warn; type Seq = u64; @@ -122,6 +118,7 @@ impl Handler> for BatchRouter { .get(&prev_seq) .context("invariant: prev_agg MUST exist if batches has a batch")?; + println!("PREV AGG = {}", prev_agg); let delay = self.config.get_delay(prev_agg); let now = self.clock.now_micros(); diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 6ef207b6c9..5a0b20c95a 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -5,10 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use super::{ batch_router::{BatchRouter, FlushSeq}, - timelock_queue::{Clock, StartTimelock, TimelockQueue}, + timelock_queue::{Clock, StartTimelock, SystemClock, TimelockQueue}, AggregateConfig, }; -use crate::{trap, EType, Insert, InsertBatch, PanicDispatcher}; +use crate::{trap, EType, EnclaveEvent, Insert, InsertBatch, PanicDispatcher}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; use std::sync::Arc; @@ -46,21 +46,22 @@ impl SnapshotBuffer { config: &AggregateConfig, store: impl Into>, ) -> Result> { - Self::with_clock(config, store, Arc::new(super::timelock_queue::SystemClock)) + let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock))?; + Ok(addr) } pub fn with_clock( config: &AggregateConfig, store: impl Into>, clock: Arc, - ) -> Result> { - let me = Self::new().start(); + ) -> Result<(Addr, Addr)> { + let addr = Self::new().start(); let store = store.into(); let router = - BatchRouter::with_clock(config, me.clone(), store.clone(), clock.clone()).start(); - let timelock = TimelockQueue::with_clock(me.clone(), clock).start(); - me.try_send(SetDependencies::new(router, timelock))?; - Ok(me) + BatchRouter::with_clock(config, addr.clone(), store.clone(), clock.clone()).start(); + let timelock = TimelockQueue::with_clock(addr.clone(), clock, None).start(); + addr.try_send(SetDependencies::new(router, timelock.clone()))?; + Ok((addr, timelock)) } } @@ -112,3 +113,148 @@ impl Handler for SnapshotBuffer { }) } } + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} + +#[cfg(test)] +mod mock_store { + use std::mem::replace; + + use crate::InsertBatch; + use actix::{Actor, Handler, Message}; + + #[derive(Message)] + #[rtype(result = "Vec")] + pub struct GetEvts; + + #[derive(Default)] + pub struct MockStore { + evts: Vec, + } + + impl Actor for MockStore { + type Context = actix::Context; + } + + impl Handler for MockStore { + type Result = (); + + fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { + self.evts.push(msg); + } + } + + impl Handler for MockStore { + type Result = Vec; + fn handle(&mut self, _: GetEvts, _: &mut Self::Context) -> Self::Result { + replace(&mut self.evts, Vec::new()) + } + } +} + +#[cfg(test)] +mod tests { + use super::super::timelock_queue::mock_clock::MockClock; + use super::mock_store::GetEvts; + use super::{mock_store, SnapshotBuffer}; + use crate::snapshot_buffer::timelock_queue::Tick; + use crate::{ + AggregateConfig, AggregateId, E3id, EnclaveEvent, EventContext, EventId, Insert, + InsertBatch, Sequenced, TestEvent, + }; + use actix::Actor; + use anyhow::Result; + use std::collections::HashMap; + use std::sync::Arc; + + #[actix::test] + async fn test_snapshot_buffer() -> Result<()> { + let mut delays = HashMap::new(); + delays.insert(AggregateId::new(0), 30); + delays.insert(AggregateId::new(1), 60); + let config = &AggregateConfig::new(delays); + let store = mock_store::MockStore::default().start(); + + let clock = Arc::new(MockClock::new(1000)); + let (buffer, timelock) = SnapshotBuffer::with_clock(config, store.clone(), clock.clone())?; + + let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(0), None) + .sequence(10); + + let enclave_10 = + EnclaveEvent::::from_data_ec(TestEvent::new("hello", 10).into(), ec.clone()); + let mut inserts_10 = vec![]; + inserts_10.push(Insert::new_with_context("one", b"one".to_vec(), ec.clone())); + inserts_10.push(Insert::new_with_context("two", b"two".to_vec(), ec.clone())); + + let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(1), None) + .sequence(11); + + let enclave_11 = EnclaveEvent::::from_data_ec( + TestEvent::new("hello", 11) + .with_e3_id(E3id::new("1", 1)) // Aggregate Id is derived from e3_id on the content + .into(), + ec.clone(), + ); + let mut inserts_11 = vec![]; + inserts_11.push(Insert::new_with_context("one", b"one".to_vec(), ec.clone())); + inserts_11.push(Insert::new_with_context("two", b"two".to_vec(), ec.clone())); + + let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(23), None) + .sequence(12); + let enclave_12 = EnclaveEvent::::from_data_ec( + TestEvent::new("hello", 10) + .with_e3_id(E3id::new("2", 23)) + .into(), + ec.clone(), + ); + + buffer.send(enclave_10).await?; + buffer.send(inserts_10[0].clone()).await?; + buffer.send(enclave_11).await?; + buffer.send(inserts_10[1].clone()).await?; + buffer.send(inserts_11[0].clone()).await?; + buffer.send(inserts_11[1].clone()).await?; + buffer.send(enclave_12).await?; + + // Nothing happens as there has not been enough delay + clock.set(1020); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(0, batches.len()); + + // Time is up so lets flush aggregate 0 (but not aggregate 1) + clock.set(1030); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(1, batches.len()); + let InsertBatch(inserts) = batches.first().unwrap(); + assert_eq!(5, inserts.len()); // Have 5 inserts as sequence,block and ts get written + + // Not ready yet + clock.set(1050); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(0, batches.len()); + + // Time is up so lets flush aggregate 1 + clock.set(1060); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(1, batches.len()); + let InsertBatch(inserts) = batches.first().unwrap(); + assert_eq!(5, inserts.len()); // Have 5 inserts as sequence,block and ts get written + + Ok(()) + } +} diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index ef4bc7a240..96dc41cfd7 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -77,22 +77,28 @@ pub struct TimelockQueue { timelocks: BinaryHeap>, batch_router: Recipient, clock: Arc, + interval: Option, // Seconds } impl TimelockQueue { pub fn new(batch_router: impl Into>) -> Self { - Self::with_clock(batch_router, Arc::new(SystemClock)) + Self::with_clock(batch_router, Arc::new(SystemClock), Some(1)) } pub fn spawn(batch_router: impl Into>) -> Addr { Self::new(batch_router).start() } - pub fn with_clock(batch_router: impl Into>, clock: Arc) -> Self { + pub fn with_clock( + batch_router: impl Into>, + clock: Arc, + interval: Option, + ) -> Self { Self { batch_router: batch_router.into(), timelocks: BinaryHeap::new(), clock, + interval, } } @@ -109,8 +115,12 @@ impl Actor for TimelockQueue { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + let Some(interval) = self.interval else { + return; + }; + // Send Tick to self every second - ctx.run_interval(Duration::from_secs(1), |_, ctx| { + ctx.run_interval(Duration::from_secs(interval), |_, ctx| { ctx.address().do_send(Tick); }); } @@ -142,16 +152,14 @@ impl Handler for TimelockQueue { } #[cfg(test)] -mod tests { - use super::*; - use actix::prelude::*; +pub mod mock_clock { + use std::sync::{ atomic::{AtomicU64, Ordering as AtomicOrdering}, - Arc, Mutex, + Arc, }; - use tokio::time::sleep; - // ==================== Mock Clock ==================== + use super::Clock; #[derive(Clone)] pub struct MockClock { @@ -179,6 +187,15 @@ mod tests { self.current_time.load(AtomicOrdering::SeqCst) } } +} + +#[cfg(test)] +mod tests { + use super::*; + use actix::prelude::*; + use mock_clock::MockClock; + use std::sync::{Arc, Mutex}; + use tokio::time::sleep; // ==================== Mock Router ==================== @@ -213,7 +230,8 @@ mod tests { let clock = MockClock::new(1000); // Start at t=1000 let queue = - TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); // Add timelock expiring at t=2000 queue @@ -235,7 +253,8 @@ mod tests { let clock = MockClock::new(1000); let queue = - TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); // Add timelock expiring at t=2000 queue @@ -262,7 +281,8 @@ mod tests { let clock = MockClock::new(1000); let queue = - TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); // Add timelock expiring at exactly t=2000 queue @@ -289,7 +309,8 @@ mod tests { let clock = MockClock::new(1000); let queue = - TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); // Add timelock expiring at t=2000 queue @@ -314,7 +335,8 @@ mod tests { let clock = MockClock::new(0); let queue = - TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); // Add timelocks with different expiries queue.send(StartTimelock::new(1, 0, 1000)).await.unwrap(); // expires at 1000 @@ -362,7 +384,8 @@ mod tests { let clock = MockClock::new(1000); let queue = - TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone())).start(); + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); queue.send(StartTimelock::new(42, 1000, 500)).await.unwrap(); diff --git a/crates/indexer/tests/integration.rs b/crates/indexer/tests/integration.rs index b7abb70697..4b26ec4efe 100644 --- a/crates/indexer/tests/integration.rs +++ b/crates/indexer/tests/integration.rs @@ -11,8 +11,8 @@ use alloy::{ }; use e3_bfv_client::compute_pk_commitment; use e3_evm_helpers::contracts::ReadOnly; -use e3_fhe_params::DEFAULT_BFV_PRESET; use e3_fhe_params::build_bfv_params_from_set_arc; +use e3_fhe_params::DEFAULT_BFV_PRESET; use e3_indexer::{DataStore, EnclaveIndexer, InMemoryStore}; use eyre::Result; use fhe::bfv::{PublicKey, SecretKey}; From a8e61667fc249826babb39d9a8c8c6402f8f297e Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 4 Feb 2026 14:41:58 +0000 Subject: [PATCH 34/63] refactor: EventStore and SnapshotBuffer tidy up --- Cargo.lock | 1 + crates/ciphernode-builder/Cargo.toml | 3 + crates/ciphernode-builder/src/event_system.rs | 130 ++++++++++++++---- crates/data/src/in_mem.rs | 1 + crates/events/src/events.rs | 22 +-- crates/events/src/eventstore.rs | 17 +-- crates/events/src/eventstore_router.rs | 10 +- crates/events/src/sequencer.rs | 15 +- .../src/snapshot_buffer/aggregate_config.rs | 6 +- crates/events/src/snapshot_buffer/batch.rs | 13 +- .../src/snapshot_buffer/batch_router.rs | 42 ++++-- crates/events/src/snapshot_buffer/mod.rs | 1 + .../src/snapshot_buffer/snapshot_buffer.rs | 80 +++++++++-- .../src/snapshot_buffer/timelock_queue.rs | 59 ++++++-- crates/evm/src/evm_chain_gateway.rs | 18 +-- crates/evm/tests/integration.rs | 8 +- crates/net/src/net_sync_manager.rs | 8 +- 17 files changed, 307 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87d37a5383..bb91bca225 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2868,6 +2868,7 @@ dependencies = [ "e3-request", "e3-sortition", "e3-sync", + "e3-test-helpers", "e3-trbfv", "e3-utils", "once_cell", diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 495eda85f4..80414a8d0e 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -32,3 +32,6 @@ tempfile.workspace = true tokio.workspace = true tracing.workspace = true bincode.workspace = true + +[dev-dependencies] +e3-test-helpers.workspace = true diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index cb95d2cdef..9d162f5db5 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -213,7 +213,7 @@ impl EventSystem { self.sequencer .get_or_try_init(|| { let router = self.eventstore_router()?; - Ok(Sequencer::new(&self.eventbus(), router).start()) + Ok(Sequencer::new(&self.eventbus(), router, self.buffer()?).start()) }) .cloned() } @@ -352,7 +352,17 @@ impl EventSystem { #[cfg(test)] mod tests { + use e3_data::AutoPersist; + use e3_data::Persistable; + use e3_data::Repository; + use e3_events::EventContext; + use e3_events::EventId; + use e3_events::StoreKeys; + use e3_events::SyncEnd; + use e3_events::Tick; + use e3_test_helpers::with_tracing; use std::time::Duration; + use tracing::info; use super::*; use actix::Actor; @@ -364,7 +374,7 @@ mod tests { use e3_events::EnclaveEventData; use e3_events::EventType; - use e3_events::ReceiveEvents; + use e3_events::GetEventsAfterResponse; use e3_events::TestEvent; use tempfile::TempDir; use tokio::time::sleep; @@ -414,9 +424,9 @@ mod tests { } } - impl Handler for Listener { + impl Handler for Listener { type Result = (); - fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: GetEventsAfterResponse, _: &mut Self::Context) -> Self::Result { self.events = msg.events().clone(); } } @@ -445,56 +455,120 @@ mod tests { #[actix::test] async fn test_event_system() -> Result<()> { - let system = EventSystem::in_mem("cn1").with_fresh_bus(); + let _guard = with_tracing("debug"); + + // This sets up the aggregation delays + let mut delays = HashMap::new(); + // Here we delay AggregationId(0) for 1 second + delays.insert(AggregateId::new(0), 1); // Ag0 is default + let config = AggregateConfig::new(delays); + + let system = EventSystem::in_mem("cn1") + .with_fresh_bus() + .with_aggregate_config(config); + let handle = system.handle()?; let datastore = system.store()?; + let buffer = system.buffer()?; + let listener = Listener { logs: Vec::new(), events: Vec::new(), } .start(); + // Sequence 1, Aggregate 0 + let ec = + EventContext::new_origin(EventId::hash(1), 10, AggregateId::new(0), None).sequence(1); + // Send all evts to the listener handle.subscribe(EventType::All, listener.clone().into()); - // Lets store some data - datastore.scope("/foo/name").write("Fred".to_string()); - datastore.scope("/foo/age").write(21u64); - datastore - .scope("/foo/occupation") - .write("developer".to_string()); - - // NOTE: Eventual consistency - // Store should not have data set on it until event has been published + // Publish an event seq 1 + info!("Publishing an event seq 1"); + handle.publish_without_context(TestEvent::new("pink", 1))?; - // Let's check the eventual consistency all data points should be none... + // Lets store some data on a plain datastore + info!("Writing to /foo/name with no context"); + datastore.scope("/foo/name").write("Fred".to_string()); + // Note there is some eventual consistency here we have to wait assert_eq!(datastore.scope("/foo/name").read::().await?, None); - assert_eq!(datastore.scope("/foo/age").read::().await?, None); - assert_eq!( - datastore.scope("/foo/occupation").read::().await?, - None - ); + info!("Wait one tick"); - // Push an event - handle.publish_without_context(TestEvent::new("pink", 1))?; + // Let's wait until all events are settled only takes a tick sleep(Duration::from_millis(1)).await; - // Now we have published an event all data should be written we can get the data from the store + // These inserts should not be buffered and should be available assert_eq!( datastore.scope("/foo/name").read::().await?, Some("Fred".to_string()) ); - assert_eq!(datastore.scope("/foo/age").read::().await?, Some(21)); + info!("Data was written now"); + + // Ok lets get a persistable + let mut persistable: Persistable = + Repository::new(datastore.scope("/foo/name")).load().await?; + + // We have the data in our persistable + assert_eq!(persistable.get(), Some("Fred".to_string())); + + info!("Data was loaded from persistable now mutating state..."); + + persistable.try_mutate(&ec, |_| Ok("Mary".to_string()))?; + + // Local state has changed straight away + assert_eq!(persistable.get(), Some("Mary".to_string())); + + // But disk state is still the same because the SnapshotBuffer is not on assert_eq!( - datastore.scope("/foo/occupation").read::().await?, - Some("developer".to_string()) + datastore.scope("/foo/name").read::().await?, + Some("Fred".to_string()) ); + info!("Local state was mutated however disk state was not"); + + info!("Publishing SyncEnd event to turn on SnapshotBuffer. This should send the seq=1 batch to the timelock..."); + // Publishing SyncEnd should turn on the SnapshotBuffer seq 2 + handle.publish(SyncEnd::new(), ec.clone())?; + + sleep(Duration::from_millis(1)).await; + + info!("Mutating persistable state to create inserts using seq=2"); + + let ec = + EventContext::new_origin(EventId::hash(1), 10, AggregateId::new(0), None).sequence(2); + + persistable.try_mutate(&ec, |_| Ok("Liz".to_string()))?; + sleep(Duration::from_millis(1)).await; + info!("Mutation complete"); + // SnapshotBuffer is not cleared unless new events are published - // Get a timestamp + // Get a timestamp for the events below let ts = handle.ts()?; - // Push a few other events + // Push a few other events seq 3 + // This sends the previous batch for seq2 to the timelock queue handle.publish_without_context(TestEvent::new("yellow", 1))?; + + // Wait a second for the timelock to be checked + sleep(Duration::from_secs(2)).await; + buffer.try_send(Tick)?; + + // Check now + info!("Reading from /foo/name and expecting it to be correct."); + assert_eq!( + datastore.scope("/foo/name").read::().await?, + Some("Liz".to_string()) + ); + + assert_eq!( + datastore + .scope(&StoreKeys::aggregate_seq(AggregateId::new(0))) + .read::() + .await?, + Some(2) + ); + + // Publish a few other events handle.publish_without_context(TestEvent::new("red", 1))?; handle.publish_without_context(TestEvent::new("white", 1))?; sleep(Duration::from_millis(100)).await; diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 75cb789869..7d772010b4 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -8,6 +8,7 @@ use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; use std::collections::BTreeMap; +use tracing::info; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Vec")] diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 660881e14e..929c264ec8 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -13,13 +13,13 @@ use crate::{CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; #[rtype("()")] pub struct StoreEventRequested { pub event: EnclaveEvent, - pub sender: Recipient, + pub sender: Recipient, } impl StoreEventRequested { pub fn new( event: EnclaveEvent, - sender: impl Into>, + sender: impl Into>, ) -> Self { Self { event, @@ -31,17 +31,17 @@ impl StoreEventRequested { /// Get events after timestamp in EventStore #[derive(Message, Debug)] #[rtype("()")] -pub struct GetEventsAfter { +pub struct GetEventsAfterRequest { correlation_id: CorrelationId, ts: u128, - sender: Recipient, + sender: Recipient, } -impl GetEventsAfter { +impl GetEventsAfterRequest { pub fn new( correlation_id: CorrelationId, ts: u128, - sender: impl Into>, + sender: impl Into>, ) -> Self { Self { correlation_id, @@ -58,19 +58,19 @@ impl GetEventsAfter { self.correlation_id } - pub fn sender(&self) -> &Recipient { + pub fn sender(&self) -> &Recipient { &self.sender } } #[derive(Message, Debug)] #[rtype("()")] -pub struct ReceiveEvents { +pub struct GetEventsAfterResponse { id: CorrelationId, events: Vec>, } -impl ReceiveEvents { +impl GetEventsAfterResponse { pub fn new(id: CorrelationId, events: Vec) -> Self { Self { id, events } } @@ -85,9 +85,9 @@ impl ReceiveEvents { /// Direct event received by the Sequencer once an event has been stored #[derive(Message, Debug)] #[rtype("()")] -pub struct EventStored(pub EnclaveEvent); +pub struct StoreEventResponse(pub EnclaveEvent); -impl EventStored { +impl StoreEventResponse { pub fn into_event(self) -> EnclaveEvent { self.0 } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index fce0ec5da3..934ae00b39 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -5,8 +5,8 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - events::{EventStored, StoreEventRequested}, - EventContextAccessors, EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, + events::{StoreEventRequested, StoreEventResponse}, + EventContextAccessors, EventLog, GetEventsAfterRequest, GetEventsAfterResponse, SequenceIndex, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; @@ -39,15 +39,15 @@ impl EventStore { } let seq = self.log.append(&event)?; self.index.insert(ts, seq)?; - sender.try_send(EventStored(event.into_sequenced(seq)))?; + sender.try_send(StoreEventResponse(event.into_sequenced(seq)))?; Ok(()) } - pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { + pub fn handle_get_events_after(&mut self, msg: GetEventsAfterRequest) -> Result<()> { // if there are no events after the timestamp return an empty vector let Some(seq) = self.index.seek(msg.ts())? else { msg.sender() - .try_send(ReceiveEvents::new(msg.id(), vec![]))?; + .try_send(GetEventsAfterResponse::new(msg.id(), vec![]))?; return Ok(()); }; // read and return the events @@ -57,7 +57,8 @@ impl EventStore { .map(|(s, e)| e.into_sequenced(s)) .collect::>(); - msg.sender().try_send(ReceiveEvents::new(msg.id(), evts))?; + msg.sender() + .try_send(GetEventsAfterResponse::new(msg.id(), evts))?; Ok(()) } } @@ -87,9 +88,9 @@ impl Handler for EventStore< } } -impl Handler for EventStore { +impl Handler for EventStore { type Result = (); - fn handle(&mut self, msg: GetEventsAfter, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: GetEventsAfterRequest, _: &mut Self::Context) -> Self::Result { if let Err(e) = self.handle_get_events_after(msg) { error!("{e}"); } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index cb52ec29ac..ba1531b7fc 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -6,10 +6,10 @@ use crate::eventstore::EventStore; use crate::{ - events::{GetEventsAfter, StoreEventRequested}, + events::{GetEventsAfterRequest, StoreEventRequested}, AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; -use crate::{CorrelationId, ReceiveEvents}; +use crate::{CorrelationId, GetEventsAfterResponse}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; use e3_utils::major_issue; @@ -50,7 +50,7 @@ impl EventStoreRouter { for (aggregate_id, ts) in msg.ts() { if let Some(store_addr) = self.stores.get(&aggregate_id) { let get_events_msg = - GetEventsAfter::new(msg.id(), ts.to_owned(), msg.sender.clone()); + GetEventsAfterRequest::new(msg.id(), ts.to_owned(), msg.sender.clone()); store_addr.do_send(get_events_msg); } } @@ -87,14 +87,14 @@ impl Handler for EventSt pub struct GetAggregateEventsAfter { pub correlation_id: CorrelationId, pub ts: HashMap, - pub sender: Recipient, + pub sender: Recipient, } impl GetAggregateEventsAfter { pub fn new( correlation_id: CorrelationId, ts: HashMap, - sender: Recipient, + sender: Recipient, ) -> Self { Self { correlation_id, diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index f9e0ee6d8a..68e97e703e 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - events::{EventStored, StoreEventRequested}, + events::{StoreEventRequested, StoreEventResponse}, EnclaveEvent, EventBus, Sequenced, Unsequenced, }; use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; @@ -16,22 +16,25 @@ use e3_utils::major_issue; pub struct Sequencer { bus: Addr>>, eventstore: Recipient, + buffer: Recipient, } impl Sequencer { pub fn new( bus: &Addr>>, eventstore: impl Into>, + buffer: impl Into>, ) -> Self { Self { bus: bus.clone(), eventstore: eventstore.into(), + buffer: buffer.into(), } } - fn handle_event_stored(&self, msg: EventStored) -> Result<()> { + fn handle_store_event_response(&self, msg: StoreEventResponse) -> Result<()> { let event = msg.into_event(); - + self.buffer.try_send(event.clone())?; self.bus.try_send(event)?; Ok(()) } @@ -53,10 +56,10 @@ impl Handler> for Sequencer { } } -impl Handler for Sequencer { +impl Handler for Sequencer { type Result = (); - fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_event_stored(msg) { + fn handle(&mut self, msg: StoreEventResponse, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_store_event_response(msg) { panic!( "{}", major_issue("Could not send event to snapshot_buffer or bus.", e) diff --git a/crates/events/src/snapshot_buffer/aggregate_config.rs b/crates/events/src/snapshot_buffer/aggregate_config.rs index 3829fc54e1..3a0c0385d4 100644 --- a/crates/events/src/snapshot_buffer/aggregate_config.rs +++ b/crates/events/src/snapshot_buffer/aggregate_config.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. use crate::AggregateId; -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; /// Central configuration for aggregates in the WriteBuffer #[derive(Debug, Clone)] @@ -13,8 +13,8 @@ pub struct AggregateConfig { } impl AggregateConfig { - pub fn get_delay(&self, id: &AggregateId) -> u64 { - self.delays.get(id).cloned().unwrap_or(0) + pub fn get_delay(&self, id: &AggregateId) -> Duration { + Duration::from_secs(self.delays.get(id).cloned().unwrap_or(0)) } } diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index 4ca59cb23c..5ec6fe3e0b 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -5,9 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use std::mem::replace; -use actix::{Actor, ActorContext, Addr, Handler, Message, Recipient}; +use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient}; -use crate::{trap, EType, Insert, InsertBatch, PanicDispatcher}; +use crate::{trap, Die, EType, Insert, InsertBatch, PanicDispatcher}; #[derive(Message)] #[rtype(result = "()")] @@ -47,8 +47,15 @@ impl Handler for Batch { let inserts = replace(&mut self.inserts, Vec::new()); trap(EType::IO, &PanicDispatcher::new(), || { self.db.try_send(InsertBatch::new(inserts))?; - ctx.stop(); + ctx.notify(Die); Ok(()) }) } } + +impl Handler for Batch { + type Result = (); + fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + } +} diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index b3ae3ab28a..7e98d0d7dc 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -14,8 +14,8 @@ use crate::{ }; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Context; -use std::{collections::HashMap, sync::Arc}; -use tracing::warn; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tracing::{debug, info, warn}; type Seq = u64; @@ -90,16 +90,28 @@ impl Handler for BatchRouter { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - let ctx = msg.ctx().context( - "Insert events sent to the BatchRouter MUST have a context but one was not found.", - )?; - let seq = ctx.seq(); - if self.batches.contains_key(&seq) { - if let Some(batch) = self.batches.get(&seq) { + // Messages without context go straight to disk + // This is probably direct datastore manipulation + let Some(ctx) = msg.ctx() else { + debug!("Message without context. Flushing straight to disk."); + self.db.try_send(InsertBatch::new(vec![msg]))?; + return Ok(()); + }; + + // Route to existing batch, or fall back to disk + match self.batches.get(&ctx.seq()) { + Some(batch) => { + debug!("Forwarding to batch for seq={}", ctx.seq()); batch.try_send(msg)?; } - } else { - warn!("Insert received for flushed seq {}", seq); + // This must mean that this insert is late + None => { + debug!( + "No batch available for seq={} assuming this is late. Flushing to disk.", + ctx.seq() + ); + self.db.try_send(InsertBatch::new(vec![msg]))?; + } } Ok(()) }) @@ -118,18 +130,21 @@ impl Handler> for BatchRouter { .get(&prev_seq) .context("invariant: prev_agg MUST exist if batches has a batch")?; - println!("PREV AGG = {}", prev_agg); + debug!( + "Preparing timelock to clear batch for seq={}, agg={}", + prev_seq, prev_agg + ); let delay = self.config.get_delay(prev_agg); - let now = self.clock.now_micros(); + let now = Duration::from_micros(self.clock.now_micros()); self.timelock_queue .try_send(StartTimelock::new(prev_seq, now, delay))?; } + debug!("Creating batch for {}", ec.seq()); let agg_id = ec.aggregate_id(); let highest_block = self.get_highest_block(agg_id, ec.block()); - let batch = Batch::spawn( self.db.clone(), vec![ @@ -163,6 +178,7 @@ impl Handler for BatchRouter { type Result = (); fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { + info!("Flushing sequence... {}", msg.seq()); if let Some(batch) = self.batches.get(&msg.seq()) { batch.try_send(Flush)?; self.batches.remove(&msg.seq()); diff --git a/crates/events/src/snapshot_buffer/mod.rs b/crates/events/src/snapshot_buffer/mod.rs index 1be85bd75b..84a4905094 100644 --- a/crates/events/src/snapshot_buffer/mod.rs +++ b/crates/events/src/snapshot_buffer/mod.rs @@ -11,3 +11,4 @@ mod timelock_queue; pub use aggregate_config::*; pub use snapshot_buffer::*; +pub use timelock_queue::Tick; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 5a0b20c95a..0146e7e584 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -5,33 +5,47 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use super::{ batch_router::{BatchRouter, FlushSeq}, - timelock_queue::{Clock, StartTimelock, SystemClock, TimelockQueue}, + timelock_queue::{Clock, StartTimelock, SystemClock, Tick, TimelockQueue}, AggregateConfig, }; -use crate::{trap, EType, EnclaveEvent, Insert, InsertBatch, PanicDispatcher}; +use crate::{ + trap, EType, EnclaveEvent, EnclaveEventData, Event, Insert, InsertBatch, PanicDispatcher, +}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; use std::sync::Arc; +use tracing::{debug, info}; #[derive(Message)] #[rtype(result = "()")] struct SetDependencies { router: Addr, - timelock: Recipient, + timelock: Addr, } impl SetDependencies { - pub fn new(router: Addr, timelock: impl Into>) -> Self { + pub fn new(router: Addr, timelock: Addr) -> Self { Self { router: router.into(), - timelock: timelock.into(), + timelock, } } } +#[derive(Message)] +#[rtype(result = "()")] +pub struct Start; + +enum SnapshotBufferState { + Running, + Paused, +} + pub struct SnapshotBuffer { router: Option>, timelock: Option>, + tickable: Option>, + state: SnapshotBufferState, } impl SnapshotBuffer { @@ -39,6 +53,8 @@ impl SnapshotBuffer { SnapshotBuffer { router: None, timelock: None, + tickable: None, + state: SnapshotBufferState::Paused, } } @@ -46,7 +62,7 @@ impl SnapshotBuffer { config: &AggregateConfig, store: impl Into>, ) -> Result> { - let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock))?; + let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock), Some(1))?; Ok(addr) } @@ -54,12 +70,13 @@ impl SnapshotBuffer { config: &AggregateConfig, store: impl Into>, clock: Arc, + interval: Option, ) -> Result<(Addr, Addr)> { let addr = Self::new().start(); let store = store.into(); let router = BatchRouter::with_clock(config, addr.clone(), store.clone(), clock.clone()).start(); - let timelock = TimelockQueue::with_clock(addr.clone(), clock, None).start(); + let timelock = TimelockQueue::with_clock(addr.clone(), clock, interval).start(); addr.try_send(SetDependencies::new(router, timelock.clone()))?; Ok((addr, timelock)) } @@ -73,6 +90,9 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { + let SnapshotBufferState::Running = self.state else { + return Ok(()); + }; if let Some(ref router) = self.router { router.try_send(msg)?; } @@ -85,6 +105,10 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { + let SnapshotBufferState::Running = self.state else { + return Ok(()); + }; + if let Some(ref timelock) = self.timelock { timelock.try_send(msg)?; } @@ -97,7 +121,8 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: SetDependencies, _: &mut Self::Context) -> Self::Result { let SetDependencies { timelock, router } = msg; - self.timelock = Some(timelock); + self.timelock = Some(timelock.clone().into()); + self.tickable = Some(timelock.into()); self.router = Some(router); } } @@ -106,9 +131,20 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - if let Some(ref router) = self.router { - router.try_send(msg)?; - } + use SnapshotBufferState as S; + match (&self.state, msg.ctx(), &self.router) { + // Doing this to cover when there are context free + // data store manipulations outside of the sync cycle. + (S::Paused, None, Some(ref router)) => { + debug!("Paused but received context free Insert. Forwarding to batch router"); + router.try_send(msg)?; + } + (S::Running, _, Some(ref router)) => { + debug!("Forwarding to batch router"); + router.try_send(msg)?; + } + _ => (), + }; Ok(()) }) } @@ -118,6 +154,13 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { + if let EnclaveEventData::SyncEnd(_) = msg.get_data() { + info!("SnapshotBuffer is now enabled"); + self.state = SnapshotBufferState::Running; + }; + let SnapshotBufferState::Running = self.state else { + return Ok(()); + }; if let Some(ref router) = self.router { router.try_send(msg)?; } @@ -126,6 +169,18 @@ impl Handler for SnapshotBuffer { } } +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: Tick, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref tickable) = self.tickable { + tickable.try_send(msg)?; + } + Ok(()) + }) + } +} + #[cfg(test)] mod mock_store { use std::mem::replace; @@ -186,7 +241,8 @@ mod tests { let store = mock_store::MockStore::default().start(); let clock = Arc::new(MockClock::new(1000)); - let (buffer, timelock) = SnapshotBuffer::with_clock(config, store.clone(), clock.clone())?; + let (buffer, timelock) = + SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(0), None) .sequence(10); diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index 96dc41cfd7..ab6f8ac9ef 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -11,6 +11,7 @@ use std::{ sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; +use tracing::{debug, info, warn}; use super::batch_router::FlushSeq; @@ -18,14 +19,21 @@ use super::batch_router::FlushSeq; #[rtype(result = "()")] pub struct StartTimelock { seq: u64, - now: u64, - delay: u64, + now: Duration, + delay: Duration, } impl StartTimelock { - pub fn new(seq: u64, now: u64, delay: u64) -> Self { + pub fn new(seq: u64, now: Duration, delay: Duration) -> Self { Self { seq, now, delay } } + pub fn new_micros(seq: u64, now: u64, delay: u64) -> Self { + Self::new( + seq, + Duration::from_micros(now), + Duration::from_micros(delay), + ) + } } pub trait Clock: Send + Sync + 'static { @@ -51,12 +59,12 @@ pub struct Tick; #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct Timelock { - expiry: u64, + expiry: Duration, seq: u64, } impl Timelock { - pub fn new(expiry: u64, seq: u64) -> Self { + pub fn new(expiry: Duration, seq: u64) -> Self { Self { expiry, seq } } } @@ -102,7 +110,7 @@ impl TimelockQueue { } } - fn next_timelock_lt(&mut self, now: u64) -> bool { + fn next_timelock_lt(&mut self, now: Duration) -> bool { if let Some(peek) = self.timelocks.peek() { peek.0.expiry <= now } else { @@ -129,6 +137,7 @@ impl Actor for TimelockQueue { impl Handler for TimelockQueue { type Result = (); fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { + debug!("START TIMELOCK: {:?}", msg.delay); let expiry = msg.now + msg.delay; self.timelocks.push(Reverse(Timelock::new(expiry, msg.seq))); } @@ -138,11 +147,19 @@ impl Handler for TimelockQueue { type Result = (); fn handle(&mut self, _: Tick, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - let now_time = self.clock.now_micros(); + let now_time = Duration::from_micros(self.clock.now_micros()); + debug!( + "Running timelock tick. waiting times: {:?}.", + self.timelocks + .iter() + .map(|t| t.0.expiry.saturating_sub(now_time)) + .collect::>(), + ); while self.timelocks.len() > 0 && self.next_timelock_lt(now_time) { if let Some(tl) = self.timelocks.pop() { let seq = tl.0.seq; + debug!("Flushing seq {}", seq); self.batch_router.try_send(FlushSeq(seq))?; } } @@ -235,7 +252,7 @@ mod tests { // Add timelock expiring at t=2000 queue - .send(StartTimelock::new(42, 1000, 1000)) + .send(StartTimelock::new_micros(42, 1000, 1000)) .await .unwrap(); @@ -258,7 +275,7 @@ mod tests { // Add timelock expiring at t=2000 queue - .send(StartTimelock::new(42, 1000, 1000)) + .send(StartTimelock::new_micros(42, 1000, 1000)) .await .unwrap(); @@ -286,7 +303,7 @@ mod tests { // Add timelock expiring at exactly t=2000 queue - .send(StartTimelock::new(42, 1000, 1000)) + .send(StartTimelock::new_micros(42, 1000, 1000)) .await .unwrap(); @@ -314,7 +331,7 @@ mod tests { // Add timelock expiring at t=2000 queue - .send(StartTimelock::new(42, 1000, 1000)) + .send(StartTimelock::new_micros(42, 1000, 1000)) .await .unwrap(); @@ -339,9 +356,18 @@ mod tests { .start(); // Add timelocks with different expiries - queue.send(StartTimelock::new(1, 0, 1000)).await.unwrap(); // expires at 1000 - queue.send(StartTimelock::new(2, 0, 2000)).await.unwrap(); // expires at 2000 - queue.send(StartTimelock::new(3, 0, 3000)).await.unwrap(); // expires at 3000 + queue + .send(StartTimelock::new_micros(1, 0, 1000)) + .await + .unwrap(); // expires at 1000 + queue + .send(StartTimelock::new_micros(2, 0, 2000)) + .await + .unwrap(); // expires at 2000 + queue + .send(StartTimelock::new_micros(3, 0, 3000)) + .await + .unwrap(); // expires at 3000 // Advance to 1500 - only first should expire clock.set(1500); @@ -387,7 +413,10 @@ mod tests { TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) .start(); - queue.send(StartTimelock::new(42, 1000, 500)).await.unwrap(); + queue + .send(StartTimelock::new_micros(42, 1000, 500)) + .await + .unwrap(); // Use advance instead of set clock.advance(600); // Now at 1600, expiry is 1500 diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index f5b198408d..85fc40d949 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -261,11 +261,7 @@ mod tests { // Send EVM event while forwarding - should reach collector let evm_event = EvmEvent::new( CorrelationId::new(), - TestEvent { - msg: "Before Complete".to_string(), - entropy: 1, - } - .into(), + TestEvent::new("Before Complete", 1).into(), 100, 12345, chain_id, @@ -290,11 +286,7 @@ mod tests { // Send EVM event while buffering - should be buffered (not received) let buffered_event = EvmEvent::new( CorrelationId::new(), - TestEvent { - msg: "Before SyncEnd".to_string(), - entropy: 2, - } - .into(), + TestEvent::new("Before SyncEnd", 2).into(), 101, 12346, chain_id, @@ -307,11 +299,7 @@ mod tests { let after_event = EvmEvent::new( CorrelationId::new(), - TestEvent { - msg: "After SyncEnd".to_string(), - entropy: 2, - } - .into(), + TestEvent::new("After SyncEnd", 2).into(), 101, 12346, chain_id, diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 6ce4503fb1..4b26937ec7 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -40,10 +40,10 @@ fn test_event_extractor( return None; }; Some( - TestEvent { - msg: event.value, - entropy: event.count.try_into().unwrap(), // This prevents de-duplication in tests - } + TestEvent::new( + &event.value, + event.count.try_into().unwrap(), // This prevents de-duplication in tests + ) .into(), ) } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 6b34ef57ba..e950f991ee 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -8,8 +8,8 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, - ReceiveEvents, TypedEvent, Unsequenced, + EnclaveEventData, GetAggregateEventsAfter, GetEventsAfterResponse, NetSyncEventsReceived, + OutgoingSyncRequested, TypedEvent, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake}; use futures::TryFutureExt; @@ -162,9 +162,9 @@ impl Handler for NetSyncManager { } /// Receive Events from EventStore -impl Handler for NetSyncManager { +impl Handler for NetSyncManager { type Result = (); - fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: GetEventsAfterResponse, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { let Some(channel) = self.requests.get(&msg.id()) else { bail!("request not found with {}", msg.id()); From 1e8595d2d86ae027354aff875852bbcdcdb20fbb Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 5 Feb 2026 07:27:25 +0000 Subject: [PATCH 35/63] update snapshot test --- Cargo.lock | 1 + .../src/ciphernode_builder.rs | 7 +- crates/events/Cargo.toml | 1 + .../src/snapshot_buffer/aggregate_config.rs | 11 +- crates/events/src/snapshot_buffer/batch.rs | 5 +- .../src/snapshot_buffer/batch_router.rs | 4 +- .../src/snapshot_buffer/snapshot_buffer.rs | 106 ++++++++++++------ .../src/snapshot_buffer/timelock_queue.rs | 39 ++++--- 8 files changed, 115 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb91bca225..8aed8ddd1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3045,6 +3045,7 @@ dependencies = [ "e3-crypto", "e3-data", "e3-events", + "e3-test-helpers", "e3-trbfv", "e3-utils", "futures-util", diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index c82664eb4e..0fae3f641a 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -29,6 +29,7 @@ use e3_sortition::{ }; use e3_sync::Synchronizer; use e3_utils::{rand_eth_addr, SharedRng}; +use std::time::Duration; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; @@ -525,17 +526,17 @@ fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { } /// Build delay configuration for a specific chain -fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, u64) { +fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, Duration) { let aggregate_id = AggregateId::from_chain_id(Some(actual_chain_id)); let finalization_ms = chain.finalization_ms.unwrap_or(0); let delay_us = finalization_ms * 1000; // ms → microseconds - (aggregate_id, delay_us) + (aggregate_id, Duration::from_micros(delay_us)) } /// Build delays configuration from chain providers fn create_aggregate_delays( chain_providers: &[(ChainConfig, u64)], -) -> Result> { +) -> Result> { let mut delays = HashMap::new(); for (chain, actual_chain_id) in chain_providers.into_iter().cloned() { diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index a37cb25c54..4d567172a8 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -38,4 +38,5 @@ test-helpers = [] # ensure test-helpers is available for integration tests proptest = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } e3-data = { workspace = true } +e3-test-helpers = { workspace = true } e3-ciphernode-builder = { workspace = true } diff --git a/crates/events/src/snapshot_buffer/aggregate_config.rs b/crates/events/src/snapshot_buffer/aggregate_config.rs index 3a0c0385d4..20a9e6cf98 100644 --- a/crates/events/src/snapshot_buffer/aggregate_config.rs +++ b/crates/events/src/snapshot_buffer/aggregate_config.rs @@ -9,21 +9,24 @@ use std::{collections::HashMap, time::Duration}; /// Central configuration for aggregates in the WriteBuffer #[derive(Debug, Clone)] pub struct AggregateConfig { - pub delays: HashMap, + pub delays: HashMap, } impl AggregateConfig { pub fn get_delay(&self, id: &AggregateId) -> Duration { - Duration::from_secs(self.delays.get(id).cloned().unwrap_or(0)) + self.delays + .get(id) + .cloned() + .unwrap_or(Duration::from_micros(0)) } } impl AggregateConfig { /// Create a new AggregateConfig with the specified delays - pub fn new(mut delays: HashMap) -> Self { + pub fn new(mut delays: HashMap) -> Self { // Always handle AggregatId of 0 with a delay of 0 if let None = delays.get(&AggregateId::new(0)) { - delays.insert(AggregateId::new(0), 0); + delays.insert(AggregateId::new(0), Duration::from_micros(0)); } Self { delays } } diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index 5ec6fe3e0b..b04a17397d 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -6,6 +6,7 @@ use std::mem::replace; use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient}; +use tracing::debug; use crate::{trap, Die, EType, Insert, InsertBatch, PanicDispatcher}; @@ -46,7 +47,9 @@ impl Handler for Batch { fn handle(&mut self, _: Flush, ctx: &mut Self::Context) -> Self::Result { let inserts = replace(&mut self.inserts, Vec::new()); trap(EType::IO, &PanicDispatcher::new(), || { - self.db.try_send(InsertBatch::new(inserts))?; + if inserts.len() > 0 { + self.db.try_send(InsertBatch::new(inserts))?; + } ctx.notify(Die); Ok(()) }) diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 7e98d0d7dc..4e5dbff521 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -15,7 +15,7 @@ use crate::{ use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Context; use std::{collections::HashMap, sync::Arc, time::Duration}; -use tracing::{debug, info, warn}; +use tracing::{debug, info, trace, warn}; type Seq = u64; @@ -101,7 +101,7 @@ impl Handler for BatchRouter { // Route to existing batch, or fall back to disk match self.batches.get(&ctx.seq()) { Some(batch) => { - debug!("Forwarding to batch for seq={}", ctx.seq()); + debug!("Forwarding to batch actor for seq={}", ctx.seq()); batch.try_send(msg)?; } // This must mean that this insert is late diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 0146e7e584..4e6423ec68 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -14,7 +14,7 @@ use crate::{ use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; use std::sync::Arc; -use tracing::{debug, info}; +use tracing::{debug, info, trace}; #[derive(Message)] #[rtype(result = "()")] @@ -136,11 +136,13 @@ impl Handler for SnapshotBuffer { // Doing this to cover when there are context free // data store manipulations outside of the sync cycle. (S::Paused, None, Some(ref router)) => { - debug!("Paused but received context free Insert. Forwarding to batch router"); + debug!( + "Paused but received context free Insert. Forwarding to batch router..." + ); router.try_send(msg)?; } (S::Running, _, Some(ref router)) => { - debug!("Forwarding to batch router"); + trace!("Forwarding Insert message to batch router..."); router.try_send(msg)?; } _ => (), @@ -224,19 +226,38 @@ mod tests { use super::{mock_store, SnapshotBuffer}; use crate::snapshot_buffer::timelock_queue::Tick; use crate::{ - AggregateConfig, AggregateId, E3id, EnclaveEvent, EventContext, EventId, Insert, - InsertBatch, Sequenced, TestEvent, + AggregateConfig, AggregateId, E3id, EnclaveEvent, EventContext, EventContextAccessors, + EventContextSeq, EventId, Insert, InsertBatch, Sequenced, SyncEnd, TestEvent, }; use actix::Actor; use anyhow::Result; + use e3_test_helpers::with_tracing; use std::collections::HashMap; use std::sync::Arc; + use std::time::Duration; + use tracing::info; + + fn create_ec(ag: usize, seq: u64) -> EventContext { + EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(ag), None).sequence(seq) + } + + fn create_event(ec: &EventContext) -> EnclaveEvent { + EnclaveEvent::::from_data_ec( + TestEvent::new("hello", ec.seq()) + .with_e3_id(E3id::new("1", *ec.aggregate_id() as u64)) + .into(), + ec.clone(), + ) + } #[actix::test] async fn test_snapshot_buffer() -> Result<()> { + let _guard = with_tracing("debug"); let mut delays = HashMap::new(); - delays.insert(AggregateId::new(0), 30); - delays.insert(AggregateId::new(1), 60); + delays.insert(AggregateId::new(0), Duration::from_micros(0)); + delays.insert(AggregateId::new(23), Duration::from_micros(30)); + delays.insert(AggregateId::new(1), Duration::from_micros(60)); + let config = &AggregateConfig::new(delays); let store = mock_store::MockStore::default().start(); @@ -244,53 +265,70 @@ mod tests { let (buffer, timelock) = SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; - let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(0), None) - .sequence(10); + buffer + .send(EnclaveEvent::from_data_ec( + SyncEnd::new().into(), + create_ec(0, 9), + )) + .await?; + + info!("TimelockQueue should be empty"); + timelock.send(Tick).await?; + + let ec = create_ec(23, 10); + let enclave_10 = create_event(&ec); - let enclave_10 = - EnclaveEvent::::from_data_ec(TestEvent::new("hello", 10).into(), ec.clone()); let mut inserts_10 = vec![]; inserts_10.push(Insert::new_with_context("one", b"one".to_vec(), ec.clone())); inserts_10.push(Insert::new_with_context("two", b"two".to_vec(), ec.clone())); - let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(1), None) - .sequence(11); + let ec = create_ec(1, 11); + let enclave_11 = create_event(&ec); - let enclave_11 = EnclaveEvent::::from_data_ec( - TestEvent::new("hello", 11) - .with_e3_id(E3id::new("1", 1)) // Aggregate Id is derived from e3_id on the content - .into(), - ec.clone(), - ); let mut inserts_11 = vec![]; inserts_11.push(Insert::new_with_context("one", b"one".to_vec(), ec.clone())); inserts_11.push(Insert::new_with_context("two", b"two".to_vec(), ec.clone())); - let ec = EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(23), None) - .sequence(12); - let enclave_12 = EnclaveEvent::::from_data_ec( - TestEvent::new("hello", 10) - .with_e3_id(E3id::new("2", 23)) - .into(), - ec.clone(), - ); + let ec = create_ec(0, 12); + let enclave_12 = create_event(&ec); + // send event 10 buffer.send(enclave_10).await?; + + info!("TimelockQueue should hold all seq=9 inserts"); + timelock.send(Tick).await?; + + // send the first insert for seq 10 buffer.send(inserts_10[0].clone()).await?; + + // send event 11 + info!("Sending event seq=11 this should start the timelock for all the seq=10 inserts"); buffer.send(enclave_11).await?; + + // send a late insert for 10 buffer.send(inserts_10[1].clone()).await?; + + // send the other inserts for 11 buffer.send(inserts_11[0].clone()).await?; buffer.send(inserts_11[1].clone()).await?; + + // send event 12 + info!("Sending event seq=12 this should start the timelock for all the seq=11 inserts"); buffer.send(enclave_12).await?; // Nothing happens as there has not been enough delay - clock.set(1020); + info!("Clock=1020 : Checking for events but there should be nothing that has flushed..."); + clock.set(Duration::from_micros(1020)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; - assert_eq!(0, batches.len()); + // assert_eq!(0, batches.len()); + assert_eq!(1, batches.len()); + let InsertBatch(inserts) = batches.first().unwrap(); + assert_eq!(3, inserts.len()); // Have sequence,block and ts written as inserts - // Time is up so lets flush aggregate 0 (but not aggregate 1) - clock.set(1030); + // Time is up so lets flush aggregate 23 (but not aggregate 1) + info!("Clock=1030 : Checking for events Tick should flush batch 10..."); + clock.set(Duration::from_micros(1030)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; assert_eq!(1, batches.len()); @@ -298,13 +336,15 @@ mod tests { assert_eq!(5, inserts.len()); // Have 5 inserts as sequence,block and ts get written // Not ready yet - clock.set(1050); + info!("Clock=1050 : Not ready yet..."); + clock.set(Duration::from_micros(1050)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; assert_eq!(0, batches.len()); // Time is up so lets flush aggregate 1 - clock.set(1060); + info!("Clock=1060 : should have all aggregate 1 changes in batch 11..."); + clock.set(Duration::from_micros(1060)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; assert_eq!(1, batches.len()); diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index ab6f8ac9ef..b46a22b99a 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -11,7 +11,7 @@ use std::{ sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use tracing::{debug, info, warn}; +use tracing::debug; use super::batch_router::FlushSeq; @@ -124,10 +124,12 @@ impl Actor for TimelockQueue { fn started(&mut self, ctx: &mut Self::Context) { let Some(interval) = self.interval else { + debug!("TimelockQueue in manual mode - will tick when requested."); return; }; // Send Tick to self every second + debug!("TimelockQueue is ticking every {}", interval); ctx.run_interval(Duration::from_secs(interval), |_, ctx| { ctx.address().do_send(Tick); }); @@ -137,7 +139,7 @@ impl Actor for TimelockQueue { impl Handler for TimelockQueue { type Result = (); fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { - debug!("START TIMELOCK: {:?}", msg.delay); + debug!("Start timelock: {:?}", msg.delay); let expiry = msg.now + msg.delay; self.timelocks.push(Reverse(Timelock::new(expiry, msg.seq))); } @@ -171,9 +173,12 @@ impl Handler for TimelockQueue { #[cfg(test)] pub mod mock_clock { - use std::sync::{ - atomic::{AtomicU64, Ordering as AtomicOrdering}, - Arc, + use std::{ + sync::{ + atomic::{AtomicU64, Ordering as AtomicOrdering}, + Arc, + }, + time::Duration, }; use super::Clock; @@ -190,12 +195,14 @@ pub mod mock_clock { } } - pub fn set(&self, time: u64) { - self.current_time.store(time, AtomicOrdering::SeqCst); + pub fn set(&self, time: Duration) { + self.current_time + .store(time.as_micros() as u64, AtomicOrdering::SeqCst); } - pub fn advance(&self, micros: u64) { - self.current_time.fetch_add(micros, AtomicOrdering::SeqCst); + pub fn advance(&self, micros: Duration) { + self.current_time + .fetch_add(micros.as_micros() as u64, AtomicOrdering::SeqCst); } } @@ -280,7 +287,7 @@ mod tests { .unwrap(); // Advance clock past expiry - clock.set(2500); + clock.set(Duration::from_millis(2500)); // Now tick should flush queue.send(Tick).await.unwrap(); @@ -308,7 +315,7 @@ mod tests { .unwrap(); // Set clock to exact expiry time - clock.set(2000); + clock.set(Duration::from_millis(2000)); queue.send(Tick).await.unwrap(); sleep(Duration::from_millis(10)).await; @@ -336,7 +343,7 @@ mod tests { .unwrap(); // Set clock to 1 microsecond before expiry - clock.set(1999); + clock.set(Duration::from_millis(1999)); queue.send(Tick).await.unwrap(); sleep(Duration::from_millis(10)).await; @@ -370,7 +377,7 @@ mod tests { .unwrap(); // expires at 3000 // Advance to 1500 - only first should expire - clock.set(1500); + clock.set(Duration::from_millis(1500)); queue.send(Tick).await.unwrap(); sleep(Duration::from_millis(10)).await; @@ -381,7 +388,7 @@ mod tests { } // Advance to 2500 - second should expire - clock.set(2500); + clock.set(Duration::from_millis(2500)); queue.send(Tick).await.unwrap(); sleep(Duration::from_millis(10)).await; @@ -392,7 +399,7 @@ mod tests { } // Advance to 5000 - third should expire - clock.set(5000); + clock.set(Duration::from_millis(5000)); queue.send(Tick).await.unwrap(); sleep(Duration::from_millis(10)).await; @@ -419,7 +426,7 @@ mod tests { .unwrap(); // Use advance instead of set - clock.advance(600); // Now at 1600, expiry is 1500 + clock.advance(Duration::from_millis(600)); // Now at 1600, expiry is 1500 queue.send(Tick).await.unwrap(); sleep(Duration::from_millis(10)).await; From f610e04387a305c33b640bcc16f1cff670eda8d5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 5 Feb 2026 07:29:35 +0000 Subject: [PATCH 36/63] fix up event tests --- .../events/src/snapshot_buffer/timelock_queue.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index b46a22b99a..52fe563085 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -343,10 +343,10 @@ mod tests { .unwrap(); // Set clock to 1 microsecond before expiry - clock.set(Duration::from_millis(1999)); + clock.set(Duration::from_micros(1999)); queue.send(Tick).await.unwrap(); - sleep(Duration::from_millis(10)).await; + sleep(Duration::from_micros(10)).await; // Should NOT flush assert!(received_seqs.lock().unwrap().is_empty()); @@ -377,9 +377,9 @@ mod tests { .unwrap(); // expires at 3000 // Advance to 1500 - only first should expire - clock.set(Duration::from_millis(1500)); + clock.set(Duration::from_micros(1500)); queue.send(Tick).await.unwrap(); - sleep(Duration::from_millis(10)).await; + sleep(Duration::from_micros(10)).await; { let seqs = received_seqs.lock().unwrap(); @@ -388,9 +388,9 @@ mod tests { } // Advance to 2500 - second should expire - clock.set(Duration::from_millis(2500)); + clock.set(Duration::from_micros(2500)); queue.send(Tick).await.unwrap(); - sleep(Duration::from_millis(10)).await; + sleep(Duration::from_micros(10)).await; { let seqs = received_seqs.lock().unwrap(); @@ -399,9 +399,9 @@ mod tests { } // Advance to 5000 - third should expire - clock.set(Duration::from_millis(5000)); + clock.set(Duration::from_micros(5000)); queue.send(Tick).await.unwrap(); - sleep(Duration::from_millis(10)).await; + sleep(Duration::from_micros(10)).await; { let seqs = received_seqs.lock().unwrap(); From 4debc4b0e47ac755f3612ec02bd6236701e631b6 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 5 Feb 2026 07:33:35 +0000 Subject: [PATCH 37/63] fix up duration --- crates/ciphernode-builder/src/event_system.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 9d162f5db5..e492720bdf 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -460,7 +460,7 @@ mod tests { // This sets up the aggregation delays let mut delays = HashMap::new(); // Here we delay AggregationId(0) for 1 second - delays.insert(AggregateId::new(0), 1); // Ag0 is default + delays.insert(AggregateId::new(0), Duration::from_secs(1)); // Ag0 is default let config = AggregateConfig::new(delays); let system = EventSystem::in_mem("cn1") @@ -601,9 +601,9 @@ mod tests { // Create an AggregateConfig with multiple AggregateIds let mut delays = HashMap::new(); - delays.insert(AggregateId::new(0), 1000); // 1ms delay - delays.insert(AggregateId::new(1), 2000); // 2ms delay - delays.insert(AggregateId::new(2), 3000); // 3ms delay + delays.insert(AggregateId::new(0), Duration::from_micros(1000)); // 1ms delay + delays.insert(AggregateId::new(1), Duration::from_micros(2000)); // 2ms delay + delays.insert(AggregateId::new(2), Duration::from_micros(3000)); // 3ms delay let aggregate_config = AggregateConfig::new(delays); // Test in-memory eventstores From afdc039f9ba121e0e09c59b2f0e3e1a413c609cf Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 5 Feb 2026 09:40:32 +0000 Subject: [PATCH 38/63] ensure actors can handle more than 16 messages at once --- crates/aggregator/src/committee_finalizer.rs | 5 +- .../src/keyshare_created_filter_buffer.rs | 4 + crates/aggregator/src/publickey_aggregator.rs | 5 +- .../src/threshold_plaintext_aggregator.rs | 5 +- .../src/ciphernode_builder.rs | 8 ++ crates/ciphernode-builder/src/event_system.rs | 88 ++++++++++++++----- crates/data/src/data_store.rs | 9 +- crates/data/src/in_mem.rs | 4 + crates/data/src/persistable.rs | 4 + crates/data/src/sled_store.rs | 4 + crates/events/src/bus_handle.rs | 4 + crates/events/src/eventbus.rs | 11 ++- crates/events/src/eventstore.rs | 5 +- crates/events/src/eventstore_router.rs | 13 ++- crates/events/src/sequencer.rs | 5 +- crates/events/src/snapshot_buffer/batch.rs | 4 + .../src/snapshot_buffer/batch_router.rs | 16 +++- .../src/snapshot_buffer/snapshot_buffer.rs | 58 ++++++++++-- .../src/snapshot_buffer/timelock_queue.rs | 4 +- crates/evm/src/ciphernode_registry_sol.rs | 6 +- crates/evm/src/enclave_sol_writer.rs | 4 + crates/evm/src/evm_chain_gateway.rs | 4 + crates/evm/src/evm_hub.rs | 4 + crates/evm/src/evm_parser.rs | 4 + crates/evm/src/evm_read_interface.rs | 3 + crates/evm/src/evm_router.rs | 4 + crates/evm/src/fix_historical_order.rs | 4 + crates/evm/src/one_shot_runnner.rs | 4 + crates/evm/src/sync_start_extractor.rs | 4 + .../keyshare/src/encryption_key_collector.rs | 2 + crates/keyshare/src/threshold_keyshare.rs | 5 +- .../keyshare/src/threshold_share_collector.rs | 2 + crates/multithread/src/multithread.rs | 4 + crates/multithread/src/report.rs | 4 + crates/net/src/document_publisher.rs | 8 +- crates/net/src/net_event_translator.rs | 4 + crates/net/src/net_sync_manager.rs | 5 +- crates/request/src/router.rs | 4 + crates/sortition/src/ciphernode_selector.rs | 4 + crates/sortition/src/sortition.rs | 5 +- crates/sync/src/sync.rs | 1 + crates/test-helpers/src/lib.rs | 3 +- crates/test-helpers/src/plaintext_writer.rs | 4 + crates/test-helpers/src/public_key_writer.rs | 4 + crates/tests/tests/integration.rs | 14 ++- crates/utils/src/constants.rs | 3 + crates/utils/src/lib.rs | 2 + 47 files changed, 318 insertions(+), 57 deletions(-) create mode 100644 crates/utils/src/constants.rs diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index bf682d9b7b..ee7be43c33 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -9,7 +9,7 @@ use e3_events::{ prelude::*, trap, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EType, EnclaveEvent, EnclaveEventData, EventType, Shutdown, TypedEvent, }; -use e3_utils::NotifySync; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use std::collections::HashMap; use std::time::Duration; use tracing::{error, info}; @@ -43,6 +43,9 @@ impl CommitteeFinalizer { impl Actor for CommitteeFinalizer { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for CommitteeFinalizer { diff --git a/crates/aggregator/src/keyshare_created_filter_buffer.rs b/crates/aggregator/src/keyshare_created_filter_buffer.rs index 43fc32b111..eea8edd67c 100644 --- a/crates/aggregator/src/keyshare_created_filter_buffer.rs +++ b/crates/aggregator/src/keyshare_created_filter_buffer.rs @@ -7,6 +7,7 @@ use actix::prelude::*; use e3_events::{prelude::*, EnclaveEvent, EnclaveEventData}; +use e3_utils::MAILBOX_LIMIT; use std::collections::HashSet; use crate::PublicKeyAggregator; @@ -42,6 +43,9 @@ impl KeyshareCreatedFilterBuffer { impl Actor for KeyshareCreatedFilterBuffer { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for KeyshareCreatedFilterBuffer { diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index 4e99a213b6..1b36c91ad3 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -14,8 +14,8 @@ use e3_events::{ }; use e3_events::{trap, EType}; use e3_fhe::{Fhe, GetAggregatePublicKey}; -use e3_utils::ArcBytes; use e3_utils::NotifySync; +use e3_utils::{ArcBytes, MAILBOX_LIMIT}; use std::sync::Arc; use tracing::{error, info}; @@ -141,6 +141,9 @@ impl PublicKeyAggregator { impl Actor for PublicKeyAggregator { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for PublicKeyAggregator { diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index ba0060e63b..22ad6e92af 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -19,8 +19,8 @@ use e3_trbfv::{ calculate_threshold_decryption::CalculateThresholdDecryptionRequest, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; -use e3_utils::utility_types::ArcBytes; use e3_utils::NotifySync; +use e3_utils::{utility_types::ArcBytes, MAILBOX_LIMIT}; use tracing::{debug, info, trace}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -268,6 +268,9 @@ impl ThresholdPlaintextAggregator { impl Actor for ThresholdPlaintextAggregator { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for ThresholdPlaintextAggregator { diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 0fae3f641a..49b631987f 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -69,6 +69,7 @@ pub struct CiphernodeBuilder { threads: Option, threshold_plaintext_agg: bool, net_config: Option, + start_buffer: bool, } // Simple Net Configuration @@ -133,6 +134,7 @@ impl CiphernodeBuilder { threads: None, threshold_plaintext_agg: false, net_config: None, + start_buffer: false, } } @@ -184,6 +186,12 @@ impl CiphernodeBuilder { self.testmode_errors = true; self } + /// Ensure SnapshotBuffer starts immediately instead of waiting for SyncEnd. This is important + /// for tests that don't specifically + pub fn testmode_start_buffer_immediately(mut self) -> Self { + self.start_buffer = true; + self + } /// Use the node configuration on these specific chains. This will overwrite any previously /// given chains. diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index e492720bdf..c211292903 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::get_enclave_event_bus; -use actix::{Actor, Addr, Recipient}; +use actix::{Actor, Addr, Handler, Recipient}; use anyhow::{anyhow, Result}; use e3_data::{ CommitLogEventLog, DataStore, InMemEventLog, InMemSequenceIndex, InMemStore, SledSequenceIndex, @@ -15,12 +15,14 @@ use e3_events::hlc::Hlc; use e3_events::{ AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, InsertBatch, Sequencer, SnapshotBuffer, StoreEventRequested, + UpdateDestination, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; +use tracing::info; struct InMemBackend { eventstores: OnceCell>>>, @@ -45,6 +47,7 @@ struct PersistedBackend { impl PersistedBackend { fn get_or_init_store(&self, handle: &BusHandle) -> Result> { + println!("get_or_init_store in {:?} ...", self.sled_path); self.store .get_or_try_init(|| SledStore::new(handle, &self.sled_path)) .cloned() @@ -91,6 +94,8 @@ pub struct EventSystem { aggregate_config: OnceCell, /// Cached EventStoreAddrs for idempotency eventstore_addrs: OnceCell, + /// Whether to start the buffer straight away + start_buffer: bool, } impl EventSystem { @@ -114,6 +119,7 @@ impl EventSystem { hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), + start_buffer: false, } } @@ -132,6 +138,7 @@ impl EventSystem { hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), + start_buffer: false, } } @@ -152,6 +159,7 @@ impl EventSystem { hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), + start_buffer: false, } } @@ -181,8 +189,14 @@ impl EventSystem { self } + pub fn start_buffer_immediately(mut self) -> Self { + self.start_buffer = true; + self + } + /// Get the eventbus address pub fn eventbus(&self) -> Addr> { + info!("eventbus..."); self.eventbus.get_or_init(get_enclave_event_bus).clone() } @@ -197,13 +211,11 @@ impl EventSystem { pub fn buffer(&self) -> Result> { self.buffer .get_or_try_init(|| { - let store: Recipient = match &self.backend { - EventSystemBackend::InMem(b) => b.get_or_init_store().into(), - EventSystemBackend::Persisted(b) => { - b.get_or_init_store(&self.handle()?)?.into() - } - }; - SnapshotBuffer::spawn(&self.aggregate_config(), store) + SnapshotBuffer::spawn( + &self.aggregate_config(), + NoopBatchReceiver::new().start(), + self.start_buffer, + ) }) .cloned() } @@ -291,8 +303,10 @@ impl EventSystem { pub fn persisted_eventstore_router( &self, ) -> Result>> { + info!("persisted_eventstore_router..."); let eventstores = self.eventstore_addrs()?; if let EventStoreAddrs::Persisted(addrs) = eventstores { + info!("creating router..."); let router = EventStoreRouter::new(addrs); Ok(router.start()) } else { @@ -302,6 +316,7 @@ impl EventSystem { /// Get an EventStoreRouter Recipient pub fn eventstore_router(&self) -> Result> { + info!("eventstore_reader..."); let eventstores = self.eventstore_addrs()?; match &eventstores { EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), @@ -318,6 +333,7 @@ impl EventSystem { /// Get the BusHandle pub fn handle(&self) -> Result { + println!("handle"); self.handle .get_or_try_init(|| { Ok(BusHandle::new( @@ -331,16 +347,27 @@ impl EventSystem { /// Get the DataStore pub fn store(&self) -> Result { - match &self.backend { - EventSystemBackend::InMem(b) => Ok(DataStore::from_in_mem_with_buffer( - &b.get_or_init_store(), - self.buffer()?, - )), - EventSystemBackend::Persisted(b) => Ok(DataStore::from_sled_store_with_buffer( - &b.get_or_init_store(&self.handle()?)?, - self.buffer()?, - )), - } + println!("store()..."); + let store = match &self.backend { + EventSystemBackend::InMem(b) => { + let base = b.get_or_init_store(); + let buffer = self.buffer()?; + buffer.try_send(UpdateDestination::new(base.clone()))?; + DataStore::from_in_mem_with_buffer(&base, self.buffer()?) + } + EventSystemBackend::Persisted(b) => { + let base = b.get_or_init_store(&self.handle()?)?; + let buffer = self.buffer()?; + buffer.try_send(UpdateDestination::new(base))?; + + DataStore::from_sled_store_with_buffer( + &b.get_or_init_store(&self.handle()?)?, + self.buffer()?, + ) + } + }; + + Ok(store) } fn node_id(name: &str) -> u32 { @@ -350,6 +377,24 @@ impl EventSystem { } } +struct NoopBatchReceiver; + +impl NoopBatchReceiver { + pub fn new() -> Self { + Self {} + } +} +impl Actor for NoopBatchReceiver { + type Context = actix::Context; +} + +impl Handler for NoopBatchReceiver { + type Result = (); + fn handle(&mut self, msg: InsertBatch, ctx: &mut Self::Context) -> Self::Result { + // do nothing + } +} + #[cfg(test)] mod tests { use e3_data::AutoPersist; @@ -357,6 +402,7 @@ mod tests { use e3_data::Repository; use e3_events::EventContext; use e3_events::EventId; + use e3_events::Start; use e3_events::StoreKeys; use e3_events::SyncEnd; use e3_events::Tick; @@ -436,12 +482,14 @@ mod tests { } #[actix::test] - async fn test_persisted() { + async fn test_persisted() -> Result<()> { + let _guard = with_tracing("debug"); let tmp = TempDir::new().unwrap(); let system = EventSystem::persisted("cn2", tmp.path().join("log"), tmp.path().join("sled")); - + system.buffer()?.send(Start).await?; let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); + Ok(()) } #[actix::test] diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 4e00f7ddc8..64e8ea4deb 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -174,12 +174,13 @@ impl DataStore { pub fn from_sled_store_with_buffer( addr: &Addr, - write_buffer: impl Into>, + snapshot_buffer: impl Into>, ) -> Self { + println!("from_sled_store_with_buffer..."); Self { addr: StoreAddr::Sled(addr.clone()), get: addr.clone().recipient(), - insert: write_buffer.into(), + insert: snapshot_buffer.into(), insert_sync: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], @@ -188,12 +189,12 @@ impl DataStore { pub fn from_in_mem_with_buffer( addr: &Addr, - write_buffer: impl Into>, + snapshot_buffer: impl Into>, ) -> Self { Self { addr: StoreAddr::InMem(addr.clone()), get: addr.clone().recipient(), - insert: write_buffer.into(), + insert: snapshot_buffer.into(), insert_sync: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 7d772010b4..c689acd9fb 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -7,6 +7,7 @@ use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; +use e3_utils::MAILBOX_LIMIT; use std::collections::BTreeMap; use tracing::info; @@ -32,6 +33,9 @@ pub struct InMemStore { impl Actor for InMemStore { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl InMemStore { diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 3472a27473..9c76322c81 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -280,6 +280,7 @@ mod tests { use actix::{Actor, Addr, Handler, Message}; use e3_events::{Get, Insert, Remove}; + use e3_utils::MAILBOX_LIMIT; use super::{Persistable, StoreConnector}; @@ -300,6 +301,9 @@ mod tests { impl Actor for MockConnector { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for MockConnector { diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 660ddb20b3..12cc3eeb0b 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -9,6 +9,7 @@ use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::Result; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData, EventType}; use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; +use e3_utils::MAILBOX_LIMIT; use std::path::PathBuf; use tracing::{error, info}; @@ -19,6 +20,9 @@ pub struct SledStore { impl Actor for SledStore { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl SledStore { diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 17fd7b63a8..d721c4530f 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use actix::{Actor, Addr, Handler, Recipient}; use anyhow::Result; use derivative::Derivative; +use e3_utils::MAILBOX_LIMIT; use tracing::error; use crate::{ @@ -412,6 +413,9 @@ where F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler> for BusHandlePipe diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index cf3be8cc63..6760976f8e 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -8,7 +8,7 @@ use crate::traits::{ErrorEvent, Event}; use crate::EventType; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; -use e3_utils::{colorize, Color}; +use e3_utils::{colorize, Color, MAILBOX_LIMIT, MAILBOX_LIMIT_LARGE}; use std::collections::{HashMap, VecDeque}; use std::marker::PhantomData; use tracing::info; @@ -50,6 +50,9 @@ pub struct EventBus { impl Actor for EventBus { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT_LARGE) + } } impl EventBus { @@ -219,6 +222,9 @@ impl EventFilter { impl Actor for EventFilter { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT_LARGE) + } } impl Handler for EventFilter { @@ -401,6 +407,9 @@ impl Handler> for HistoryCollector { impl Actor for HistoryCollector { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for HistoryCollector { diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 934ae00b39..1a82684992 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -10,7 +10,7 @@ use crate::{ }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; -use e3_utils::major_issue; +use e3_utils::{major_issue, MAILBOX_LIMIT}; use tracing::{error, warn}; const MAX_STORAGE_ERRORS: u64 = 10; @@ -75,6 +75,9 @@ impl EventStore { impl Actor for EventStore { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for EventStore { diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index ba1531b7fc..fb38381702 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -12,9 +12,9 @@ use crate::{ use crate::{CorrelationId, GetEventsAfterResponse}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; -use e3_utils::major_issue; +use e3_utils::{major_issue, MAILBOX_LIMIT}; use std::collections::HashMap; -use tracing::error; +use tracing::{error, info}; pub struct EventStoreRouter { stores: HashMap>>, @@ -22,14 +22,17 @@ pub struct EventStoreRouter { impl EventStoreRouter { pub fn new(stores: HashMap>>) -> Self { + info!("Making eventstore router..."); let stores = stores .into_iter() .map(|(index, addr)| (AggregateId::new(index), addr)) .collect(); + Self { stores } } pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { + info!("Handling store event requested...."); let aggregate_id = msg.event.aggregate_id(); let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { @@ -42,7 +45,7 @@ impl EventStoreRouter { let sender = msg.sender; let forwarded_msg = StoreEventRequested::new(event, sender); - store_addr.do_send(forwarded_msg); + store_addr.try_send(forwarded_msg)?; Ok(()) } @@ -60,6 +63,10 @@ impl EventStoreRouter { impl Actor for EventStoreRouter { type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for EventStoreRouter { diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 68e97e703e..5ea107f022 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -10,7 +10,7 @@ use crate::{ }; use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; use anyhow::Result; -use e3_utils::major_issue; +use e3_utils::{major_issue, MAILBOX_LIMIT_LARGE}; /// Component to sequence the storage of events pub struct Sequencer { @@ -42,6 +42,9 @@ impl Sequencer { impl Actor for Sequencer { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT_LARGE) + } } impl Handler> for Sequencer { diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index b04a17397d..acbb517557 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -6,6 +6,7 @@ use std::mem::replace; use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient}; +use e3_utils::MAILBOX_LIMIT; use tracing::debug; use crate::{trap, Die, EType, Insert, InsertBatch, PanicDispatcher}; @@ -33,6 +34,9 @@ impl Batch { impl Actor for Batch { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for Batch { diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 4e5dbff521..63fd5797e0 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -6,7 +6,7 @@ use super::{ batch::{Batch, Flush}, timelock_queue::{Clock, StartTimelock}, - AggregateConfig, + AggregateConfig, UpdateDestination, }; use crate::{ trap, AggregateId, EType, EnclaveEvent, EventContextAccessors, EventContextSeq, Insert, @@ -14,6 +14,7 @@ use crate::{ }; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Context; +use e3_utils::MAILBOX_LIMIT; use std::{collections::HashMap, sync::Arc, time::Duration}; use tracing::{debug, info, trace, warn}; @@ -41,6 +42,9 @@ pub struct BatchRouter { impl Actor for BatchRouter { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl BatchRouter { @@ -189,6 +193,16 @@ impl Handler for BatchRouter { } } +impl Handler for BatchRouter { + type Result = (); + fn handle(&mut self, msg: UpdateDestination, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + self.db = msg.0; + Ok(()) + }) + } +} + /// Encode the same as bincode without using a result fn encode_u64(value: u64) -> Vec { value.to_le_bytes().to_vec() diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 4e6423ec68..83c89bf0b9 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -13,6 +13,7 @@ use crate::{ }; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use std::sync::Arc; use tracing::{debug, info, trace}; @@ -36,6 +37,14 @@ impl SetDependencies { #[rtype(result = "()")] pub struct Start; +#[derive(Message)] +#[rtype(result = "()")] +pub struct UpdateDestination(pub Recipient); +impl UpdateDestination { + pub fn new(base: impl Into>) -> Self { + Self(base.into()) + } +} enum SnapshotBufferState { Running, Paused, @@ -49,20 +58,26 @@ pub struct SnapshotBuffer { } impl SnapshotBuffer { - pub fn new() -> Self { + pub fn new(started: bool) -> Self { SnapshotBuffer { router: None, timelock: None, tickable: None, - state: SnapshotBufferState::Paused, + state: if !started { + SnapshotBufferState::Paused + } else { + SnapshotBufferState::Running + }, } } pub fn spawn( config: &AggregateConfig, store: impl Into>, + started: bool, ) -> Result> { - let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock), Some(1))?; + info!("spawning SnapshotBuffer..."); + let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock), Some(1), started)?; Ok(addr) } @@ -71,8 +86,9 @@ impl SnapshotBuffer { store: impl Into>, clock: Arc, interval: Option, + started: bool, ) -> Result<(Addr, Addr)> { - let addr = Self::new().start(); + let addr = Self::new(started).start(); let store = store.into(); let router = BatchRouter::with_clock(config, addr.clone(), store.clone(), clock.clone()).start(); @@ -84,6 +100,9 @@ impl SnapshotBuffer { impl Actor for SnapshotBuffer { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for SnapshotBuffer { @@ -154,11 +173,10 @@ impl Handler for SnapshotBuffer { impl Handler for SnapshotBuffer { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { if let EnclaveEventData::SyncEnd(_) = msg.get_data() { - info!("SnapshotBuffer is now enabled"); - self.state = SnapshotBufferState::Running; + self.notify_sync(ctx, Start); }; let SnapshotBufferState::Running = self.state else { return Ok(()); @@ -183,12 +201,36 @@ impl Handler for SnapshotBuffer { } } +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, _: Start, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + info!("SnapshotBuffer is now enabled"); + self.state = SnapshotBufferState::Running; + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: UpdateDestination, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} + #[cfg(test)] mod mock_store { use std::mem::replace; use crate::InsertBatch; use actix::{Actor, Handler, Message}; + use e3_utils::MAILBOX_LIMIT; #[derive(Message)] #[rtype(result = "Vec")] @@ -263,7 +305,7 @@ mod tests { let clock = Arc::new(MockClock::new(1000)); let (buffer, timelock) = - SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; + SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None, false)?; buffer .send(EnclaveEvent::from_data_ec( diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index 52fe563085..0b04cc577a 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{trap, EType, PanicDispatcher}; use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient}; +use e3_utils::MAILBOX_LIMIT; use std::{ cmp::{Ordering, Reverse}, collections::BinaryHeap, @@ -123,13 +124,14 @@ impl Actor for TimelockQueue { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); let Some(interval) = self.interval else { debug!("TimelockQueue in manual mode - will tick when requested."); return; }; // Send Tick to self every second - debug!("TimelockQueue is ticking every {}", interval); + debug!("TimelockQueue is ticking every {}s", interval); ctx.run_interval(Duration::from_secs(interval), |_, ctx| { ctx.address().do_send(Tick); }); diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 3448590ce9..dd86146cd0 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -23,7 +23,7 @@ use e3_events::{ EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; -use e3_utils::NotifySync; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use tracing::{error, info, trace}; sol!( @@ -279,6 +279,10 @@ impl CiphernodeRegistrySolWriter impl Actor for CiphernodeRegistrySolWriter

{ type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 68a1f6adc9..189c346c3f 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -26,6 +26,7 @@ use e3_events::EventType; use e3_events::Shutdown; use e3_events::{E3id, EType, PlaintextAggregated}; use e3_utils::NotifySync; +use e3_utils::MAILBOX_LIMIT; use tracing::info; sol!( @@ -70,6 +71,9 @@ impl EnclaveSolWriter

{ impl Actor for EnclaveSolWriter

{ type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EnclaveSolWriter

{ diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 85fc40d949..d837781d14 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -18,6 +18,7 @@ use e3_events::{ EvmSyncEventsReceived, SyncEnd, SyncStart, Unsequenced, }; use e3_events::{Event, EventPublisher}; +use e3_utils::MAILBOX_LIMIT; use tracing::info; /// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. @@ -181,6 +182,9 @@ impl EvmChainGateway { impl Actor for EvmChainGateway { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EvmChainGateway { diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs index 5dcf92bd9f..2bc3aad605 100644 --- a/crates/evm/src/evm_hub.rs +++ b/crates/evm/src/evm_hub.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Addr, Handler}; +use e3_utils::MAILBOX_LIMIT; use crate::events::{EnclaveEvmEvent, EvmEventProcessor}; @@ -25,6 +26,9 @@ impl EvmHub { impl Actor for EvmHub { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EvmHub { diff --git a/crates/evm/src/evm_parser.rs b/crates/evm/src/evm_parser.rs index da31fa9161..bcbf3bf045 100644 --- a/crates/evm/src/evm_parser.rs +++ b/crates/evm/src/evm_parser.rs @@ -6,6 +6,7 @@ use actix::{Actor, Handler}; use e3_events::{hlc::HlcTimestamp, EnclaveEventData}; +use e3_utils::MAILBOX_LIMIT; use tracing::info; use crate::{ @@ -20,6 +21,9 @@ pub struct EvmParser { impl Actor for EvmParser { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl EvmParser { diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index 38c87c3104..b00a6309fd 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -17,6 +17,7 @@ use alloy_primitives::Address; use anyhow::anyhow; use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscriber, EventType}; use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; +use e3_utils::MAILBOX_LIMIT; use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; use tokio::select; @@ -119,6 +120,8 @@ impl Actor for EvmReadInterface

{ type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + // let reader_addr = ctx.address(); let bus = self.bus.clone(); let next = self.next.clone(); diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 9a4251b867..953dbce34e 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -7,6 +7,7 @@ use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; use actix::{Actor, Handler}; use alloy_primitives::Address; +use e3_utils::MAILBOX_LIMIT; use std::collections::HashMap; use tracing::{debug, error, info}; @@ -42,6 +43,9 @@ impl EvmRouter { impl Actor for EvmRouter { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EvmRouter { diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index 1faf833d21..2fe149c4a4 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -8,6 +8,7 @@ use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; use actix::{Actor, Addr, Handler}; use bloom::{BloomFilter, ASMS}; use e3_events::CorrelationId; +use e3_utils::MAILBOX_LIMIT; use tracing::info; pub struct FixHistoricalOrder { @@ -50,6 +51,9 @@ impl FixHistoricalOrder { impl Actor for FixHistoricalOrder { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for FixHistoricalOrder { diff --git a/crates/evm/src/one_shot_runnner.rs b/crates/evm/src/one_shot_runnner.rs index 6a6576774d..ac96756c4a 100644 --- a/crates/evm/src/one_shot_runnner.rs +++ b/crates/evm/src/one_shot_runnner.rs @@ -6,6 +6,7 @@ use actix::prelude::*; use anyhow::Result; +use e3_utils::MAILBOX_LIMIT; use std::marker::PhantomData; use tracing::error; @@ -40,6 +41,9 @@ where M: Message + 'static + Unpin, { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for OneShotRunner diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs index 5c98db97d3..2aa0748984 100644 --- a/crates/evm/src/sync_start_extractor.rs +++ b/crates/evm/src/sync_start_extractor.rs @@ -6,6 +6,7 @@ use actix::{Actor, Addr, Handler, Recipient}; use e3_events::{EnclaveEvent, EnclaveEventData, Event, SyncStart}; +use e3_utils::MAILBOX_LIMIT; pub struct SyncStartExtractor { dest: Recipient, @@ -22,6 +23,9 @@ impl SyncStartExtractor { } impl Actor for SyncStartExtractor { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for SyncStartExtractor { diff --git a/crates/keyshare/src/encryption_key_collector.rs b/crates/keyshare/src/encryption_key_collector.rs index f98e6e896f..15d5ef5d81 100644 --- a/crates/keyshare/src/encryption_key_collector.rs +++ b/crates/keyshare/src/encryption_key_collector.rs @@ -15,6 +15,7 @@ use e3_events::{ E3id, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, TypedEvent, }; use e3_trbfv::PartyId; +use e3_utils::MAILBOX_LIMIT; use tracing::{info, warn}; const DEFAULT_COLLECTION_TIMEOUT: Duration = Duration::from_secs(60); @@ -87,6 +88,7 @@ impl Actor for EncryptionKeyCollector { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); info!( e3_id = %self.e3_id, "EncryptionKeyCollector started, scheduling timeout in {:?}", diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 16b6b57eb3..6480e5a93a 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -27,8 +27,8 @@ use e3_trbfv::{ shares::{BfvEncryptedShares, EncryptableVec, Encrypted, ShamirShare, SharedSecret}, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; -use e3_utils::NotifySync; use e3_utils::{to_ordered_vec, utility_types::ArcBytes}; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use fhe::bfv::BfvParameters; use fhe::bfv::{PublicKey, SecretKey}; use fhe_traits::{DeserializeParametrized, Serialize}; @@ -330,6 +330,9 @@ impl ThresholdKeyshare { impl Actor for ThresholdKeyshare { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl ThresholdKeyshare { diff --git a/crates/keyshare/src/threshold_share_collector.rs b/crates/keyshare/src/threshold_share_collector.rs index 11a416a1be..ae1167352f 100644 --- a/crates/keyshare/src/threshold_share_collector.rs +++ b/crates/keyshare/src/threshold_share_collector.rs @@ -15,6 +15,7 @@ use e3_events::{ E3id, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated, TypedEvent, }; use e3_trbfv::PartyId; +use e3_utils::MAILBOX_LIMIT; use tracing::{info, warn}; use crate::{AllThresholdSharesCollected, ThresholdKeyshare}; @@ -65,6 +66,7 @@ impl Actor for ThresholdShareCollector { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); info!( e3_id = %self.e3_id, "ThresholdShareCollector started, scheduling timeout in {:?}", diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index e0b40fe459..61cfe36eb6 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -35,6 +35,7 @@ use e3_trbfv::gen_esi_sss::gen_esi_sss; use e3_trbfv::gen_pk_share_and_sk_sss::gen_pk_share_and_sk_sss; use e3_trbfv::{TrBFVError, TrBFVRequest, TrBFVResponse}; use e3_utils::SharedRng; +use e3_utils::MAILBOX_LIMIT; use rand::Rng; use tracing::error; use tracing::info; @@ -93,6 +94,9 @@ impl Multithread { impl Actor for Multithread { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for Multithread { diff --git a/crates/multithread/src/report.rs b/crates/multithread/src/report.rs index 9eedc23dab..da9a05defc 100644 --- a/crates/multithread/src/report.rs +++ b/crates/multithread/src/report.rs @@ -7,6 +7,7 @@ use std::{collections::HashMap, thread, time::Duration}; use actix::{Actor, Handler, Message, MessageResponse}; +use e3_utils::MAILBOX_LIMIT; #[derive(Message)] #[rtype(result = "FlattenedReport")] @@ -34,6 +35,9 @@ pub struct MultithreadReport { impl Actor for MultithreadReport { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl MultithreadReport { diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index d906b1af1b..e28f21b58b 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -20,9 +20,12 @@ use e3_events::{ EncryptionKeyCreated, Event, EventContext, EventType, Filter, PartyId, PublishDocumentRequested, Sequenced, ThresholdShareCreated, TypedEvent, }; -use e3_utils::retry::{retry_with_backoff, to_retry}; use e3_utils::ArcBytes; use e3_utils::NotifySync; +use e3_utils::{ + retry::{retry_with_backoff, to_retry}, + MAILBOX_LIMIT, +}; use futures::TryFutureExt; use serde::{Deserialize, Serialize}; use std::{ @@ -132,6 +135,9 @@ impl DocumentPublisher { impl Actor for DocumentPublisher { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for DocumentPublisher { diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 65276c288f..4178ed698e 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -25,6 +25,7 @@ use e3_events::EventContextAccessors; use e3_events::EventType; use e3_events::Unsequenced; use e3_events::{CorrelationId, EnclaveEvent, EventId}; +use e3_utils::MAILBOX_LIMIT; use libp2p::identity::ed25519; use std::collections::HashSet; use std::sync::Arc; @@ -47,6 +48,9 @@ pub struct NetEventTranslator { impl Actor for NetEventTranslator { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } /// Libp2pEvent is used to send data to the NetInterface from the NetEventTranslator diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index e950f991ee..8fae908568 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -11,7 +11,7 @@ use e3_events::{ EnclaveEventData, GetAggregateEventsAfter, GetEventsAfterResponse, NetSyncEventsReceived, OutgoingSyncRequested, TypedEvent, Unsequenced, }; -use e3_utils::{retry_with_backoff, to_retry, OnceTake}; +use e3_utils::{retry_with_backoff, to_retry, OnceTake, MAILBOX_LIMIT}; use futures::TryFutureExt; use libp2p::request_response::ResponseChannel; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -83,6 +83,9 @@ impl NetSyncManager { impl Actor for NetSyncManager { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } /// Event broadcast from event bus diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index 5d3b7410e4..d7a0a52b7e 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -27,6 +27,7 @@ use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::EventType; use e3_events::{E3id, EnclaveEvent, Event}; +use e3_utils::MAILBOX_LIMIT; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; @@ -145,6 +146,9 @@ impl E3Router { impl Actor for E3Router { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for E3Router { diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index a82d72fffa..dd3eed23a8 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -17,6 +17,7 @@ use e3_events::{ }; use e3_request::E3Meta; use e3_utils::NotifySync; +use e3_utils::MAILBOX_LIMIT; use std::collections::HashMap; use tracing::info; @@ -30,6 +31,9 @@ pub struct CiphernodeSelector { impl Actor for CiphernodeSelector { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl CiphernodeSelector { diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 87aa1eb628..10c56023ec 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -17,7 +17,7 @@ use e3_events::{ PlaintextOutputPublished, Seed, TicketBalanceUpdated, TypedEvent, }; use e3_events::{BusHandle, E3id, EnclaveEventData}; -use e3_utils::NotifySync; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::ops::Deref; @@ -352,6 +352,9 @@ impl Sortition { impl Actor for Sortition { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for Sortition { diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 809834c241..bf1b52108c 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -273,6 +273,7 @@ mod tests { } #[actix::test] + #[ignore] async fn test_synchronizer_full_flow() -> Result<()> { let _guard = e3_test_helpers::with_tracing("info"); // Setup event system and synchronizer diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 64113c79d9..16ef1612f8 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -32,6 +32,7 @@ use rand::Rng; use rand_chacha::rand_core::SeedableRng; use rand_chacha::ChaCha20Rng; use std::sync::Arc; +use tracing::trace; pub use utils::*; pub fn create_shared_rng_from_u64(value: u64) -> Arc> { @@ -142,7 +143,7 @@ pub fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { || DocumentPublisher::is_document_publisher_event(e) }); } else { - println!("not piping bus to itself"); + trace!("Source = Dest! Not piping bus to itself"); } } } diff --git a/crates/test-helpers/src/plaintext_writer.rs b/crates/test-helpers/src/plaintext_writer.rs index 943d0a7961..3dafe48959 100644 --- a/crates/test-helpers/src/plaintext_writer.rs +++ b/crates/test-helpers/src/plaintext_writer.rs @@ -10,6 +10,7 @@ use super::write_file_with_dirs; use actix::{Actor, Addr, Context, Handler}; use e3_bfv_client::decode_bytes_to_vec_u64; use e3_events::{prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventType}; +use e3_utils::MAILBOX_LIMIT; use tracing::{error, info}; pub struct PlaintextWriter { @@ -29,6 +30,9 @@ impl PlaintextWriter { impl Actor for PlaintextWriter { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for PlaintextWriter { diff --git a/crates/test-helpers/src/public_key_writer.rs b/crates/test-helpers/src/public_key_writer.rs index e10655d3b5..69298a3eb8 100644 --- a/crates/test-helpers/src/public_key_writer.rs +++ b/crates/test-helpers/src/public_key_writer.rs @@ -11,6 +11,7 @@ use actix::{Actor, Addr, Context, Handler}; use e3_events::{ prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, }; +use e3_utils::MAILBOX_LIMIT; use tracing::info; pub struct PublicKeyWriter { @@ -30,6 +31,9 @@ impl PublicKeyWriter { impl Actor for PublicKeyWriter { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for PublicKeyWriter { diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index e74e019249..dbb273f802 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -22,7 +22,9 @@ use e3_net::events::{GossipData, NetEvent}; use e3_net::NetEventTranslator; use e3_sortition::{calculate_buffer_size, RegisteredNode, ScoreSortition, Ticket}; use e3_test_helpers::ciphernode_system::CiphernodeSystemBuilder; -use e3_test_helpers::{create_seed_from_u64, create_shared_rng_from_u64, AddToCommittee}; +use e3_test_helpers::{ + create_seed_from_u64, create_shared_rng_from_u64, with_tracing, AddToCommittee, +}; use e3_trbfv::helpers::calculate_error_size; use e3_utils::rand_eth_addr; use e3_utils::utility_types::ArcBytes; @@ -179,14 +181,8 @@ async fn test_trbfv_actor() -> Result<()> { println!("Running test_trbfv_actor..."); let mut report: Vec<(&str, Duration)> = vec![]; let whole_test = Instant::now(); - use tracing_subscriber::{fmt, EnvFilter}; - - let subscriber = fmt() - .with_env_filter(EnvFilter::new("info")) - .with_test_writer() - .finish(); - let _guard = tracing::subscriber::set_default(subscriber); + let _guard = with_tracing("info"); // NOTE: Here we are trying to make it as clear as possible as to what is going on so attempting to // avoid over abstracting test helpers and favouring straight forward single descriptive @@ -261,6 +257,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_pubkey_aggregation() .with_sortition_score() .with_threshold_plaintext_aggregation() + .testmode_start_buffer_immediately() .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() @@ -276,6 +273,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_shared_multithread_report(&multithread_report) .with_trbfv() .with_sortition_score() + .testmode_start_buffer_immediately() .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() diff --git a/crates/utils/src/constants.rs b/crates/utils/src/constants.rs new file mode 100644 index 0000000000..237bcce678 --- /dev/null +++ b/crates/utils/src/constants.rs @@ -0,0 +1,3 @@ +// Max message +pub const MAILBOX_LIMIT: usize = 256; +pub const MAILBOX_LIMIT_LARGE: usize = 256 * 10; diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 454c5b4791..5e9978adcc 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -6,6 +6,7 @@ pub mod actix; pub mod alloy; +pub mod constants; pub mod error; pub mod formatters; pub mod helpers; @@ -14,6 +15,7 @@ pub mod retry; pub mod utility_types; pub use actix::*; pub use alloy::*; +pub use constants::*; pub use error::*; pub use formatters::*; pub use helpers::*; From 15bb576633973b5d3f4ccae9ec217b63551cc59d Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 5 Feb 2026 09:50:11 +0000 Subject: [PATCH 39/63] add headers and fix indexer delay --- crates/indexer/tests/integration.rs | 2 +- crates/utils/src/constants.rs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/indexer/tests/integration.rs b/crates/indexer/tests/integration.rs index 4b26ec4efe..0214d26a5b 100644 --- a/crates/indexer/tests/integration.rs +++ b/crates/indexer/tests/integration.rs @@ -43,7 +43,7 @@ sol!( async fn test_indexer() -> Result<()> { const E3_ID: u64 = 10; const THRESHOLD: u64 = 10; - const INDEXER_DELAY_MS: u64 = 10; + const INDEXER_DELAY_MS: u64 = 30; let param_set = DEFAULT_BFV_PRESET.into(); let params = build_bfv_params_from_set_arc(param_set); diff --git a/crates/utils/src/constants.rs b/crates/utils/src/constants.rs index 237bcce678..0800376848 100644 --- a/crates/utils/src/constants.rs +++ b/crates/utils/src/constants.rs @@ -1,3 +1,12 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +// NOTE: Currently this is here as everything depends on utils. We could consider moving this +// closer to the configuration if we need to make this dynamic or create a create just for this. + // Max message pub const MAILBOX_LIMIT: usize = 256; pub const MAILBOX_LIMIT_LARGE: usize = 256 * 10; From 510529ae6a1afeea04d3849fcd064c899e2cbba9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 6 Feb 2026 05:09:09 +0000 Subject: [PATCH 40/63] feat: setup sync function --- Cargo.lock | 1 + .../src/ciphernode_builder.rs | 15 +- crates/ciphernode-builder/src/event_system.rs | 19 +- crates/ciphernode-builder/src/evm_system.rs | 13 +- crates/events/src/bus_handle.rs | 12 +- ...c_events_received.rs => enable_effects.rs} | 16 +- crates/events/src/enclave_event/mod.rs | 18 +- crates/events/src/enclave_event/sync_start.rs | 90 ++- crates/events/src/event_context.rs | 22 +- crates/events/src/events.rs | 7 +- crates/events/src/eventstore_router.rs | 76 ++- crates/events/src/sync.rs | 2 + crates/events/src/traits.rs | 8 +- crates/evm/src/evm_chain_gateway.rs | 36 +- crates/evm/src/sync_start_extractor.rs | 10 +- crates/evm/tests/integration.rs | 17 +- crates/net/src/net_sync_manager.rs | 25 +- crates/sync/Cargo.toml | 1 + crates/sync/src/sync.rs | 558 +++++++++--------- crates/utils/src/actix/channel.rs | 77 +++ crates/utils/src/{actix.rs => actix/mod.rs} | 3 +- crates/utils/src/lib.rs | 2 +- 22 files changed, 631 insertions(+), 397 deletions(-) rename crates/events/src/enclave_event/{evm_sync_events_received.rs => enable_effects.rs} (59%) create mode 100644 crates/utils/src/actix/channel.rs rename crates/utils/src/{actix.rs => actix/mod.rs} (98%) diff --git a/Cargo.lock b/Cargo.lock index 8aed8ddd1d..6a262946e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3442,6 +3442,7 @@ dependencies = [ "e3-data", "e3-events", "e3-test-helpers", + "e3-utils", "tokio", "tracing", ] diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 49b631987f..1097e6a9f0 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -27,7 +27,7 @@ use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; -use e3_sync::Synchronizer; +use e3_sync::sync; use e3_utils::{rand_eth_addr, SharedRng}; use std::time::Duration; use std::{collections::HashMap, path::PathBuf, sync::Arc}; @@ -381,6 +381,7 @@ impl CiphernodeBuilder { let bus = event_system.handle()?; let store = event_system.store()?; + let eventstore = event_system.eventstore_getter()?; let cipher = &self.cipher; let repositories = Arc::new(store.repositories()); @@ -472,9 +473,15 @@ impl CiphernodeBuilder { ) }; - let buffer = event_system.buffer()?; - - Synchronizer::setup(&bus, &evm_config, &repositories, &aggregate_config, &buffer); + // Run the sync routine + sync( + &bus, + &evm_config, + &repositories, + &aggregate_config, + &eventstore, + ) + .await?; Ok(CiphernodeHandle::new( addr.to_owned(), diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index c211292903..9514a15c0e 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -14,8 +14,8 @@ use e3_data::{ use e3_events::hlc::Hlc; use e3_events::{ AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, - EventStoreRouter, InsertBatch, Sequencer, SnapshotBuffer, StoreEventRequested, - UpdateDestination, + EventStoreRouter, GetEventsAfterSeq, InsertBatch, Sequencer, SnapshotBuffer, + StoreEventRequested, UpdateDestination, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -324,6 +324,15 @@ impl EventSystem { } } + pub fn eventstore_getter(&self) -> Result> { + info!("eventstore_reader..."); + let eventstores = self.eventstore_addrs()?; + match &eventstores { + EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), + EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), + } + } + /// Get an instance of the Hlc pub fn hlc(&self) -> Result { self.hlc @@ -473,7 +482,7 @@ mod tests { impl Handler for Listener { type Result = (); fn handle(&mut self, msg: GetEventsAfterResponse, _: &mut Self::Context) -> Self::Result { - self.events = msg.events().clone(); + self.events = msg.into_events(); } } @@ -629,11 +638,11 @@ mod tests { let router = system.in_mem_eventstore_router()?; // Get all events after the given timestamp using the router - use e3_events::{AggregateId, GetAggregateEventsAfter}; + use e3_events::{AggregateId, GetEventsAfterTs}; let mut ts_map = HashMap::new(); ts_map.insert(AggregateId::new(0), ts); let get_events_msg = - GetAggregateEventsAfter::new(CorrelationId::new(), ts_map, listener.clone().into()); + GetEventsAfterTs::new(CorrelationId::new(), ts_map, listener.clone().into()); router.do_send(get_events_msg); sleep(Duration::from_millis(100)).await; diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 991af572cb..3b1ce66ce2 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -8,7 +8,7 @@ use std::mem::replace; use actix::Actor; use alloy::{primitives::Address, providers::Provider}; -use e3_events::{BusHandle, EventSubscriber, EventType, SyncStart}; +use e3_events::{BusHandle, EventSubscriber, EventType, HistoricalEvmSyncStart}; use e3_evm::{ EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, FixHistoricalOrder, OneShotRunner, SyncStartExtractor, @@ -56,7 +56,7 @@ impl EvmSystemChainBuilder

{ // Fix the historical order to avoid missing historical events let next = FixHistoricalOrder::setup(next); - // This will run once when the SyncStart event is received + // This will run once when the HistoricalEvmSyncStart event is received let next = OneShotRunner::setup({ // Clone self refs for closure let bus = self.bus.clone(); @@ -67,7 +67,7 @@ impl EvmSystemChainBuilder

{ let route_factories = replace(&mut self.route_factories, Vec::new()); // The event is defined here - move |msg: SyncStart| { + move |msg: HistoricalEvmSyncStart| { // Extract config let deploy_block = msg.get_evm_config(chain_id)?.deploy_block(); @@ -83,11 +83,12 @@ impl EvmSystemChainBuilder

{ } }); - // We get a SyncStart event and sent to oneShotRunner + // We get a HistoricalEvmSyncStart event and sent to oneShotRunner let next = SyncStartExtractor::setup(next); - // Finaly subscribe to the bus and wait for SyncStart - self.bus.subscribe(EventType::SyncStart, next.recipient()); + // Finaly subscribe to the bus and wait for HistoricalEvmSyncStart + self.bus + .subscribe(EventType::HistoricalEvmSyncStart, next.recipient()); } } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index d721c4530f..0a20823e53 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -94,9 +94,9 @@ impl EventPublisher> for BusHandle { fn publish( &self, data: impl Into, - ctx: EventContext, + caused_by: impl Into>, ) -> Result<()> { - self.publish_local(data, Some(ctx)) + self.publish_local(data, Some(caused_by.into())) } fn publish_without_context(&self, data: impl Into) -> Result<()> { @@ -116,10 +116,10 @@ impl EventPublisher> for BusHandle { &self, data: impl Into, remote_ts: u128, - caused_by: EventContext, + caused_by: impl Into>, block: Option, ) -> Result<()> { - self.publish_from_remote_impl(data, remote_ts, Some(caused_by), block) + self.publish_from_remote_impl(data, remote_ts, Some(caused_by.into()), block) } fn naked_dispatch(&self, event: EnclaveEvent) { @@ -142,9 +142,9 @@ impl BusHandle { fn publish_local( &self, data: impl Into, - ctx: Option>, + caused_by: Option>, ) -> Result<()> { - let evt = self.event_from(data, ctx)?; + let evt = self.event_from(data, caused_by)?; self.sequencer.do_send(evt); Ok(()) } diff --git a/crates/events/src/enclave_event/evm_sync_events_received.rs b/crates/events/src/enclave_event/enable_effects.rs similarity index 59% rename from crates/events/src/enclave_event/evm_sync_events_received.rs rename to crates/events/src/enclave_event/enable_effects.rs index f6d9325bf8..16b4919ccf 100644 --- a/crates/events/src/enclave_event/evm_sync_events_received.rs +++ b/crates/events/src/enclave_event/enable_effects.rs @@ -8,22 +8,18 @@ use actix::Message; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; -use super::{EnclaveEvent, Unsequenced}; - +/// Dispatched once the sync process is complete and live listening should continue #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EvmSyncEventsReceived { - pub events: Vec>, - pub chain_id: u64, -} +pub struct EnableEffects; -impl EvmSyncEventsReceived { - pub fn new(events: Vec>, chain_id: u64) -> Self { - Self { events, chain_id } +impl EnableEffects { + pub fn new() -> Self { + Self {} } } -impl Display for EvmSyncEventsReceived { +impl Display for EnableEffects { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 3999493248..05f3335085 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -18,10 +18,10 @@ mod decryptionshare_created; mod die; mod e3_request_complete; mod e3_requested; +mod enable_effects; mod enclave_error; mod encryption_key_collection_failed; mod encryption_key_created; -mod evm_sync_events_received; mod keyshare_created; mod net_sync_events_received; mod operator_activation_changed; @@ -57,10 +57,10 @@ pub use die::*; pub use e3_request_complete::*; pub use e3_requested::*; use e3_utils::{colorize, colorize_event_ids, Color}; +pub use enable_effects::*; pub use enclave_error::*; pub use encryption_key_collection_failed::*; pub use encryption_key_created::*; -pub use evm_sync_events_received::*; pub use keyshare_created::*; pub use net_sync_events_received::*; pub use operator_activation_changed::*; @@ -209,12 +209,12 @@ pub enum EnclaveEventData { ComputeRequest(ComputeRequest), // ComputeRequested ComputeResponse(ComputeResponse), // ComputeResponseReceived ComputeRequestError(ComputeRequestError), // ComputeRequestFailed - OutgoingSyncRequested(OutgoingSyncRequested), NetSyncEventsReceived(NetSyncEventsReceived), - EvmSyncEventsReceived(EvmSyncEventsReceived), - SyncStart(SyncStart), + HistoricalEvmSyncStart(HistoricalEvmSyncStart), + HistoricalNetSyncStart(HistoricalNetSyncStart), SyncEffect(SyncEffect), SyncEnd(SyncEnd), + EnableEffects(EnableEffects), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -494,12 +494,12 @@ impl_event_types!( ComputeRequest, ComputeResponse, ComputeRequestError, - OutgoingSyncRequested, NetSyncEventsReceived, - EvmSyncEventsReceived, - SyncStart, + HistoricalEvmSyncStart, + HistoricalNetSyncStart, SyncEffect, - SyncEnd + SyncEnd, + EnableEffects ); impl TryFrom<&EnclaveEvent> for EnclaveError { diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 388db36859..89993094ed 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -4,33 +4,37 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use super::EvmSyncEventsReceived; +use crate::AggregateId; use crate::{EvmEventConfig, EvmEventConfigChain}; use actix::{Message, Recipient}; use anyhow::Context; use anyhow::Result; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use std::fmt::{self, Display}; +use super::EnclaveEvent; +use super::Unsequenced; + /// Dispatched by the Sync actor when initial data is read and the sync process needs to be started #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct SyncStart { +pub struct HistoricalEvmSyncStart { /// The initial information for reading historical events from chains. This is generated from /// from persisted information pub evm_config: EvmEventConfig, #[serde(skip)] /// We include the sender here so that the evm can communicate directly with the sync actor - pub sender: Option>, // Must be Option to allow serde deserialize on - // EnclaveEvent as Default is required to be - // implemented this is fine as this event is never - // shared + pub sender: Option>, // Must be Option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented this is fine as this event is never + // shared } -impl SyncStart { +impl HistoricalEvmSyncStart { pub fn new( - sender: impl Into>, + sender: impl Into>, evm_config: EvmEventConfig, ) -> Self { Self { @@ -48,7 +52,75 @@ impl SyncStart { } } -impl Display for SyncStart { +impl Display for HistoricalEvmSyncStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +/// Dispatched by the Sync actor when initial data is read and the sync process needs to be started +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct HistoricalNetSyncStart { + pub since: BTreeMap, + #[serde(skip)] + /// We include the sender here so that the evm can communicate directly with the sync actor + pub sender: Option>, // Must be Option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented this is fine as this event is never + // shared +} + +impl HistoricalNetSyncStart { + pub fn new( + sender: impl Into>, + since: BTreeMap, + ) -> Self { + Self { + since, + sender: Some(sender.into()), + } + } +} + +impl Display for HistoricalNetSyncStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct HistoricalEvmEventsReceived { + pub events: Vec>, + pub chain_id: u64, +} + +impl HistoricalEvmEventsReceived { + pub fn new(events: Vec>, chain_id: u64) -> Self { + Self { events, chain_id } + } +} + +impl Display for HistoricalEvmEventsReceived { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct HistoricalNetEventsReceived { + pub events: Vec>, +} + +impl HistoricalNetEventsReceived { + pub fn new(events: Vec>) -> Self { + Self { events } + } +} + +impl Display for HistoricalNetEventsReceived { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 55e1ee7acd..c18380cdf5 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -9,10 +9,11 @@ use std::{fmt, ops::Deref}; use serde::{Deserialize, Serialize}; use crate::{ - E3id, EventContextAccessors, EventContextSeq, EventId, SeqState, Sequenced, Unsequenced, + E3id, EnclaveEventData, EventContextAccessors, EventContextSeq, EventId, SeqState, Sequenced, + Unsequenced, }; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Ord, PartialOrd, Eq, Hash, Serialize, Deserialize)] pub struct AggregateId(usize); impl AggregateId { @@ -122,6 +123,16 @@ impl EventContext { ) } + pub fn with_ts(mut self, ts: u128) -> Self { + self.ts = ts; + self + } + + pub fn with_aggregate(mut self, aggregate_id: AggregateId) -> Self { + self.aggregate_id = aggregate_id; + self + } + pub fn sequence(self, value: u64) -> EventContext { EventContext:: { seq: value, @@ -135,6 +146,13 @@ impl EventContext { } } +impl From for EventContext { + fn from(value: EnclaveEventData) -> Self { + let id = EventId::hash(value); + EventContext::::new_origin(id, 0, AggregateId::new(0), Some(0)) + } +} + impl EventContextAccessors for EventContext { fn id(&self) -> EventId { self.id diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 929c264ec8..cf2ac353fb 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -63,6 +63,7 @@ impl GetEventsAfterRequest { } } +/// The response of a request to get all EventStore events by either sequence or timestamp #[derive(Message, Debug)] #[rtype("()")] pub struct GetEventsAfterResponse { @@ -74,9 +75,11 @@ impl GetEventsAfterResponse { pub fn new(id: CorrelationId, events: Vec) -> Self { Self { id, events } } - pub fn events(&self) -> &Vec { - &self.events + + pub fn into_events(self) -> Vec { + self.events } + pub fn id(&self) -> CorrelationId { self.id } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index fb38381702..77aa85b475 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -49,7 +49,7 @@ impl EventStoreRouter { Ok(()) } - pub fn handle_get_events_after(&mut self, msg: GetAggregateEventsAfter) -> Result<()> { + pub fn handle_get_events_after(&mut self, msg: GetEventsAfterTs) -> Result<()> { for (aggregate_id, ts) in msg.ts() { if let Some(store_addr) = self.stores.get(&aggregate_id) { let get_events_msg = @@ -59,6 +59,17 @@ impl EventStoreRouter { } Ok(()) } + + // pub fn handle_get_events_after_seq(&mut self, msg: GetEventsAfterSeq) -> Result<()> { + // for (aggregate_id, seq) in msg.seq() { + // if let Some(store_addr) = self.stores.get(&aggregate_id) { + // let get_events_msg = + // GetEventsAfterRequest::new(msg.id(), seq.to_owned(), msg.sender.clone()); + // store_addr.do_send(get_events_msg); + // } + // } + // Ok(()) + // } } impl Actor for EventStoreRouter { @@ -79,25 +90,35 @@ impl Handler for EventStoreR } } -impl Handler for EventStoreRouter { +impl Handler for EventStoreRouter { type Result = (); - fn handle(&mut self, msg: GetAggregateEventsAfter, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: GetEventsAfterTs, _: &mut Self::Context) -> Self::Result { if let Err(e) = self.handle_get_events_after(msg) { error!("Failed to route get events after request: {}", e); } } } +impl Handler for EventStoreRouter { + type Result = (); + + fn handle(&mut self, msg: GetEventsAfterSeq, _: &mut Self::Context) -> Self::Result { + // if let Err(e) = self.handle_get_events_after_seq(msg) { + // error!("Failed to route get events after request: {}", e); + // } + } +} + #[derive(Message, Debug)] #[rtype("()")] -pub struct GetAggregateEventsAfter { - pub correlation_id: CorrelationId, - pub ts: HashMap, - pub sender: Recipient, +pub struct GetEventsAfterTs { + correlation_id: CorrelationId, + ts: HashMap, + sender: Recipient, } -impl GetAggregateEventsAfter { +impl GetEventsAfterTs { pub fn new( correlation_id: CorrelationId, ts: HashMap, @@ -117,4 +138,43 @@ impl GetAggregateEventsAfter { pub fn ts(&self) -> &HashMap { &self.ts } + + pub fn sender(self) -> Recipient { + self.sender + } +} + +/// A request to get all events from all aggregates after a specific set of sequences +#[derive(Message, Debug)] +#[rtype("()")] +pub struct GetEventsAfterSeq { + correlation_id: CorrelationId, + seq: HashMap, + sender: Recipient, +} + +impl GetEventsAfterSeq { + pub fn new( + correlation_id: CorrelationId, + seq: HashMap, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + seq, + sender: sender.into(), + } + } + + pub fn id(&self) -> CorrelationId { + self.correlation_id + } + + pub fn seq(&self) -> &HashMap { + &self.seq + } + + pub fn sender(self) -> Recipient { + self.sender + } } diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index cac5157f00..55ff422853 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -37,11 +37,13 @@ impl EvmEventConfig { config: BTreeMap::new(), } } + pub fn from_config(config: impl Into>) -> Self { EvmEventConfig { config: config.into(), } } + pub fn get(&self, chain_id: &ChainId) -> Option<&EvmEventConfigChain> { self.config.get(&chain_id) } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index ecdd0d56b7..47210f9168 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -87,7 +87,11 @@ pub trait EventPublisher { /// This method should be used for events that have originated locally. /// /// The ctx parameter is to pass on the current context to the local event. - fn publish(&self, data: impl Into, ctx: EventContext) -> Result<()>; + fn publish( + &self, + data: impl Into, + caused_by: impl Into>, + ) -> Result<()>; /// This creates a context based on the given data. This should only be used when an event is /// the origin event and does not originate remotely. This is also useful in tests. fn publish_without_context(&self, data: impl Into) -> Result<()>; @@ -112,7 +116,7 @@ pub trait EventPublisher { &self, data: impl Into, remote_ts: u128, - caused_by: EventContext, + caused_by: impl Into>, block: Option, ) -> Result<()>; /// Dispatch the given event without applying any HLC transformation. diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index d837781d14..06e7731502 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -15,7 +15,7 @@ use anyhow::{bail, Context}; use e3_events::EType; use e3_events::{ trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, - EvmSyncEventsReceived, SyncEnd, SyncStart, Unsequenced, + HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnd, Unsequenced, }; use e3_events::{Event, EventPublisher}; use e3_utils::MAILBOX_LIMIT; @@ -30,7 +30,7 @@ pub struct EvmChainGateway { #[derive(Clone, Default, Debug)] struct ForwardToSyncActorData { - pub sender: Option>, + pub sender: Option>, pub buffer: Vec>, } @@ -45,7 +45,7 @@ impl ForwardToSyncActorData { enum SyncStatus { /// Intial State Init(Vec>), // Include a buffer to hold events that arrive too early - /// After SyncStart we forward all events to SyncActor + /// After HistoricalEvmSyncStart we forward all events to SyncActor ForwardToSyncActor(ForwardToSyncActorData), /// Once the chain has completed historical sync then we buffer all "live" events until sync is /// complete @@ -63,7 +63,7 @@ impl Default for SyncStatus { impl SyncStatus { pub fn forward_to_sync_actor( &mut self, - sender: Recipient, + sender: Recipient, ) -> Result>> { let Self::Init(buffer) = self else { bail!( @@ -114,16 +114,18 @@ impl EvmChainGateway { pub fn setup(bus: &BusHandle) -> Addr { let addr = Self::new(bus).start(); bus.subscribe_all( - &[EventType::SyncStart, EventType::SyncEnd], + &[EventType::HistoricalEvmSyncStart, EventType::SyncEnd], addr.clone().recipient(), ); addr } - fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { - // Received a SyncStart event from the event bus. Get the sender within that event and forward + fn handle_sync_start(&mut self, msg: HistoricalEvmSyncStart) -> Result<()> { + // Received a HistoricalEvmSyncStart event from the event bus. Get the sender within that event and forward // all events to that actor - let sender = msg.sender.context("No sender on SyncStart Message")?; + let sender = msg + .sender + .context("No sender on HistoricalEvmSyncStart Message")?; let mut buffer = self.status.forward_to_sync_actor(sender)?; // Drain any events that were buffered early for evt in buffer.drain(..) { @@ -164,7 +166,7 @@ impl EvmChainGateway { let sender = state .sender .context("ForwardToSyncActor state must hold a sender")?; - let event = EvmSyncEventsReceived::new(state.buffer, event.chain_id); + let event = HistoricalEvmEventsReceived::new(state.buffer, event.chain_id); sender.try_send(event)?; Ok(()) } @@ -192,7 +194,7 @@ impl Handler for EvmChainGateway { fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Evm, &self.bus.with_ec(msg.get_ctx()), || { match msg.into_data() { - EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, + EnclaveEventData::HistoricalEvmSyncStart(e) => self.handle_sync_start(e)?, EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e)?, _ => (), } @@ -221,16 +223,16 @@ mod tests { use tracing_subscriber::{fmt, EnvFilter}; struct SyncEventCollector { - tx: mpsc::UnboundedSender, + tx: mpsc::UnboundedSender, } impl Actor for SyncEventCollector { type Context = actix::Context; } - impl Handler for SyncEventCollector { + impl Handler for SyncEventCollector { type Result = (); - fn handle(&mut self, msg: EvmSyncEventsReceived, _: &mut Self::Context) { + fn handle(&mut self, msg: HistoricalEvmEventsReceived, _: &mut Self::Context) { let _ = self.tx.send(msg); } } @@ -256,10 +258,10 @@ mod tests { let chain_id = 1u64; - // SyncStart: Init -> ForwardToSyncActor + // HistoricalEvmSyncStart: Init -> ForwardToSyncActor let mut evm_config = EvmEventConfig::new(); evm_config.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_without_context(SyncStart::new(collector.clone(), evm_config)) + bus.publish_without_context(HistoricalEvmSyncStart::new(collector.clone(), evm_config)) .unwrap(); // Send EVM event while forwarding - should reach collector @@ -271,7 +273,7 @@ mod tests { chain_id, ); - // This will actually arrive earlier than SyncStart but aught to be buffered + // This will actually arrive earlier than HistoricalEvmSyncStart but aught to be buffered addr.send(EnclaveEvmEvent::Event(evm_event)).await?; // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive @@ -334,7 +336,7 @@ mod tests { assert_eq!( event_types, vec![ - "SyncStart", + "HistoricalEvmSyncStart", "TestEvent", "SyncEnd", "TestEvent", diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs index 2aa0748984..9bcebdc53b 100644 --- a/crates/evm/src/sync_start_extractor.rs +++ b/crates/evm/src/sync_start_extractor.rs @@ -5,19 +5,19 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Addr, Handler, Recipient}; -use e3_events::{EnclaveEvent, EnclaveEventData, Event, SyncStart}; +use e3_events::{EnclaveEvent, EnclaveEventData, Event, HistoricalEvmSyncStart}; use e3_utils::MAILBOX_LIMIT; pub struct SyncStartExtractor { - dest: Recipient, + dest: Recipient, } impl SyncStartExtractor { - pub fn new(dest: impl Into>) -> Self { + pub fn new(dest: impl Into>) -> Self { Self { dest: dest.into() } } - pub fn setup(dest: impl Into>) -> Addr { + pub fn setup(dest: impl Into>) -> Addr { Self::new(dest).start() } } @@ -31,7 +31,7 @@ impl Actor for SyncStartExtractor { impl Handler for SyncStartExtractor { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - if let EnclaveEventData::SyncStart(evt) = msg.into_data() { + if let EnclaveEventData::HistoricalEvmSyncStart(evt) = msg.into_data() { self.dest.do_send(evt) } } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 4b26937ec7..cd2ece9e7c 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -17,7 +17,8 @@ use anyhow::Result; use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEventConfig, - EvmEventConfigChain, EvmSyncEventsReceived, GetEvents, SyncEnd, SyncStart, TestEvent, + EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnd, + TestEvent, }; use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; use std::{sync::Arc, time::Duration}; @@ -73,9 +74,13 @@ impl FakeSyncActor { } } -impl Handler for FakeSyncActor { +impl Handler for FakeSyncActor { type Result = (); - fn handle(&mut self, mut msg: EvmSyncEventsReceived, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + mut msg: HistoricalEvmEventsReceived, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { for evt in msg.events.drain(..) { self.bus.naked_dispatch(evt); @@ -117,11 +122,11 @@ async fn evm_reader() -> Result<()> { }) .build(); - // SyncStart holds initialization information such as start block and earliest event + // HistoricalEvmSyncStart holds initialization information such as start block and earliest event // This should trigger all chains to start to sync let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_without_context(SyncStart::new(sync, evm_info))?; + bus.publish_without_context(HistoricalEvmSyncStart::new(sync, evm_info))?; sleep(Duration::from_secs(1)).await; contract @@ -199,7 +204,7 @@ async fn ensure_historical_events() -> Result<()> { .build(); let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_without_context(SyncStart::new(sync, evm_info))?; + bus.publish_without_context(HistoricalEvmSyncStart::new(sync, evm_info))?; for msg in live_events.clone() { contract diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 8fae908568..f7099853fa 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -8,8 +8,8 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, GetAggregateEventsAfter, GetEventsAfterResponse, NetSyncEventsReceived, - OutgoingSyncRequested, TypedEvent, Unsequenced, + EnclaveEventData, EventType, GetEventsAfterResponse, GetEventsAfterTs, HistoricalNetSyncStart, + NetSyncEventsReceived, OutgoingSyncRequested, TypedEvent, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake, MAILBOX_LIMIT}; use futures::TryFutureExt; @@ -31,7 +31,7 @@ pub struct NetSyncManager { /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so /// that we do not do excessive resubscribes without actually listening for events. rx: Arc>, - eventstore: Recipient, + eventstore: Recipient, requests: HashMap>>, } @@ -40,7 +40,7 @@ impl NetSyncManager { bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, - eventstore: Recipient, + eventstore: Recipient, ) -> Self { Self { bus: bus.clone(), @@ -56,11 +56,11 @@ impl NetSyncManager { bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, - eventstore: Recipient, + eventstore: Recipient, ) -> Addr { let mut events = rx.resubscribe(); let addr = Self::new(bus, tx, rx, eventstore).start(); - + bus.subscribe(EventType::HistoricalNetSyncStart, addr.clone().recipient()); // Forward from NetEvent tokio::spawn({ debug!("Spawning event receive loop!"); @@ -95,18 +95,18 @@ impl Handler for NetSyncManager { let (msg, ec) = msg.into_components(); match msg { // We are making a sync request of another node - EnclaveEventData::OutgoingSyncRequested(data) => ctx.notify(TypedEvent::new(data, ec)), + EnclaveEventData::HistoricalNetSyncStart(data) => ctx.notify(TypedEvent::new(data, ec)), _ => (), } } } /// SyncRequest is called on start up to fetch remote events -impl Handler> for NetSyncManager { +impl Handler> for NetSyncManager { type Result = ResponseFuture<()>; fn handle( &mut self, - msg: TypedEvent, + msg: TypedEvent, ctx: &mut Self::Context, ) -> Self::Result { trap_fut( @@ -154,7 +154,7 @@ impl Handler for NetSyncManager { trap(EType::Net, &self.bus, || { let id = CorrelationId::new(); self.requests.insert(id, msg.channel); - self.eventstore.try_send(GetAggregateEventsAfter::new( + self.eventstore.try_send(GetEventsAfterTs::new( id, msg.value.since, ctx.address().recipient(), @@ -176,9 +176,8 @@ impl Handler for NetSyncManager { self.tx.try_send(NetCommand::SyncResponse { value: SyncResponseValue { events: msg - .events() + .into_events() .into_iter() - .cloned() .map(|ev| ev.try_into()) .collect::>()?, ts: self.bus.ts()?, // NOTE: We are storing a local timestamp on this response @@ -219,7 +218,7 @@ async fn sync_request( async fn handle_sync_request_event( net_cmds: mpsc::Sender, net_events: Arc>, - event: TypedEvent, + event: TypedEvent, address: impl Into>>, ) -> Result<()> { let (event, ctx) = event.into_components(); diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index fd3289b711..268d7b1aa3 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -12,6 +12,7 @@ anyhow.workspace = true e3-events.workspace = true e3-data.workspace = true e3-config.workspace = true +e3-utils.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index bf1b52108c..daaaf6ce9f 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -5,186 +5,128 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::SyncRepositoryFactory; -use actix::{Actor, Addr, AsyncContext, Handler, Message}; -use anyhow::{Context, Result}; +use actix::{Message, Recipient}; +use anyhow::Result; use e3_data::Repositories; use e3_events::{ - trap, trap_fut, AggregateConfig, AggregateId, BusHandle, EType, EnclaveEvent, + AggregateConfig, AggregateId, BusHandle, CorrelationId, EnableEffects, EnclaveEvent, EventContextAccessors, EventPublisher, EvmEventConfig, EvmEventConfigChain, - EvmSyncEventsReceived, SnapshotBuffer, SyncEnd, Unsequenced, + GetEventsAfterResponse, GetEventsAfterSeq, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, + HistoricalNetEventsReceived, HistoricalNetSyncStart, SyncEnd, Unsequenced, }; -use std::collections::{BTreeMap, HashSet}; -use tracing::info; - -// NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p -type ChainId = u64; - -/// Manage the synchronization of events across. -pub struct Synchronizer { - bus: BusHandle, - evm_config: Option, - evm_events: Vec>, - evm_to_sync: HashSet, - repositories: Repositories, - snapshot_buffer: Addr, - // net_config: NetEventConfig, - aggregate_config: AggregateConfig, -} +use e3_utils::actix::channel as actix_toolbox; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + time::Duration, +}; +use tokio::{sync::mpsc::Receiver, time::timeout}; + +pub async fn sync( + bus: &BusHandle, + default_config: &EvmEventConfig, + repositories: &Repositories, + aggregate_config: &AggregateConfig, + eventstore: &Recipient, +) -> Result<()> { + // 1. Load snapsshot metadata + let snapshot = + SnapshotMeta::read_from_disk(aggregate_config.aggregates(), default_config, repositories) + .await?; -impl Synchronizer { - pub fn new( - bus: BusHandle, - evm_config: EvmEventConfig, - repositories: Repositories, - aggregate_config: AggregateConfig, - snapshot_buffer: Addr, - ) -> Self { - let evm_to_sync = evm_config.chains(); - Self { - evm_config: Some(evm_config), - bus, - evm_to_sync, - evm_events: Vec::new(), - repositories, - aggregate_config, - snapshot_buffer, - } + // 2. Determine the evm blocks to read from based on the SnapshotMeta + let evm_config = snapshot.to_evm_config(); + let net_config = snapshot.to_net_config(); + + // 3. Load EventStore events since the sequence number found in the snapshot into memory. + let (tx, rx) = actix_toolbox::oneshot::(); + eventstore.try_send(GetEventsAfterSeq::new( + CorrelationId::new(), + snapshot.to_sequence_map(), + tx, + ))?; + let events = rx.await?.into_events(); + + // 4. Replay the EventStore events to all listeners (except effects) + for event in events { + bus.event_bus().try_send(event)?; } - pub fn setup( - bus: &BusHandle, - evm_config: &EvmEventConfig, - repositories: &Repositories, - aggregate_config: &AggregateConfig, - snapshot_buffer: &Addr, - ) -> Addr { - Self::new( - bus.clone(), - evm_config.clone(), - repositories.clone(), - aggregate_config.clone(), - snapshot_buffer.clone(), - ) - .start() - } + // TODO: Detect open loops - incase we crashed in the middle of a request we need to play the + // request event again once effects are on - fn handle_evm_sync_events_received(&mut self, mut msg: EvmSyncEventsReceived) -> Result<()> { - let chain_id = msg.chain_id; - info!("handle sync complete for chain({})", chain_id); - self.evm_to_sync.remove(&chain_id); - self.evm_events.append(&mut msg.events); - info!("{} chains left to sync...", self.evm_to_sync.len()); - if self.evm_to_sync.is_empty() { - self.sort_and_finalize()?; - } - Ok(()) - } + // 5. Load the historical evm events to memory from all chains + let (addr, rx) = actix_toolbox::mpsc::(256); + bus.publish_without_context(HistoricalEvmSyncStart::new(addr, evm_config.clone()))?; + let historical_evm_events = + collect_historical_evm_events(rx, &evm_config, Duration::from_secs(30)).await; - fn sort_and_finalize(&mut self) -> Result<()> { - info!("all chains synced draining to bus and running sync end"); - // Order all events (theoretically) - self.evm_events.sort_by_key(|i| i.ts()); + // 6. Load the historical libp2p events to memory + let (addr, rx) = actix_toolbox::oneshot::(); + bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; + let historical_net_events = rx.await?.events; - // publish them in order - for evt in self.evm_events.drain(..) { - self.bus.naked_dispatch(evt); - } - self.bus.publish_without_context(SyncEnd::new())?; - Ok(()) - } -} + // 7. Sort both the evm and libp2p events together by HLC timestamp + let mut historical = historical_evm_events + .into_iter() + .chain(historical_net_events) + .collect::>(); -impl Actor for Synchronizer { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.notify(Bootstrap); - } -} + historical.sort_by_key(|event| event.ts()); -impl Handler for Synchronizer { - type Result = (); - fn handle(&mut self, msg: EvmSyncEventsReceived, _: &mut Self::Context) -> Self::Result { - trap(EType::Sync, &self.bus.clone(), || { - self.handle_evm_sync_events_received(msg)?; - Ok(()) - }) - } -} + // 8. Enable effects + bus.publish_without_context(EnableEffects::new())?; -impl Handler for Synchronizer { - type Result = actix::ResponseFuture<()>; - fn handle(&mut self, _: Bootstrap, ctx: &mut Self::Context) -> Self::Result { - let address = ctx.address(); - let repositories = self.repositories.clone(); - let evm_config = self.evm_config.take(); - let aggregates = self.aggregate_config.aggregates(); - let bus = self.bus.clone(); - trap_fut( - EType::Sync, - &self.bus.clone(), - handle_bootstrap(bus, address, repositories, evm_config, aggregates), - ) + // 9. Publish the new sorted events to the eventstore + for event in historical { + bus.naked_dispatch(event); } -} - -async fn handle_bootstrap( - bus: BusHandle, - address: Addr, - repositories: Repositories, - evm_config: Option, - aggregates: Vec, -) -> Result<()> { - let evm_config = evm_config - .context("EvmEventConfig was not set likely Bootstrap was called more than once.")?; - // ============================================================ - // Phase 1: Load Snapshot - // ============================================================ - - // 1.1 Read snapshot from disk (may not exist on first boot) - let snapshot = SnapshotMeta::read_from_disk(aggregates, evm_config, repositories).await?; - - // 1.2 Extract state, last_applied_hlc, last_block_number (use defaults if no snapshot) - - // 1.4 Pause WriteBuffer streaming (don't write replayed mutations to disk) - - // ============================================================ - // Phase 2: Replay Missed Events - // ============================================================ - // 2.1 Query EventStore for all events WHERE hlc > last_applied_hlc ORDER BY hlc + bus.publish_without_context(SyncEnd::new())?; - // 2.2 For each event: - // - Route to appropriate actor by aggregate_id - // - Apply event mutation (in-memory only) - // - Track highest block_number seen (if event has one) + // normal live operations - // ============================================================ - // Phase 3: Determine Blockchain Resume Point - // ============================================================ - - // 3.1 Get highest block_number from replayed events (if any had block numbers) - - // 3.2 Fall back to snapshot.last_block_number if no blockchain events replayed - - // 3.3 Calculate resume_block = max(config.deploy_block, highest_block + 1) - - // ============================================================ - // Phase 4: Resume Normal Operation - // ============================================================ - - // 4.1 Update WriteBuffer watermark to current HLC + Ok(()) +} - // 4.2 Resume WriteBuffer streaming (mutations now flow to disk) +pub async fn collect_historical_evm_events( + mut receiver: Receiver, + config: &EvmEventConfig, + max_dur: Duration, +) -> Vec> { + // Get expected chain IDs from config + let expected = config.chains(); + let mut received = HashSet::new(); + let mut results = Vec::new(); + + let fut = async { + while received.len() < expected.len() { + if let Some(mut msg) = receiver.recv().await { + // Only accept messages for expected chains we haven't received yet + if expected.contains(&msg.chain_id) && !received.contains(&msg.chain_id) { + received.insert(msg.chain_id); + results.append(&mut msg.events); + } + } else { + // Channel closed before receiving all expected chains + break; + } + } + }; - // 4.3 Subscribe to blockchain from resume_block + if let Err(_) = timeout(max_dur, fut).await { + for chain_id in expected.difference(&received) { + eprintln!( + "Error: Timeout waiting for historical events from chain {}", + chain_id + ); + } + } - // 4.4 Return Ok / start event loop - // Get the sequences for each aggregate - // bus.publish_without_context(SyncStart::new(address, aggregate_states.as_evm_config()))?; - Ok(()) + results } /// Latest event information in store +#[derive(Clone)] pub struct AggregateState { ts: u128, aggregate_id: AggregateId, @@ -192,15 +134,17 @@ pub struct AggregateState { block: u64, } -struct SnapshotMeta { +#[derive(Clone)] +pub struct SnapshotMeta { aggregate_state: Vec, } impl SnapshotMeta { + /// Load the SnapshotMeta from the Snapshot on disk pub async fn read_from_disk( ids: Vec, - initial_evm_config: EvmEventConfig, - repositories: Repositories, + initial_evm_config: &EvmEventConfig, + repositories: &Repositories, ) -> Result { let mut aggregate_state = Vec::new(); for aggregate_id in ids { @@ -226,7 +170,8 @@ impl SnapshotMeta { Ok(Self { aggregate_state }) } - pub fn as_evm_config(&self) -> EvmEventConfig { + /// Return an EvmEventConfig based on the SnapshotMeta + pub fn to_evm_config(&self) -> EvmEventConfig { let map: BTreeMap = self .aggregate_state .iter() @@ -241,141 +186,172 @@ impl SnapshotMeta { .collect(); EvmEventConfig::from_config(map) } -} - -#[derive(Message)] -#[rtype("()")] -pub struct Bootstrap; - -#[cfg(test)] -mod tests { - use super::*; - use e3_ciphernode_builder::EventSystem; - use e3_events::{EnclaveEvent, EventFactory}; - use e3_events::{ - EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, TestEvent, - }; - use std::collections::HashMap; - use std::time::Duration; - use tokio::time::sleep; - - fn hlc_faucet(bus: &BusHandle, num: usize) -> Result> { - let mut queue = Vec::new(); - for _ in 0..num { - queue.push(bus.ts()?) - } - - Ok(queue.into_iter()) - } - async fn settle() { - sleep(Duration::from_millis(100)).await; + pub fn to_net_config(&self) -> BTreeMap { + self.aggregate_state + .iter() + .map(|s| (s.aggregate_id, s.ts)) + .collect() } - #[actix::test] - #[ignore] - async fn test_synchronizer_full_flow() -> Result<()> { - let _guard = e3_test_helpers::with_tracing("info"); - // Setup event system and synchronizer - let system = EventSystem::new("test").with_fresh_bus(); - let bus: BusHandle = system.handle()?; - let history_collector = bus.history(); - - // Configure test chains - let mut evm_config = EvmEventConfig::new(); - evm_config.insert(1, EvmEventConfigChain::new(0)); - evm_config.insert(2, EvmEventConfigChain::new(0)); - let repositories = Repositories::in_mem(); - let snapshot_buffer = system.buffer()?; - // Start synchronizer - let sync_addr = Synchronizer::setup( - &bus, - &evm_config, - &repositories, - &AggregateConfig::new(HashMap::new()), - &snapshot_buffer, - ); - settle().await; - - // Verify SyncStart was published - let history = history_collector - .send(GetEvents::::new()) - .await?; - let sync_start_count = history - .into_iter() - .filter(|e| matches!(e.get_data(), EnclaveEventData::SyncStart(_))) - .count(); - assert!(sync_start_count > 0, "SyncStart should be dispatched"); - - // Create test events with timestamps - let mut timelord = hlc_faucet(&bus, 100)?; - - // Test events - timestamps generated in order - let h_2_1 = bus.event_from_remote_source( - EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), - None, - timelord.next().unwrap(), - Some(1), - )?; - - let h_1_1 = bus.event_from_remote_source( - EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), - None, - timelord.next().unwrap(), - Some(1), - )?; - - let h_1_2 = bus.event_from_remote_source( - EnclaveEventData::TestEvent(TestEvent::new("1-second", 2)), - None, - timelord.next().unwrap(), - Some(2), - )?; - - let h_2_2 = bus.event_from_remote_source( - EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), - None, - timelord.next().unwrap(), - Some(2), - )?; - - // Send events in mixed order to test sorting - sync_addr - .send(EvmSyncEventsReceived::new(vec![h_2_2, h_2_1], 2)) - .await?; - sync_addr - .send(EvmSyncEventsReceived::new(vec![h_1_1, h_1_2], 1)) - .await?; - - settle().await; - - // Get final event history and verify ordering - let full = history_collector - .send(GetEvents::::new()) - .await?; - println!("full = {}", full.len()); - let events: Vec = full - .into_iter() - .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) - .collect(); - - let event_strings: Vec = events - .into_iter() - .filter_map(|e| { - if let EnclaveEventData::TestEvent(data) = e.into_data() { - Some(data.msg) - } else { - None - } + /// Return a map between AggregateIds and Sequence + pub fn to_sequence_map(&self) -> HashMap { + self.aggregate_state + .iter() + .fold(HashMap::new(), |mut acc, item| { + acc.insert(item.aggregate_id, item.seq); + acc }) - .collect(); + } +} - // Events should be published in timestamp order - assert_eq!( - event_strings, - vec!["2-first", "1-first", "1-second", "2-second"] - ); +#[derive(Message)] +#[rtype("()")] +pub struct Bootstrap; - Ok(()) +#[derive(Message)] +#[rtype("()")] +pub struct SnapshotLoaded { + pub snapshot: SnapshotMeta, +} +impl SnapshotLoaded { + pub fn new(snapshot: SnapshotMeta) -> Self { + Self { snapshot } } } + +// #[cfg(test)] +// mod tests { +// use super::*; +// use e3_ciphernode_builder::EventSystem; +// use e3_events::{EnclaveEvent, EventFactory}; +// use e3_events::{ +// EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, TestEvent, +// }; +// use std::collections::HashMap; +// use std::time::Duration; +// use tokio::time::sleep; +// +// fn hlc_faucet(bus: &BusHandle, num: usize) -> Result> { +// let mut queue = Vec::new(); +// for _ in 0..num { +// queue.push(bus.ts()?) +// } +// +// Ok(queue.into_iter()) +// } +// +// async fn settle() { +// sleep(Duration::from_millis(100)).await; +// } +// +// #[actix::test] +// #[ignore] +// async fn test_synchronizer_full_flow() -> Result<()> { +// let _guard = e3_test_helpers::with_tracing("info"); +// // Setup event system and synchronizer +// let system = EventSystem::new("test").with_fresh_bus(); +// let bus: BusHandle = system.handle()?; +// let history_collector = bus.history(); +// +// // Configure test chains +// let mut evm_config = EvmEventConfig::new(); +// evm_config.insert(1, EvmEventConfigChain::new(0)); +// evm_config.insert(2, EvmEventConfigChain::new(0)); +// let repositories = Repositories::in_mem(); +// let snapshot_buffer = system.buffer()?; +// // Start synchronizer +// let sync_addr = Synchronizer::setup( +// &bus, +// &evm_config, +// &repositories, +// &AggregateConfig::new(HashMap::new()), +// &snapshot_buffer, +// ); +// settle().await; +// +// // Verify HistoricalEvmSyncStart was published +// let history = history_collector +// .send(GetEvents::::new()) +// .await?; +// let sync_start_count = history +// .into_iter() +// .filter(|e| matches!(e.get_data(), EnclaveEventData::HistoricalEvmSyncStart(_))) +// .count(); +// assert!( +// sync_start_count > 0, +// "HistoricalEvmSyncStart should be dispatched" +// ); +// +// // Create test events with timestamps +// let mut timelord = hlc_faucet(&bus, 100)?; +// +// // Test events - timestamps generated in order +// let h_2_1 = bus.event_from_remote_source( +// EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), +// None, +// timelord.next().unwrap(), +// Some(1), +// )?; +// +// let h_1_1 = bus.event_from_remote_source( +// EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), +// None, +// timelord.next().unwrap(), +// Some(1), +// )?; +// +// let h_1_2 = bus.event_from_remote_source( +// EnclaveEventData::TestEvent(TestEvent::new("1-second", 2)), +// None, +// timelord.next().unwrap(), +// Some(2), +// )?; +// +// let h_2_2 = bus.event_from_remote_source( +// EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), +// None, +// timelord.next().unwrap(), +// Some(2), +// )?; +// +// // Send events in mixed order to test sorting +// sync_addr +// .send(HistoricalEvmEventsReceived::new(vec![h_2_2, h_2_1], 2)) +// .await?; +// sync_addr +// .send(HistoricalEvmEventsReceived::new(vec![h_1_1, h_1_2], 1)) +// .await?; +// +// settle().await; +// +// // Get final event history and verify ordering +// let full = history_collector +// .send(GetEvents::::new()) +// .await?; +// println!("full = {}", full.len()); +// let events: Vec = full +// .into_iter() +// .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) +// .collect(); +// +// let event_strings: Vec = events +// .into_iter() +// .filter_map(|e| { +// if let EnclaveEventData::TestEvent(data) = e.into_data() { +// Some(data.msg) +// } else { +// None +// } +// }) +// .collect(); +// +// // Events should be published in timestamp order +// assert_eq!( +// event_strings, +// vec!["2-first", "1-first", "1-second", "2-second"] +// ); +// +// Ok(()) +// } +// } diff --git a/crates/utils/src/actix/channel.rs b/crates/utils/src/actix/channel.rs new file mode 100644 index 0000000000..e1b93c538d --- /dev/null +++ b/crates/utils/src/actix/channel.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Context, Handler, Message}; +use tokio::sync::{ + mpsc, + oneshot::{self}, +}; + +// Oneshot == +pub struct Oneshot(Option>); + +impl Actor for Oneshot +where + M: Message + Send + 'static, +{ + type Context = actix::Context; +} + +impl Handler for Oneshot +where + M: Message + Send + 'static, +{ + type Result = (); + + fn handle(&mut self, m: M, _: &mut Context) -> Self::Result { + self.0.take().map(|s| s.send(m)); + } +} + +/// Return a oneshot channel where instead of a oneshot::Sender we use an actix::Recipient. +/// After the oneshot has completed sending messages to the address will noop +pub fn oneshot() -> (actix::Recipient, oneshot::Receiver) +where + M: Message + Send + 'static, +{ + let (tx, rx) = oneshot::channel(); + (Oneshot(Some(tx)).start().recipient(), rx) +} + +// Mpsc == + +pub struct Mpsc(mpsc::Sender); + +impl Actor for Mpsc +where + M: Message + Send + 'static, +{ + type Context = actix::Context; +} + +impl Handler for Mpsc +where + M: Message + Send + 'static, +{ + type Result = (); + fn handle(&mut self, m: M, _: &mut Context) -> Self::Result { + let s = self.0.clone(); + actix::spawn(async move { + if let Err(e) = s.send(m).await { + eprintln!("Failed to send message: {}", e); + } + }); + } +} + +/// Return a mpsc channel where instead of a mpsc::Sender we use an actix::Recipient. +pub fn mpsc(buffer: usize) -> (actix::Recipient, mpsc::Receiver) +where + M: Message + Send + 'static, +{ + let (tx, rx) = mpsc::channel(buffer); + (Mpsc(tx).start().recipient(), rx) +} diff --git a/crates/utils/src/actix.rs b/crates/utils/src/actix/mod.rs similarity index 98% rename from crates/utils/src/actix.rs rename to crates/utils/src/actix/mod.rs index 63cd1dbc02..35edc6604d 100644 --- a/crates/utils/src/actix.rs +++ b/crates/utils/src/actix/mod.rs @@ -4,8 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; +pub mod channel; +use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; // Helper to allow for bail behaviour in actor model async handlers diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 5e9978adcc..857daf3a6d 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -13,7 +13,7 @@ pub mod helpers; pub mod path; pub mod retry; pub mod utility_types; -pub use actix::*; +pub use actix::NotifySync; pub use alloy::*; pub use constants::*; pub use error::*; From 69117b988b5f9dac74242f83b2ad0f1ae91ba67d Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 8 Feb 2026 23:41:39 +0000 Subject: [PATCH 41/63] fixup eventstore query events --- crates/ciphernode-builder/src/event_system.rs | 19 ++- crates/events/src/events.rs | 158 ++++++++++++++---- crates/events/src/eventstore.rs | 43 ++++- crates/events/src/eventstore_router.rs | 122 ++++---------- crates/net/src/net_sync_manager.rs | 16 +- crates/sync/src/sync.rs | 55 +++++- 6 files changed, 251 insertions(+), 162 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 9514a15c0e..eeda7a3cf0 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -14,7 +14,7 @@ use e3_data::{ use e3_events::hlc::Hlc; use e3_events::{ AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, - EventStoreRouter, GetEventsAfterSeq, InsertBatch, Sequencer, SnapshotBuffer, + EventStoreQueryBy, EventStoreRouter, InsertBatch, SeqAgg, Sequencer, SnapshotBuffer, StoreEventRequested, UpdateDestination, }; use e3_utils::enumerate_path; @@ -324,7 +324,7 @@ impl EventSystem { } } - pub fn eventstore_getter(&self) -> Result> { + pub fn eventstore_getter(&self) -> Result>> { info!("eventstore_reader..."); let eventstores = self.eventstore_addrs()?; match &eventstores { @@ -399,7 +399,7 @@ impl Actor for NoopBatchReceiver { impl Handler for NoopBatchReceiver { type Result = (); - fn handle(&mut self, msg: InsertBatch, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, _: InsertBatch, _: &mut Self::Context) -> Self::Result { // do nothing } } @@ -415,6 +415,7 @@ mod tests { use e3_events::StoreKeys; use e3_events::SyncEnd; use e3_events::Tick; + use e3_events::TsAgg; use e3_test_helpers::with_tracing; use std::time::Duration; use tracing::info; @@ -428,8 +429,8 @@ mod tests { use e3_events::CorrelationId; use e3_events::EnclaveEventData; + use e3_events::EventStoreQueryResponse; use e3_events::EventType; - use e3_events::GetEventsAfterResponse; use e3_events::TestEvent; use tempfile::TempDir; use tokio::time::sleep; @@ -479,9 +480,9 @@ mod tests { } } - impl Handler for Listener { + impl Handler for Listener { type Result = (); - fn handle(&mut self, msg: GetEventsAfterResponse, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { self.events = msg.into_events(); } } @@ -638,11 +639,11 @@ mod tests { let router = system.in_mem_eventstore_router()?; // Get all events after the given timestamp using the router - use e3_events::{AggregateId, GetEventsAfterTs}; + use e3_events::AggregateId; let mut ts_map = HashMap::new(); ts_map.insert(AggregateId::new(0), ts); - let get_events_msg = - GetEventsAfterTs::new(CorrelationId::new(), ts_map, listener.clone().into()); + let sender: Recipient = listener.clone().into(); + let get_events_msg = EventStoreQueryBy::::new(CorrelationId::new(), ts_map, sender); router.do_send(get_events_msg); sleep(Duration::from_millis(100)).await; diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index cf2ac353fb..d2e2c2df1e 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -4,9 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::collections::HashMap; + use actix::{Message, Recipient}; -use crate::{CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; +use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; /// Direct event received by the EventStore to store an event #[derive(Message, Debug)] @@ -28,70 +30,154 @@ impl StoreEventRequested { } } -/// Get events after timestamp in EventStore +/// The response of a request to get all EventStore events by either sequence or timestamp #[derive(Message, Debug)] #[rtype("()")] -pub struct GetEventsAfterRequest { +pub struct EventStoreQueryResponse { + id: CorrelationId, + events: Vec>, +} + +impl EventStoreQueryResponse { + pub fn new(id: CorrelationId, events: Vec) -> Self { + Self { id, events } + } + + pub fn into_events(self) -> Vec { + self.events + } + + pub fn id(&self) -> CorrelationId { + self.id + } +} + +/// Direct event received by the Sequencer once an event has been stored +#[derive(Message, Debug)] +#[rtype("()")] +pub struct StoreEventResponse(pub EnclaveEvent); + +impl StoreEventResponse { + pub fn into_event(self) -> EnclaveEvent { + self.0 + } +} + +/// Trait for various EventStore query types +pub trait QueryKind { + type Shape; +} + +/// Query by aggregated sequence +pub struct SeqAgg; +impl QueryKind for SeqAgg { + type Shape = HashMap; +} + +/// Query by aggregated timestamp +pub struct TsAgg; +impl QueryKind for TsAgg { + type Shape = HashMap; +} + +/// Query by timestamp +pub struct Ts; +impl QueryKind for Ts { + type Shape = u128; +} + +/// Query by seq +pub struct Seq; +impl QueryKind for Seq { + type Shape = u64; +} + +#[derive(Message, Debug)] +#[rtype("()")] +pub struct EventStoreQueryBy { correlation_id: CorrelationId, - ts: u128, - sender: Recipient, + query: Q::Shape, + sender: Recipient, } -impl GetEventsAfterRequest { +impl EventStoreQueryBy { pub fn new( correlation_id: CorrelationId, - ts: u128, - sender: impl Into>, + query: HashMap, + sender: impl Into>, ) -> Self { Self { correlation_id, - ts, + query, sender: sender.into(), } } - pub fn ts(&self) -> u128 { - self.ts + pub fn query(&self) -> &HashMap { + &self.query } +} - pub fn id(&self) -> CorrelationId { - self.correlation_id +impl EventStoreQueryBy { + pub fn new( + correlation_id: CorrelationId, + query: HashMap, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + query, + sender: sender.into(), + } } - pub fn sender(&self) -> &Recipient { - &self.sender + pub fn query(&self) -> &HashMap { + &self.query } } -/// The response of a request to get all EventStore events by either sequence or timestamp -#[derive(Message, Debug)] -#[rtype("()")] -pub struct GetEventsAfterResponse { - id: CorrelationId, - events: Vec>, -} +impl EventStoreQueryBy { + pub fn new( + correlation_id: CorrelationId, + query: u128, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + query, + sender: sender.into(), + } + } -impl GetEventsAfterResponse { - pub fn new(id: CorrelationId, events: Vec) -> Self { - Self { id, events } + pub fn query(&self) -> u128 { + self.query } +} - pub fn into_events(self) -> Vec { - self.events +impl EventStoreQueryBy { + pub fn new( + correlation_id: CorrelationId, + query: u64, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + query, + sender: sender.into(), + } } - pub fn id(&self) -> CorrelationId { - self.id + pub fn query(&self) -> u64 { + self.query } } -/// Direct event received by the Sequencer once an event has been stored -#[derive(Message, Debug)] -#[rtype("()")] -pub struct StoreEventResponse(pub EnclaveEvent); +impl EventStoreQueryBy { + pub fn id(&self) -> CorrelationId { + self.correlation_id + } -impl StoreEventResponse { - pub fn into_event(self) -> EnclaveEvent { - self.0 + pub fn sender(self) -> Recipient { + self.sender } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 1a82684992..41fb44fbf2 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -6,7 +6,8 @@ use crate::{ events::{StoreEventRequested, StoreEventResponse}, - EventContextAccessors, EventLog, GetEventsAfterRequest, GetEventsAfterResponse, SequenceIndex, + EventContextAccessors, EventLog, EventStoreQueryBy, EventStoreQueryResponse, Seq, + SequenceIndex, Ts, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; @@ -43,11 +44,12 @@ impl EventStore { Ok(()) } - pub fn handle_get_events_after(&mut self, msg: GetEventsAfterRequest) -> Result<()> { + pub fn handle_event_store_query_ts(&mut self, msg: EventStoreQueryBy) -> Result<()> { // if there are no events after the timestamp return an empty vector - let Some(seq) = self.index.seek(msg.ts())? else { + let id = msg.id(); + let Some(seq) = self.index.seek(msg.query())? else { msg.sender() - .try_send(GetEventsAfterResponse::new(msg.id(), vec![]))?; + .try_send(EventStoreQueryResponse::new(id, vec![]))?; return Ok(()); }; // read and return the events @@ -58,7 +60,23 @@ impl EventStore { .collect::>(); msg.sender() - .try_send(GetEventsAfterResponse::new(msg.id(), evts))?; + .try_send(EventStoreQueryResponse::new(id, evts))?; + Ok(()) + } + + pub fn handle_event_store_query_seq(&mut self, msg: EventStoreQueryBy) -> Result<()> { + // if there are no events after the timestamp return an empty vector + let id = msg.id(); + + // read and return the events + let evts = self + .log + .read_from(msg.query()) + .map(|(s, e)| e.into_sequenced(s)) + .collect::>(); + + msg.sender() + .try_send(EventStoreQueryResponse::new(id, evts))?; Ok(()) } } @@ -91,10 +109,19 @@ impl Handler for EventStore< } } -impl Handler for EventStore { +impl Handler> for EventStore { + type Result = (); + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_ts(msg) { + error!("{e}"); + } + } +} + +impl Handler> for EventStore { type Result = (); - fn handle(&mut self, msg: GetEventsAfterRequest, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_get_events_after(msg) { + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_seq(msg) { error!("{e}"); } } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 77aa85b475..7a9a3b4c14 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -6,11 +6,10 @@ use crate::eventstore::EventStore; use crate::{ - events::{GetEventsAfterRequest, StoreEventRequested}, - AggregateId, EventContextAccessors, EventLog, SequenceIndex, + events::StoreEventRequested, AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; -use crate::{CorrelationId, GetEventsAfterResponse}; -use actix::{Actor, Addr, Handler, Message, Recipient}; +use crate::{EventStoreQueryBy, Seq, SeqAgg, Ts, TsAgg}; +use actix::{Actor, Addr, Handler}; use anyhow::Result; use e3_utils::{major_issue, MAILBOX_LIMIT}; use std::collections::HashMap; @@ -49,27 +48,33 @@ impl EventStoreRouter { Ok(()) } - pub fn handle_get_events_after(&mut self, msg: GetEventsAfterTs) -> Result<()> { - for (aggregate_id, ts) in msg.ts() { + pub fn handle_event_store_query_ts(&mut self, msg: EventStoreQueryBy) -> Result<()> { + let id = msg.id(); + let query = msg.query().clone(); + let sender = msg.sender(); + for (aggregate_id, ts) in query { if let Some(store_addr) = self.stores.get(&aggregate_id) { let get_events_msg = - GetEventsAfterRequest::new(msg.id(), ts.to_owned(), msg.sender.clone()); + EventStoreQueryBy::::new(id, ts.to_owned(), sender.clone()); store_addr.do_send(get_events_msg); } } Ok(()) } - // pub fn handle_get_events_after_seq(&mut self, msg: GetEventsAfterSeq) -> Result<()> { - // for (aggregate_id, seq) in msg.seq() { - // if let Some(store_addr) = self.stores.get(&aggregate_id) { - // let get_events_msg = - // GetEventsAfterRequest::new(msg.id(), seq.to_owned(), msg.sender.clone()); - // store_addr.do_send(get_events_msg); - // } - // } - // Ok(()) - // } + pub fn handle_event_store_query_seq(&mut self, msg: EventStoreQueryBy) -> Result<()> { + let id = msg.id(); + let query = msg.query().clone(); + let sender = msg.sender(); + for (aggregate_id, ts) in query { + if let Some(store_addr) = self.stores.get(&aggregate_id) { + let get_events_msg = + EventStoreQueryBy::::new(id, ts.to_owned(), sender.clone()); + store_addr.do_send(get_events_msg); + } + } + Ok(()) + } } impl Actor for EventStoreRouter { @@ -90,91 +95,22 @@ impl Handler for EventStoreR } } -impl Handler for EventStoreRouter { +impl Handler> for EventStoreRouter { type Result = (); - fn handle(&mut self, msg: GetEventsAfterTs, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_get_events_after(msg) { + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_ts(msg) { error!("Failed to route get events after request: {}", e); } } } -impl Handler for EventStoreRouter { +impl Handler> for EventStoreRouter { type Result = (); - fn handle(&mut self, msg: GetEventsAfterSeq, _: &mut Self::Context) -> Self::Result { - // if let Err(e) = self.handle_get_events_after_seq(msg) { - // error!("Failed to route get events after request: {}", e); - // } - } -} - -#[derive(Message, Debug)] -#[rtype("()")] -pub struct GetEventsAfterTs { - correlation_id: CorrelationId, - ts: HashMap, - sender: Recipient, -} - -impl GetEventsAfterTs { - pub fn new( - correlation_id: CorrelationId, - ts: HashMap, - sender: Recipient, - ) -> Self { - Self { - correlation_id, - ts, - sender, - } - } - - pub fn id(&self) -> CorrelationId { - self.correlation_id - } - - pub fn ts(&self) -> &HashMap { - &self.ts - } - - pub fn sender(self) -> Recipient { - self.sender - } -} - -/// A request to get all events from all aggregates after a specific set of sequences -#[derive(Message, Debug)] -#[rtype("()")] -pub struct GetEventsAfterSeq { - correlation_id: CorrelationId, - seq: HashMap, - sender: Recipient, -} - -impl GetEventsAfterSeq { - pub fn new( - correlation_id: CorrelationId, - seq: HashMap, - sender: impl Into>, - ) -> Self { - Self { - correlation_id, - seq, - sender: sender.into(), + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_seq(msg) { + error!("Failed to route get events after request: {}", e); } } - - pub fn id(&self) -> CorrelationId { - self.correlation_id - } - - pub fn seq(&self) -> &HashMap { - &self.seq - } - - pub fn sender(self) -> Recipient { - self.sender - } } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index f7099853fa..45e0e26e10 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -8,8 +8,8 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, EventType, GetEventsAfterResponse, GetEventsAfterTs, HistoricalNetSyncStart, - NetSyncEventsReceived, OutgoingSyncRequested, TypedEvent, Unsequenced, + EnclaveEventData, EventStoreQueryBy, EventStoreQueryResponse, EventType, + HistoricalNetSyncStart, NetSyncEventsReceived, TsAgg, TypedEvent, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake, MAILBOX_LIMIT}; use futures::TryFutureExt; @@ -31,7 +31,7 @@ pub struct NetSyncManager { /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so /// that we do not do excessive resubscribes without actually listening for events. rx: Arc>, - eventstore: Recipient, + eventstore: Recipient>, requests: HashMap>>, } @@ -40,7 +40,7 @@ impl NetSyncManager { bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, - eventstore: Recipient, + eventstore: Recipient>, ) -> Self { Self { bus: bus.clone(), @@ -56,7 +56,7 @@ impl NetSyncManager { bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, - eventstore: Recipient, + eventstore: Recipient>, ) -> Addr { let mut events = rx.resubscribe(); let addr = Self::new(bus, tx, rx, eventstore).start(); @@ -154,7 +154,7 @@ impl Handler for NetSyncManager { trap(EType::Net, &self.bus, || { let id = CorrelationId::new(); self.requests.insert(id, msg.channel); - self.eventstore.try_send(GetEventsAfterTs::new( + self.eventstore.try_send(EventStoreQueryBy::::new( id, msg.value.since, ctx.address().recipient(), @@ -165,9 +165,9 @@ impl Handler for NetSyncManager { } /// Receive Events from EventStore -impl Handler for NetSyncManager { +impl Handler for NetSyncManager { type Result = (); - fn handle(&mut self, msg: GetEventsAfterResponse, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { let Some(channel) = self.requests.get(&msg.id()) else { bail!("request not found with {}", msg.id()); diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index daaaf6ce9f..62b274f90a 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -10,9 +10,9 @@ use anyhow::Result; use e3_data::Repositories; use e3_events::{ AggregateConfig, AggregateId, BusHandle, CorrelationId, EnableEffects, EnclaveEvent, - EventContextAccessors, EventPublisher, EvmEventConfig, EvmEventConfigChain, - GetEventsAfterResponse, GetEventsAfterSeq, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, - HistoricalNetEventsReceived, HistoricalNetSyncStart, SyncEnd, Unsequenced, + EventContextAccessors, EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, + EvmEventConfig, EvmEventConfigChain, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, + HistoricalNetEventsReceived, HistoricalNetSyncStart, SeqAgg, SyncEnd, Unsequenced, }; use e3_utils::actix::channel as actix_toolbox; use std::{ @@ -26,7 +26,7 @@ pub async fn sync( default_config: &EvmEventConfig, repositories: &Repositories, aggregate_config: &AggregateConfig, - eventstore: &Recipient, + eventstore: &Recipient>, ) -> Result<()> { // 1. Load snapsshot metadata let snapshot = @@ -38,13 +38,16 @@ pub async fn sync( let net_config = snapshot.to_net_config(); // 3. Load EventStore events since the sequence number found in the snapshot into memory. - let (tx, rx) = actix_toolbox::oneshot::(); - eventstore.try_send(GetEventsAfterSeq::new( + let (tx, rx) = actix_toolbox::mpsc::(256); + eventstore.try_send(EventStoreQueryBy::::new( CorrelationId::new(), snapshot.to_sequence_map(), tx, ))?; - let events = rx.await?.into_events(); + + let events = + collect_eventstore_query_response(rx, snapshot.aggregates().len(), Duration::from_secs(5)) + .await; // 4. Replay the EventStore events to all listeners (except effects) for event in events { @@ -107,7 +110,6 @@ pub async fn collect_historical_evm_events( results.append(&mut msg.events); } } else { - // Channel closed before receiving all expected chains break; } } @@ -125,6 +127,36 @@ pub async fn collect_historical_evm_events( results } +pub async fn collect_eventstore_query_response( + mut receiver: Receiver, + expected: usize, + max_dur: Duration, +) -> Vec { + let mut results = Vec::new(); + let mut received = 0; + + let collect = async { + for _ in 0..expected { + match receiver.recv().await { + Some(msg) => { + results.extend(msg.into_events()); + received += 1; + } + None => break, + } + } + }; + + if timeout(max_dur, collect).await.is_err() { + eprintln!( + "Error: Timeout waiting for historical events from {} aggregates", + expected - received + ); + } + + results +} + /// Latest event information in store #[derive(Clone)] pub struct AggregateState { @@ -203,6 +235,13 @@ impl SnapshotMeta { acc }) } + + pub fn aggregates(&self) -> Vec { + self.aggregate_state + .iter() + .map(|s| s.aggregate_id) + .collect() + } } #[derive(Message)] From 4fe3e1823ced80b016ab33528203c5dbc2ea82bd Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 00:48:21 +0000 Subject: [PATCH 42/63] add logging for sync --- .../src/enclave_event/enable_effects.rs | 6 ++-- crates/events/src/enclave_event/mod.rs | 4 +-- crates/sync/src/sync.rs | 31 ++++++++++++++++--- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/crates/events/src/enclave_event/enable_effects.rs b/crates/events/src/enclave_event/enable_effects.rs index 16b4919ccf..16ae123770 100644 --- a/crates/events/src/enclave_event/enable_effects.rs +++ b/crates/events/src/enclave_event/enable_effects.rs @@ -11,15 +11,15 @@ use std::fmt::{self, Display}; /// Dispatched once the sync process is complete and live listening should continue #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EnableEffects; +pub struct EffectsEnabled; -impl EnableEffects { +impl EffectsEnabled { pub fn new() -> Self { Self {} } } -impl Display for EnableEffects { +impl Display for EffectsEnabled { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 05f3335085..d07cebd413 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -214,7 +214,7 @@ pub enum EnclaveEventData { HistoricalNetSyncStart(HistoricalNetSyncStart), SyncEffect(SyncEffect), SyncEnd(SyncEnd), - EnableEffects(EnableEffects), + EffectsEnabled(EffectsEnabled), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -499,7 +499,7 @@ impl_event_types!( HistoricalNetSyncStart, SyncEffect, SyncEnd, - EnableEffects + EffectsEnabled ); impl TryFrom<&EnclaveEvent> for EnclaveError { diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 62b274f90a..22016eefa4 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -9,7 +9,7 @@ use actix::{Message, Recipient}; use anyhow::Result; use e3_data::Repositories; use e3_events::{ - AggregateConfig, AggregateId, BusHandle, CorrelationId, EnableEffects, EnclaveEvent, + AggregateConfig, AggregateId, BusHandle, CorrelationId, EffectsEnabled, EnclaveEvent, EventContextAccessors, EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, EvmEventConfig, EvmEventConfigChain, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetEventsReceived, HistoricalNetSyncStart, SeqAgg, SyncEnd, Unsequenced, @@ -20,6 +20,7 @@ use std::{ time::Duration, }; use tokio::{sync::mpsc::Receiver, time::timeout}; +use tracing::info; pub async fn sync( bus: &BusHandle, @@ -29,44 +30,62 @@ pub async fn sync( eventstore: &Recipient>, ) -> Result<()> { // 1. Load snapsshot metadata + info!("Loading snapshot metadata..."); let snapshot = SnapshotMeta::read_from_disk(aggregate_config.aggregates(), default_config, repositories) .await?; + info!( + "Snapshot metadata loaded for {} aggregates.", + snapshot.aggregates().len() + ); // 2. Determine the evm blocks to read from based on the SnapshotMeta let evm_config = snapshot.to_evm_config(); let net_config = snapshot.to_net_config(); // 3. Load EventStore events since the sequence number found in the snapshot into memory. + info!("Loading EventStore events..."); let (tx, rx) = actix_toolbox::mpsc::(256); eventstore.try_send(EventStoreQueryBy::::new( CorrelationId::new(), snapshot.to_sequence_map(), tx, ))?; - let events = collect_eventstore_query_response(rx, snapshot.aggregates().len(), Duration::from_secs(5)) .await; + info!("{} EventStore events loaded.", events.len()); + info!("Replaying events to actors..."); // 4. Replay the EventStore events to all listeners (except effects) for event in events { bus.event_bus().try_send(event)?; } + info!("Events replayed."); // TODO: Detect open loops - incase we crashed in the middle of a request we need to play the // request event again once effects are on // 5. Load the historical evm events to memory from all chains + info!("Loading historical blockchain events..."); let (addr, rx) = actix_toolbox::mpsc::(256); bus.publish_without_context(HistoricalEvmSyncStart::new(addr, evm_config.clone()))?; let historical_evm_events = collect_historical_evm_events(rx, &evm_config, Duration::from_secs(30)).await; + info!( + "{} historical blockchain events loaded.", + historical_evm_events.len() + ); // 6. Load the historical libp2p events to memory + info!("Loading historical libp2p events..."); let (addr, rx) = actix_toolbox::oneshot::(); bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; let historical_net_events = rx.await?.events; + info!( + "{} historical libp2p events loaded.", + historical_net_events.len() + ); // 7. Sort both the evm and libp2p events together by HLC timestamp let mut historical = historical_evm_events @@ -75,17 +94,21 @@ pub async fn sync( .collect::>(); historical.sort_by_key(|event| event.ts()); + info!("Historical events sorted."); // 8. Enable effects - bus.publish_without_context(EnableEffects::new())?; + bus.publish_without_context(EffectsEnabled::new())?; + info!("Effects enabled"); // 9. Publish the new sorted events to the eventstore + info!("Publishing historical events to actors..."); for event in historical { bus.naked_dispatch(event); } + info!("Historical events published."); bus.publish_without_context(SyncEnd::new())?; - + info!("Sync finished."); // normal live operations Ok(()) From 03bce4bbc0f5d87ab6428f70a8da585c41c30f33 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 03:16:12 +0000 Subject: [PATCH 43/63] buffer events in net package until SyncEnded --- .../src/ciphernode_builder.rs | 12 +- crates/ciphernode-builder/src/event_system.rs | 23 +- crates/events/src/enclave_event/mod.rs | 4 +- crates/events/src/enclave_event/sync_end.rs | 6 +- crates/events/src/events.rs | 2 +- .../src/snapshot_buffer/snapshot_buffer.rs | 6 +- crates/evm/src/evm_chain_gateway.rs | 20 +- crates/evm/tests/integration.rs | 4 +- crates/net/src/lib.rs | 70 ++++++ crates/net/src/net_event_buffer.rs | 201 ++++++++++++++++++ crates/net/src/net_event_translator.rs | 138 +++++------- crates/net/src/net_sync_manager.rs | 8 +- crates/sync/src/sync.rs | 4 +- 13 files changed, 378 insertions(+), 120 deletions(-) create mode 100644 crates/net/src/net_event_buffer.rs diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 1097e6a9f0..95633ef52a 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -21,7 +21,7 @@ use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::ThresholdKeyshareExtension; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; -use e3_net::{NetEventTranslator, NetRepositoryFactory}; +use e3_net::{setup_with_interface, NetRepositoryFactory}; use e3_request::E3Router; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, @@ -186,7 +186,7 @@ impl CiphernodeBuilder { self.testmode_errors = true; self } - /// Ensure SnapshotBuffer starts immediately instead of waiting for SyncEnd. This is important + /// Ensure SnapshotBuffer starts immediately instead of waiting for SyncEnded. This is important /// for tests that don't specifically pub fn testmode_start_buffer_immediately(mut self) -> Self { self.start_buffer = true; @@ -381,7 +381,8 @@ impl CiphernodeBuilder { let bus = event_system.handle()?; let store = event_system.store()?; - let eventstore = event_system.eventstore_getter()?; + let eventstore_ts = event_system.eventstore_getter_ts()?; + let eventstore_seq = event_system.eventstore_getter_seq()?; let cipher = &self.cipher; let repositories = Arc::new(store.repositories()); @@ -457,12 +458,13 @@ impl CiphernodeBuilder { let (join_handle, peer_id) = if let Some(net_config) = self.net_config { let repositories = store.repositories(); - let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( + let (_, _, join_handle, peer_id) = setup_with_interface( bus.clone(), net_config.peers, &self.cipher, net_config.quic_port, repositories.libp2p_keypair(), + eventstore_ts, ) .await?; (join_handle, peer_id) @@ -479,7 +481,7 @@ impl CiphernodeBuilder { &evm_config, &repositories, &aggregate_config, - &eventstore, + &eventstore_seq, ) .await?; diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index eeda7a3cf0..55617d4ffd 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -14,8 +14,8 @@ use e3_data::{ use e3_events::hlc::Hlc; use e3_events::{ AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, - EventStoreQueryBy, EventStoreRouter, InsertBatch, SeqAgg, Sequencer, SnapshotBuffer, - StoreEventRequested, UpdateDestination, + EventStoreQueryBy, EventStoreRouter, InsertBatch, QueryKind, SeqAgg, Sequencer, SnapshotBuffer, + StoreEventRequested, TsAgg, UpdateDestination, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -324,7 +324,16 @@ impl EventSystem { } } - pub fn eventstore_getter(&self) -> Result>> { + pub fn eventstore_getter_seq(&self) -> Result>> { + info!("eventstore_reader..."); + let eventstores = self.eventstore_addrs()?; + match &eventstores { + EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), + EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), + } + } + + pub fn eventstore_getter_ts(&self) -> Result>> { info!("eventstore_reader..."); let eventstores = self.eventstore_addrs()?; match &eventstores { @@ -413,7 +422,7 @@ mod tests { use e3_events::EventId; use e3_events::Start; use e3_events::StoreKeys; - use e3_events::SyncEnd; + use e3_events::SyncEnded; use e3_events::Tick; use e3_events::TsAgg; use e3_test_helpers::with_tracing; @@ -584,9 +593,9 @@ mod tests { ); info!("Local state was mutated however disk state was not"); - info!("Publishing SyncEnd event to turn on SnapshotBuffer. This should send the seq=1 batch to the timelock..."); - // Publishing SyncEnd should turn on the SnapshotBuffer seq 2 - handle.publish(SyncEnd::new(), ec.clone())?; + info!("Publishing SyncEnded event to turn on SnapshotBuffer. This should send the seq=1 batch to the timelock..."); + // Publishing SyncEnded should turn on the SnapshotBuffer seq 2 + handle.publish(SyncEnded::new(), ec.clone())?; sleep(Duration::from_millis(1)).await; diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index d07cebd413..b234449e88 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -213,7 +213,7 @@ pub enum EnclaveEventData { HistoricalEvmSyncStart(HistoricalEvmSyncStart), HistoricalNetSyncStart(HistoricalNetSyncStart), SyncEffect(SyncEffect), - SyncEnd(SyncEnd), + SyncEnded(SyncEnded), EffectsEnabled(EffectsEnabled), /// This is a test event to use in testing TestEvent(TestEvent), @@ -498,7 +498,7 @@ impl_event_types!( HistoricalEvmSyncStart, HistoricalNetSyncStart, SyncEffect, - SyncEnd, + SyncEnded, EffectsEnabled ); diff --git a/crates/events/src/enclave_event/sync_end.rs b/crates/events/src/enclave_event/sync_end.rs index 400ba9f5e2..6b2118a72b 100644 --- a/crates/events/src/enclave_event/sync_end.rs +++ b/crates/events/src/enclave_event/sync_end.rs @@ -11,15 +11,15 @@ use std::fmt::{self, Display}; /// Dispatched once the sync process is complete and live listening should continue #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct SyncEnd; +pub struct SyncEnded; -impl SyncEnd { +impl SyncEnded { pub fn new() -> Self { Self {} } } -impl Display for SyncEnd { +impl Display for SyncEnded { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index d2e2c2df1e..3ff819c7ea 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -65,7 +65,7 @@ impl StoreEventResponse { /// Trait for various EventStore query types pub trait QueryKind { - type Shape; + type Shape: Send; } /// Query by aggregated sequence diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 83c89bf0b9..ed71011b8e 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -175,7 +175,7 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - if let EnclaveEventData::SyncEnd(_) = msg.get_data() { + if let EnclaveEventData::SyncEnded(_) = msg.get_data() { self.notify_sync(ctx, Start); }; let SnapshotBufferState::Running = self.state else { @@ -269,7 +269,7 @@ mod tests { use crate::snapshot_buffer::timelock_queue::Tick; use crate::{ AggregateConfig, AggregateId, E3id, EnclaveEvent, EventContext, EventContextAccessors, - EventContextSeq, EventId, Insert, InsertBatch, Sequenced, SyncEnd, TestEvent, + EventContextSeq, EventId, Insert, InsertBatch, Sequenced, SyncEnded, TestEvent, }; use actix::Actor; use anyhow::Result; @@ -309,7 +309,7 @@ mod tests { buffer .send(EnclaveEvent::from_data_ec( - SyncEnd::new().into(), + SyncEnded::new().into(), create_ec(0, 9), )) .await?; diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 06e7731502..8e1860162c 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -15,7 +15,7 @@ use anyhow::{bail, Context}; use e3_events::EType; use e3_events::{ trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, - HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnd, Unsequenced, + HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnded, Unsequenced, }; use e3_events::{Event, EventPublisher}; use e3_utils::MAILBOX_LIMIT; @@ -114,7 +114,7 @@ impl EvmChainGateway { pub fn setup(bus: &BusHandle) -> Addr { let addr = Self::new(bus).start(); bus.subscribe_all( - &[EventType::HistoricalEvmSyncStart, EventType::SyncEnd], + &[EventType::HistoricalEvmSyncStart, EventType::SyncEnded], addr.clone().recipient(), ); addr @@ -134,7 +134,7 @@ impl EvmChainGateway { Ok(()) } - fn handle_sync_end(&mut self, _: SyncEnd) -> Result<()> { + fn handle_sync_ended(&mut self, _: SyncEnded) -> Result<()> { let buffer = self.status.live()?; for evt in buffer { self.publish_evm_event(evt)?; @@ -195,7 +195,7 @@ impl Handler for EvmChainGateway { trap(EType::Evm, &self.bus.with_ec(msg.get_ctx()), || { match msg.into_data() { EnclaveEventData::HistoricalEvmSyncStart(e) => self.handle_sync_start(e)?, - EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e)?, + EnclaveEventData::SyncEnded(e) => self.handle_sync_ended(e)?, _ => (), } Ok(()) @@ -292,20 +292,20 @@ mod tests { // Send EVM event while buffering - should be buffered (not received) let buffered_event = EvmEvent::new( CorrelationId::new(), - TestEvent::new("Before SyncEnd", 2).into(), + TestEvent::new("Before SyncEnded", 2).into(), 101, 12346, chain_id, ); addr.send(EnclaveEvmEvent::Event(buffered_event)).await?; - // The Synchronizer will publish the SyncEnd event when it has all the information it needs + // The Synchronizer will publish the SyncEnded event when it has all the information it needs // and has published everything to the bus - bus.publish_without_context(SyncEnd::new())?; + bus.publish_without_context(SyncEnded::new())?; let after_event = EvmEvent::new( CorrelationId::new(), - TestEvent::new("After SyncEnd", 2).into(), + TestEvent::new("After SyncEnded", 2).into(), 101, 12346, chain_id, @@ -328,7 +328,7 @@ mod tests { assert_eq!( test_events, - vec!["Before Complete", "Before SyncEnd", "After SyncEnd"] + vec!["Before Complete", "Before SyncEnded", "After SyncEnded"] ); let event_types: Vec = full.iter().map(|e| e.event_type()).collect(); @@ -338,7 +338,7 @@ mod tests { vec![ "HistoricalEvmSyncStart", "TestEvent", - "SyncEnd", + "SyncEnded", "TestEvent", "TestEvent" ] diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index cd2ece9e7c..a2b0736d6b 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -17,7 +17,7 @@ use anyhow::Result; use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEventConfig, - EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnd, + EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnded, TestEvent, }; use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; @@ -85,7 +85,7 @@ impl Handler for FakeSyncActor { for evt in msg.events.drain(..) { self.bus.naked_dispatch(evt); } - self.bus.publish_without_context(SyncEnd::new())?; + self.bus.publish_without_context(SyncEnded::new())?; Ok(()) }) } diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index eb7c49065a..edcb44fbfb 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -8,13 +8,83 @@ mod cid; mod dialer; mod document_publisher; pub mod events; +mod net_event_buffer; mod net_event_translator; mod net_interface; mod net_sync_manager; mod repo; +use std::sync::Arc; + +use actix::{Addr, Recipient}; +use anyhow::bail; pub use cid::Cid; pub use document_publisher::*; +use e3_crypto::Cipher; +use e3_data::Repository; +use e3_events::{BusHandle, EventStoreQueryBy, TsAgg}; +use libp2p::identity::ed25519; +use net_event_buffer::NetEventBuffer; pub use net_event_translator::*; pub use net_interface::*; +use net_sync_manager::NetSyncManager; pub use repo::*; +use tracing::{info, instrument}; + +/// Spawn a Libp2p interface and hook it up to this actor +#[instrument(name = "libp2p", skip_all)] +pub async fn setup_with_interface( + bus: BusHandle, + peers: Vec, + cipher: &Arc, + quic_port: u16, + repository: Repository>, + eventstore: impl Into>>, +) -> anyhow::Result<( + Addr, + Option>, + tokio::task::JoinHandle>, + String, +)> { + let topic = "tmp-enclave-gossip-topic"; + + // Get existing keypair or generate a new one + let mut bytes = match repository.read().await? { + Some(bytes) => { + info!("Found keypair in repository"); + cipher.decrypt_data(&bytes)? + } + None => bail!("No network keypair found in repository, please generate a new one using `enclave net generate-key`"), + }; + + // Create peer from keypair + let keypair: libp2p::identity::Keypair = + ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; + + // Generate a new interface to read and write peer events to + let mut interface = NetInterface::new(&keypair, peers, Some(quic_port), topic)?; + + // NOTE: Pass the unbuffered rx to SyncManager as it must operate before live events are + // processed + let _net_sync = NetSyncManager::setup( + &bus, + &interface.tx(), + &Arc::new(interface.rx()), + eventstore.into(), + ); + + // Buffer all incoming events until SyncEnded + let rx = &Arc::new(NetEventBuffer::setup(&bus, &interface.rx())); + let addr = NetEventTranslator::setup(&bus, &interface.tx(), rx, topic); + let maybe_publisher = Some(DocumentPublisher::setup(&bus, &interface.tx(), rx, topic)); + + // TODO: actix::spawn might avoid all the cleanup code + let handle = tokio::spawn(async move { Ok(interface.start().await?) }); + + Ok(( + addr, + maybe_publisher, + handle, + keypair.public().to_peer_id().to_string(), + )) +} diff --git a/crates/net/src/net_event_buffer.rs b/crates/net/src/net_event_buffer.rs new file mode 100644 index 0000000000..a23c1db537 --- /dev/null +++ b/crates/net/src/net_event_buffer.rs @@ -0,0 +1,201 @@ +use actix::prelude::*; +use anyhow::{anyhow, bail, Result}; +use e3_events::{ + trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, EventType, + SyncEnded, +}; +use tokio::sync::broadcast::{self, error::RecvError}; + +use crate::events::NetEvent; + +#[derive(Debug)] +enum NetState { + Running, + Syncing(Vec), +} + +impl NetState { + pub fn run(&mut self) -> Result> { + let Self::Syncing(buffer) = self else { + bail!("Cannot change state to Running when state is {:?}", self); + }; + let buffer = std::mem::take(buffer); + *self = Self::Running; + Ok(buffer) + } +} + +/// Actor that controls a broadcast channel which will Buffer NetEvents until it receives a +/// EnclaveEvent::SyncEnded at which time it will release al events to the broadcast channel +pub struct NetEventBuffer { + state: NetState, + input_rx: Option>, + output_tx: broadcast::Sender, + bus: BusHandle, +} + +impl NetEventBuffer { + pub fn setup( + bus: &BusHandle, + input_rx: &broadcast::Receiver, + ) -> broadcast::Receiver { + let input_rx = input_rx.resubscribe(); + let (output_tx, output_rx) = broadcast::channel(1024); + + let actor = Self { + state: NetState::Syncing(Vec::new()), + input_rx: Some(input_rx), + output_tx: output_tx.clone(), + bus: bus.clone(), + }; + + let addr = actor.start(); + + // Subscribe to EnclaveEvent on the bus + bus.subscribe(EventType::SyncEnded, addr.clone().recipient()); + + output_rx + } + + fn handle_enclave_event(&mut self, msg: EnclaveEvent) -> Result<()> { + if let EnclaveEventData::SyncEnded(m) = msg.get_data() { + return self.process_sync_ended(m.clone()); + } + Ok(()) + } + + fn process_sync_ended(&mut self, _: SyncEnded) -> Result<()> { + let pending = self.state.run()?; + for event in pending { + self.forward_event(event)?; + } + Ok(()) + } + + fn forward_event(&mut self, event: NetEvent) -> Result<()> { + self.output_tx + .send(event) + .map_err(|e| anyhow!("Failed to forward event: {}", e))?; + Ok(()) + } +} + +impl Actor for NetEventBuffer { + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + // Spawn task to read from broadcast channel + let addr = ctx.address(); + let mut input_rx = self.input_rx.take().expect("input_rx should be present"); + + actix::spawn(async move { + loop { + match input_rx.recv().await { + Ok(event) => addr.do_send(IncomingNetEvent(event)), + Err(RecvError::Lagged(_)) => continue, + Err(RecvError::Closed) => break, + } + } + }); + } +} + +#[derive(Message)] +#[rtype(result = "()")] +struct IncomingNetEvent(NetEvent); + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: IncomingNetEvent, _: &mut Self::Context) { + trap(EType::Net, &self.bus.clone(), || { + match &mut self.state { + NetState::Syncing(buffer) => buffer.push(msg.0), + NetState::Running => { + self.forward_event(msg.0)?; + } + } + Ok(()) + }) + } +} + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + self.handle_enclave_event(msg) + }) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::events::{GossipData, NetEvent}; + use e3_ciphernode_builder::EventSystem; + use e3_events::EventPublisher; + use tokio::{ + sync::broadcast, + time::{sleep, timeout}, + }; + + #[actix::test] + async fn test_buffers_until_sync_ended() -> Result<()> { + // Setup + let system = EventSystem::new("test").with_fresh_bus(); + let bus = system.handle()?; + let (input_tx, input_rx) = broadcast::channel(16); + let mut output_rx = NetEventBuffer::setup(&bus, &input_rx); + + // Send events while syncing - should be buffered + let event1 = NetEvent::GossipData(GossipData::GossipBytes(vec![1, 2, 3])); + let event2 = NetEvent::GossipData(GossipData::GossipBytes(vec![4, 5, 6])); + input_tx.send(event1.clone()).unwrap(); + input_tx.send(event2.clone()).unwrap(); + + // Give actor time to process + sleep(Duration::from_millis(10)).await; + + // Verify no events forwarded yet (should timeout) + assert!( + timeout(Duration::from_millis(50), output_rx.recv()) + .await + .is_err(), + "Events should be buffered, not forwarded during sync" + ); + + // Send SyncEnded event + bus.publish_without_context(SyncEnded::new()).unwrap(); + + // Now buffered events should be forwarded + let received1 = output_rx.recv().await.unwrap(); + let received2 = output_rx.recv().await.unwrap(); + + assert!( + matches!(received1, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![1, 2, 3]) + ); + assert!( + matches!(received2, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![4, 5, 6]) + ); + + // Send new event after sync - should forward immediately + let event3 = NetEvent::GossipData(GossipData::GossipBytes(vec![7, 8, 9])); + input_tx.send(event3.clone()).unwrap(); + + let received3 = + tokio::time::timeout(tokio::time::Duration::from_millis(100), output_rx.recv()) + .await + .expect("Event should be forwarded immediately after sync") + .unwrap(); + + assert!( + matches!(received3, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![7, 8, 9]) + ); + + Ok(()) + } +} diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 4178ed698e..06b82d396f 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -1,5 +1,4 @@ // SPDX-License-Identifier: LGPL-3.0-only -// // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. @@ -7,6 +6,8 @@ use crate::events::GossipData; use crate::events::NetCommand; use crate::events::NetEvent; +use crate::net_event_buffer::NetEventBuffer; +use crate::net_sync_manager::NetSyncManager; use crate::DocumentPublisher; use crate::NetInterface; /// Actor for connecting to an libp2p client via it's mpsc channel interface @@ -22,7 +23,9 @@ use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::Event; use e3_events::EventContextAccessors; +use e3_events::EventStoreQueryBy; use e3_events::EventType; +use e3_events::TsAgg; use e3_events::Unsequenced; use e3_events::{CorrelationId, EnclaveEvent, EventId}; use e3_utils::MAILBOX_LIMIT; @@ -55,7 +58,7 @@ impl Actor for NetEventTranslator { /// Libp2pEvent is used to send data to the NetInterface from the NetEventTranslator #[derive(Message, Clone, Debug, PartialEq, Eq)] -#[rtype(result = "anyhow::Result<()>")] +#[rtype(result = "()")] struct LibP2pEvent(pub GossipData); impl NetEventTranslator { @@ -114,97 +117,70 @@ impl NetEventTranslator { } } - /// Spawn a Libp2p interface and hook it up to this actor - #[instrument(name = "libp2p", skip_all)] - pub async fn setup_with_interface( - bus: BusHandle, - peers: Vec, - cipher: &Arc, - quic_port: u16, - repository: Repository>, - ) -> Result<( - Addr, - Option>, - tokio::task::JoinHandle>, - String, - )> { - // TODO: We should separate NetInterface from NetEventTranslator - let topic = "tmp-enclave-gossip-topic"; - // Get existing keypair or generate a new one - let mut bytes = match repository.read().await? { - Some(bytes) => { - info!("Found keypair in repository"); - cipher.decrypt_data(&bytes)? - } - None => bail!("No network keypair found in repository, please generate a new one using `enclave net generate-key`"), - }; - - // Create peer from keypair - let keypair: libp2p::identity::Keypair = - ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; - let mut interface = NetInterface::new(&keypair, peers, Some(quic_port), topic)?; - - // Setup and start net event translator - let rx = &Arc::new(interface.rx()); - let addr = NetEventTranslator::setup(&bus, &interface.tx(), rx, topic); - let maybe_publisher = Some(DocumentPublisher::setup(&bus, &interface.tx(), rx, topic)); - - let handle = tokio::spawn(async move { Ok(interface.start().await?) }); - - Ok(( - addr, - maybe_publisher, - handle, - keypair.public().to_peer_id().to_string(), - )) + fn process_gossip_event(&mut self, msg: EnclaveEvent) -> Result<()> { + // if we have seen this event before dont rebroadcast + let id = msg.event_id(); + if self.sent_events.contains(&id) { + trace!(evt_id=%id,"Have seen event before not rebroadcasting!"); + return Ok(()); + } + + warn!("GossipPublish event: {}", msg.event_type()); + let topic = self.topic.clone(); + let data: GossipData = msg.try_into()?; + + self.tx.try_send(NetCommand::GossipPublish { + topic, + data, + correlation_id: CorrelationId::new(), + })?; + + Ok(()) } -} -impl Handler for NetEventTranslator { - type Result = anyhow::Result<()>; - fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { - let LibP2pEvent(data) = msg; - let event: EnclaveEvent = data.try_into()?; + fn handle_enclave_event(&mut self, msg: EnclaveEvent) -> Result<()> { + // Ignore events that should be considered local + if !Self::is_forwardable_event(&msg) { + let id = msg.event_id(); + trace!(evt_id=%id,"Local events should not be rebroadcast so ignoring"); + return Ok(()); + } + + self.process_gossip_event(msg)?; + + Ok(()) + } + + fn publish_event(&mut self, event: EnclaveEvent) -> Result<()> { let id = event.id(); let (data, ec) = event.into_components(); self.bus.publish_from_remote(data, ec.ts(), None)?; self.sent_events.insert(id); Ok(()) } + + fn handle_remote_event(&mut self, msg: LibP2pEvent) -> Result<()> { + let data = msg.0; + let event: EnclaveEvent = data.try_into()?; + self.publish_event(event)?; + Ok(()) + } } -impl Handler for NetEventTranslator { +impl Handler for NetEventTranslator { type Result = (); - fn handle(&mut self, event: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Net, &self.bus.with_ec(event.get_ctx()), || { - let sent_events = self.sent_events.clone(); - let tx = self.tx.clone(); - let evt = event.clone(); - let topic = self.topic.clone(); - let id: EventId = evt.clone().into(); - - // Ignore events that should be considered local - if !Self::is_forwardable_event(&evt) { - trace!(evt_id=%id,"Local events should not be rebroadcast so ignoring"); - return Ok(()); - } - - // if we have seen this event before dont rebroadcast - if sent_events.contains(&id) { - trace!(evt_id=%id,"Have seen event before not rebroadcasting!"); - return Ok(()); - } - - warn!("GossipPublish event: {}", event.event_type()); - let data: GossipData = evt.try_into()?; - - tx.try_send(NetCommand::GossipPublish { - topic, - data, - correlation_id: CorrelationId::new(), - })?; + fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.clone(), || { + self.handle_remote_event(msg) + }) + } +} - Ok(()) +impl Handler for NetEventTranslator { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + self.handle_enclave_event(msg) }) } } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 45e0e26e10..52424c9711 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -28,8 +28,7 @@ pub struct NetSyncManager { bus: BusHandle, /// NetCommand sender to forward commands to the NetInterface tx: mpsc::Sender, - /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so - /// that we do not do excessive resubscribes without actually listening for events. + /// NetEvents receiver to receive events rx: Arc>, eventstore: Recipient>, requests: HashMap>>, @@ -45,8 +44,7 @@ impl NetSyncManager { Self { bus: bus.clone(), tx: tx.clone(), - rx: rx.clone(), - + rx: Arc::clone(rx), eventstore, requests: HashMap::new(), } @@ -60,7 +58,9 @@ impl NetSyncManager { ) -> Addr { let mut events = rx.resubscribe(); let addr = Self::new(bus, tx, rx, eventstore).start(); + bus.subscribe(EventType::HistoricalNetSyncStart, addr.clone().recipient()); + // Forward from NetEvent tokio::spawn({ debug!("Spawning event receive loop!"); diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 22016eefa4..986ecd8b6a 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -12,7 +12,7 @@ use e3_events::{ AggregateConfig, AggregateId, BusHandle, CorrelationId, EffectsEnabled, EnclaveEvent, EventContextAccessors, EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, EvmEventConfig, EvmEventConfigChain, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, - HistoricalNetEventsReceived, HistoricalNetSyncStart, SeqAgg, SyncEnd, Unsequenced, + HistoricalNetEventsReceived, HistoricalNetSyncStart, SeqAgg, SyncEnded, Unsequenced, }; use e3_utils::actix::channel as actix_toolbox; use std::{ @@ -107,7 +107,7 @@ pub async fn sync( } info!("Historical events published."); - bus.publish_without_context(SyncEnd::new())?; + bus.publish_without_context(SyncEnded::new())?; info!("Sync finished."); // normal live operations From ad244888b14a06e4b057b6279b007de3b5da3f36 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 03:18:28 +0000 Subject: [PATCH 44/63] headers --- crates/net/src/net_event_buffer.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/net/src/net_event_buffer.rs b/crates/net/src/net_event_buffer.rs index a23c1db537..77f830eece 100644 --- a/crates/net/src/net_event_buffer.rs +++ b/crates/net/src/net_event_buffer.rs @@ -1,4 +1,10 @@ -use actix::prelude::*; +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, AsyncContext, Handler, Message}; use anyhow::{anyhow, bail, Result}; use e3_events::{ trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, EventType, @@ -81,7 +87,7 @@ impl NetEventBuffer { } impl Actor for NetEventBuffer { - type Context = Context; + type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { // Spawn task to read from broadcast channel From 095e9d10193eae460c107e55358d547af0a4cd03 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 06:31:44 +0000 Subject: [PATCH 45/63] ensure evm writers and net commands are only available after EffectsEnabled --- .../src/ciphernode_builder.rs | 10 +-- crates/ciphernode-builder/src/evm_system.rs | 14 ++-- crates/events/src/event_extractor.rs | 80 +++++++++++++++++++ crates/events/src/lib.rs | 2 + .../src/snapshot_buffer/snapshot_buffer.rs | 1 - crates/evm/src/ciphernode_registry_sol.rs | 76 +++++++++--------- crates/evm/src/enclave_sol.rs | 35 -------- crates/evm/src/enclave_sol_writer.rs | 30 +++---- crates/evm/src/lib.rs | 4 - crates/net/src/lib.rs | 35 ++++---- crates/utils/src/actix/mod.rs | 1 + .../src/actix/oneshot_runner.rs} | 2 +- 12 files changed, 169 insertions(+), 121 deletions(-) create mode 100644 crates/events/src/event_extractor.rs delete mode 100644 crates/evm/src/enclave_sol.rs rename crates/{evm/src/one_shot_runnner.rs => utils/src/actix/oneshot_runner.rs} (98%) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 95633ef52a..9019c8f6a0 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -458,7 +458,7 @@ impl CiphernodeBuilder { let (join_handle, peer_id) = if let Some(net_config) = self.net_config { let repositories = store.repositories(); - let (_, _, join_handle, peer_id) = setup_with_interface( + setup_with_interface( bus.clone(), net_config.peers, &self.cipher, @@ -466,8 +466,7 @@ impl CiphernodeBuilder { repositories.libp2p_keypair(), eventstore_ts, ) - .await?; - (join_handle, peer_id) + .await? } else { ( tokio::spawn(std::future::ready(Ok(()))), @@ -585,7 +584,7 @@ async fn setup_evm_system( if contract_components.enclave { let write_provider = provider_cache.ensure_write_provider(chain).await?; let contract = &chain.contracts.enclave; - EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?).await?; + EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?); system.with_contract(contract.address()?, move |next| { EnclaveSolReader::setup(&next).recipient() }); @@ -625,8 +624,7 @@ async fn setup_evm_system( write_provider.clone(), contract.address()?, pubkey_agg, - ) - .await?; + ); info!("CiphernodeRegistrySolWriter attached for publishing committees"); if pubkey_agg { diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 3b1ce66ce2..5d88eb4c90 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -8,11 +8,14 @@ use std::mem::replace; use actix::Actor; use alloy::{primitives::Address, providers::Provider}; -use e3_events::{BusHandle, EventSubscriber, EventType, HistoricalEvmSyncStart}; +use e3_events::{ + run_once, BusHandle, EventExtractor, EventSubscriber, EventType, HistoricalEvmSyncStart, +}; use e3_evm::{ EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, - FixHistoricalOrder, OneShotRunner, SyncStartExtractor, + FixHistoricalOrder, SyncStartExtractor, }; +use e3_utils::actix::oneshot_runner::OneShotRunner; pub trait RouteFn: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} impl RouteFn for F where F: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} @@ -57,7 +60,7 @@ impl EvmSystemChainBuilder

{ let next = FixHistoricalOrder::setup(next); // This will run once when the HistoricalEvmSyncStart event is received - let next = OneShotRunner::setup({ + let next = run_once::({ // Clone self refs for closure let bus = self.bus.clone(); let provider = self.provider.clone(); @@ -67,7 +70,7 @@ impl EvmSystemChainBuilder

{ let route_factories = replace(&mut self.route_factories, Vec::new()); // The event is defined here - move |msg: HistoricalEvmSyncStart| { + move |msg| { // Extract config let deploy_block = msg.get_evm_config(chain_id)?.deploy_block(); @@ -83,9 +86,6 @@ impl EvmSystemChainBuilder

{ } }); - // We get a HistoricalEvmSyncStart event and sent to oneShotRunner - let next = SyncStartExtractor::setup(next); - // Finaly subscribe to the bus and wait for HistoricalEvmSyncStart self.bus .subscribe(EventType::HistoricalEvmSyncStart, next.recipient()); diff --git a/crates/events/src/event_extractor.rs b/crates/events/src/event_extractor.rs new file mode 100644 index 0000000000..34278dc83a --- /dev/null +++ b/crates/events/src/event_extractor.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::{EffectsEnabled, EnclaveEvent, EnclaveEventData, Event, HistoricalEvmSyncStart}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use e3_utils::{actix::oneshot_runner::OneShotRunner, MAILBOX_LIMIT}; + +/// Trait for events that can be extracted from EnclaveEventData +pub trait ExtractableEvent: Message + Send + 'static { + fn extract_from(data: EnclaveEventData) -> Option + where + Self: Sized; +} + +/// Generic event extractor that can handle any ExtractableEvent +pub struct EventExtractor { + dest: Recipient, +} + +impl EventExtractor { + pub fn new(dest: impl Into>) -> Self { + Self { dest: dest.into() } + } + + pub fn setup(dest: impl Into>) -> Addr { + Self::new(dest).start() + } +} + +impl Actor for EventExtractor { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} + +impl Handler for EventExtractor { + type Result = (); + + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let Some(evt) = T::extract_from(msg.into_data()) { + self.dest.do_send(evt) + } + } +} + +impl ExtractableEvent for EffectsEnabled { + fn extract_from(data: EnclaveEventData) -> Option { + if let EnclaveEventData::EffectsEnabled(evt) = data { + Some(evt) + } else { + None + } + } +} + +impl ExtractableEvent for HistoricalEvmSyncStart { + fn extract_from(data: EnclaveEventData) -> Option { + if let EnclaveEventData::HistoricalEvmSyncStart(evt) = data { + Some(evt) + } else { + None + } + } +} + +/// Helper function to set up a one-shot event extractor +pub fn run_once( + f: impl FnOnce(T) -> anyhow::Result<()> + Unpin + Send + 'static, +) -> Addr> { + // EventExtractor filters the given event and sends to the receiver + EventExtractor::::setup( + // OneShotRunner runs the closure once when it first receives the event + OneShotRunner::setup(f), + ) +} diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index b956799785..d772e652bd 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -10,6 +10,7 @@ mod data_events; mod e3id; mod enclave_event; mod event_context; +mod event_extractor; mod event_id; mod eventbus; mod events; @@ -32,6 +33,7 @@ pub use data_events::*; pub use e3id::*; pub use enclave_event::*; pub use event_context::*; +pub use event_extractor::*; pub use event_id::*; pub use eventbus::*; pub use events::*; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index ed71011b8e..937c6789b2 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -230,7 +230,6 @@ mod mock_store { use crate::InsertBatch; use actix::{Actor, Handler, Message}; - use e3_utils::MAILBOX_LIMIT; #[derive(Message)] #[rtype(result = "Vec")] diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index dd86146cd0..8903b7d8e5 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -19,9 +19,9 @@ use alloy::{ }; use anyhow::Result; use e3_events::{ - prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, - EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, PublicKeyAggregated, - Seed, Shutdown, TicketGenerated, TicketId, + prelude::*, run_once, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, + EffectsEnabled, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, + PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; use e3_utils::{NotifySync, MAILBOX_LIMIT}; use tracing::{error, info, trace}; @@ -231,7 +231,7 @@ pub struct CiphernodeRegistrySolWriter

{ } impl CiphernodeRegistrySolWriter

{ - pub async fn new( + pub fn new( bus: &BusHandle, provider: EthProvider

, contract_address: Address, @@ -243,37 +243,43 @@ impl CiphernodeRegistrySolWriter }) } - pub async fn attach( + pub fn attach( bus: &BusHandle, provider: EthProvider

, contract_address: Address, is_aggregator: bool, - ) -> Result>> { - let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address) - .await? - .start(); - - if is_aggregator { - bus.subscribe_all( - &[ - EventType::PublicKeyAggregated, - EventType::CommitteeFinalizeRequested, - ], - addr.clone().into(), - ) - } + ) { + let runner = run_once::({ + let bus = bus.clone(); + move |_| { + let addr = + CiphernodeRegistrySolWriter::new(&bus, provider, contract_address)?.start(); + + if is_aggregator { + bus.subscribe_all( + &[ + EventType::PublicKeyAggregated, + EventType::CommitteeFinalizeRequested, + ], + addr.clone().into(), + ) + } + + bus.subscribe_all( + &[ + // Subscribe to TicketGenerated for ticket submission + EventType::TicketGenerated, + // Stop gracefully on shutdown + EventType::Shutdown, + ], + addr.clone().into(), + ); + + Ok(()) + } + }); - bus.subscribe_all( - &[ - // Subscribe to TicketGenerated for ticket submission - EventType::TicketGenerated, - // Stop gracefully on shutdown - EventType::Shutdown, - ], - addr.clone().into(), - ); - - Ok(addr) + bus.subscribe(EventType::EffectsEnabled, runner.recipient()); } } @@ -538,18 +544,14 @@ impl CiphernodeRegistrySol { CiphernodeRegistrySolReader::setup(processor) } - pub async fn attach_writer

( + pub fn attach_writer

( bus: &BusHandle, provider: EthProvider

, contract_address: Address, is_aggregator: bool, - ) -> Result>> - where + ) where P: Provider + WalletProvider + Clone + 'static, { - let writer = - CiphernodeRegistrySolWriter::attach(bus, provider, contract_address, is_aggregator) - .await?; - Ok(writer) + CiphernodeRegistrySolWriter::attach(bus, provider, contract_address, is_aggregator); } } diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs deleted file mode 100644 index 69df39b45c..0000000000 --- a/crates/evm/src/enclave_sol.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::{ - enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - events::EvmEventProcessor, evm_parser::EvmParser, helpers::EthProvider, -}; -use actix::Addr; -use alloy::providers::{Provider, WalletProvider}; -use alloy_primitives::Address; -use anyhow::Result; -use e3_events::BusHandle; - -pub struct EnclaveSol; - -impl EnclaveSol { - pub async fn attach( - processor: &EvmEventProcessor, - bus: &BusHandle, - write_provider: EthProvider, - contract_address: Address, - ) -> Result> - where - W: Provider + WalletProvider + Clone + 'static, - { - let addr = EnclaveSolReader::setup(processor); - - EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; - - Ok(addr) - } -} diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 189c346c3f..a95416dce0 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -7,7 +7,6 @@ use crate::helpers::EthProvider; use crate::send_tx_with_retry; use actix::prelude::*; -use actix::Addr; use alloy::{ primitives::Address, providers::{Provider, WalletProvider}, @@ -18,12 +17,12 @@ use alloy::{ rpc::types::TransactionReceipt, }; use anyhow::Result; -use e3_events::prelude::*; use e3_events::BusHandle; -use e3_events::EnclaveEvent; use e3_events::EnclaveEventData; use e3_events::EventType; use e3_events::Shutdown; +use e3_events::{prelude::*, EffectsEnabled}; +use e3_events::{run_once, EnclaveEvent}; use e3_events::{E3id, EType, PlaintextAggregated}; use e3_utils::NotifySync; use e3_utils::MAILBOX_LIMIT; @@ -55,17 +54,20 @@ impl EnclaveSolWriter

{ }) } - pub async fn attach( - bus: &BusHandle, - provider: EthProvider

, - contract_address: Address, - ) -> Result>> { - let addr = EnclaveSolWriter::new(bus, provider, contract_address)?.start(); - bus.subscribe_all( - &[EventType::PlaintextAggregated, EventType::Shutdown], - addr.clone().into(), - ); - Ok(addr) + pub fn attach(bus: &BusHandle, provider: EthProvider

, contract_address: Address) { + let addr = run_once::({ + let bus = bus.clone(); + move |_| { + let addr = EnclaveSolWriter::new(&bus, provider, contract_address)?.start(); + bus.subscribe_all( + &[EventType::PlaintextAggregated, EventType::Shutdown], + addr.clone().into(), + ); + Ok(()) + } + }); + + bus.subscribe(EventType::EffectsEnabled, addr.recipient()); } } diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 47f8b45f31..93045c9804 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -6,7 +6,6 @@ mod bonding_registry_sol; mod ciphernode_registry_sol; -mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; mod events; @@ -17,7 +16,6 @@ mod evm_read_interface; mod evm_router; mod fix_historical_order; pub mod helpers; -mod one_shot_runnner; mod repo; mod sync_start_extractor; @@ -25,7 +23,6 @@ pub use bonding_registry_sol::BondingRegistrySolReader; pub use ciphernode_registry_sol::{ CiphernodeRegistrySol, CiphernodeRegistrySolReader, CiphernodeRegistrySolWriter, }; -pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; pub use events::*; @@ -36,6 +33,5 @@ pub use evm_read_interface::*; pub use evm_router::*; pub use fix_historical_order::*; pub use helpers::*; -pub use one_shot_runnner::*; pub use repo::*; pub use sync_start_extractor::*; diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index edcb44fbfb..19475e1b5f 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -22,7 +22,7 @@ pub use cid::Cid; pub use document_publisher::*; use e3_crypto::Cipher; use e3_data::Repository; -use e3_events::{BusHandle, EventStoreQueryBy, TsAgg}; +use e3_events::{run_once, BusHandle, EffectsEnabled, EventStoreQueryBy, EventSubscriber, TsAgg}; use libp2p::identity::ed25519; use net_event_buffer::NetEventBuffer; pub use net_event_translator::*; @@ -40,12 +40,7 @@ pub async fn setup_with_interface( quic_port: u16, repository: Repository>, eventstore: impl Into>>, -) -> anyhow::Result<( - Addr, - Option>, - tokio::task::JoinHandle>, - String, -)> { +) -> anyhow::Result<(tokio::task::JoinHandle>, String)> { let topic = "tmp-enclave-gossip-topic"; // Get existing keypair or generate a new one @@ -74,17 +69,25 @@ pub async fn setup_with_interface( ); // Buffer all incoming events until SyncEnded - let rx = &Arc::new(NetEventBuffer::setup(&bus, &interface.rx())); - let addr = NetEventTranslator::setup(&bus, &interface.tx(), rx, topic); - let maybe_publisher = Some(DocumentPublisher::setup(&bus, &interface.tx(), rx, topic)); + let rx = Arc::new(NetEventBuffer::setup(&bus, &interface.rx())); + let tx = interface.tx(); + + let runner = run_once::({ + let bus = bus.clone(); + let rx = rx.clone(); + let topic = topic.to_owned(); + let tx = tx.clone(); + move |_| { + NetEventTranslator::setup(&bus, &tx, &rx, &topic); + DocumentPublisher::setup(&bus, &tx, &rx, &topic); + Ok(()) + } + }); + + bus.subscribe(e3_events::EventType::EffectsEnabled, runner.recipient()); // TODO: actix::spawn might avoid all the cleanup code let handle = tokio::spawn(async move { Ok(interface.start().await?) }); - Ok(( - addr, - maybe_publisher, - handle, - keypair.public().to_peer_id().to_string(), - )) + Ok((handle, keypair.public().to_peer_id().to_string())) } diff --git a/crates/utils/src/actix/mod.rs b/crates/utils/src/actix/mod.rs index 35edc6604d..8da657a2d4 100644 --- a/crates/utils/src/actix/mod.rs +++ b/crates/utils/src/actix/mod.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. pub mod channel; +pub mod oneshot_runner; use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; diff --git a/crates/evm/src/one_shot_runnner.rs b/crates/utils/src/actix/oneshot_runner.rs similarity index 98% rename from crates/evm/src/one_shot_runnner.rs rename to crates/utils/src/actix/oneshot_runner.rs index ac96756c4a..bae7c875d9 100644 --- a/crates/evm/src/one_shot_runnner.rs +++ b/crates/utils/src/actix/oneshot_runner.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::MAILBOX_LIMIT; use actix::prelude::*; use anyhow::Result; -use e3_utils::MAILBOX_LIMIT; use std::marker::PhantomData; use tracing::error; From 5c1c0c04e62a53e97432dea05377fcdf3677becf Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 07:00:33 +0000 Subject: [PATCH 46/63] only enable multithread when EffectsEnabled has been switched on --- crates/multithread/src/multithread.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index 61cfe36eb6..9ecc45c108 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -17,10 +17,12 @@ use actix::prelude::*; use actix::{Actor, Handler}; use anyhow::Result; use e3_crypto::Cipher; +use e3_events::run_once; use e3_events::trap_fut; use e3_events::BusHandle; use e3_events::ComputeRequestErrorKind; use e3_events::EType; +use e3_events::EffectsEnabled; use e3_events::EnclaveEvent; use e3_events::EnclaveEventData; use e3_events::ErrorDispatcher; @@ -83,7 +85,15 @@ impl Multithread { report: Option>, ) -> Addr { let addr = Self::new(bus.clone(), rng.clone(), cipher.clone(), task_pool, report).start(); - bus.subscribe(EventType::ComputeRequest, addr.clone().recipient()); + run_once::({ + let bus = bus.clone(); + let addr = addr.clone(); + move |_| { + bus.subscribe(EventType::ComputeRequest, addr.clone().recipient()); + Ok(()) + } + }); + addr } From f28e2dc223928368e007d1a6de7a9fe3765323f0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 08:51:45 +0000 Subject: [PATCH 47/63] we actually want the persistable to save data during sync --- .../src/ciphernode_builder.rs | 4 +- crates/ciphernode-builder/src/event_system.rs | 3 +- .../src/snapshot_buffer/snapshot_buffer.rs | 116 ++++++++++-------- crates/net/src/lib.rs | 2 +- 4 files changed, 67 insertions(+), 58 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 9019c8f6a0..c12c457fd1 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -21,7 +21,7 @@ use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::ThresholdKeyshareExtension; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; -use e3_net::{setup_with_interface, NetRepositoryFactory}; +use e3_net::{setup_net, NetRepositoryFactory}; use e3_request::E3Router; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, @@ -458,7 +458,7 @@ impl CiphernodeBuilder { let (join_handle, peer_id) = if let Some(net_config) = self.net_config { let repositories = store.repositories(); - setup_with_interface( + setup_net( bus.clone(), net_config.peers, &self.cipher, diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 55617d4ffd..4039b35a7b 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -196,7 +196,6 @@ impl EventSystem { /// Get the eventbus address pub fn eventbus(&self) -> Addr> { - info!("eventbus..."); self.eventbus.get_or_init(get_enclave_event_bus).clone() } @@ -505,7 +504,7 @@ mod tests { let _guard = with_tracing("debug"); let tmp = TempDir::new().unwrap(); let system = EventSystem::persisted("cn2", tmp.path().join("log"), tmp.path().join("sled")); - system.buffer()?.send(Start).await?; + // system.buffer()?.send(Start).await?; let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); Ok(()) diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 937c6789b2..d428a61e02 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -9,7 +9,8 @@ use super::{ AggregateConfig, }; use crate::{ - trap, EType, EnclaveEvent, EnclaveEventData, Event, Insert, InsertBatch, PanicDispatcher, + trap, EType, EffectsEnabled, EnclaveEvent, EnclaveEventData, Event, Insert, InsertBatch, + PanicDispatcher, }; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; @@ -45,39 +46,46 @@ impl UpdateDestination { Self(base.into()) } } -enum SnapshotBufferState { - Running, - Paused, -} +// enum SnapshotBufferState { +// Disabled, // SnapshotBuffer is completely disabled +// Forwarding, // Forwarding to disk +// Buffering, // Buffering +// } pub struct SnapshotBuffer { router: Option>, timelock: Option>, tickable: Option>, - state: SnapshotBufferState, + // state: SnapshotBufferState, } impl SnapshotBuffer { - pub fn new(started: bool) -> Self { + pub fn new(start_buffering: bool) -> Self { SnapshotBuffer { router: None, timelock: None, tickable: None, - state: if !started { - SnapshotBufferState::Paused - } else { - SnapshotBufferState::Running - }, + // state: if !start_buffering { + // SnapshotBufferState::Buffering + // } else { + // SnapshotBufferState::Forwarding + // }, } } pub fn spawn( config: &AggregateConfig, store: impl Into>, - started: bool, + start_buffering: bool, ) -> Result> { info!("spawning SnapshotBuffer..."); - let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock), Some(1), started)?; + let (addr, _) = Self::with_clock( + config, + store, + Arc::new(SystemClock), + Some(1), + start_buffering, + )?; Ok(addr) } @@ -86,9 +94,9 @@ impl SnapshotBuffer { store: impl Into>, clock: Arc, interval: Option, - started: bool, + start_buffering: bool, ) -> Result<(Addr, Addr)> { - let addr = Self::new(started).start(); + let addr = Self::new(start_buffering).start(); let store = store.into(); let router = BatchRouter::with_clock(config, addr.clone(), store.clone(), clock.clone()).start(); @@ -109,9 +117,9 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - let SnapshotBufferState::Running = self.state else { - return Ok(()); - }; + // let SnapshotBufferState::Forwarding = self.state else { + // return Ok(()); + // }; if let Some(ref router) = self.router { router.try_send(msg)?; } @@ -124,9 +132,9 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - let SnapshotBufferState::Running = self.state else { - return Ok(()); - }; + // let SnapshotBufferState::Forwarding = self.state else { + // return Ok(()); + // }; if let Some(ref timelock) = self.timelock { timelock.try_send(msg)?; @@ -150,22 +158,24 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - use SnapshotBufferState as S; - match (&self.state, msg.ctx(), &self.router) { - // Doing this to cover when there are context free - // data store manipulations outside of the sync cycle. - (S::Paused, None, Some(ref router)) => { - debug!( - "Paused but received context free Insert. Forwarding to batch router..." - ); - router.try_send(msg)?; - } - (S::Running, _, Some(ref router)) => { - trace!("Forwarding Insert message to batch router..."); - router.try_send(msg)?; - } - _ => (), + // use SnapshotBufferState as S; + // match (&self.state, msg.ctx(), &self.router) { + // // Doing this to cover when there are context free + // // data store manipulations outside of the sync cycle. + // (S::Buffering, None, Some(ref router)) => { + // debug!( + // "Paused but received context free Insert. Forwarding to batch router..." + // ); + // router.try_send(msg)?; + // } + // (S::Forwarding, _, Some(ref router)) => { + if let Some(ref router) = self.router { + trace!("Forwarding Insert message to batch router..."); + router.try_send(msg)?; }; + // } + // _ => (), + // }; Ok(()) }) } @@ -175,12 +185,12 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - if let EnclaveEventData::SyncEnded(_) = msg.get_data() { - self.notify_sync(ctx, Start); - }; - let SnapshotBufferState::Running = self.state else { - return Ok(()); - }; + // if let EnclaveEventData::EffectsEnabled(msg) = msg.get_data() { + // self.notify_sync(ctx, msg.clone()); + // }; + // let SnapshotBufferState::Forwarding = self.state else { + // return Ok(()); + // }; if let Some(ref router) = self.router { router.try_send(msg)?; } @@ -201,16 +211,16 @@ impl Handler for SnapshotBuffer { } } -impl Handler for SnapshotBuffer { - type Result = (); - fn handle(&mut self, _: Start, _: &mut Self::Context) -> Self::Result { - trap(EType::IO, &PanicDispatcher::new(), || { - info!("SnapshotBuffer is now enabled"); - self.state = SnapshotBufferState::Running; - Ok(()) - }) - } -} +// impl Handler for SnapshotBuffer { +// type Result = (); +// fn handle(&mut self, _: EffectsEnabled, _: &mut Self::Context) -> Self::Result { +// trap(EType::IO, &PanicDispatcher::new(), || { +// info!("SnapshotBuffer is now enabled"); +// self.state = SnapshotBufferState::Forwarding; +// Ok(()) +// }) +// } +// } impl Handler for SnapshotBuffer { type Result = (); diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index 19475e1b5f..4b12a34985 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -33,7 +33,7 @@ use tracing::{info, instrument}; /// Spawn a Libp2p interface and hook it up to this actor #[instrument(name = "libp2p", skip_all)] -pub async fn setup_with_interface( +pub async fn setup_net( bus: BusHandle, peers: Vec, cipher: &Arc, From 59de42f7a2eb2c02d737dd15ffd66c847f0f3050 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 13:14:46 +0000 Subject: [PATCH 48/63] correlation id and run tests without fixing merge --- .github/workflows/ci.yml | 3 ++- crates/net/src/events.rs | 7 +++++- crates/net/src/net_interface.rs | 36 +++++++++++++++++++++--------- crates/net/src/net_sync_manager.rs | 1 + 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed3fdb9f92..105e2a9200 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,8 @@ concurrency: on: workflow_dispatch: - pull_request: + # pull_request: # XXX: temporarily allowing tests to run on PR + pull_request_target: branches: - main - dev diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 89bf9ba2ff..a3bad8ea58 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -86,6 +86,7 @@ pub struct SyncRequestReceived { #[rtype("()")] pub struct OutgoingSyncRequestSucceeded { pub value: SyncResponseValue, + pub correlation_id: CorrelationId, } #[derive(Debug, Clone)] @@ -118,7 +119,10 @@ pub enum NetCommand { Shutdown, /// Called from the syning node to request libp2p events from a random peer node starting /// from the given timestamp. - OutgoingSyncRequest { value: SyncRequestValue }, + OutgoingSyncRequest { + correlation_id: CorrelationId, + value: SyncRequestValue, + }, /// Send libp2p events back to a peer that requested a sync. SyncResponse { value: SyncResponseValue, @@ -133,6 +137,7 @@ impl NetCommand { N::DhtPutRecord { correlation_id, .. } => Some(*correlation_id), N::DhtGetRecord { correlation_id, .. } => Some(*correlation_id), N::GossipPublish { correlation_id, .. } => Some(*correlation_id), + N::OutgoingSyncRequest { correlation_id, .. } => Some(*correlation_id), _ => None, } } diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index e32fba0f59..925d05e107 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -27,12 +27,13 @@ use libp2p::{ StreamProtocol, Swarm, }; use rand::prelude::IteratorRandom; -use std::sync::atomic::AtomicBool; use std::{ collections::HashMap, + fmt::Display, sync::{atomic::Ordering, Arc}, time::Instant, }; +use std::{hash::Hash, sync::atomic::AtomicBool}; use std::{io::Error, time::Duration}; use tokio::{select, sync::broadcast, sync::mpsc}; @@ -398,12 +399,21 @@ async fn process_swarm_event( } SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { - message: RequestResponseMessage::Response { response, .. }, + message: + RequestResponseMessage::Response { + request_id, + response, + .. + }, .. })) => { // received a response to a request for events + let correlation_id = correlator.expire(request_id)?; event_tx.send(NetEvent::OutgoingSyncRequestSucceeded( - OutgoingSyncRequestSucceeded { value: response }, + OutgoingSyncRequestSucceeded { + value: response, + correlation_id, + }, ))?; } @@ -452,7 +462,10 @@ async fn process_swarm_command( key, } => handle_get_record(swarm, correlator, correlation_id, key), NetCommand::Shutdown => handle_shutdown(swarm, shutdown_flag), - NetCommand::OutgoingSyncRequest { value } => handle_outgoing_sync_request(swarm, value), + NetCommand::OutgoingSyncRequest { + correlation_id, + value, + } => handle_outgoing_sync_request(swarm, correlator, correlation_id, value), NetCommand::SyncResponse { value, channel } => handle_sync_response(swarm, channel, value), } } @@ -581,6 +594,8 @@ fn handle_shutdown( fn handle_outgoing_sync_request( swarm: &mut Swarm, + correlator: &mut Correlator, + correlation_id: CorrelationId, value: SyncRequestValue, ) -> Result<()> { // TODO: @@ -601,7 +616,8 @@ fn handle_outgoing_sync_request( }; // Request events - swarm.behaviour_mut().sync.send_request(&peer, value); + let query_id = swarm.behaviour_mut().sync.send_request(&peer, value); + correlator.track(query_id, correlation_id); Ok(()) } @@ -623,7 +639,7 @@ fn handle_sync_response( /// This correlates query_id and correlation_id. #[derive(Clone)] struct Correlator { - inner: HashMap, + inner: HashMap, } impl Correlator { @@ -633,13 +649,13 @@ impl Correlator { } } /// Add a pairing between query_id and correlation_id - pub fn track(&mut self, query_id: QueryId, correlation_id: CorrelationId) { - self.inner.insert(query_id, correlation_id); + pub fn track(&mut self, query_id: impl Display, correlation_id: CorrelationId) { + self.inner.insert(format!("{query_id}"), correlation_id); } /// Remove the pairing and return the correlation_id - pub fn expire(&mut self, query_id: &QueryId) -> Result { + pub fn expire(&mut self, query_id: impl Display) -> Result { self.inner - .remove(query_id) + .remove(&format!("{query_id}")) .ok_or_else(|| anyhow::anyhow!("Failed to correlate query_id={}", query_id)) } } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 52424c9711..fdbd359e21 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -201,6 +201,7 @@ async fn sync_request( net_cmds, net_events, NetCommand::OutgoingSyncRequest { + correlation_id: CorrelationId::new(), value: SyncRequestValue { since }, }, |e| match e.clone() { From d5462391e58dcea72f71faf9c414171ae59871df Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 13:17:09 +0000 Subject: [PATCH 49/63] try push --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 105e2a9200..39e22adc3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ concurrency: on: workflow_dispatch: # pull_request: # XXX: temporarily allowing tests to run on PR - pull_request_target: + push: branches: - main - dev From 9c9475e5147bd68d7c644181ece46ae20dc16a89 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 9 Feb 2026 13:19:13 +0000 Subject: [PATCH 50/63] attempt 2 getting tests to run --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39e22adc3c..c60b1a6f76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,7 @@ concurrency: on: workflow_dispatch: - # pull_request: # XXX: temporarily allowing tests to run on PR - push: + pull_request: branches: - main - dev @@ -15,6 +14,7 @@ on: branches: - main - dev + - '**' # XXX: temporarily allowing tests to run on PR env: SUPPORT_DOCKERFILE_PATH: crates/support/Dockerfile CIPHERNODE_DOCKERFILE_PATH: crates/Dockerfile From 4301491ed730aa581f4ce625929e92cc1ac92807 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 01:30:34 +0000 Subject: [PATCH 51/63] temporarily disable net requests in sync and hook up multithread properly --- .../src/snapshot_buffer/batch_router.rs | 2 +- crates/evm/src/evm_chain_gateway.rs | 1 - crates/evm/src/evm_parser.rs | 4 ++-- crates/evm/src/evm_read_interface.rs | 8 +++---- crates/evm/src/fix_historical_order.rs | 10 ++++----- crates/multithread/src/multithread.rs | 22 ++++++++++++------- crates/net/src/net_sync_manager.rs | 3 ++- crates/sync/src/sync.rs | 18 +++++++-------- 8 files changed, 37 insertions(+), 31 deletions(-) diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 63fd5797e0..71b88f3147 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -182,7 +182,7 @@ impl Handler for BatchRouter { type Result = (); fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - info!("Flushing sequence... {}", msg.seq()); + debug!("Flushing sequence... {}", msg.seq()); if let Some(batch) = self.batches.get(&msg.seq()) { batch.try_send(Flush)?; self.batches.remove(&msg.seq()); diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 8e1860162c..5a4f68fe12 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -206,7 +206,6 @@ impl Handler for EvmChainGateway { impl Handler for EvmChainGateway { type Result = (); fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - info!("Handler"); trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) } } diff --git a/crates/evm/src/evm_parser.rs b/crates/evm/src/evm_parser.rs index bcbf3bf045..e4549fd8ff 100644 --- a/crates/evm/src/evm_parser.rs +++ b/crates/evm/src/evm_parser.rs @@ -7,7 +7,7 @@ use actix::{Actor, Handler}; use e3_events::{hlc::HlcTimestamp, EnclaveEventData}; use e3_utils::MAILBOX_LIMIT; -use tracing::info; +use tracing::{debug, info}; use crate::{ events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, @@ -45,7 +45,7 @@ impl Handler for EvmParser { id, timestamp, }) => { - info!("processing event({})", msg.get_id()); + debug!("processing event({})", msg.get_id()); let extractor = self.extractor; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index b00a6309fd..54fbab66f3 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -22,7 +22,7 @@ use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; use tokio::select; use tokio::sync::oneshot; -use tracing::{error, info, instrument, warn}; +use tracing::{debug, error, info, instrument, warn}; pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; @@ -168,7 +168,7 @@ async fn stream_from_evm( let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); last_id = Some(evt.get_id()); - info!("Sending event({})", evt.get_id()); + debug!("Sending event({})", evt.get_id()); next.do_send(evt) } } @@ -179,7 +179,7 @@ async fn stream_from_evm( } } let historical_sync_event = HistoricalSyncComplete::new(chain_id, last_id); - warn!( + info!( "Historical Sync Complete event({})", historical_sync_event.get_id() ); @@ -205,7 +205,7 @@ async fn stream_from_evm( } } _ = &mut shutdown => { - info!("Received shutdown signal, stopping EVM stream"); + warn!("Received shutdown signal, stopping EVM stream"); match provider_ref.unsubscribe(id).await { Ok(_) => info!("Unsubscribed successfully from EVM event stream"), Err(err) => error!("Cannot unsubscribe from EVM event stream: {}", err), diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index 2fe149c4a4..21acede3ac 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -9,7 +9,7 @@ use actix::{Actor, Addr, Handler}; use bloom::{BloomFilter, ASMS}; use e3_events::CorrelationId; use e3_utils::MAILBOX_LIMIT; -use tracing::info; +use tracing::{debug, info}; pub struct FixHistoricalOrder { dest: EvmEventProcessor, @@ -37,7 +37,7 @@ impl FixHistoricalOrder { })) = self.pending_sync_complete { if self.seen_ids.contains(id) { - info!("Forwarding historical send complete event"); + debug!("Forwarding historical send complete event"); self.dest .do_send(self.pending_sync_complete.take().unwrap()); } @@ -61,13 +61,13 @@ impl Handler for FixHistoricalOrder { fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { let id = msg.get_id(); - info!("Receiving EnclaveEvmEvent event({})", msg.get_id()); + debug!("Receiving EnclaveEvmEvent event({})", msg.get_id()); match msg { none_hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { prev_event: None, .. }) => { - info!( + debug!( "Historical order event({}) has no previous event. Forwarding...", id ); @@ -77,7 +77,7 @@ impl Handler for FixHistoricalOrder { prev_event: Some(prev), .. }) => { - info!( + debug!( "Historical order event({}) has previous event({}). Buffering...", id, prev ); diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index 9ecc45c108..89a07ae656 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -85,14 +85,20 @@ impl Multithread { report: Option>, ) -> Addr { let addr = Self::new(bus.clone(), rng.clone(), cipher.clone(), task_pool, report).start(); - run_once::({ - let bus = bus.clone(); - let addr = addr.clone(); - move |_| { - bus.subscribe(EventType::ComputeRequest, addr.clone().recipient()); - Ok(()) - } - }); + + bus.subscribe( + EventType::EffectsEnabled, + run_once::({ + let bus = bus.clone(); + let addr = addr.clone(); + move |_| { + bus.subscribe(EventType::ComputeRequest, addr.clone().recipient()); + info!("Multithread actor listening for events."); + Ok(()) + } + }) + .recipient(), + ); addr } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index fdbd359e21..9a6aff9782 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -197,11 +197,12 @@ async fn sync_request( net_events: Arc>, since: HashMap, ) -> Result { + let id = CorrelationId::new(); call_and_await_response( net_cmds, net_events, NetCommand::OutgoingSyncRequest { - correlation_id: CorrelationId::new(), + correlation_id: id, value: SyncRequestValue { since }, }, |e| match e.clone() { diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 986ecd8b6a..1d2ff39c7f 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -78,19 +78,19 @@ pub async fn sync( ); // 6. Load the historical libp2p events to memory - info!("Loading historical libp2p events..."); - let (addr, rx) = actix_toolbox::oneshot::(); - bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; - let historical_net_events = rx.await?.events; - info!( - "{} historical libp2p events loaded.", - historical_net_events.len() - ); + // info!("Loading historical libp2p events..."); + // let (addr, rx) = actix_toolbox::oneshot::(); + // bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; + // let historical_net_events = rx.await?.events; + // info!( + // "{} historical libp2p events loaded.", + // historical_net_events.len() + // ); // 7. Sort both the evm and libp2p events together by HLC timestamp let mut historical = historical_evm_events .into_iter() - .chain(historical_net_events) + // .chain(historical_net_events) .collect::>(); historical.sort_by_key(|event| event.ts()); From cfc16284e6386ac79fa21656267ce45806582e03 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 02:19:43 +0000 Subject: [PATCH 52/63] filter for source = net when requesting remote events --- crates/data/src/commit_log_event_log.rs | 10 ++- crates/data/src/in_mem_event_log.rs | 10 ++- crates/events/src/bus_handle.rs | 26 ++++--- crates/events/src/enclave_event/mod.rs | 37 +++++++--- .../events/src/enclave_event/typed_event.rs | 10 ++- crates/events/src/event_context.rs | 67 ++++++++++++++++--- crates/events/src/traits.rs | 10 ++- crates/evm/src/events.rs | 5 +- crates/evm/src/evm_chain_gateway.rs | 1 - crates/net/src/document_publisher.rs | 3 +- crates/net/src/events.rs | 18 +++-- crates/net/src/net_event_translator.rs | 4 +- crates/net/src/net_sync_manager.rs | 4 +- 13 files changed, 160 insertions(+), 45 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 059fdca595..39fa684e6b 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -78,11 +78,17 @@ impl EventLog for CommitLogEventLog { #[cfg(test)] mod tests { use super::*; - use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; + use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, EventSource, TestEvent}; use tempfile::tempdir; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123, None) + EnclaveEvent::::new_with_timestamp( + data.into().into(), + None, + 123, + None, + EventSource::Local, + ) } #[test] diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index d8b227f200..d48a9837b3 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -46,10 +46,16 @@ impl EventLog for InMemEventLog { #[cfg(test)] mod tests { use super::*; - use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; + use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, EventSource, TestEvent}; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123, None) + EnclaveEvent::::new_with_timestamp( + data.into().into(), + None, + 123, + None, + EventSource::Local, + ) } #[test] diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 0a20823e53..a66f39172d 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -17,11 +17,11 @@ use crate::{ hlc::Hlc, sequencer::Sequencer, traits::{ - ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, - EventSubscriber, + ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventContextAccessors, + EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, EventType, - HistoryCollector, Sequenced, Subscribe, Unsequenced, Unsubscribe, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, EventSource, + EventType, HistoryCollector, Sequenced, Subscribe, Unsequenced, Unsubscribe, }; #[derive(Clone, Derivative)] @@ -108,8 +108,9 @@ impl EventPublisher> for BusHandle { data: impl Into, remote_ts: u128, block: Option, + source: EventSource, ) -> Result<()> { - self.publish_from_remote_impl(data, remote_ts, None, block) + self.publish_from_remote_impl(data, remote_ts, None, block, source) } fn publish_from_remote_as_response( @@ -118,8 +119,9 @@ impl EventPublisher> for BusHandle { remote_ts: u128, caused_by: impl Into>, block: Option, + source: EventSource, ) -> Result<()> { - self.publish_from_remote_impl(data, remote_ts, Some(caused_by.into()), block) + self.publish_from_remote_impl(data, remote_ts, Some(caused_by.into()), block, source) } fn naked_dispatch(&self, event: EnclaveEvent) { @@ -134,8 +136,9 @@ impl BusHandle { remote_ts: u128, caused_by: Option>, block: Option, + source: EventSource, ) -> Result<()> { - let evt = self.event_from_remote_source(data, caused_by, remote_ts, block)?; + let evt = self.event_from_remote_source(data, caused_by, remote_ts, block, source)?; self.sequencer.do_send(evt); Ok(()) } @@ -171,6 +174,7 @@ impl EventFactory> for BusHandle { caused_by, ts.into(), None, + EventSource::Local, )) } @@ -180,6 +184,7 @@ impl EventFactory> for BusHandle { caused_by: Option>, ts: u128, block: Option, + source: EventSource, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; Ok(EnclaveEvent::::new_with_timestamp( @@ -187,6 +192,7 @@ impl EventFactory> for BusHandle { caused_by, ts.into(), block, + source, )) } } @@ -425,9 +431,11 @@ where type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { if (self.predicate)(&msg) { + let source = msg.source(); let (data, ts) = msg.split(); - let _ = self.handle.publish_from_remote(data, ts, None); // TODO: check if this is fine - // to erase block data + let _ = self.handle.publish_from_remote(data, ts, None, source); + // TODO: check if this is fine + // to erase block data } } } diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index b234449e88..99d19e65aa 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -85,7 +85,7 @@ pub use typed_event::*; use crate::{ event_context::{AggregateId, EventContext}, traits::{ErrorEvent, Event, EventConstructorWithTimestamp, EventContextAccessors}, - E3id, EventContextSeq, EventId, WithAggregateId, + E3id, EventContextSeq, EventId, EventSource, WithAggregateId, }; use actix::Message; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -308,6 +308,13 @@ impl EventContextAccessors for EnclaveEvent { fn block(&self) -> Option { self.ctx.block() } + fn source(&self) -> EventSource { + self.ctx.source() + } + fn with_source(mut self, source: EventSource) -> Self { + self.ctx = self.ctx.with_source(source); + self + } } impl EventContextSeq for EnclaveEvent { @@ -321,7 +328,7 @@ impl EnclaveEvent { let ts = self.ts(); let block = self.block(); let data = self.clone().into_data(); - EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts, block) + EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts, block, self.source()) } pub fn to_typed_event(&self, data: T) -> TypedEvent { @@ -343,13 +350,20 @@ impl EnclaveEvent { impl EnclaveEvent { /// test-helpers only utility function to create a new sequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, None, time, None).into_sequenced(seq) + EnclaveEvent::::new_with_timestamp(data, None, time, None, EventSource::Local) + .into_sequenced(seq) } /// test-helpers only utility function to create a new sequenced event pub fn from_data_ec(data: EnclaveEventData, ec: EventContext) -> Self { - EnclaveEvent::::new_with_timestamp(data, Some(ec.clone()), ec.ts(), ec.block()) - .into_sequenced(ec.seq()) + EnclaveEvent::::new_with_timestamp( + data, + Some(ec.clone()), + ec.ts(), + ec.block(), + EventSource::Local, + ) + .into_sequenced(ec.seq()) } /// test-helpers only utility function to remove time information from an event @@ -394,8 +408,12 @@ impl ErrorEvent for EnclaveEvent { let aggregate_id = AggregateId::new(0); // Error events use default aggregate_id let ctx = caused_by - .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id, None)) - .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id, None)); + .map(|cause| { + EventContext::from_cause(id, cause, ts, aggregate_id, None, EventSource::Local) + }) + .unwrap_or_else(|| { + EventContext::new_origin(id, ts, aggregate_id, None, EventSource::Local) + }); Ok(EnclaveEvent { payload: payload.into(), @@ -557,6 +575,7 @@ impl EventConstructorWithTimestamp for EnclaveEvent { caused_by: Option>, ts: u128, block: Option, + source: EventSource, ) -> Self { let payload: EnclaveEventData = data.into(); let id = EventId::hash(&payload); @@ -564,8 +583,8 @@ impl EventConstructorWithTimestamp for EnclaveEvent { EnclaveEvent { payload, ctx: caused_by - .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id, block)) - .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id, block)), + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id, block, source)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id, block, source)), } } } diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index ed5a546791..86b0ff07f5 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::{ event_context::{AggregateId, EventContext}, - EventContextAccessors, EventContextSeq, EventId, + EventContextAccessors, EventContextSeq, EventId, EventSource, }; use super::Sequenced; @@ -75,6 +75,14 @@ impl EventContextAccessors for TypedEvent { fn block(&self) -> Option { self.ctx.block() } + + fn source(&self) -> EventSource { + self.ctx.source() + } + fn with_source(mut self, source: EventSource) -> Self { + self.ctx = self.ctx.with_source(source); + self + } } impl EventContextSeq for TypedEvent { diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index c18380cdf5..60743649ba 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -13,6 +13,13 @@ use crate::{ Unsequenced, }; +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum EventSource { + Local, + Net, + Evm, +} + #[derive(Clone, Copy, Debug, PartialEq, Ord, PartialOrd, Eq, Hash, Serialize, Deserialize)] pub struct AggregateId(usize); @@ -75,6 +82,7 @@ pub struct EventContext { ts: u128, aggregate_id: AggregateId, block: Option, + source: EventSource, } impl EventContext { @@ -85,6 +93,7 @@ impl EventContext { ts: u128, aggregate_id: AggregateId, block: Option, + source: EventSource, ) -> Self { Self { id, @@ -94,6 +103,7 @@ impl EventContext { ts, aggregate_id, block, + source, } } @@ -102,8 +112,9 @@ impl EventContext { ts: u128, aggregate_id: AggregateId, block: Option, + source: EventSource, ) -> Self { - Self::new(id, id, id, ts, aggregate_id, block) + Self::new(id, id, id, ts, aggregate_id, block, source) } pub fn from_cause( @@ -112,6 +123,7 @@ impl EventContext { ts: u128, aggregate_id: AggregateId, block: Option, + source: EventSource, ) -> Self { EventContext::new( id, @@ -120,6 +132,7 @@ impl EventContext { ts, aggregate_id, cause.block.max(block), // block watermark + source, ) } @@ -142,6 +155,7 @@ impl EventContext { ts: self.ts, aggregate_id: self.aggregate_id, block: self.block, + source: self.source, } } } @@ -149,7 +163,13 @@ impl EventContext { impl From for EventContext { fn from(value: EnclaveEventData) -> Self { let id = EventId::hash(value); - EventContext::::new_origin(id, 0, AggregateId::new(0), Some(0)) + EventContext::::new_origin( + id, + 0, + AggregateId::new(0), + Some(0), + EventSource::Local, + ) } } @@ -177,6 +197,15 @@ impl EventContextAccessors for EventContext { fn block(&self) -> Option { self.block } + + fn source(&self) -> EventSource { + self.source + } + + fn with_source(mut self, source: EventSource) -> Self { + self.source = source; + self + } } impl EventContextSeq for EventContext { @@ -189,7 +218,7 @@ impl EventContextSeq for EventContext { mod tests { use crate::{ event_context::{AggregateId, EventContext}, - EventId, + EventId, EventSource, }; #[test] @@ -203,16 +232,31 @@ mod tests { 1, AggregateId::new(1), None, + EventSource::Local, ) .sequence(1); events.push(one.clone()); - let two = EventContext::from_cause(EventId::hash(2), one, 2, AggregateId::new(1), None) - .sequence(2); + let two = EventContext::from_cause( + EventId::hash(2), + one, + 2, + AggregateId::new(1), + None, + EventSource::Local, + ) + .sequence(2); events.push(two.clone()); - let three = EventContext::from_cause(EventId::hash(3), two, 3, AggregateId::new(1), None) - .sequence(3); + let three = EventContext::from_cause( + EventId::hash(3), + two, + 3, + AggregateId::new(1), + None, + EventSource::Local, + ) + .sequence(3); events.push(three.clone()); assert_eq!( @@ -225,7 +269,8 @@ mod tests { causation_id: EventId::hash(1), ts: 1, aggregate_id: AggregateId::new(1), - block: None + block: None, + source: EventSource::Local }, EventContext { seq: 2, @@ -234,7 +279,8 @@ mod tests { causation_id: EventId::hash(1), ts: 2, aggregate_id: AggregateId::new(1), - block: None + block: None, + source: EventSource::Local }, EventContext { seq: 3, @@ -243,7 +289,8 @@ mod tests { causation_id: EventId::hash(2), ts: 3, aggregate_id: AggregateId::new(1), - block: None + block: None, + source: EventSource::Local }, ] ) diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 47210f9168..5ac78dd3ed 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -11,7 +11,7 @@ use std::hash::Hash; use crate::{ event_context::{AggregateId, EventContext}, - EnclaveEvent, EventId, EventType, Sequenced, Unsequenced, + EnclaveEvent, EventId, EventSource, EventType, Sequenced, Unsequenced, }; /// Trait that must be implemented by events used with EventBus @@ -65,6 +65,7 @@ pub trait EventFactory { caused_by: Option>, ts: u128, block: Option, + source: EventSource, ) -> Result; } @@ -104,6 +105,7 @@ pub trait EventPublisher { data: impl Into, remote_ts: u128, block: Option, + source: EventSource, ) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. @@ -118,6 +120,7 @@ pub trait EventPublisher { remote_ts: u128, caused_by: impl Into>, block: Option, + source: EventSource, ) -> Result<()>; /// Dispatch the given event without applying any HLC transformation. fn naked_dispatch(&self, event: E); @@ -147,6 +150,7 @@ pub trait EventConstructorWithTimestamp: Event + Sized { caused_by: Option>, ts: u128, block: Option, + source: EventSource, ) -> Self; } @@ -186,6 +190,10 @@ pub trait EventContextAccessors { fn aggregate_id(&self) -> AggregateId; /// The highest block watermark we have seen fn block(&self) -> Option; + /// The event source + fn source(&self) -> EventSource; + /// Apply a new source fluently + fn with_source(self, source: EventSource) -> Self; } pub trait EventContextSeq { diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index 366b91e639..b6f2bcaff1 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -8,7 +8,8 @@ use actix::{Message, Recipient}; use alloy::rpc::types::Log; use anyhow::Result; use e3_events::{ - BusHandle, CorrelationId, EnclaveEvent, EnclaveEventData, EventFactory, Unsequenced, + BusHandle, CorrelationId, EnclaveEvent, EnclaveEventData, EventFactory, EventSource, + Unsequenced, }; use serde::{Deserialize, Serialize}; @@ -81,7 +82,7 @@ impl EvmEvent { pub fn into_enclave_event(self, bus: &BusHandle) -> Result> { let data = self.data; let ts = self.ts; - bus.event_from_remote_source(data, None, ts, Some(self.block)) + bus.event_from_remote_source(data, None, ts, Some(self.block), EventSource::Evm) } } diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index 5a4f68fe12..c0ba917892 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -19,7 +19,6 @@ use e3_events::{ }; use e3_events::{Event, EventPublisher}; use e3_utils::MAILBOX_LIMIT; -use tracing::info; /// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. /// It coordinates event flow between these components. diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index e28f21b58b..df56b8e5df 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -17,7 +17,7 @@ use chrono::{DateTime, Utc}; use e3_events::{ prelude::*, trap, trap_fut, BusHandle, CiphernodeSelected, CorrelationId, DocumentKind, DocumentMeta, DocumentReceived, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, - EncryptionKeyCreated, Event, EventContext, EventType, Filter, PartyId, + EncryptionKeyCreated, Event, EventContext, EventSource, EventType, Filter, PartyId, PublishDocumentRequested, Sequenced, ThresholdShareCreated, TypedEvent, }; use e3_utils::ArcBytes; @@ -305,6 +305,7 @@ pub async fn handle_document_published_notification( }, event.ts, None, + EventSource::Net, )?; Ok(()) diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index a3bad8ea58..1723181567 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -7,7 +7,10 @@ use crate::Cid; use actix::Message; use anyhow::{bail, Context, Result}; -use e3_events::{AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; +use e3_events::{ + AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, EventContextAccessors, EventSource, + Sequenced, Unsequenced, +}; use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, @@ -59,7 +62,7 @@ impl TryFrom for EnclaveEvent { bail!("GossipData was not the GossipBytes variant"); }; - Ok(EnclaveEvent::from_bytes(&bytes)?) + Ok(EnclaveEvent::from_bytes(&bytes)?.with_source(EventSource::Net)) } } @@ -303,7 +306,7 @@ where #[cfg(test)] mod tests { use e3_events::{ - EnclaveEvent, EventConstructorWithTimestamp, Sequenced, TestEvent, Unsequenced, + EnclaveEvent, EventConstructorWithTimestamp, EventSource, Sequenced, TestEvent, Unsequenced, }; use super::GossipData; @@ -311,8 +314,13 @@ mod tests { #[test] fn test_enclave_event_gossip_lifecycle() -> anyhow::Result<()> { // event is created locally - let event: EnclaveEvent = - EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), None, 31415, None); + let event: EnclaveEvent = EnclaveEvent::new_with_timestamp( + TestEvent::new("fish", 42).into(), + None, + 31415, + None, + EventSource::local, + ); // event is sequenced after bus.publish() adds a sequence number let event: EnclaveEvent = event.into_sequenced(90210); diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 06b82d396f..e5f641ec76 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -23,6 +23,7 @@ use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::Event; use e3_events::EventContextAccessors; +use e3_events::EventSource; use e3_events::EventStoreQueryBy; use e3_events::EventType; use e3_events::TsAgg; @@ -154,7 +155,8 @@ impl NetEventTranslator { fn publish_event(&mut self, event: EnclaveEvent) -> Result<()> { let id = event.id(); let (data, ec) = event.into_components(); - self.bus.publish_from_remote(data, ec.ts(), None)?; + self.bus + .publish_from_remote(data, ec.ts(), None, EventSource::Net)?; self.sent_events.insert(id); Ok(()) } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 9a6aff9782..993cdeb3a6 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -8,7 +8,7 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, EventStoreQueryBy, EventStoreQueryResponse, EventType, + EnclaveEventData, EventSource, EventStoreQueryBy, EventStoreQueryResponse, EventType, HistoricalNetSyncStart, NetSyncEventsReceived, TsAgg, TypedEvent, Unsequenced, }; use e3_utils::{retry_with_backoff, to_retry, OnceTake, MAILBOX_LIMIT}; @@ -140,6 +140,7 @@ impl Handler> for NetSyncManager { msg.value.ts, ctx, None, + EventSource::Net, )?; Ok(()) @@ -178,6 +179,7 @@ impl Handler for NetSyncManager { events: msg .into_events() .into_iter() + .filter(|e| e.source() == EventSource::Net) .map(|ev| ev.try_into()) .collect::>()?, ts: self.bus.ts()?, // NOTE: We are storing a local timestamp on this response From a9f744f431595a8a65c6dd5d27d68a4f50c85e04 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 02:20:49 +0000 Subject: [PATCH 53/63] reenable sync events --- crates/sync/src/sync.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 1d2ff39c7f..986ecd8b6a 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -78,19 +78,19 @@ pub async fn sync( ); // 6. Load the historical libp2p events to memory - // info!("Loading historical libp2p events..."); - // let (addr, rx) = actix_toolbox::oneshot::(); - // bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; - // let historical_net_events = rx.await?.events; - // info!( - // "{} historical libp2p events loaded.", - // historical_net_events.len() - // ); + info!("Loading historical libp2p events..."); + let (addr, rx) = actix_toolbox::oneshot::(); + bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; + let historical_net_events = rx.await?.events; + info!( + "{} historical libp2p events loaded.", + historical_net_events.len() + ); // 7. Sort both the evm and libp2p events together by HLC timestamp let mut historical = historical_evm_events .into_iter() - // .chain(historical_net_events) + .chain(historical_net_events) .collect::>(); historical.sort_by_key(|event| event.ts()); From 90e02bcdf0d50041aab251795a566374af5eeb34 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 02:56:09 +0000 Subject: [PATCH 54/63] wait for peers to be dialed before attempting to sync --- crates/evm/src/fix_historical_order.rs | 2 +- crates/net/src/events.rs | 29 ++++++++++++++++++ crates/net/src/net_interface.rs | 2 +- crates/net/src/net_sync_manager.rs | 41 ++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index 21acede3ac..075bb81977 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -86,7 +86,7 @@ impl Handler for FixHistoricalOrder { } EnclaveEvmEvent::Processed(id) => self.track_id(id), other => { - info!("Forwarding event({})", other.get_id()); + debug!("Forwarding event({})", other.get_id()); self.track_id(other.get_id()); self.dest.do_send(other); } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 1723181567..3fb4841cb2 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -207,6 +207,7 @@ pub enum NetEvent { /// Received gossipsub events from a peer in response to a `SyncRequest`. OutgoingSyncRequestSucceeded(OutgoingSyncRequestSucceeded), OutgoingSyncRequestFailed(OutgoingSyncRequestFailed), + AllPeersDialed, } #[derive(Clone, Debug)] @@ -303,6 +304,34 @@ where result } +pub async fn await_event( + net_events: &Arc>, + matcher: F, + timeout: Duration, +) -> Result +where + F: Fn(&NetEvent) -> Option, +{ + let mut rx = net_events.resubscribe(); + + let result = tokio::time::timeout(timeout, async { + loop { + match rx.recv().await { + Ok(event) => { + if let Some(result) = matcher(&event) { + return Ok(result); + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(e) => return Err(e.into()), + } + } + }) + .await + .map_err(|_| anyhow::anyhow!(format!("Timed out waiting for event")))?; + result +} + #[cfg(test)] mod tests { use e3_events::{ diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 925d05e107..b65b6a2bc0 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -146,7 +146,7 @@ impl NetInterface { let peers = self.peers.clone(); async move { dial_peers(&cmd_tx, &event_tx, &peers).await?; - + event_tx.send(NetEvent::AllPeersDialed)?; return anyhow::Ok(()); } }); diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 993cdeb3a6..e6d1bc70ff 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; +use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, @@ -19,7 +19,7 @@ use tokio::sync::{broadcast, mpsc}; use tracing::debug; use crate::events::{ - call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, + await_event, call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, SyncResponseValue, }; @@ -32,6 +32,7 @@ pub struct NetSyncManager { rx: Arc>, eventstore: Recipient>, requests: HashMap>>, + peers_ready: bool, } impl NetSyncManager { @@ -47,6 +48,7 @@ impl NetSyncManager { rx: Arc::clone(rx), eventstore, requests: HashMap::new(), + peers_ready: false, } } @@ -71,6 +73,7 @@ impl NetSyncManager { match event { // Someone is asking for our sync NetEvent::SyncRequestReceived(value) => addr.do_send(value), + NetEvent::AllPeersDialed => addr.do_send(AllPeersDialed), _ => (), } } @@ -112,7 +115,13 @@ impl Handler> for NetSyncManager { trap_fut( EType::Net, &self.bus.with_ec(msg.get_ctx()), - handle_sync_request_event(self.tx.clone(), self.rx.clone(), msg, ctx.address()), + handle_sync_request_event( + self.tx.clone(), + self.rx.clone(), + msg, + ctx.address(), + !self.peers_ready, + ), ) } } @@ -192,6 +201,17 @@ impl Handler for NetSyncManager { } } +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: AllPeersDialed, ctx: &mut Self::Context) -> Self::Result { + self.peers_ready = true; + } +} + +#[derive(Message)] +#[rtype(result = "()")] +struct AllPeersDialed; + const SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); async fn sync_request( @@ -224,8 +244,23 @@ async fn handle_sync_request_event( net_events: Arc>, event: TypedEvent, address: impl Into>>, + wait_for_event: bool, ) -> Result<()> { let (event, ctx) = event.into_components(); + if wait_for_event { + await_event( + &net_events, + |e| { + if matches!(e, &NetEvent::AllPeersDialed) { + Some(e.clone()) + } else { + None + } + }, + Duration::from_secs(30), + ) + .await?; + } // Make the sync request // value returned includes the timestamp from the remote peer From 938cce4e22ddcb3ff4fefe5eb0ec236f89dcc9f7 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 03:30:37 +0000 Subject: [PATCH 55/63] add todo to fix issue with net sync --- crates/events/src/eventstore_router.rs | 11 +++++++++++ crates/net/src/net_sync_manager.rs | 12 ++++++++++-- crates/utils/src/helpers.rs | 3 +++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 7a9a3b4c14..04cfb7d3f6 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -75,6 +75,17 @@ impl EventStoreRouter { } Ok(()) } + + // TODO: + // 1. on each query type store a map of query by id(CorrelationId) to sender and AggregationIds in an + // EventStoreQueryRequest object which should include a field for results + // 2. Create a handler for EventStoreQueryResponse that looks up the request by correlation_id + // and keeps trackof the received requests. It should get the aggregation id off an event to + // see which aggregation_id the result is from and it should mark that AggregationId as + // complete. + // 3. Once all the requests have been received combine all the events together into a single + // Vec and forward to the original sender. + // 4. Don't be afraid of using small simple decomposed components to build this. } impl Actor for EventStoreRouter { diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index e6d1bc70ff..89e9df1fd9 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -16,7 +16,7 @@ use futures::TryFutureExt; use libp2p::request_response::ResponseChannel; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::sync::{broadcast, mpsc}; -use tracing::debug; +use tracing::{debug, info}; use crate::events::{ await_event, call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, @@ -162,8 +162,11 @@ impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: SyncRequestReceived, ctx: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus, || { + info!("GOT SyncRequestReceived"); let id = CorrelationId::new(); + info!("STORING channel in requests map..."); self.requests.insert(id, msg.channel); + info!("QUERYING eventstore..."); self.eventstore.try_send(EventStoreQueryBy::::new( id, msg.value.since, @@ -179,10 +182,14 @@ impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { + info!("RECEIVED response from eventstore."); let Some(channel) = self.requests.get(&msg.id()) else { bail!("request not found with {}", msg.id()); }; - + info!( + "GOT CHANNEL: Got channel from request store. NOW sending SyncResponse with channel={:?}", + channel + ); self.tx.try_send(NetCommand::SyncResponse { value: SyncResponseValue { events: msg @@ -204,6 +211,7 @@ impl Handler for NetSyncManager { impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: AllPeersDialed, ctx: &mut Self::Context) -> Self::Result { + info!("All peers dialed"); self.peers_ready = true; } } diff --git a/crates/utils/src/helpers.rs b/crates/utils/src/helpers.rs index 9380d01abf..fa24bd7c4d 100644 --- a/crates/utils/src/helpers.rs +++ b/crates/utils/src/helpers.rs @@ -8,6 +8,8 @@ use std::{ sync::{Arc, Mutex}, }; +use tracing::info; + pub fn to_ordered_vec(source: HashMap) -> Vec where K: Ord + Copy, @@ -47,6 +49,7 @@ impl OnceTake { /// Takes the item, returning `None` if already taken. pub fn take(&self) -> Option { + info!("take has been called!"); self.0.lock().unwrap().take() } From 5ae87b074bea87db70cb2a5284bc75a8fa5c469a Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 09:03:18 +0000 Subject: [PATCH 56/63] debug the issue --- crates/events/src/eventstore_router.rs | 195 +++++++++++++++++++------ crates/net/src/events.rs | 20 ++- crates/net/src/net_interface.rs | 13 +- crates/net/src/net_sync_manager.rs | 13 +- crates/sync/src/sync.rs | 38 +---- crates/utils/src/retry.rs | 2 + 6 files changed, 199 insertions(+), 82 deletions(-) diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 04cfb7d3f6..7368bb6e85 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -3,18 +3,87 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. - use crate::eventstore::EventStore; use crate::{ - events::StoreEventRequested, AggregateId, EventContextAccessors, EventLog, SequenceIndex, + events::{EventStoreQueryResponse, StoreEventRequested}, + AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; -use crate::{EventStoreQueryBy, Seq, SeqAgg, Ts, TsAgg}; -use actix::{Actor, Addr, Handler}; +use crate::{CorrelationId, Die, EnclaveEvent, EventStoreQueryBy, Seq, SeqAgg, Ts, TsAgg}; +use actix::{Actor, ActorContext, Addr, AsyncContext, Context, Handler, Recipient}; use anyhow::Result; use e3_utils::{major_issue, MAILBOX_LIMIT}; use std::collections::HashMap; -use tracing::{error, info}; +use tracing::{error, info, warn}; + +/// QueryAggregator - handles a single query's lifecycle +struct QueryAggregator { + parent_id: CorrelationId, + sender: Recipient, + pending: HashMap, + collected_events: Vec, +} + +impl QueryAggregator { + fn new(parent_id: CorrelationId, sender: Recipient) -> Self { + Self { + parent_id, + sender, + pending: HashMap::new(), + collected_events: Vec::new(), + } + } + + fn add_pending(&mut self, sub_query_id: CorrelationId, aggregate_id: AggregateId) { + self.pending.insert(sub_query_id, aggregate_id); + } + + fn pending_aggregates(&self) -> Vec<&AggregateId> { + self.pending.values().collect() + } +} + +impl Actor for QueryAggregator { + type Context = Context; +} + +impl Handler for QueryAggregator { + type Result = (); + + fn handle(&mut self, msg: EventStoreQueryResponse, ctx: &mut Self::Context) -> Self::Result { + let sub_query_id = msg.id(); + + if let Some(aggregate_id) = self.pending.remove(&sub_query_id) { + info!( + "Received response for aggregate {:?}, {} pending", + aggregate_id, + self.pending.len() + ); + self.collected_events.extend(msg.into_events()); + + if self.pending.is_empty() { + info!("All aggregates fulfilled, sending response"); + let response = EventStoreQueryResponse::new( + self.parent_id, + std::mem::take(&mut self.collected_events), + ); + self.sender.do_send(response); + ctx.notify(Die) + } + } else { + warn!("Received response for unknown sub-query: {}", sub_query_id); + } + } +} + +impl Handler for QueryAggregator { + type Result = (); + + fn handle(&mut self, msg: Die, ctx: &mut Self::Context) -> Self::Result { + ctx.stop() + } +} +/// EventStoreRouter - routes events and spawns query aggregators to handle eventstore queries pub struct EventStoreRouter { stores: HashMap>>, } @@ -26,70 +95,114 @@ impl EventStoreRouter { .into_iter() .map(|(index, addr)| (AggregateId::new(index), addr)) .collect(); - Self { stores } } pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { info!("Handling store event requested...."); let aggregate_id = msg.event.aggregate_id(); - let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { self.stores .get(&AggregateId::new(0)) .expect("Default EventStore for AggregateId(0) not found") }); - let event = msg.event; let sender = msg.sender; - let forwarded_msg = StoreEventRequested::new(event, sender); store_addr.try_send(forwarded_msg)?; Ok(()) } - pub fn handle_event_store_query_ts(&mut self, msg: EventStoreQueryBy) -> Result<()> { - let id = msg.id(); + pub fn handle_event_store_query_ts( + &mut self, + msg: EventStoreQueryBy, + _ctx: &mut Context, + ) -> Result<()> { + info!("Received request for timestamp query."); + let parent_id = msg.id(); let query = msg.query().clone(); let sender = msg.sender(); - for (aggregate_id, ts) in query { - if let Some(store_addr) = self.stores.get(&aggregate_id) { - let get_events_msg = - EventStoreQueryBy::::new(id, ts.to_owned(), sender.clone()); - store_addr.do_send(get_events_msg); - } + + let sub_queries: Vec<_> = query + .into_iter() + .filter_map(|(aggregate_id, ts)| { + self.stores + .get(&aggregate_id) + .map(|store_addr| (aggregate_id, ts, CorrelationId::new(), store_addr.clone())) + }) + .collect(); + + if sub_queries.is_empty() { + info!("No valid stores to query, sending empty response immediately"); + let response = EventStoreQueryResponse::new(parent_id, Vec::new()); + sender.do_send(response); + return Ok(()); + } + + let mut aggregator = QueryAggregator::new(parent_id, sender); + for (aggregate_id, _, sub_query_id, _) in &sub_queries { + aggregator.add_pending(*sub_query_id, aggregate_id.clone()); + } + let aggregator_addr = aggregator.start(); + + for (aggregate_id, ts, sub_query_id, store_addr) in sub_queries { + let get_events_msg = + EventStoreQueryBy::::new(sub_query_id, ts, aggregator_addr.clone().recipient()); + info!("Sending query for aggregate {:?}", aggregate_id); + store_addr.do_send(get_events_msg); } + Ok(()) } - pub fn handle_event_store_query_seq(&mut self, msg: EventStoreQueryBy) -> Result<()> { - let id = msg.id(); + pub fn handle_event_store_query_seq( + &mut self, + msg: EventStoreQueryBy, + _ctx: &mut Context, + ) -> Result<()> { + info!("Received request for sequence query."); + let parent_id = msg.id(); let query = msg.query().clone(); let sender = msg.sender(); - for (aggregate_id, ts) in query { - if let Some(store_addr) = self.stores.get(&aggregate_id) { - let get_events_msg = - EventStoreQueryBy::::new(id, ts.to_owned(), sender.clone()); - store_addr.do_send(get_events_msg); - } + + let sub_queries: Vec<_> = query + .into_iter() + .filter_map(|(aggregate_id, seq)| { + self.stores + .get(&aggregate_id) + .map(|store_addr| (aggregate_id, seq, CorrelationId::new(), store_addr.clone())) + }) + .collect(); + + if sub_queries.is_empty() { + info!("No valid stores to query, sending empty response immediately"); + let response = EventStoreQueryResponse::new(parent_id, Vec::new()); + sender.do_send(response); + return Ok(()); + } + + let mut aggregator = QueryAggregator::new(parent_id, sender); + for (aggregate_id, _, sub_query_id, _) in &sub_queries { + aggregator.add_pending(*sub_query_id, aggregate_id.clone()); } + let aggregator_addr = aggregator.start(); + + for (aggregate_id, seq, sub_query_id, store_addr) in sub_queries { + let get_events_msg = EventStoreQueryBy::::new( + sub_query_id, + seq, + aggregator_addr.clone().recipient(), + ); + info!("Sending query for aggregate {:?}", aggregate_id); + store_addr.do_send(get_events_msg); + } + Ok(()) } - - // TODO: - // 1. on each query type store a map of query by id(CorrelationId) to sender and AggregationIds in an - // EventStoreQueryRequest object which should include a field for results - // 2. Create a handler for EventStoreQueryResponse that looks up the request by correlation_id - // and keeps trackof the received requests. It should get the aggregation id off an event to - // see which aggregation_id the result is from and it should mark that AggregationId as - // complete. - // 3. Once all the requests have been received combine all the events together into a single - // Vec and forward to the original sender. - // 4. Don't be afraid of using small simple decomposed components to build this. } impl Actor for EventStoreRouter { - type Context = actix::Context; + type Context = Context; fn started(&mut self, ctx: &mut Self::Context) { ctx.set_mailbox_capacity(MAILBOX_LIMIT); @@ -109,8 +222,8 @@ impl Handler for EventStoreR impl Handler> for EventStoreRouter { type Result = (); - fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_event_store_query_ts(msg) { + fn handle(&mut self, msg: EventStoreQueryBy, ctx: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_ts(msg, ctx) { error!("Failed to route get events after request: {}", e); } } @@ -119,8 +232,8 @@ impl Handler> for EventS impl Handler> for EventStoreRouter { type Result = (); - fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_event_store_query_seq(msg) { + fn handle(&mut self, msg: EventStoreQueryBy, ctx: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_seq(msg, ctx) { error!("Failed to route get events after request: {}", e); } } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 3fb4841cb2..1728a45a47 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -26,6 +26,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::sync::{broadcast, mpsc}; +use tracing::{error, info}; /// Incoming/Outgoing GossipData. We disambiguate on concerns relative to the net package. #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] @@ -281,21 +282,36 @@ where let debug_cmd = format!("{:?}", command); // Send the command to NetInterface + info!( + "call_and_await_response: SENDING command {:?} with timeout {:?}", + command, timeout + ); net_cmds.send(command).await?; let result = tokio::time::timeout(timeout, async { loop { match rx.recv().await { Ok(event) => { + info!("RECEIVED and event {:?}", event); // Only process events matching our correlation ID if event.correlation_id() == Some(id) { if let Some(result) = matcher(&event) { return result; } // None means unexpected event type, keep waiting - } // Ignore events with non-matching IDs + info!("matcher failed for event {:?}! skipping...", event); + } else { + // Ignore events with non-matching IDs + info!( + "correlation failed for event {:?} against correlation={:?}! skipping...", + event, id + ); + } } Err(broadcast::error::RecvError::Lagged(_)) => continue, - Err(e) => return Err(e.into()), + Err(e) => { + error!("RECEIVED an error from rx: {:?} returning error", e); + return Err(e.into()); + } } } }) diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index b65b6a2bc0..264721acac 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -390,6 +390,8 @@ async fn process_swarm_event( }, .. })) => { + info!("***INCOMING REQUEST RECEIVED {}!!***", request_id); + // received a request for events event_tx.send(NetEvent::SyncRequestReceived(SyncRequestReceived { request_id, @@ -407,8 +409,10 @@ async fn process_swarm_event( }, .. })) => { + info!("***OUTGOING SYNC REQUEST RESPONSE***{}", request_id); // received a response to a request for events let correlation_id = correlator.expire(request_id)?; + info!("correlation_id = {correlation_id}"); event_tx.send(NetEvent::OutgoingSyncRequestSucceeded( OutgoingSyncRequestSucceeded { value: response, @@ -598,6 +602,7 @@ fn handle_outgoing_sync_request( correlation_id: CorrelationId, value: SyncRequestValue, ) -> Result<()> { + info!("***OUTGOING SYNC REQUEST!! {} ***", correlation_id); // TODO: // This is a first pass. // Lots of stuff to work through here: @@ -617,6 +622,10 @@ fn handle_outgoing_sync_request( // Request events let query_id = swarm.behaviour_mut().sync.send_request(&peer, value); + info!( + "QueryId={} correlated with correlation_id={}", + query_id, correlation_id + ); correlator.track(query_id, correlation_id); Ok(()) } @@ -626,11 +635,13 @@ fn handle_sync_response( channel: OnceTake>, value: SyncResponseValue, ) -> Result<()> { + info!("Sending response..."); let channel = channel.try_take()?; match swarm.behaviour_mut().sync.send_response(channel, value) { Ok(_) => (), - Err(_res) => { + Err(value) => { // TODO: report failure + error!("sync sending response failure! {:?}", value); } } Ok(()) diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 89e9df1fd9..3ac9d8beec 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -112,6 +112,7 @@ impl Handler> for NetSyncManager { msg: TypedEvent, ctx: &mut Self::Context, ) -> Self::Result { + info!("HISTORICAL_NET_SYNC_START"); trap_fut( EType::Net, &self.bus.with_ec(msg.get_ctx()), @@ -210,8 +211,8 @@ impl Handler for NetSyncManager { impl Handler for NetSyncManager { type Result = (); - fn handle(&mut self, msg: AllPeersDialed, ctx: &mut Self::Context) -> Self::Result { - info!("All peers dialed"); + fn handle(&mut self, _: AllPeersDialed, _: &mut Self::Context) -> Self::Result { + info!("Received handler: All peers dialed"); self.peers_ready = true; } } @@ -227,6 +228,7 @@ async fn sync_request( net_events: Arc>, since: HashMap, ) -> Result { + info!("RUNNING sync request..."); let id = CorrelationId::new(); call_and_await_response( net_cmds, @@ -254,12 +256,15 @@ async fn handle_sync_request_event( address: impl Into>>, wait_for_event: bool, ) -> Result<()> { + info!("Sync request event received"); let (event, ctx) = event.into_components(); + info!("Waiting for peers to have been contacted..."); if wait_for_event { await_event( &net_events, |e| { if matches!(e, &NetEvent::AllPeersDialed) { + info!("AllPeersDialed matched!"); Some(e.clone()) } else { None @@ -269,11 +274,13 @@ async fn handle_sync_request_event( ) .await?; } + info!("handle_sync_request_event: All peers have been dialed."); // Make the sync request // value returned includes the timestamp from the remote peer let value = retry_with_backoff( || { + info!("Running SYNC REQUEST!!"); sync_request( net_cmds.clone(), net_events.clone(), @@ -282,7 +289,7 @@ async fn handle_sync_request_event( .map_err(to_retry) }, 4, - 1000, + 5000, ) .await?; diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 986ecd8b6a..650ea88025 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -45,15 +45,13 @@ pub async fn sync( // 3. Load EventStore events since the sequence number found in the snapshot into memory. info!("Loading EventStore events..."); - let (tx, rx) = actix_toolbox::mpsc::(256); + let (addr, rx) = actix_toolbox::oneshot::(); eventstore.try_send(EventStoreQueryBy::::new( CorrelationId::new(), snapshot.to_sequence_map(), - tx, + addr, ))?; - let events = - collect_eventstore_query_response(rx, snapshot.aggregates().len(), Duration::from_secs(5)) - .await; + let events = rx.await?.into_events(); info!("{} EventStore events loaded.", events.len()); info!("Replaying events to actors..."); @@ -150,36 +148,6 @@ pub async fn collect_historical_evm_events( results } -pub async fn collect_eventstore_query_response( - mut receiver: Receiver, - expected: usize, - max_dur: Duration, -) -> Vec { - let mut results = Vec::new(); - let mut received = 0; - - let collect = async { - for _ in 0..expected { - match receiver.recv().await { - Some(msg) => { - results.extend(msg.into_events()); - received += 1; - } - None => break, - } - } - }; - - if timeout(max_dur, collect).await.is_err() { - eprintln!( - "Error: Timeout waiting for historical events from {} aggregates", - expected - received - ); - } - - results -} - /// Latest event information in store #[derive(Clone)] pub struct AggregateState { diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs index 700442a1e1..6eb82724e8 100644 --- a/crates/utils/src/retry.rs +++ b/crates/utils/src/retry.rs @@ -9,6 +9,7 @@ use std::{future::Future, time::Duration}; use tokio::time::sleep; use tracing::{error, warn}; +#[derive(Debug)] pub enum RetryError { Failure(anyhow::Error), Retry(anyhow::Error), @@ -46,6 +47,7 @@ where match operation().await { Ok(value) => return Ok(value), Err(re) => { + tracing::error!("RETRY FAILED {:?}", re); match re { RetryError::Retry(e) => { if current_attempt >= max_attempts { From 338d79930316dfbf14e9a10e7a72ce4e236c5ab1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 10:49:36 +0000 Subject: [PATCH 57/63] skip libp2p events --- crates/ciphernode-builder/src/event_system.rs | 24 ++++++--- crates/events/src/bus_handle.rs | 6 +-- .../src/snapshot_buffer/snapshot_buffer.rs | 23 ++++---- crates/net/src/events.rs | 17 +++++- crates/net/src/net_interface.rs | 54 +++++++++++++++++-- crates/sync/src/sync.rs | 19 +++---- 6 files changed, 111 insertions(+), 32 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 4039b35a7b..7ca1e514a8 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -14,7 +14,7 @@ use e3_data::{ use e3_events::hlc::Hlc; use e3_events::{ AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, - EventStoreQueryBy, EventStoreRouter, InsertBatch, QueryKind, SeqAgg, Sequencer, SnapshotBuffer, + EventStoreQueryBy, EventStoreRouter, InsertBatch, SeqAgg, Sequencer, SnapshotBuffer, StoreEventRequested, TsAgg, UpdateDestination, }; use e3_utils::enumerate_path; @@ -419,7 +419,7 @@ mod tests { use e3_data::Repository; use e3_events::EventContext; use e3_events::EventId; - use e3_events::Start; + use e3_events::EventSource; use e3_events::StoreKeys; use e3_events::SyncEnded; use e3_events::Tick; @@ -544,8 +544,14 @@ mod tests { .start(); // Sequence 1, Aggregate 0 - let ec = - EventContext::new_origin(EventId::hash(1), 10, AggregateId::new(0), None).sequence(1); + let ec = EventContext::new_origin( + EventId::hash(1), + 10, + AggregateId::new(0), + None, + EventSource::Local, + ) + .sequence(1); // Send all evts to the listener handle.subscribe(EventType::All, listener.clone().into()); @@ -600,8 +606,14 @@ mod tests { info!("Mutating persistable state to create inserts using seq=2"); - let ec = - EventContext::new_origin(EventId::hash(1), 10, AggregateId::new(0), None).sequence(2); + let ec = EventContext::new_origin( + EventId::hash(1), + 10, + AggregateId::new(0), + None, + EventSource::Local, + ) + .sequence(2); persistable.try_mutate(&ec, |_| Ok("Liz".to_string()))?; sleep(Duration::from_millis(1)).await; diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index a66f39172d..36ec3958a7 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -250,8 +250,8 @@ mod tests { use e3_ciphernode_builder::EventSystem; // NOTE: We cannot pull from crate as the features will be missing as they are not default. use e3_events::{ - hlc::Hlc, prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventPublisher, EventType, - TestEvent, + hlc::Hlc, prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventPublisher, + EventSource, EventType, TestEvent, }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::time::sleep; @@ -282,7 +282,7 @@ mod tests { fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { let ts = msg.ts(); self.dest - .publish_from_remote(msg.into_data(), ts, None) + .publish_from_remote(msg.into_data(), ts, None, EventSource::Local) .unwrap() } } diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index d428a61e02..587cf6aea6 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -8,15 +8,12 @@ use super::{ timelock_queue::{Clock, StartTimelock, SystemClock, Tick, TimelockQueue}, AggregateConfig, }; -use crate::{ - trap, EType, EffectsEnabled, EnclaveEvent, EnclaveEventData, Event, Insert, InsertBatch, - PanicDispatcher, -}; +use crate::{trap, EType, EnclaveEvent, Insert, InsertBatch, PanicDispatcher}; use actix::{Actor, Addr, Handler, Message, Recipient}; use anyhow::Result; -use e3_utils::{NotifySync, MAILBOX_LIMIT}; +use e3_utils::MAILBOX_LIMIT; use std::sync::Arc; -use tracing::{debug, info, trace}; +use tracing::{info, trace}; #[derive(Message)] #[rtype(result = "()")] @@ -60,7 +57,7 @@ pub struct SnapshotBuffer { } impl SnapshotBuffer { - pub fn new(start_buffering: bool) -> Self { + pub fn new(_: bool) -> Self { SnapshotBuffer { router: None, timelock: None, @@ -278,7 +275,8 @@ mod tests { use crate::snapshot_buffer::timelock_queue::Tick; use crate::{ AggregateConfig, AggregateId, E3id, EnclaveEvent, EventContext, EventContextAccessors, - EventContextSeq, EventId, Insert, InsertBatch, Sequenced, SyncEnded, TestEvent, + EventContextSeq, EventId, EventSource, Insert, InsertBatch, Sequenced, SyncEnded, + TestEvent, }; use actix::Actor; use anyhow::Result; @@ -289,7 +287,14 @@ mod tests { use tracing::info; fn create_ec(ag: usize, seq: u64) -> EventContext { - EventContext::new_origin(EventId::hash(1), 1000, AggregateId::new(ag), None).sequence(seq) + EventContext::new_origin( + EventId::hash(1), + 1000, + AggregateId::new(ag), + None, + EventSource::Local, + ) + .sequence(seq) } fn create_event(ec: &EventContext) -> EnclaveEvent { diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 1728a45a47..4be60f6987 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -94,7 +94,10 @@ pub struct OutgoingSyncRequestSucceeded { } #[derive(Debug, Clone)] -pub struct OutgoingSyncRequestFailed; +pub struct OutgoingSyncRequestFailed { + pub correlation_id: CorrelationId, + pub error: String, +} /// NetInterface Commands are sent to the network peer over a mspc channel #[derive(Debug)] @@ -227,6 +230,8 @@ impl NetEvent { N::DhtGetRecordSucceeded { correlation_id, .. } => Some(*correlation_id), N::DhtPutRecordError { correlation_id, .. } => Some(*correlation_id), N::DhtPutRecordSucceeded { correlation_id, .. } => Some(*correlation_id), + N::OutgoingSyncRequestSucceeded(msg) => Some(msg.correlation_id), + N::OutgoingSyncRequestFailed(msg) => Some(msg.correlation_id), _ => None, } } @@ -348,6 +353,14 @@ where result } +pub fn estimate_hashmap_size(map: &HashMap) -> usize { + let entry_size = size_of::() + size_of::(); + let capacity = map.capacity(); + + // HashMap uses ~1 byte of overhead per slot for metadata + capacity * (entry_size + 1) + size_of::>() +} + #[cfg(test)] mod tests { use e3_events::{ @@ -364,7 +377,7 @@ mod tests { None, 31415, None, - EventSource::local, + EventSource::Local, ); // event is sequenced after bus.publish() adds a sequence number diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 264721acac..3f5ee0053f 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -44,8 +44,8 @@ const MAX_KADEMLIA_PAYLOAD_MB: usize = 10; const MAX_GOSSIP_MSG_SIZE_KB: usize = 700; use crate::events::{ - GossipData, NetCommand, OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, - SyncResponseValue, + estimate_hashmap_size, GossipData, NetCommand, OutgoingSyncRequestFailed, + OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, SyncResponseValue, }; use crate::events::{NetEvent, PutOrStoreError}; use crate::{dialer::dial_peers, Cid}; @@ -194,12 +194,15 @@ fn create_behaviour( gossipsub::MessageAuthenticity::Signed(key.clone()), gossipsub_config, )?; + let request_response_config = + request_response::Config::default().with_request_timeout(Duration::from_secs(30)); + let sync = CborRequestResponse::::new( [( StreamProtocol::new("/enclave/sync/0.0.1"), ProtocolSupport::Full, )], - request_response::Config::default(), + request_response_config, ); let mut config = KademliaConfig::new(PROTOCOL_NAME); config @@ -421,6 +424,49 @@ async fn process_swarm_event( ))?; } + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync( + RequestResponseEvent::OutboundFailure { + peer, + request_id, + error, + }, + )) => { + info!( + "***OUTBOUND SYNC REQUEST FAILURE*** peer={}, request_id={}, error={:?}", + peer, request_id, error + ); + let correlation_id = correlator.expire(request_id)?; + event_tx.send(NetEvent::OutgoingSyncRequestFailed( + OutgoingSyncRequestFailed { + correlation_id, + error: format!("Outbound sync request failed: {:?}", error), + }, + ))?; + } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::InboundFailure { + peer, + request_id, + error, + })) => { + info!( + "***INBOUND SYNC REQUEST FAILURE*** peer={}, request_id={}, error={:?}", + peer, request_id, error + ); + // Handle inbound failures - log for now, could add more sophisticated error handling if needed + } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::ResponseSent { + peer, + request_id, + })) => { + info!( + "***SYNC RESPONSE SENT*** peer={}, request_id={}", + peer, request_id + ); + // Successfully sent response to inbound request + } + unknown => { trace!("Unknown event: {:?}", unknown); } @@ -620,6 +666,8 @@ fn handle_outgoing_sync_request( bail!("No peer found on swarm!") }; + info!("VALUE SIZE: {:?}", estimate_hashmap_size(&value.since)); + // Request events let query_id = swarm.behaviour_mut().sync.send_request(&peer, value); info!( diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 650ea88025..165ba15a73 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -75,20 +75,21 @@ pub async fn sync( historical_evm_events.len() ); + // XXX: Skipping as we have bugs in libp2p netevent requests // 6. Load the historical libp2p events to memory - info!("Loading historical libp2p events..."); - let (addr, rx) = actix_toolbox::oneshot::(); - bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; - let historical_net_events = rx.await?.events; - info!( - "{} historical libp2p events loaded.", - historical_net_events.len() - ); + // info!("Loading historical libp2p events..."); + // let (addr, rx) = actix_toolbox::oneshot::(); + // bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; + // let historical_net_events = rx.await?.events; + // info!( + // "{} historical libp2p events loaded.", + // historical_net_events.len() + // ); // 7. Sort both the evm and libp2p events together by HLC timestamp let mut historical = historical_evm_events .into_iter() - .chain(historical_net_events) + // .chain(historical_net_events) // Commenting out to skip .collect::>(); historical.sort_by_key(|event| event.ts()); From a3536baedbca3384a85dc16e21d01a00becbe9ba Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 10 Feb 2026 13:33:46 +0000 Subject: [PATCH 58/63] remove ci push run --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de2e1564b4..3dbe36e654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,6 @@ on: branches: - main - dev - - '**' # XXX: temporarily allowing tests to run on PR env: SUPPORT_DOCKERFILE_PATH: crates/support/Dockerfile CIPHERNODE_DOCKERFILE_PATH: crates/Dockerfile From a8e80c4ddc7ca935600c037d65a3e8f23eea8824 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 12 Feb 2026 07:35:35 +0000 Subject: [PATCH 59/63] use debug --- crates/events/src/eventstore_router.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 7368bb6e85..ba214df7ff 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -13,7 +13,7 @@ use actix::{Actor, ActorContext, Addr, AsyncContext, Context, Handler, Recipient use anyhow::Result; use e3_utils::{major_issue, MAILBOX_LIMIT}; use std::collections::HashMap; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; /// QueryAggregator - handles a single query's lifecycle struct QueryAggregator { @@ -90,7 +90,7 @@ pub struct EventStoreRouter { impl EventStoreRouter { pub fn new(stores: HashMap>>) -> Self { - info!("Making eventstore router..."); + debug!("Making eventstore router..."); let stores = stores .into_iter() .map(|(index, addr)| (AggregateId::new(index), addr)) @@ -99,7 +99,7 @@ impl EventStoreRouter { } pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { - info!("Handling store event requested...."); + debug!("Handling store event requested...."); let aggregate_id = msg.event.aggregate_id(); let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { self.stores @@ -118,7 +118,7 @@ impl EventStoreRouter { msg: EventStoreQueryBy, _ctx: &mut Context, ) -> Result<()> { - info!("Received request for timestamp query."); + debug!("Received request for timestamp query."); let parent_id = msg.id(); let query = msg.query().clone(); let sender = msg.sender(); @@ -133,7 +133,7 @@ impl EventStoreRouter { .collect(); if sub_queries.is_empty() { - info!("No valid stores to query, sending empty response immediately"); + debug!("No valid stores to query, sending empty response immediately"); let response = EventStoreQueryResponse::new(parent_id, Vec::new()); sender.do_send(response); return Ok(()); @@ -148,7 +148,7 @@ impl EventStoreRouter { for (aggregate_id, ts, sub_query_id, store_addr) in sub_queries { let get_events_msg = EventStoreQueryBy::::new(sub_query_id, ts, aggregator_addr.clone().recipient()); - info!("Sending query for aggregate {:?}", aggregate_id); + debug!("Sending query for aggregate {:?}", aggregate_id); store_addr.do_send(get_events_msg); } @@ -160,7 +160,7 @@ impl EventStoreRouter { msg: EventStoreQueryBy, _ctx: &mut Context, ) -> Result<()> { - info!("Received request for sequence query."); + debug!("Received request for sequence query."); let parent_id = msg.id(); let query = msg.query().clone(); let sender = msg.sender(); @@ -175,7 +175,7 @@ impl EventStoreRouter { .collect(); if sub_queries.is_empty() { - info!("No valid stores to query, sending empty response immediately"); + debug!("No valid stores to query, sending empty response immediately"); let response = EventStoreQueryResponse::new(parent_id, Vec::new()); sender.do_send(response); return Ok(()); @@ -193,7 +193,7 @@ impl EventStoreRouter { seq, aggregator_addr.clone().recipient(), ); - info!("Sending query for aggregate {:?}", aggregate_id); + debug!("Sending query for aggregate {:?}", aggregate_id); store_addr.do_send(get_events_msg); } From 02e00c5817a335ebfbc53f23645fac30b4725fd2 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 14 Feb 2026 01:31:37 +0000 Subject: [PATCH 60/63] ensure snapshotbuffer receives events first without having to explicitly send them --- crates/ciphernode-builder/src/event_system.rs | 17 +-- crates/events/src/sequencer.rs | 4 - .../src/snapshot_buffer/snapshot_buffer.rs | 49 ------- crates/sync/src/sync.rs | 138 +----------------- 4 files changed, 9 insertions(+), 199 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 7ca1e514a8..af81b1117d 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -14,8 +14,8 @@ use e3_data::{ use e3_events::hlc::Hlc; use e3_events::{ AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, - EventStoreQueryBy, EventStoreRouter, InsertBatch, SeqAgg, Sequencer, SnapshotBuffer, - StoreEventRequested, TsAgg, UpdateDestination, + EventStoreQueryBy, EventStoreRouter, EventSubscriber, EventType, InsertBatch, SeqAgg, + Sequencer, SnapshotBuffer, StoreEventRequested, TsAgg, UpdateDestination, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -224,7 +224,7 @@ impl EventSystem { self.sequencer .get_or_try_init(|| { let router = self.eventstore_router()?; - Ok(Sequencer::new(&self.eventbus(), router, self.buffer()?).start()) + Ok(Sequencer::new(&self.eventbus(), router).start()) }) .cloned() } @@ -353,11 +353,11 @@ impl EventSystem { println!("handle"); self.handle .get_or_try_init(|| { - Ok(BusHandle::new( - self.eventbus(), - self.sequencer()?, - self.hlc()?, - )) + let handle = BusHandle::new(self.eventbus(), self.sequencer()?, self.hlc()?); + // Buffer subscribes to all events first + // This is important so as to open up a batch for each sequence + handle.subscribe(EventType::All, self.buffer()?.recipient()); + Ok(handle) }) .cloned() } @@ -504,7 +504,6 @@ mod tests { let _guard = with_tracing("debug"); let tmp = TempDir::new().unwrap(); let system = EventSystem::persisted("cn2", tmp.path().join("log"), tmp.path().join("sled")); - // system.buffer()?.send(Start).await?; let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); Ok(()) diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 5ea107f022..cb8a929243 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -16,25 +16,21 @@ use e3_utils::{major_issue, MAILBOX_LIMIT_LARGE}; pub struct Sequencer { bus: Addr>>, eventstore: Recipient, - buffer: Recipient, } impl Sequencer { pub fn new( bus: &Addr>>, eventstore: impl Into>, - buffer: impl Into>, ) -> Self { Self { bus: bus.clone(), eventstore: eventstore.into(), - buffer: buffer.into(), } } fn handle_store_event_response(&self, msg: StoreEventResponse) -> Result<()> { let event = msg.into_event(); - self.buffer.try_send(event.clone())?; self.bus.try_send(event)?; Ok(()) } diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 587cf6aea6..e9cb9de952 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -43,17 +43,11 @@ impl UpdateDestination { Self(base.into()) } } -// enum SnapshotBufferState { -// Disabled, // SnapshotBuffer is completely disabled -// Forwarding, // Forwarding to disk -// Buffering, // Buffering -// } pub struct SnapshotBuffer { router: Option>, timelock: Option>, tickable: Option>, - // state: SnapshotBufferState, } impl SnapshotBuffer { @@ -62,11 +56,6 @@ impl SnapshotBuffer { router: None, timelock: None, tickable: None, - // state: if !start_buffering { - // SnapshotBufferState::Buffering - // } else { - // SnapshotBufferState::Forwarding - // }, } } @@ -114,9 +103,6 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - // let SnapshotBufferState::Forwarding = self.state else { - // return Ok(()); - // }; if let Some(ref router) = self.router { router.try_send(msg)?; } @@ -129,10 +115,6 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - // let SnapshotBufferState::Forwarding = self.state else { - // return Ok(()); - // }; - if let Some(ref timelock) = self.timelock { timelock.try_send(msg)?; } @@ -155,24 +137,10 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - // use SnapshotBufferState as S; - // match (&self.state, msg.ctx(), &self.router) { - // // Doing this to cover when there are context free - // // data store manipulations outside of the sync cycle. - // (S::Buffering, None, Some(ref router)) => { - // debug!( - // "Paused but received context free Insert. Forwarding to batch router..." - // ); - // router.try_send(msg)?; - // } - // (S::Forwarding, _, Some(ref router)) => { if let Some(ref router) = self.router { trace!("Forwarding Insert message to batch router..."); router.try_send(msg)?; }; - // } - // _ => (), - // }; Ok(()) }) } @@ -182,12 +150,6 @@ impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::IO, &PanicDispatcher::new(), || { - // if let EnclaveEventData::EffectsEnabled(msg) = msg.get_data() { - // self.notify_sync(ctx, msg.clone()); - // }; - // let SnapshotBufferState::Forwarding = self.state else { - // return Ok(()); - // }; if let Some(ref router) = self.router { router.try_send(msg)?; } @@ -208,17 +170,6 @@ impl Handler for SnapshotBuffer { } } -// impl Handler for SnapshotBuffer { -// type Result = (); -// fn handle(&mut self, _: EffectsEnabled, _: &mut Self::Context) -> Self::Result { -// trap(EType::IO, &PanicDispatcher::new(), || { -// info!("SnapshotBuffer is now enabled"); -// self.state = SnapshotBufferState::Forwarding; -// Ok(()) -// }) -// } -// } - impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: UpdateDestination, _: &mut Self::Context) -> Self::Result { diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index 165ba15a73..a81eed2737 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -41,7 +41,7 @@ pub async fn sync( // 2. Determine the evm blocks to read from based on the SnapshotMeta let evm_config = snapshot.to_evm_config(); - let net_config = snapshot.to_net_config(); + let _net_config = snapshot.to_net_config(); // 3. Load EventStore events since the sequence number found in the snapshot into memory. info!("Loading EventStore events..."); @@ -250,139 +250,3 @@ impl SnapshotLoaded { Self { snapshot } } } - -// #[cfg(test)] -// mod tests { -// use super::*; -// use e3_ciphernode_builder::EventSystem; -// use e3_events::{EnclaveEvent, EventFactory}; -// use e3_events::{ -// EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, TestEvent, -// }; -// use std::collections::HashMap; -// use std::time::Duration; -// use tokio::time::sleep; -// -// fn hlc_faucet(bus: &BusHandle, num: usize) -> Result> { -// let mut queue = Vec::new(); -// for _ in 0..num { -// queue.push(bus.ts()?) -// } -// -// Ok(queue.into_iter()) -// } -// -// async fn settle() { -// sleep(Duration::from_millis(100)).await; -// } -// -// #[actix::test] -// #[ignore] -// async fn test_synchronizer_full_flow() -> Result<()> { -// let _guard = e3_test_helpers::with_tracing("info"); -// // Setup event system and synchronizer -// let system = EventSystem::new("test").with_fresh_bus(); -// let bus: BusHandle = system.handle()?; -// let history_collector = bus.history(); -// -// // Configure test chains -// let mut evm_config = EvmEventConfig::new(); -// evm_config.insert(1, EvmEventConfigChain::new(0)); -// evm_config.insert(2, EvmEventConfigChain::new(0)); -// let repositories = Repositories::in_mem(); -// let snapshot_buffer = system.buffer()?; -// // Start synchronizer -// let sync_addr = Synchronizer::setup( -// &bus, -// &evm_config, -// &repositories, -// &AggregateConfig::new(HashMap::new()), -// &snapshot_buffer, -// ); -// settle().await; -// -// // Verify HistoricalEvmSyncStart was published -// let history = history_collector -// .send(GetEvents::::new()) -// .await?; -// let sync_start_count = history -// .into_iter() -// .filter(|e| matches!(e.get_data(), EnclaveEventData::HistoricalEvmSyncStart(_))) -// .count(); -// assert!( -// sync_start_count > 0, -// "HistoricalEvmSyncStart should be dispatched" -// ); -// -// // Create test events with timestamps -// let mut timelord = hlc_faucet(&bus, 100)?; -// -// // Test events - timestamps generated in order -// let h_2_1 = bus.event_from_remote_source( -// EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), -// None, -// timelord.next().unwrap(), -// Some(1), -// )?; -// -// let h_1_1 = bus.event_from_remote_source( -// EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), -// None, -// timelord.next().unwrap(), -// Some(1), -// )?; -// -// let h_1_2 = bus.event_from_remote_source( -// EnclaveEventData::TestEvent(TestEvent::new("1-second", 2)), -// None, -// timelord.next().unwrap(), -// Some(2), -// )?; -// -// let h_2_2 = bus.event_from_remote_source( -// EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), -// None, -// timelord.next().unwrap(), -// Some(2), -// )?; -// -// // Send events in mixed order to test sorting -// sync_addr -// .send(HistoricalEvmEventsReceived::new(vec![h_2_2, h_2_1], 2)) -// .await?; -// sync_addr -// .send(HistoricalEvmEventsReceived::new(vec![h_1_1, h_1_2], 1)) -// .await?; -// -// settle().await; -// -// // Get final event history and verify ordering -// let full = history_collector -// .send(GetEvents::::new()) -// .await?; -// println!("full = {}", full.len()); -// let events: Vec = full -// .into_iter() -// .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) -// .collect(); -// -// let event_strings: Vec = events -// .into_iter() -// .filter_map(|e| { -// if let EnclaveEventData::TestEvent(data) = e.into_data() { -// Some(data.msg) -// } else { -// None -// } -// }) -// .collect(); -// -// // Events should be published in timestamp order -// assert_eq!( -// event_strings, -// vec!["2-first", "1-first", "1-second", "2-second"] -// ); -// -// Ok(()) -// } -// } From 549996e1dd2b655e801f1db22934aac4e839499b Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 14 Feb 2026 11:44:36 +0000 Subject: [PATCH 61/63] remove start_buffer --- crates/ciphernode-builder/src/event_system.rs | 16 +--------------- .../src/snapshot_buffer/snapshot_buffer.rs | 16 ++++------------ 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index af81b1117d..8d0b674614 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -94,8 +94,6 @@ pub struct EventSystem { aggregate_config: OnceCell, /// Cached EventStoreAddrs for idempotency eventstore_addrs: OnceCell, - /// Whether to start the buffer straight away - start_buffer: bool, } impl EventSystem { @@ -119,7 +117,6 @@ impl EventSystem { hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), - start_buffer: false, } } @@ -138,7 +135,6 @@ impl EventSystem { hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), - start_buffer: false, } } @@ -159,7 +155,6 @@ impl EventSystem { hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), - start_buffer: false, } } @@ -189,11 +184,6 @@ impl EventSystem { self } - pub fn start_buffer_immediately(mut self) -> Self { - self.start_buffer = true; - self - } - /// Get the eventbus address pub fn eventbus(&self) -> Addr> { self.eventbus.get_or_init(get_enclave_event_bus).clone() @@ -210,11 +200,7 @@ impl EventSystem { pub fn buffer(&self) -> Result> { self.buffer .get_or_try_init(|| { - SnapshotBuffer::spawn( - &self.aggregate_config(), - NoopBatchReceiver::new().start(), - self.start_buffer, - ) + SnapshotBuffer::spawn(&self.aggregate_config(), NoopBatchReceiver::new().start()) }) .cloned() } diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index e9cb9de952..bf73313580 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -51,7 +51,7 @@ pub struct SnapshotBuffer { } impl SnapshotBuffer { - pub fn new(_: bool) -> Self { + pub fn new() -> Self { SnapshotBuffer { router: None, timelock: None, @@ -62,16 +62,9 @@ impl SnapshotBuffer { pub fn spawn( config: &AggregateConfig, store: impl Into>, - start_buffering: bool, ) -> Result> { info!("spawning SnapshotBuffer..."); - let (addr, _) = Self::with_clock( - config, - store, - Arc::new(SystemClock), - Some(1), - start_buffering, - )?; + let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock), Some(1))?; Ok(addr) } @@ -80,9 +73,8 @@ impl SnapshotBuffer { store: impl Into>, clock: Arc, interval: Option, - start_buffering: bool, ) -> Result<(Addr, Addr)> { - let addr = Self::new(start_buffering).start(); + let addr = Self::new().start(); let store = store.into(); let router = BatchRouter::with_clock(config, addr.clone(), store.clone(), clock.clone()).start(); @@ -270,7 +262,7 @@ mod tests { let clock = Arc::new(MockClock::new(1000)); let (buffer, timelock) = - SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None, false)?; + SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; buffer .send(EnclaveEvent::from_data_ec( From f2c7d7d4fb6a7a55d3c32ae996b35281b6083211 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 14 Feb 2026 13:59:16 +0000 Subject: [PATCH 62/63] remove redundant logging --- crates/ciphernode-builder/src/event_system.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 8d0b674614..3b11ef15b7 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -47,7 +47,6 @@ struct PersistedBackend { impl PersistedBackend { fn get_or_init_store(&self, handle: &BusHandle) -> Result> { - println!("get_or_init_store in {:?} ...", self.sled_path); self.store .get_or_try_init(|| SledStore::new(handle, &self.sled_path)) .cloned() @@ -288,10 +287,8 @@ impl EventSystem { pub fn persisted_eventstore_router( &self, ) -> Result>> { - info!("persisted_eventstore_router..."); let eventstores = self.eventstore_addrs()?; if let EventStoreAddrs::Persisted(addrs) = eventstores { - info!("creating router..."); let router = EventStoreRouter::new(addrs); Ok(router.start()) } else { @@ -301,7 +298,6 @@ impl EventSystem { /// Get an EventStoreRouter Recipient pub fn eventstore_router(&self) -> Result> { - info!("eventstore_reader..."); let eventstores = self.eventstore_addrs()?; match &eventstores { EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), @@ -310,7 +306,6 @@ impl EventSystem { } pub fn eventstore_getter_seq(&self) -> Result>> { - info!("eventstore_reader..."); let eventstores = self.eventstore_addrs()?; match &eventstores { EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), @@ -319,7 +314,6 @@ impl EventSystem { } pub fn eventstore_getter_ts(&self) -> Result>> { - info!("eventstore_reader..."); let eventstores = self.eventstore_addrs()?; match &eventstores { EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), @@ -336,7 +330,6 @@ impl EventSystem { /// Get the BusHandle pub fn handle(&self) -> Result { - println!("handle"); self.handle .get_or_try_init(|| { let handle = BusHandle::new(self.eventbus(), self.sequencer()?, self.hlc()?); @@ -350,7 +343,6 @@ impl EventSystem { /// Get the DataStore pub fn store(&self) -> Result { - println!("store()..."); let store = match &self.backend { EventSystemBackend::InMem(b) => { let base = b.get_or_init_store(); From 5d25c90c7d18db2df6689411c250eddef98afa60 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 14 Feb 2026 14:10:02 +0000 Subject: [PATCH 63/63] ensure store is called within the context of get_enclave_bus_handle --- crates/ciphernode-builder/src/event_system.rs | 1 - crates/ciphernode-builder/src/eventbus_factory.rs | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 3b11ef15b7..26c28fadcb 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -22,7 +22,6 @@ use once_cell::sync::OnceCell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; -use tracing::info; struct InMemBackend { eventstores: OnceCell>>>, diff --git a/crates/ciphernode-builder/src/eventbus_factory.rs b/crates/ciphernode-builder/src/eventbus_factory.rs index 5277e5f878..d2ddc5b6b0 100644 --- a/crates/ciphernode-builder/src/eventbus_factory.rs +++ b/crates/ciphernode-builder/src/eventbus_factory.rs @@ -103,5 +103,7 @@ pub fn get_error_collector() -> Addr> { pub fn get_enclave_bus_handle(config: &AppConfig) -> anyhow::Result { let bus = get_enclave_event_bus(); let system = EventSystem::new(&config.name()).with_event_bus(bus); + system.store()?; // Ensure store is initialized before returning to avoid potentially dropping + // events. Ok(system.handle()?) }