diff --git a/Cargo.lock b/Cargo.lock index e4a230145b..0cffd27f15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1143,9 +1143,9 @@ dependencies = [ [[package]] name = "alloy-trie" -version = "0.9.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7fd448ab0a017de542de1dcca7a58e7019fe0e7a34ed3f9543ebddf6aceffa" +checksum = "428aa0f0e0658ff091f8f667c406e034b431cb10abd39de4f507520968acc499" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -1154,7 +1154,6 @@ dependencies = [ "nybbles", "serde", "smallvec", - "thiserror 2.0.18", "tracing", ] @@ -2199,9 +2198,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] @@ -3102,6 +3101,7 @@ dependencies = [ "e3-request", "e3-sortition", "e3-sync", + "e3-test-helpers", "e3-trbfv", "e3-utils", "e3-zk-prover", @@ -3214,6 +3214,7 @@ dependencies = [ "async-trait", "bincode 1.3.3", "commitlog", + "e3-config", "e3-events", "e3-utils", "once_cell", @@ -3282,6 +3283,7 @@ dependencies = [ "e3-data", "e3-events", "e3-fhe-params", + "e3-test-helpers", "e3-trbfv", "e3-utils", "futures-util", @@ -3317,6 +3319,7 @@ dependencies = [ "e3-evm", "e3-fhe-params", "e3-sortition", + "e3-test-helpers", "e3-trbfv", "e3-utils", "futures-util", @@ -3508,6 +3511,7 @@ dependencies = [ "async-trait", "bincode 1.3.3", "chrono", + "derivative", "e3-ciphernode-builder", "e3-config", "e3-crypto", @@ -3642,7 +3646,11 @@ dependencies = [ "actix", "anyhow", "e3-ciphernode-builder", + "e3-config", + "e3-data", "e3-events", + "e3-test-helpers", + "e3-utils", "tokio", "tracing", ] @@ -3679,6 +3687,7 @@ dependencies = [ "rand_chacha 0.3.1", "tokio", "tracing", + "tracing-subscriber", ] [[package]] @@ -3766,6 +3775,7 @@ dependencies = [ "derivative", "rand 0.8.5", "rand_chacha 0.3.1", + "regex", "serde", "tokio", "tracing", @@ -4233,9 +4243,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" dependencies = [ "crc32fast", "miniz_oxide", @@ -4875,7 +4885,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.6", + "webpki-roots 1.0.5", ] [[package]] @@ -4909,13 +4919,14 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "base64", "bytes", "futures-channel", + "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", @@ -4925,7 +4936,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.2", - "system-configuration 0.7.0", + "system-configuration", "tokio", "tower-service", "tracing", @@ -5092,7 +5103,7 @@ dependencies = [ "netlink-proto", "netlink-sys", "rtnetlink", - "system-configuration 0.6.1", + "system-configuration", "tokio", "windows", ] @@ -5241,9 +5252,9 @@ dependencies = [ [[package]] name = "interprocess" -version = "2.3.1" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bf2b0e0785c5394a7392f66d7c4fb9c653633c29b27a932280da3cb344c66a" +checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d" dependencies = [ "doctest-file", "futures-core", @@ -6088,9 +6099,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" @@ -7039,9 +7050,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" dependencies = [ "memchr", "ucd-trie", @@ -7057,16 +7068,6 @@ dependencies = [ "indexmap 2.13.0", ] -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset 0.5.7", - "indexmap 2.13.0", -] - [[package]] name = "petgraph" version = "0.8.3" @@ -7405,7 +7406,7 @@ dependencies = [ "rand 0.9.2", "rand_chacha 0.9.0", "rand_xorshift 0.4.0", - "regex-syntax 0.8.9", + "regex-syntax 0.8.8", "rusty-fork", "tempfile", "unarray", @@ -7463,7 +7464,7 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph 0.7.1", + "petgraph 0.6.5", "prettyplease", "prost 0.13.5", "prost-types 0.13.5", @@ -7911,8 +7912,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.14", - "regex-syntax 0.8.9", + "regex-automata 0.4.13", + "regex-syntax 0.8.8", ] [[package]] @@ -7926,20 +7927,20 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.9", + "regex-syntax 0.8.8", ] [[package]] name = "regex-lite" -version = "0.1.9" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" @@ -7949,9 +7950,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.9" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" @@ -7998,7 +7999,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.6", + "webpki-roots 1.0.5", ] [[package]] @@ -8301,9 +8302,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.23" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "safe-proc-macro2" @@ -9058,17 +9059,6 @@ dependencies = [ "system-configuration-sys", ] -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags 2.10.0", - "core-foundation", - "system-configuration-sys", -] - [[package]] name = "system-configuration-sys" version = "0.6.0" @@ -9647,9 +9637,9 @@ dependencies = [ [[package]] name = "tracing-test-macro" -version = "0.2.6" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", "syn 2.0.114", @@ -9721,9 +9711,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.23" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" @@ -10049,14 +10039,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.6", + "webpki-roots 1.0.5", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" dependencies = [ "rustls-pki-types", ] @@ -10578,18 +10568,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.39" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.39" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0" dependencies = [ "proc-macro2", "quote", diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index 6ecda11931..ba06ce1044 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -7,9 +7,9 @@ use actix::prelude::*; use e3_events::{ prelude::*, trap, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EType, - EnclaveEvent, EnclaveEventData, EventType, Shutdown, + EnclaveEvent, EnclaveEventData, EventType, Shutdown, TypedEvent, }; -use e3_utils::NotifySync; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use std::collections::HashMap; use std::time::Duration; use tracing::{error, info}; @@ -43,28 +43,39 @@ impl CommitteeFinalizer { impl Actor for CommitteeFinalizer { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for CommitteeFinalizer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::CommitteeRequested(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::CommitteeRequested(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } } -impl Handler for CommitteeFinalizer { +impl Handler> for CommitteeFinalizer { type Result = (); - fn handle(&mut self, msg: CommitteeRequested, ctx: &mut Self::Context) -> Self::Result { + // TODO: Remove all async from this function. Remove reliance on e3_evm package. Add unit test. + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let ec = msg.get_ctx().clone(); let e3_id = msg.e3_id.clone(); let committee_deadline = msg.committee_deadline; const FINALIZATION_BUFFER_SECONDS: u64 = 1; - let e3_id_for_log = e3_id.clone(); let fut = async move { match e3_evm::helpers::get_current_timestamp().await { @@ -113,10 +124,10 @@ impl Handler for CommitteeFinalizer { move |act, _ctx| { info!(e3_id = %e3_id_clone, "Dispatching CommitteeFinalizeRequested event"); - trap(EType::Sortition, &act.bus.clone(), || { + trap(EType::Sortition, &act.bus.with_ec(&ec), || { bus.publish(CommitteeFinalizeRequested { e3_id: e3_id_clone.clone(), - })?; + },ec)?; Ok(()) }); diff --git a/crates/aggregator/src/keyshare_created_filter_buffer.rs b/crates/aggregator/src/keyshare_created_filter_buffer.rs index 43fc32b111..eea8edd67c 100644 --- a/crates/aggregator/src/keyshare_created_filter_buffer.rs +++ b/crates/aggregator/src/keyshare_created_filter_buffer.rs @@ -7,6 +7,7 @@ use actix::prelude::*; use e3_events::{prelude::*, EnclaveEvent, EnclaveEventData}; +use e3_utils::MAILBOX_LIMIT; use std::collections::HashSet; use crate::PublicKeyAggregator; @@ -42,6 +43,9 @@ impl KeyshareCreatedFilterBuffer { impl Actor for KeyshareCreatedFilterBuffer { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for KeyshareCreatedFilterBuffer { diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index bfdf3b0195..1b36c91ad3 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -9,13 +9,13 @@ use anyhow::Result; use e3_bfv_client::client::compute_pk_commitment; use e3_data::Persistable; use e3_events::{ - prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, KeyshareCreated, OrderedSet, - PublicKeyAggregated, Seed, + prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, EventContext, + KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, Sequenced, TypedEvent, }; use e3_events::{trap, EType}; use e3_fhe::{Fhe, GetAggregatePublicKey}; -use e3_utils::ArcBytes; use e3_utils::NotifySync; +use e3_utils::{ArcBytes, MAILBOX_LIMIT}; use std::sync::Arc; use tracing::{error, info}; @@ -88,8 +88,13 @@ impl PublicKeyAggregator { } } - pub fn add_keyshare(&mut self, keyshare: ArcBytes, node: String) -> Result<()> { - self.state.try_mutate(|mut state| { + pub fn add_keyshare( + &mut self, + keyshare: ArcBytes, + node: String, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(&ec, |mut state| { let PublicKeyAggregatorState::Collecting { threshold_n, keyshares, @@ -119,8 +124,8 @@ impl PublicKeyAggregator { }) } - pub fn set_pubkey(&mut self, pubkey: Vec) -> Result<()> { - self.state.try_mutate(|mut state| { + pub fn set_pubkey(&mut self, pubkey: Vec, ec: &EventContext) -> Result<()> { + self.state.try_mutate(ec, |mut state| { let PublicKeyAggregatorState::Computing { keyshares, nodes } = &mut state else { return Ok(state); }; @@ -136,87 +141,103 @@ impl PublicKeyAggregator { impl Actor for PublicKeyAggregator { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - match msg.into_data() { - EnclaveEventData::KeyshareCreated(data) => self.notify_sync(ctx, data)?, - EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), - _ => (), - }; - Ok(()) - }); + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::KeyshareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + _ => (), + }; } } -impl Handler for PublicKeyAggregator { - type Result = Result<()>; - - fn handle(&mut self, event: KeyshareCreated, ctx: &mut Self::Context) -> Self::Result { - let e3_id = event.e3_id.clone(); - let pubkey = event.pubkey.clone(); - let node = event.node.clone(); - - if e3_id != self.e3_id { - error!("Wrong e3_id sent to aggregator. This should not happen."); - return Ok(()); - } +impl Handler> for PublicKeyAggregator { + type Result = (); - self.add_keyshare(pubkey, node)?; + fn handle( + &mut self, + event: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (event, ec) = event.into_components(); + trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { + let e3_id = event.e3_id.clone(); + let pubkey = event.pubkey.clone(); + let node = event.node.clone(); + + if e3_id != self.e3_id { + error!("Wrong e3_id sent to aggregator. This should not happen."); + return Ok(()); + } - if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { - self.notify_sync( - ctx, - ComputeAggregate { - keyshares: keyshares.clone(), - e3_id, - }, - )? - } + self.add_keyshare(pubkey, node, &ec)?; + + if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { + self.notify_sync( + ctx, + TypedEvent::new( + ComputeAggregate { + keyshares: keyshares.clone(), + e3_id, + }, + ec, + ), + ) + } - Ok(()) + Ok(()) + }) } } -impl Handler for PublicKeyAggregator { - type Result = Result<()>; - - fn handle(&mut self, msg: ComputeAggregate, _: &mut Self::Context) -> Self::Result { - info!("Computing Aggregate PublicKey..."); - let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { - keyshares: msg.keyshares, - })?; - - let public_key_hash = compute_pk_commitment( - pubkey.clone(), - self.fhe.params.degree(), - self.fhe.params.plaintext(), - self.fhe.params.moduli().to_vec(), - )?; - - // Update the local state - self.set_pubkey(pubkey)?; - - if let Some(PublicKeyAggregatorState::Complete { - public_key: pubkey, - nodes, - .. - }) = self.state.get() - { - info!("Notifying network of PublicKey"); - info!("Sending PublicKeyAggregated..."); - let event = PublicKeyAggregated { - pubkey, - public_key_hash, - e3_id: msg.e3_id, +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { + info!("Computing Aggregate PublicKey..."); + let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { + keyshares: msg.keyshares, + })?; + + let public_key_hash = compute_pk_commitment( + pubkey.clone(), + self.fhe.params.degree(), + self.fhe.params.plaintext(), + self.fhe.params.moduli().to_vec(), + )?; + + // Update the local state + self.set_pubkey(pubkey, &ec)?; + + if let Some(PublicKeyAggregatorState::Complete { + public_key: pubkey, nodes, - }; - self.bus.publish(event)?; - } - Ok(()) + .. + }) = self.state.get() + { + info!("Notifying network of PublicKey"); + info!("Sending PublicKeyAggregated..."); + let event = PublicKeyAggregated { + pubkey, + public_key_hash, + e3_id: msg.e3_id, + nodes, + }; + self.bus.publish(event, ec)?; + } + Ok(()) + }) } } diff --git a/crates/aggregator/src/repo.rs b/crates/aggregator/src/repo.rs index 769c7da7d2..303bd38e93 100644 --- a/crates/aggregator/src/repo.rs +++ b/crates/aggregator/src/repo.rs @@ -4,9 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use crate::{PublicKeyAggregatorState, ThresholdPlaintextAggregatorState}; diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index 1c0fb0d5c2..13aad3b4db 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -12,16 +12,16 @@ use e3_data::Persistable; use e3_events::{ prelude::*, trap, BusHandle, ComputeRequest, ComputeResponse, ComputeResponseKind, CorrelationId, DecryptionshareCreated, Die, E3id, EType, EnclaveEvent, EnclaveEventData, - PlaintextAggregated, Seed, + EventContext, PlaintextAggregated, Seed, Sequenced, TypedEvent, }; -use e3_sortition::{GetNodesForE3, Sortition}; +use e3_sortition::{E3CommitteeContainsRequest, E3CommitteeContainsResponse, Sortition}; use e3_trbfv::{ calculate_threshold_decryption::CalculateThresholdDecryptionRequest, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; -use e3_utils::utility_types::ArcBytes; use e3_utils::NotifySync; -use tracing::{debug, error, info, trace}; +use e3_utils::{utility_types::ArcBytes, MAILBOX_LIMIT}; +use tracing::{debug, info, trace}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct Collecting { @@ -145,8 +145,13 @@ impl ThresholdPlaintextAggregator { } } - pub fn add_share(&mut self, party_id: u64, share: Vec) -> Result<()> { - self.state.try_mutate(|state| { + pub fn add_share( + &mut self, + party_id: u64, + share: Vec, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(ec, |state| { info!("Adding share for party_id={}", party_id); let current: Collecting = state.clone().try_into()?; let ciphertext_output = current.ciphertext_output; @@ -181,8 +186,12 @@ impl ThresholdPlaintextAggregator { }) } - pub fn set_decryption(&mut self, decrypted: Vec) -> Result<()> { - self.state.try_mutate(|mut state| { + pub fn set_decryption( + &mut self, + decrypted: Vec, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(ec, |mut state| { let ThresholdPlaintextAggregatorState::Computing(Computing { shares, .. }) = &mut state else { return Ok(state.clone()); @@ -196,7 +205,8 @@ impl ThresholdPlaintextAggregator { }) } - pub fn handle_compute_aggregate(&mut self, msg: ComputeAggregate) -> Result<()> { + pub fn handle_compute_aggregate(&mut self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("create_calculate_threshold_decryption_event..."); let e3_id = self.e3_id.clone(); @@ -221,11 +231,12 @@ impl ThresholdPlaintextAggregator { CorrelationId::new(), e3_id, ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } - pub fn handle_compute_response(&mut self, msg: ComputeResponse) -> Result<()> { + pub fn handle_compute_response(&mut self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); ensure!( msg.e3_id == self.e3_id, "PlaintextAggregator should never receive incorrect e3_id msgs" @@ -243,7 +254,7 @@ impl ThresholdPlaintextAggregator { // Update the local state let plaintext = response.plaintext; - self.set_decryption(plaintext.clone())?; + self.set_decryption(plaintext.clone(), &ec)?; // Dispatch the PlaintextAggregated event let event = PlaintextAggregated { @@ -252,107 +263,142 @@ impl ThresholdPlaintextAggregator { }; info!("Dispatching plaintext event {:?}", event); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } } impl Actor for ThresholdPlaintextAggregator { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for ThresholdPlaintextAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::DecryptionshareCreated(data) => ctx.notify(data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::DecryptionshareCreated(data) => ctx.notify(TypedEvent::new(data, ec)), EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), - EnclaveEventData::ComputeResponse(data) => self.notify_sync(ctx, data), + EnclaveEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for ThresholdPlaintextAggregator { - type Result = ResponseActFuture>; +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = + self.state.get() + else { + debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); + return Ok(()); + }; + let node = msg.node.clone(); + let e3_id = msg.e3_id.clone(); + let request = E3CommitteeContainsRequest::new(e3_id, node, msg, ctx.address()); + self.sortition.try_send(request)?; + Ok(()) + }, + ) + } +} - fn handle(&mut self, event: DecryptionshareCreated, _: &mut Self::Context) -> Self::Result { - let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = - self.state.get() - else { - debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); - return Box::pin(fut::ready(Ok(()))); - }; - info!(event=?event, "Processing DecryptionShareCreated..."); - let address = event.node.clone(); - let party_id = event.party_id; - let e3_id = event.e3_id.clone(); - let decryption_share = event.decryption_share.clone(); - - Box::pin( - self.sortition - .send(GetNodesForE3 { - e3_id: e3_id.clone(), - chain_id: e3_id.chain_id(), - }) - .into_actor(self) - .map(move |res, act, ctx| { - let nodes = res?; - - if !nodes.contains(&address) { - trace!("Node {} not found in finalized committee", address); - return Ok(()); - } - - if e3_id != act.e3_id { - error!("Wrong e3_id sent to aggregator. This should not happen."); - return Ok(()); - } - - // Trust the party_id from the event - it's based on CommitteeFinalized order - // which is the authoritative source of truth for party IDs - act.add_share(party_id, decryption_share)?; - - if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { - threshold_m, - threshold_n, - shares, - ciphertext_output, +impl Handler>> + for ThresholdPlaintextAggregator +{ + type Result = (); + fn handle( + &mut self, + msg: E3CommitteeContainsResponse>, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let e3_id = &msg.e3_id; + if *e3_id != self.e3_id { + bail!("Wrong e3_id sent to aggregator. This should not happen.") + }; + + if !msg.is_found_in_committee() { + trace!("Node {} not found in finalized committee", &msg.node); + return Ok(()); + }; + + // Trust the party_id from the event - it's based on CommitteeFinalized order + // which is the authoritative source of truth for party IDs + let ( + DecryptionshareCreated { + party_id, + decryption_share, .. - })) = act.state.get() - { - act.notify_sync( - ctx, + }, + ec, + ) = msg.into_inner().into_components(); + + self.add_share(party_id, decryption_share, &ec)?; + + if let Some(ThresholdPlaintextAggregatorState::Computing(Computing { + threshold_m, + threshold_n, + shares, + ciphertext_output, + .. + })) = self.state.get() + { + self.notify_sync( + ctx, + TypedEvent::new( ComputeAggregate { shares: shares.clone(), ciphertext_output: ciphertext_output.clone(), threshold_m, threshold_n, }, - ) - } - - Ok(()) - }), + ec, + ), + ) + } + Ok(()) + }, ) } } -impl Handler for ThresholdPlaintextAggregator { +impl Handler> for ThresholdPlaintextAggregator { type Result = (); - fn handle(&mut self, msg: ComputeAggregate, _: &mut Self::Context) -> Self::Result { - trap(EType::PlaintextAggregation, &self.bus.clone(), || { - self.handle_compute_aggregate(msg) - }) + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_aggregate(msg), + ) } } -impl Handler for ThresholdPlaintextAggregator { +impl Handler> for ThresholdPlaintextAggregator { type Result = (); - fn handle(&mut self, msg: ComputeResponse, _: &mut Self::Context) -> Self::Result { - trap(EType::PlaintextAggregation, &self.bus.clone(), || { - self.handle_compute_response(msg) - }) + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg), + ) } } diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index e30c2dba3e..0e86c91666 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -34,3 +34,6 @@ tempfile.workspace = true tokio.workspace = true tracing.workspace = true bincode.workspace = true + +[dev-dependencies] +e3-test-helpers.workspace = true diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 4c0dc5ecd7..34aa511e09 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -4,7 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::event_system::AggregateConfig; use crate::{CiphernodeHandle, EventSystem, EvmSystemChainBuilder, ProviderCache, WriteEnabled}; use actix::{Actor, Addr}; use alloy::signers::local::PrivateKeySigner; @@ -15,22 +14,25 @@ use e3_aggregator::CommitteeFinalizer; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; use e3_data::{InMemStore, RepositoriesFactory}; -use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EvmEventConfig}; +use e3_events::{ + AggregateConfig, AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EvmEventConfig, +}; use e3_evm::{BondingRegistrySolReader, CiphernodeRegistrySolReader, EnclaveSolWriter}; use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_fhe_params::BfvPreset; use e3_keyshare::ext::ThresholdKeyshareExtension; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; -use e3_net::{NetEventTranslator, NetRepositoryFactory}; +use e3_net::{setup_net, NetRepositoryFactory}; use e3_request::E3Router; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; -use e3_sync::Synchronizer; +use e3_sync::sync; use e3_utils::{rand_eth_addr, SharedRng}; use e3_zk_prover::{setup_zk_actors, ZkBackend}; +use std::time::Duration; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; @@ -72,6 +74,7 @@ pub struct CiphernodeBuilder { threshold_plaintext_agg: bool, zk_backend: Option, net_config: Option, + start_buffer: bool, } // Simple Net Configuration @@ -137,6 +140,7 @@ impl CiphernodeBuilder { testmode_signer: None, threshold_plaintext_agg: false, net_config: None, + start_buffer: false, zk_backend: None, } } @@ -189,6 +193,12 @@ impl CiphernodeBuilder { self.testmode_errors = true; self } + /// Ensure SnapshotBuffer starts immediately instead of waiting for SyncEnded. This is important + /// for tests that don't specifically + pub fn testmode_start_buffer_immediately(mut self) -> Self { + self.start_buffer = true; + self + } /// Use the node configuration on these specific chains. This will overwrite any previously /// given chains. @@ -380,7 +390,7 @@ impl CiphernodeBuilder { if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { EventSystem::persisted(&addr, log_path, kv_path) .with_event_bus(local_bus) - .with_aggregate_config(aggregate_config) + .with_aggregate_config(aggregate_config.clone()) } else { if let Some(ref store) = self.in_mem_store { EventSystem::in_mem_from_store(&addr, store) @@ -395,6 +405,8 @@ impl CiphernodeBuilder { let bus = event_system.handle()?; let store = event_system.store()?; + let eventstore_ts = event_system.eventstore_getter_ts()?; + let eventstore_seq = event_system.eventstore_getter_seq()?; let cipher = &self.cipher; let repositories = Arc::new(store.repositories()); @@ -481,15 +493,15 @@ impl CiphernodeBuilder { let (join_handle, peer_id) = if let Some(net_config) = self.net_config { let repositories = store.repositories(); - let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( + setup_net( bus.clone(), net_config.peers, &self.cipher, net_config.quic_port, repositories.libp2p_keypair(), + eventstore_ts, ) - .await?; - (join_handle, peer_id) + .await? } else { ( tokio::spawn(std::future::ready(Ok(()))), @@ -497,7 +509,15 @@ impl CiphernodeBuilder { ) }; - Synchronizer::setup(&bus, evm_config); // TODO: add net config if required + // Run the sync routine + sync( + &bus, + &evm_config, + &repositories, + &aggregate_config, + &eventstore_seq, + ) + .await?; Ok(CiphernodeHandle::new( addr.to_owned(), @@ -569,17 +589,17 @@ fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { } /// Build delay configuration for a specific chain -fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, u64) { - let aggregate_id = AggregateId::new(actual_chain_id as usize); +fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, Duration) { + let aggregate_id = AggregateId::from_chain_id(Some(actual_chain_id)); let finalization_ms = chain.finalization_ms.unwrap_or(0); let delay_us = finalization_ms * 1000; // ms → microseconds - (aggregate_id, delay_us) + (aggregate_id, Duration::from_micros(delay_us)) } /// Build delays configuration from chain providers fn create_aggregate_delays( chain_providers: &[(ChainConfig, u64)], -) -> Result> { +) -> Result> { let mut delays = HashMap::new(); for (chain, actual_chain_id) in chain_providers.into_iter().cloned() { @@ -611,7 +631,7 @@ async fn setup_evm_system( if contract_components.enclave { let write_provider = provider_cache.ensure_write_provider(chain).await?; let contract = &chain.contracts.enclave; - EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?).await?; + EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?); system.with_contract(contract.address()?, move |next| { EnclaveSolReader::setup(&next).recipient() }); @@ -651,8 +671,7 @@ async fn setup_evm_system( write_provider.clone(), contract.address()?, pubkey_agg, - ) - .await?; + ); info!("CiphernodeRegistrySolWriter attached for publishing committees"); if pubkey_agg { diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 921f845423..26c28fadcb 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -5,16 +5,17 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::get_enclave_event_bus; -use actix::{Actor, Addr, Recipient}; +use actix::{Actor, Addr, Handler, Recipient}; use anyhow::{anyhow, Result}; use e3_data::{ - CommitLogEventLog, DataStore, ForwardTo, InMemEventLog, InMemSequenceIndex, InMemStore, - InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, + CommitLogEventLog, DataStore, InMemEventLog, InMemSequenceIndex, InMemStore, SledSequenceIndex, + SledStore, }; use e3_events::hlc::Hlc; use e3_events::{ - BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, EventType, - Sequencer, StoreEventRequested, + AggregateConfig, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, + EventStoreQueryBy, EventStoreRouter, EventSubscriber, EventType, InsertBatch, SeqAgg, + Sequencer, SnapshotBuffer, StoreEventRequested, TsAgg, UpdateDestination, }; use e3_utils::enumerate_path; use once_cell::sync::OnceCell; @@ -22,13 +23,19 @@ use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; -pub use e3_data::AggregateConfig; - struct InMemBackend { eventstores: OnceCell>>>, store: OnceCell>, } +impl InMemBackend { + fn get_or_init_store(&self) -> Addr { + self.store + .get_or_init(|| InMemStore::new(true).start()) + .clone() + } +} + /// Hold the Persistent EventStore instance and SledStore struct PersistedBackend { log_path: PathBuf, @@ -37,6 +44,14 @@ struct PersistedBackend { store: OnceCell>, } +impl PersistedBackend { + fn get_or_init_store(&self, handle: &BusHandle) -> Result> { + self.store + .get_or_try_init(|| SledStore::new(handle, &self.sled_path)) + .cloned() + } +} + /// An EventSystemBackend is holding the potentially persistent structures for the system enum EventSystemBackend { InMem(InMemBackend), @@ -64,15 +79,13 @@ pub struct EventSystem { /// EventSystem backend either persisted or in memory backend: EventSystemBackend, /// WriteBuffer for batching inserts from actors into a snapshot - buffer: OnceCell>, + buffer: OnceCell>, /// EventSystem Sequencer sequencer: OnceCell>, /// EventSystem eventbus eventbus: OnceCell>>, /// EventSystem BusHandle handle: OnceCell, - /// A OnceLock that is used to indicate whether the system is wired to write snapshots - wired: OnceCell<()>, /// Hlc override hlc: OnceCell, /// Central configuration for aggregates, including delays and other settings @@ -99,7 +112,6 @@ impl EventSystem { sequencer: OnceCell::new(), eventbus: OnceCell::new(), handle: OnceCell::new(), - wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), @@ -118,7 +130,6 @@ impl EventSystem { sequencer: OnceCell::new(), eventbus: OnceCell::new(), handle: OnceCell::new(), - wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), @@ -139,7 +150,6 @@ impl EventSystem { sequencer: OnceCell::new(), eventbus: OnceCell::new(), handle: OnceCell::new(), - wired: OnceCell::new(), hlc: OnceCell::new(), aggregate_config: OnceCell::new(), eventstore_addrs: OnceCell::new(), @@ -185,16 +195,12 @@ impl EventSystem { } /// Get the buffer address - pub fn buffer(&self) -> Addr { - let buffer = self - .buffer - .get_or_init(|| { - let config = self.aggregate_config(); - WriteBuffer::with_config(config).start() + pub fn buffer(&self) -> Result> { + self.buffer + .get_or_try_init(|| { + SnapshotBuffer::spawn(&self.aggregate_config(), NoopBatchReceiver::new().start()) }) - .clone(); - self.wire_if_ready(); - buffer + .cloned() } /// Get the sequencer address @@ -202,7 +208,7 @@ impl EventSystem { self.sequencer .get_or_try_init(|| { let router = self.eventstore_router()?; - Ok(Sequencer::new(&self.eventbus(), router, self.buffer()).start()) + Ok(Sequencer::new(&self.eventbus(), router).start()) }) .cloned() } @@ -298,6 +304,22 @@ impl EventSystem { } } + pub fn eventstore_getter_seq(&self) -> Result>> { + let eventstores = self.eventstore_addrs()?; + match &eventstores { + EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), + EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), + } + } + + pub fn eventstore_getter_ts(&self) -> Result>> { + let eventstores = self.eventstore_addrs()?; + match &eventstores { + EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), + EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), + } + } + /// Get an instance of the Hlc pub fn hlc(&self) -> Result { self.hlc @@ -309,11 +331,11 @@ impl EventSystem { pub fn handle(&self) -> Result { self.handle .get_or_try_init(|| { - Ok(BusHandle::new( - self.eventbus(), - self.sequencer()?, - self.hlc()?, - )) + let handle = BusHandle::new(self.eventbus(), self.sequencer()?, self.hlc()?); + // Buffer subscribes to all events first + // This is important so as to open up a batch for each sequence + handle.subscribe(EventType::All, self.buffer()?.recipient()); + Ok(handle) }) .cloned() } @@ -322,48 +344,24 @@ impl EventSystem { pub fn store(&self) -> Result { let store = match &self.backend { EventSystemBackend::InMem(b) => { - let addr = b - .store - .get_or_init(|| InMemStore::new(true).start()) - .clone(); - DataStore::from_in_mem(&addr, &self.buffer()) + let base = b.get_or_init_store(); + let buffer = self.buffer()?; + buffer.try_send(UpdateDestination::new(base.clone()))?; + DataStore::from_in_mem_with_buffer(&base, self.buffer()?) } EventSystemBackend::Persisted(b) => { - let addr = b - .store - .get_or_try_init(|| { - let handle = self.handle()?; - SledStore::new(&handle, &b.sled_path) - })? - .clone(); - DataStore::from_sled_store(&addr, &self.buffer()) + let base = b.get_or_init_store(&self.handle()?)?; + let buffer = self.buffer()?; + buffer.try_send(UpdateDestination::new(base))?; + + DataStore::from_sled_store_with_buffer( + &b.get_or_init_store(&self.handle()?)?, + self.buffer()?, + ) } }; - self.wire_if_ready(); - Ok(store) - } - - // We need to ensure that once the buffer and store are created they are connected so that - // inserts are sent between the two actors. This internal function ensures this happens. - fn wire_if_ready(&self) { - let buffer = match self.buffer.get() { - Some(b) => b, - None => return, - }; - - let store: Option> = match &self.backend { - EventSystemBackend::InMem(b) => b.store.get().cloned().map(Into::into), - EventSystemBackend::Persisted(b) => b.store.get().cloned().map(Into::into), - }; - let Some(store) = store else { - return; - }; - - // Now we know both are ready, so initialization will succeed - self.wired.get_or_init(|| { - buffer.do_send(ForwardTo::new(store)); - }); + Ok(store) } fn node_id(name: &str) -> u32 { @@ -373,9 +371,39 @@ impl EventSystem { } } +struct NoopBatchReceiver; + +impl NoopBatchReceiver { + pub fn new() -> Self { + Self {} + } +} +impl Actor for NoopBatchReceiver { + type Context = actix::Context; +} + +impl Handler for NoopBatchReceiver { + type Result = (); + fn handle(&mut self, _: InsertBatch, _: &mut Self::Context) -> Self::Result { + // do nothing + } +} + #[cfg(test)] mod tests { + use e3_data::AutoPersist; + use e3_data::Persistable; + use e3_data::Repository; + use e3_events::EventContext; + use e3_events::EventId; + use e3_events::EventSource; + use e3_events::StoreKeys; + use e3_events::SyncEnded; + use e3_events::Tick; + use e3_events::TsAgg; + use e3_test_helpers::with_tracing; use std::time::Duration; + use tracing::info; use super::*; use actix::Actor; @@ -386,7 +414,8 @@ mod tests { use e3_events::CorrelationId; use e3_events::EnclaveEventData; - use e3_events::ReceiveEvents; + use e3_events::EventStoreQueryResponse; + use e3_events::EventType; use e3_events::TestEvent; use tempfile::TempDir; use tokio::time::sleep; @@ -436,10 +465,10 @@ mod tests { } } - impl Handler for Listener { + impl Handler for Listener { type Result = (); - fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { - self.events = msg.events().clone(); + fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { + self.events = msg.into_events(); } } @@ -448,15 +477,13 @@ mod tests { } #[actix::test] - async fn test_persisted() { + async fn test_persisted() -> Result<()> { + let _guard = with_tracing("debug"); let tmp = TempDir::new().unwrap(); let system = EventSystem::persisted("cn2", tmp.path().join("log"), tmp.path().join("sled")); - let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); - - // Wiring happened automatically - assert!(system.wired.get().is_some()); + Ok(()) } #[actix::test] @@ -466,65 +493,138 @@ mod tests { let _handle = system.handle().expect("Failed to get handle"); system.store().expect("Failed to get store"); - - // Wiring happened automatically - assert!(system.wired.get().is_some()); } #[actix::test] async fn test_event_system() -> Result<()> { - let system = EventSystem::in_mem("cn1").with_fresh_bus(); + let _guard = with_tracing("debug"); + + // This sets up the aggregation delays + let mut delays = HashMap::new(); + // Here we delay AggregationId(0) for 1 second + delays.insert(AggregateId::new(0), Duration::from_secs(1)); // Ag0 is default + let config = AggregateConfig::new(delays); + + let system = EventSystem::in_mem("cn1") + .with_fresh_bus() + .with_aggregate_config(config); + let handle = system.handle()?; let datastore = system.store()?; + let buffer = system.buffer()?; + let listener = Listener { logs: Vec::new(), events: Vec::new(), } .start(); + // Sequence 1, Aggregate 0 + let ec = EventContext::new_origin( + EventId::hash(1), + 10, + AggregateId::new(0), + None, + EventSource::Local, + ) + .sequence(1); + // Send all evts to the listener handle.subscribe(EventType::All, listener.clone().into()); - // Lets store some data + // Publish an event seq 1 + info!("Publishing an event seq 1"); + handle.publish_without_context(TestEvent::new("pink", 1))?; + + // Lets store some data on a plain datastore + info!("Writing to /foo/name with no context"); datastore.scope("/foo/name").write("Fred".to_string()); - datastore.scope("/foo/age").write(21u64); - datastore - .scope("/foo/occupation") - .write("developer".to_string()); + // Note there is some eventual consistency here we have to wait + assert_eq!(datastore.scope("/foo/name").read::().await?, None); + info!("Wait one tick"); - // NOTE: Eventual consistency - // Store should not have data set on it until event has been published + // Let's wait until all events are settled only takes a tick + sleep(Duration::from_millis(1)).await; - // Let's check the eventual consistency all data points should be none... - assert_eq!(datastore.scope("/foo/name").read::().await?, None); - assert_eq!(datastore.scope("/foo/age").read::().await?, None); + // These inserts should not be buffered and should be available assert_eq!( - datastore.scope("/foo/occupation").read::().await?, - None + datastore.scope("/foo/name").read::().await?, + Some("Fred".to_string()) ); + info!("Data was written now"); - // Push an event - handle.publish(TestEvent::new("pink", 1))?; - sleep(Duration::from_millis(1)).await; + // Ok lets get a persistable + let mut persistable: Persistable = + Repository::new(datastore.scope("/foo/name")).load().await?; + + // We have the data in our persistable + assert_eq!(persistable.get(), Some("Fred".to_string())); + + info!("Data was loaded from persistable now mutating state..."); - // Now we have published an event all data should be written we can get the data from the store + persistable.try_mutate(&ec, |_| Ok("Mary".to_string()))?; + + // Local state has changed straight away + assert_eq!(persistable.get(), Some("Mary".to_string())); + + // But disk state is still the same because the SnapshotBuffer is not on assert_eq!( datastore.scope("/foo/name").read::().await?, Some("Fred".to_string()) ); - assert_eq!(datastore.scope("/foo/age").read::().await?, Some(21)); + info!("Local state was mutated however disk state was not"); + + info!("Publishing SyncEnded event to turn on SnapshotBuffer. This should send the seq=1 batch to the timelock..."); + // Publishing SyncEnded should turn on the SnapshotBuffer seq 2 + handle.publish(SyncEnded::new(), ec.clone())?; + + sleep(Duration::from_millis(1)).await; + + info!("Mutating persistable state to create inserts using seq=2"); + + let ec = EventContext::new_origin( + EventId::hash(1), + 10, + AggregateId::new(0), + None, + EventSource::Local, + ) + .sequence(2); + + persistable.try_mutate(&ec, |_| Ok("Liz".to_string()))?; + sleep(Duration::from_millis(1)).await; + info!("Mutation complete"); + // SnapshotBuffer is not cleared unless new events are published + + // Get a timestamp for the events below + let ts = handle.ts()?; + + // Push a few other events seq 3 + // This sends the previous batch for seq2 to the timelock queue + handle.publish_without_context(TestEvent::new("yellow", 1))?; + + // Wait a second for the timelock to be checked + sleep(Duration::from_secs(2)).await; + buffer.try_send(Tick)?; + + // Check now + info!("Reading from /foo/name and expecting it to be correct."); assert_eq!( - datastore.scope("/foo/occupation").read::().await?, - Some("developer".to_string()) + datastore.scope("/foo/name").read::().await?, + Some("Liz".to_string()) ); - // Get a timestamp - let ts = handle.ts()?; + assert_eq!( + datastore + .scope(&StoreKeys::aggregate_seq(AggregateId::new(0))) + .read::() + .await?, + Some(2) + ); - // Push a few other events - handle.publish(TestEvent::new("yellow", 1))?; - handle.publish(TestEvent::new("red", 1))?; - handle.publish(TestEvent::new("white", 1))?; + // Publish a few other events + handle.publish_without_context(TestEvent::new("red", 1))?; + handle.publish_without_context(TestEvent::new("white", 1))?; sleep(Duration::from_millis(100)).await; // Get the event logs from the listener @@ -535,11 +635,11 @@ mod tests { let router = system.in_mem_eventstore_router()?; // Get all events after the given timestamp using the router - use e3_events::{AggregateId, GetAggregateEventsAfter}; + use e3_events::AggregateId; let mut ts_map = HashMap::new(); ts_map.insert(AggregateId::new(0), ts); - let get_events_msg = - GetAggregateEventsAfter::new(CorrelationId::new(), ts_map, listener.clone().into()); + let sender: Recipient = listener.clone().into(); + let get_events_msg = EventStoreQueryBy::::new(CorrelationId::new(), ts_map, sender); router.do_send(get_events_msg); sleep(Duration::from_millis(100)).await; @@ -555,9 +655,9 @@ mod tests { // Create an AggregateConfig with multiple AggregateIds let mut delays = HashMap::new(); - delays.insert(AggregateId::new(0), 1000); // 1ms delay - delays.insert(AggregateId::new(1), 2000); // 2ms delay - delays.insert(AggregateId::new(2), 3000); // 3ms delay + delays.insert(AggregateId::new(0), Duration::from_micros(1000)); // 1ms delay + delays.insert(AggregateId::new(1), Duration::from_micros(2000)); // 2ms delay + delays.insert(AggregateId::new(2), Duration::from_micros(3000)); // 3ms delay let aggregate_config = AggregateConfig::new(delays); // Test in-memory eventstores diff --git a/crates/ciphernode-builder/src/eventbus_factory.rs b/crates/ciphernode-builder/src/eventbus_factory.rs index 5277e5f878..d2ddc5b6b0 100644 --- a/crates/ciphernode-builder/src/eventbus_factory.rs +++ b/crates/ciphernode-builder/src/eventbus_factory.rs @@ -103,5 +103,7 @@ pub fn get_error_collector() -> Addr> { pub fn get_enclave_bus_handle(config: &AppConfig) -> anyhow::Result { let bus = get_enclave_event_bus(); let system = EventSystem::new(&config.name()).with_event_bus(bus); + system.store()?; // Ensure store is initialized before returning to avoid potentially dropping + // events. Ok(system.handle()?) } diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 0c44393463..5d88eb4c90 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -8,11 +8,14 @@ use std::mem::replace; use actix::Actor; use alloy::{primitives::Address, providers::Provider}; -use e3_events::{BusHandle, EventSubscriber, EventType, SyncStart}; +use e3_events::{ + run_once, BusHandle, EventExtractor, EventSubscriber, EventType, HistoricalEvmSyncStart, +}; use e3_evm::{ EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, - FixHistoricalOrder, OneShotRunner, SyncStartExtractor, + FixHistoricalOrder, SyncStartExtractor, }; +use e3_utils::actix::oneshot_runner::OneShotRunner; pub trait RouteFn: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} impl RouteFn for F where F: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} @@ -48,32 +51,61 @@ impl EvmSystemChainBuilder

{ } pub fn build(&mut self) { - let gateway = FixHistoricalOrder::setup(EvmChainGateway::setup(&self.bus)); - let runner = SyncStartExtractor::setup(OneShotRunner::setup({ + // Think about the following in reverse order + + // Gateway is the final step before connecting to the bus + let next = EvmChainGateway::setup(&self.bus); + + // Fix the historical order to avoid missing historical events + let next = FixHistoricalOrder::setup(next); + + // This will run once when the HistoricalEvmSyncStart event is received + let next = run_once::({ + // Clone self refs for closure let bus = self.bus.clone(); let provider = self.provider.clone(); - let gateway = gateway.clone(); let chain_id = self.chain_id; - // Only gets consumed once so fine to do this + + // Only gets consumed once so fine to use replace to clean out route_factories let route_factories = replace(&mut self.route_factories, Vec::new()); - move |msg: SyncStart| { - let config = msg.get_evm_config(chain_id)?; - let gateway = gateway.recipient(); - let mut router = EvmRouter::new(); - - for (address, route_fn) in route_factories { - let processor = route_fn(gateway.clone()); - router = router.add_route(address, &processor); - } - - router = router.add_fallback(&gateway); - let filters = - Filters::from_routing_table(router.get_routing_table(), config.deploy_block()); - let router = router.start(); - EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); + + // The event is defined here + move |msg| { + // Extract config + let deploy_block = msg.get_evm_config(chain_id)?.deploy_block(); + + // Pass next to the router + let router = configure_router(next, route_factories); + + // Extract filters from the router + let filters = filters_from_router(&router, deploy_block); + + // Setup and start the read interface and the router + EvmReadInterface::setup(&provider, router.start(), &bus, filters); Ok(()) } - })); - self.bus.subscribe(EventType::SyncStart, runner.recipient()); + }); + + // Finaly subscribe to the bus and wait for HistoricalEvmSyncStart + self.bus + .subscribe(EventType::HistoricalEvmSyncStart, next.recipient()); } } + +/// Setup a router with a fallback and route factories all forwarding to next +fn configure_router( + next: impl Into, + route_factories: Vec<(Address, Box)>, +) -> EvmRouter { + let next = next.into(); + let mut router = EvmRouter::new().add_fallback(&next); + for (address, route_fn) in route_factories { + let processor = route_fn(next.clone()); + router = router.add_route(address, &processor); + } + router +} + +fn filters_from_router(router: &EvmRouter, deploy_block: u64) -> Filters { + Filters::from_routing_table(router.get_routing_table(), deploy_block) +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 5c8b7199f9..2e18c6f92d 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -10,11 +10,9 @@ pub mod contract; pub mod load_config; pub mod paths_engine; pub mod rpc; -mod store_keys; pub mod validation; mod yaml; pub use app_config::*; pub use contract::*; pub use rpc::*; -pub use store_keys::*; diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index 59c4aa6987..895bfd7bb8 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/gnosisguild/enclave/crates/data" actix = { workspace = true } e3-events = { workspace = true } e3-utils = { workspace = true } +e3-config = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } sled = { workspace = true } diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 46dc04dea9..39fa684e6b 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -78,11 +78,17 @@ impl EventLog for CommitLogEventLog { #[cfg(test)] mod tests { use super::*; - use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; + use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, EventSource, TestEvent}; use tempfile::tempdir; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) + EnclaveEvent::::new_with_timestamp( + data.into().into(), + None, + 123, + None, + EventSource::Local, + ) } #[test] diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index d4516f858f..64e8ea4deb 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -6,12 +6,13 @@ use std::borrow::Cow; -use crate::{Get, Insert, InsertSync, Remove, WriteBuffer}; -use crate::{InMemStore, IntoKey, SledStore}; +use crate::{InMemStore, SledStore}; use actix::{Addr, Recipient}; use anyhow::anyhow; use anyhow::Context; use anyhow::Result; +use e3_events::IntoKey; +use e3_events::{Get, Insert, InsertSync, Remove}; use serde::{Deserialize, Serialize}; use tracing::error; @@ -171,22 +172,51 @@ impl DataStore { } } - pub fn from_sled_store(addr: &Addr, write_buffer: &Addr) -> Self { + pub fn from_sled_store_with_buffer( + addr: &Addr, + snapshot_buffer: impl Into>, + ) -> Self { + println!("from_sled_store_with_buffer..."); Self { addr: StoreAddr::Sled(addr.clone()), get: addr.clone().recipient(), - insert: write_buffer.clone().recipient(), + insert: snapshot_buffer.into(), insert_sync: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], } } - pub fn from_in_mem(addr: &Addr, write_buffer: &Addr) -> Self { + pub fn from_in_mem_with_buffer( + addr: &Addr, + snapshot_buffer: impl Into>, + ) -> Self { Self { addr: StoreAddr::InMem(addr.clone()), get: addr.clone().recipient(), - insert: write_buffer.clone().recipient(), + insert: snapshot_buffer.into(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + } + } + + pub fn from_in_mem(addr: &Addr) -> Self { + Self { + addr: StoreAddr::InMem(addr.clone()), + get: addr.clone().recipient(), + insert: addr.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + } + } + + pub fn from_sled_store(addr: &Addr) -> Self { + Self { + addr: StoreAddr::Sled(addr.clone()), + get: addr.clone().recipient(), + insert: addr.clone().recipient(), insert_sync: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index ed0f52f60f..c689acd9fb 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -4,10 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertBatch, InsertSync, Remove}; use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; +use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; +use e3_utils::MAILBOX_LIMIT; use std::collections::BTreeMap; +use tracing::info; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Vec")] @@ -31,6 +33,9 @@ pub struct InMemStore { impl Actor for InMemStore { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl InMemStore { diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index a283ded98d..d48a9837b3 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -46,10 +46,16 @@ impl EventLog for InMemEventLog { #[cfg(test)] mod tests { use super::*; - use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; + use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, EventSource, TestEvent}; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) + EnclaveEvent::::new_with_timestamp( + data.into().into(), + None, + 123, + None, + EventSource::Local, + ) } #[test] diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 83827a3aa4..31f8b63bff 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -6,11 +6,9 @@ mod commit_log_event_log; mod data_store; -mod events; mod in_mem; mod in_mem_event_log; mod in_mem_sequence_index; -mod into_key; mod persistable; mod repositories; mod repository; @@ -19,15 +17,12 @@ mod sled_sequence_index; mod sled_store; mod sled_utils; mod snapshot; -mod write_buffer; pub use commit_log_event_log::*; pub use data_store::*; -pub use events::*; pub use in_mem::*; pub use in_mem_event_log::*; pub use in_mem_sequence_index::*; -pub use into_key::IntoKey; pub use persistable::*; pub use repositories::*; pub use repository::*; @@ -35,4 +30,3 @@ pub use sled_db::*; pub use sled_sequence_index::*; pub use sled_store::*; pub use snapshot::*; -pub use write_buffer::*; diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 7247a252b0..9c76322c81 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -3,17 +3,18 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, Remove, Repository}; +use crate::Repository; use actix::Recipient; use anyhow::*; use async_trait::async_trait; -use e3_events::{EventContext, EventContextManager, Sequenced}; +use e3_events::{EventContext, EventContextManager, Get, Insert, Remove, Sequenced}; use serde::{de::DeserializeOwned, Serialize}; pub trait PersistableData: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} impl PersistableData for T where T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} -/// AutoPersist enables a repository to generate a persistable container +/// AutoPersist enables a repository to generate a persistable container. This is not a database and +/// should not be thought of as a database. This is for creating actor snapshots. #[async_trait] pub trait AutoPersist where @@ -193,10 +194,25 @@ where } /// Mutate the content if available or return an error - pub fn try_mutate(&mut self, mutator: F) -> Result<()> + pub fn try_mutate_without_context(&mut self, mutator: F) -> Result<()> where F: FnOnce(T) -> Result, { + self.try_mutate_impl(mutator, None) + } + + pub fn try_mutate(&mut self, ctx: &EventContext, mutator: F) -> Result<()> + where + F: FnOnce(T) -> Result, + { + self.try_mutate_impl(mutator, Some(ctx.clone())) + } + + fn try_mutate_impl(&mut self, mutator: F, ctx: Option>) -> Result<()> + where + F: FnOnce(T) -> Result, + { + self.ctx = ctx; // Set the context let content = self.data.clone().ok_or(anyhow!("Data has not been set"))?; self.data = Some(mutator(content)?); self.write_to_store(); @@ -251,8 +267,11 @@ impl EventContextManager for Persistable { self.ctx.clone() } - fn set_ctx(&mut self, value: &EventContext) { - self.ctx = Some(value.clone()) + fn set_ctx(&mut self, value: C) + where + C: Into>, + { + self.ctx = Some(value.into().clone()) } } @@ -260,7 +279,8 @@ impl EventContextManager for Persistable { mod tests { use actix::{Actor, Addr, Handler, Message}; - use crate::{Get, Insert, Remove}; + use e3_events::{Get, Insert, Remove}; + use e3_utils::MAILBOX_LIMIT; use super::{Persistable, StoreConnector}; @@ -281,6 +301,9 @@ mod tests { impl Actor for MockConnector { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for MockConnector { diff --git a/crates/data/src/repositories.rs b/crates/data/src/repositories.rs index 4b89ef6b8c..63cce29663 100644 --- a/crates/data/src/repositories.rs +++ b/crates/data/src/repositories.rs @@ -3,9 +3,10 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::{DataStore, InMemStore, Repository}; +use actix::Actor; -use crate::{DataStore, Repository}; - +#[derive(Clone)] pub struct Repositories { pub store: DataStore, } @@ -27,6 +28,10 @@ impl Repositories { pub fn new(store: DataStore) -> Self { Repositories { store } } + + pub fn in_mem() -> Self { + DataStore::from_in_mem(&InMemStore::new(false).start()).into() + } } impl From> for Repositories { diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 641ea39797..72e508d425 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -4,15 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::sled_utils::{clear_all_caches, get_or_open_db_tree}; use anyhow::{Context, Result}; +use e3_events::{Get, Insert, Remove}; use sled::{transaction::ConflictableTransactionError, Tree}; use std::path::PathBuf; -use crate::{ - sled_utils::{clear_all_caches, get_or_open_db_tree}, - Get, Insert, Remove, -}; - pub struct SledDb { db: Tree, } diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 6bf35bea46..12cc3eeb0b 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,10 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertBatch, InsertSync, Remove, SledDb}; +use crate::SledDb; use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::Result; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData, EventType}; +use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; +use e3_utils::MAILBOX_LIMIT; use std::path::PathBuf; use tracing::{error, info}; @@ -18,6 +20,9 @@ pub struct SledStore { impl Actor for SledStore { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl SledStore { diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs deleted file mode 100644 index c144b33403..0000000000 --- a/crates/data/src/write_buffer.rs +++ /dev/null @@ -1,266 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::{Actor, Handler, Message, Recipient}; -use e3_events::hlc::HlcTimestamp; -use e3_events::{AggregateId, CommitSnapshot, EventContextAccessors}; -use std::{ - collections::HashMap, - time::{SystemTime, UNIX_EPOCH}, -}; - -use crate::{Insert, InsertBatch}; - -/// Central configuration for aggregates in the WriteBuffer -#[derive(Debug, Clone)] -pub struct AggregateConfig { - pub delays: HashMap, -} - -impl AggregateConfig { - /// Create a new AggregateConfig with the specified delays - pub fn new(mut delays: HashMap) -> Self { - // Always handle AggregatId of 0 with a delay of 0 - if let None = delays.get(&AggregateId::new(0)) { - delays.insert(AggregateId::new(0), 0); - } - Self { delays } - } - - /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured - pub fn indexed_ids(&self) -> Vec { - self.delays.keys().map(|id| id.to_usize()).collect() - } -} - -#[derive(Debug)] -struct AggregateBuffer { - buffer: Vec, -} - -impl AggregateBuffer { - fn new() -> Self { - Self { buffer: Vec::new() } - } -} - -pub struct WriteBuffer { - /// Destination recipient for batched inserts - dest: Option>, - /// Per-aggregate buffers for organizing inserts - aggregate_buffers: HashMap, - /// Per-aggregate wait time configuration - config: AggregateConfig, -} - -impl Actor for WriteBuffer { - type Context = actix::Context; -} - -impl WriteBuffer { - pub fn new() -> Self { - Self { - dest: None, - aggregate_buffers: HashMap::new(), - config: AggregateConfig::new(HashMap::new()), - } - } - - pub fn with_config(config: AggregateConfig) -> Self { - Self { - dest: None, - aggregate_buffers: HashMap::new(), - config, - } - } - - fn handle_insert(&mut self, msg: Insert) { - let aggregate_id = if let Some(event_ctx) = msg.ctx() { - event_ctx.aggregate_id().clone() - } else { - AggregateId::new(0) - }; - - let agg_buffer = self - .aggregate_buffers - .entry(aggregate_id) - .or_insert_with(|| AggregateBuffer::new()); - agg_buffer.buffer.push(msg.clone()); - } - - fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { - // Store the sequence number as an Insert message so snapshots hold the most recent event - // they were created against - self.handle_insert(Insert::new( - &format!("//aggregate_seq/{}", msg.aggregate_id()), - msg.seq().to_le_bytes().to_vec(), // Same as bincode avoiding result - )); - - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|_| std::time::Duration::from_secs(0)) - .as_micros() as u64; - - if let Some(ref dest) = self.dest { - let (updated_buffers, expired_inserts) = - process_expired_inserts(&self.aggregate_buffers, &self.config.delays, now); - if !expired_inserts.is_empty() { - let batch = InsertBatch::new(expired_inserts); - dest.do_send(batch); - } - - self.aggregate_buffers = updated_buffers; - } - } -} - -impl Handler for WriteBuffer { - type Result = (); - fn handle(&mut self, msg: ForwardTo, _: &mut Self::Context) -> Self::Result { - self.dest = Some(msg.dest()) - } -} - -impl Handler for WriteBuffer { - type Result = (); - - fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - self.handle_insert(msg) - } -} - -fn process_expired_inserts( - aggregate_buffers: &HashMap, - config: &HashMap, - now: u64, -) -> (HashMap, Vec) { - let mut updated_buffers = HashMap::new(); - let mut all_expired_inserts = Vec::new(); - - for (aggregate_id, agg_buffer) in aggregate_buffers { - let delay_micros = config.get(aggregate_id).copied().unwrap_or(0); - let cutoff_time = now.saturating_sub(delay_micros); - let mut expired_inserts = Vec::new(); - let mut remaining_inserts = Vec::new(); - - for insert in &agg_buffer.buffer { - if let Some(ctx) = insert.ctx() { - let event_wall_time = HlcTimestamp::wall_time(ctx.ts()); - if event_wall_time < cutoff_time { - expired_inserts.push(insert.clone()); - } else { - remaining_inserts.push(insert.clone()); - } - } else { - // If there is no context just flush it - expired_inserts.push(insert.clone()); - } - } - - all_expired_inserts.extend(expired_inserts); - - if !remaining_inserts.is_empty() { - let mut new_agg_buffer = AggregateBuffer::new(); - new_agg_buffer.buffer = remaining_inserts; - updated_buffers.insert(aggregate_id.clone(), new_agg_buffer); - } - } - - (updated_buffers, all_expired_inserts) -} - -impl Handler for WriteBuffer { - type Result = (); - - fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { - self.handle_commit_snapshot(msg) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::events::Insert; - use e3_events::{hlc::HlcTimestamp, EventContext, EventId}; - - #[test] - fn test_process_expired_inserts() { - let aggregate_id = AggregateId::new(1); - - // Create test inserts with different timestamps (in microseconds) - // Create proper HlcTimestamps and encode them to u128 - let old_hlc = HlcTimestamp::new(500_000, 0, 1); // 0.5 seconds ago - let new_hlc = HlcTimestamp::new(3_000_000, 0, 2); // 3 seconds from epoch - - let old_ctx = EventContext::new( - EventId::hash(1), - EventId::hash(1), - EventId::hash(1), - old_hlc.into(), - aggregate_id.clone(), - ) - .sequence(1); - - let new_ctx = EventContext::new( - EventId::hash(2), - EventId::hash(2), - EventId::hash(2), - new_hlc.into(), - aggregate_id.clone(), - ) - .sequence(2); - - let old_insert = Insert::new_with_context("old_key", b"old_value".to_vec(), old_ctx); - let new_insert = Insert::new_with_context("new_key", b"new_value".to_vec(), new_ctx); - let insert_no_ctx = Insert::new("no_ctx_key", b"no_ctx_value".to_vec()); - - // Set up aggregate buffer with mixed inserts - let mut agg_buffer = AggregateBuffer::new(); - agg_buffer.buffer.push(old_insert.clone()); - agg_buffer.buffer.push(new_insert.clone()); - agg_buffer.buffer.push(insert_no_ctx.clone()); - - let mut aggregate_buffers = HashMap::new(); - aggregate_buffers.insert(aggregate_id.clone(), agg_buffer); - - // Set config with 1 second delay - let mut delays = HashMap::new(); - delays.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds - let config = AggregateConfig::new(delays); - - // Use current time of 2 seconds, so old insert (0.5s) and insert without context should expire, - // new insert (3s) should remain - let now = 2_000_000; // 2 seconds in microseconds - - let (updated_buffers, expired_inserts) = - process_expired_inserts(&aggregate_buffers, &config.delays, now); - - // Verify expired inserts (old insert and insert without context) - assert_eq!(expired_inserts.len(), 2); - assert!(expired_inserts.contains(&old_insert)); - assert!(expired_inserts.contains(&insert_no_ctx)); - - // Verify remaining inserts in buffer - assert_eq!(updated_buffers.len(), 1); - let remaining_buffer = updated_buffers.get(&aggregate_id).unwrap(); - assert_eq!(remaining_buffer.buffer.len(), 1); - assert!(remaining_buffer.buffer.contains(&new_insert)); - } -} - -#[derive(Message)] -#[rtype("()")] -pub struct ForwardTo(Recipient); - -impl ForwardTo { - pub fn new(dest: impl Into>) -> Self { - Self(dest.into()) - } - - pub fn dest(self) -> Recipient { - self.0 - } -} diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 1b8f96e818..353ced3c75 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -22,7 +22,7 @@ pub async fn listen_for_shutdown(node: CiphernodeHandle) { info!("SIGTERM received, initiating graceful shutdown..."); // Stop the actor system - match bus.publish(Shutdown){ + match bus.publish_without_context(Shutdown){ Ok(_) => (), Err(e) => error!("Shutdown failed to publish! {e}") } diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index 4e75b4136d..716e437a08 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -39,4 +39,5 @@ test-helpers = [] # ensure test-helpers is available for integration tests proptest = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } e3-data = { workspace = true } +e3-test-helpers = { workspace = true } e3-ciphernode-builder = { workspace = true } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 1863054021..36ec3958a7 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use actix::{Actor, Addr, Handler, Recipient}; use anyhow::Result; use derivative::Derivative; +use e3_utils::MAILBOX_LIMIT; use tracing::error; use crate::{ @@ -16,11 +17,11 @@ use crate::{ hlc::Hlc, sequencer::Sequencer, traits::{ - ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, - EventSubscriber, + ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventContextAccessors, + EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, EventType, - HistoryCollector, Sequenced, Subscribe, Unsequenced, Unsubscribe, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, EventSource, + EventType, HistoryCollector, Sequenced, Subscribe, Unsequenced, Unsubscribe, }; #[derive(Clone, Derivative)] @@ -57,7 +58,7 @@ impl BusHandle { EventBus::>::history(&self.event_bus) } - /// Access the sequencer to internally dispatch am event to + /// Access the sequencer to internally dispatch an event to pub fn sequencer(&self) -> &Addr { &self.sequencer } @@ -81,19 +82,46 @@ impl BusHandle { let pipe = BusHandlePipe::new(other.to_owned(), predicate).start(); self.subscribe(EventType::All, pipe.into()); } + + pub fn with_ec(&self, ec: &EventContext) -> Self { + let mut bus = self.clone(); + bus.set_ctx(ec.clone()); + bus + } } impl EventPublisher> for BusHandle { - fn publish(&self, data: impl Into) -> Result<()> { - let evt = self.event_from(data, self.get_ctx())?; - self.sequencer.do_send(evt); - Ok(()) + fn publish( + &self, + data: impl Into, + caused_by: impl Into>, + ) -> Result<()> { + self.publish_local(data, Some(caused_by.into())) } - fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { - let evt = self.event_from_remote_source(data, self.get_ctx(), ts)?; - self.sequencer.do_send(evt); - Ok(()) + fn publish_without_context(&self, data: impl Into) -> Result<()> { + self.publish_local(data, None) + } + + fn publish_from_remote( + &self, + data: impl Into, + remote_ts: u128, + block: Option, + source: EventSource, + ) -> Result<()> { + self.publish_from_remote_impl(data, remote_ts, None, block, source) + } + + fn publish_from_remote_as_response( + &self, + data: impl Into, + remote_ts: u128, + caused_by: impl Into>, + block: Option, + source: EventSource, + ) -> Result<()> { + self.publish_from_remote_impl(data, remote_ts, Some(caused_by.into()), block, source) } fn naked_dispatch(&self, event: EnclaveEvent) { @@ -101,6 +129,30 @@ impl EventPublisher> for BusHandle { } } +impl BusHandle { + fn publish_from_remote_impl( + &self, + data: impl Into, + remote_ts: u128, + caused_by: Option>, + block: Option, + source: EventSource, + ) -> Result<()> { + let evt = self.event_from_remote_source(data, caused_by, remote_ts, block, source)?; + self.sequencer.do_send(evt); + Ok(()) + } + fn publish_local( + &self, + data: impl Into, + caused_by: Option>, + ) -> Result<()> { + let evt = self.event_from(data, caused_by)?; + self.sequencer.do_send(evt); + Ok(()) + } +} + impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { match self.event_from_error(err_type, error, self.get_ctx()) { @@ -121,6 +173,8 @@ impl EventFactory> for BusHandle { data.into(), caused_by, ts.into(), + None, + EventSource::Local, )) } @@ -129,12 +183,16 @@ impl EventFactory> for BusHandle { data: impl Into, caused_by: Option>, ts: u128, + block: Option, + source: EventSource, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), caused_by, ts.into(), + block, + source, )) } } @@ -175,8 +233,11 @@ impl EventSubscriber> for BusHandle { } impl EventContextManager for BusHandle { - fn set_ctx(&mut self, value: &EventContext) { - self.ctx = Some(value.clone()); + fn set_ctx(&mut self, value: C) + where + C: Into>, + { + self.ctx = Some(value.into().clone()); } fn get_ctx(&self) -> Option> { self.ctx.clone() @@ -189,8 +250,8 @@ mod tests { use e3_ciphernode_builder::EventSystem; // NOTE: We cannot pull from crate as the features will be missing as they are not default. use e3_events::{ - hlc::Hlc, prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventPublisher, EventType, - TestEvent, + hlc::Hlc, prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventPublisher, + EventSource, EventType, TestEvent, }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::time::sleep; @@ -220,7 +281,9 @@ mod tests { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { let ts = msg.ts(); - self.dest.publish_from_remote(msg.into_data(), ts).unwrap() + self.dest + .publish_from_remote(msg.into_data(), ts, None, EventSource::Local) + .unwrap() } } @@ -276,13 +339,13 @@ mod tests { bus_c.subscribe(EventType::All, saver.clone().into()); // Publish events in causal order across buses - bus_a.publish(TestEvent::new("one", 1))?; + bus_a.publish_without_context(TestEvent::new("one", 1))?; sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish(TestEvent::new("two", 2))?; + bus_b.publish_without_context(TestEvent::new("two", 2))?; sleep(Duration::from_millis(5)).await; // next tick - bus_a.publish(TestEvent::new("three", 3))?; + bus_a.publish_without_context(TestEvent::new("three", 3))?; sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish(TestEvent::new("four", 4))?; + bus_b.publish_without_context(TestEvent::new("four", 4))?; sleep(Duration::from_millis(50)).await; // next tick // Get events @@ -356,6 +419,9 @@ where F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler> for BusHandlePipe @@ -365,8 +431,11 @@ where type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { if (self.predicate)(&msg) { + let source = msg.source(); let (data, ts) = msg.split(); - let _ = self.handle.publish_from_remote(data, ts); + let _ = self.handle.publish_from_remote(data, ts, None, source); + // TODO: check if this is fine + // to erase block data } } } diff --git a/crates/data/src/events.rs b/crates/events/src/data_events.rs similarity index 97% rename from crates/data/src/events.rs rename to crates/events/src/data_events.rs index 6f36b8c53c..5964049eaa 100644 --- a/crates/data/src/events.rs +++ b/crates/events/src/data_events.rs @@ -4,10 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::IntoKey; +use crate::{EventContext, IntoKey, Sequenced}; use actix::Message; use anyhow::Result; -use e3_events::{EventContext, Sequenced}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] diff --git a/crates/events/src/enclave_event/decryptionshare_created.rs b/crates/events/src/enclave_event/decryptionshare_created.rs index 0df8177adb..1916db7068 100644 --- a/crates/events/src/enclave_event/decryptionshare_created.rs +++ b/crates/events/src/enclave_event/decryptionshare_created.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[rtype(result = "anyhow::Result<()>")] +#[rtype(result = "()")] pub struct DecryptionshareCreated { pub party_id: u64, pub decryption_share: Vec, // per index depending on what is required for the diff --git a/crates/events/src/enclave_event/evm_sync_events_received.rs b/crates/events/src/enclave_event/enable_effects.rs similarity index 63% rename from crates/events/src/enclave_event/evm_sync_events_received.rs rename to crates/events/src/enclave_event/enable_effects.rs index 0fcf1e3347..16ae123770 100644 --- a/crates/events/src/enclave_event/evm_sync_events_received.rs +++ b/crates/events/src/enclave_event/enable_effects.rs @@ -8,21 +8,18 @@ use actix::Message; use serde::{Deserialize, Serialize}; use std::fmt::{self, Display}; -use super::{EnclaveEvent, Unsequenced}; - +/// Dispatched once the sync process is complete and live listening should continue #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EvmSyncEventsReceived { - pub events: Vec>, -} +pub struct EffectsEnabled; -impl EvmSyncEventsReceived { - pub fn new(events: Vec>) -> Self { - Self { events } +impl EffectsEnabled { + pub fn new() -> Self { + Self {} } } -impl Display for EvmSyncEventsReceived { +impl Display for EffectsEnabled { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 5f9100ded3..547aca60b5 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::Message; +use e3_utils::major_issue; use serde::{Deserialize, Serialize}; use std::{ fmt::{self, Display}, @@ -14,6 +15,8 @@ use std::{ use crate::{BusHandle, ErrorDispatcher}; +use super::{EnclaveEvent, Unsequenced}; + pub trait FromError { fn from_error(err_type: EType, error: impl Into) -> Self; } @@ -45,6 +48,7 @@ pub enum EType { Data, Event, Computation, + DocumentPublishing, } impl EnclaveError { @@ -67,7 +71,7 @@ impl FromError for EnclaveError { /// Function to run a closure that returns a result. If result is an Err variant it is trapped and /// sent to the bus as an ErrorEvent -pub fn trap(err_type: EType, bus: &BusHandle, runner: F) +pub fn trap(err_type: EType, bus: &impl ErrorDispatcher>, runner: F) where F: FnOnce() -> anyhow::Result<()>, { @@ -94,3 +98,48 @@ where } }) } + +// The following dispatchers should be used where you don't have the BusHandle available. + +// A struct that panics on errors +pub struct PanicDispatcher; + +impl PanicDispatcher { + pub fn new() -> Self { + Self {} + } +} + +impl ErrorDispatcher> for PanicDispatcher { + fn err(&self, _: EType, error: impl Into) { + panic!("{}", major_issue("Failure!", error)); + } +} + +// A struct that warns on errors +pub struct WarningDispatcher; +impl WarningDispatcher { + pub fn new() -> Self { + Self {} + } +} +impl ErrorDispatcher> for WarningDispatcher { + fn err(&self, err_type: EType, error: impl Into) { + tracing::warn!("{:?} Failure! {}", err_type, error.into()); + } +} + +// A struct that logs errors on errors +// Avoid using this over BusHandle +pub struct LogErrorDispatcher; +impl LogErrorDispatcher { + pub fn new() -> Self { + Self {} + } +} + +impl ErrorDispatcher> for LogErrorDispatcher { + fn err(&self, err_type: EType, error: impl Into) { + tracing::error!("{:?} Failure! {}", err_type, error.into()); + } +} diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index e8d0ff7d33..1f660a7faa 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -20,12 +20,12 @@ mod e3_failed; mod e3_request_complete; mod e3_requested; mod e3_stage_changed; +mod enable_effects; mod enclave_error; mod encryption_key_collection_failed; mod encryption_key_created; mod encryption_key_pending; mod encryption_key_received; -mod evm_sync_events_received; mod keyshare_created; mod net_sync_events_received; mod operator_activation_changed; @@ -64,13 +64,13 @@ pub use e3_failed::*; pub use e3_request_complete::*; pub use e3_requested::*; pub use e3_stage_changed::*; -use e3_utils::{colorize, Color}; +use e3_utils::{colorize, colorize_event_ids, Color}; +pub use enable_effects::*; pub use enclave_error::*; pub use encryption_key_collection_failed::*; pub use encryption_key_created::*; pub use encryption_key_pending::*; pub use encryption_key_received::*; -pub use evm_sync_events_received::*; pub use keyshare_created::*; pub use net_sync_events_received::*; pub use operator_activation_changed::*; @@ -97,7 +97,7 @@ pub use typed_event::*; use crate::{ event_context::{AggregateId, EventContext}, traits::{ErrorEvent, Event, EventConstructorWithTimestamp, EventContextAccessors}, - E3id, EventContextSeq, EventId, WithAggregateId, + E3id, EventContextSeq, EventId, EventSource, WithAggregateId, }; use actix::Message; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -228,10 +228,11 @@ pub enum EnclaveEventData { SignedProofFailed(SignedProofFailed), OutgoingSyncRequested(OutgoingSyncRequested), NetSyncEventsReceived(NetSyncEventsReceived), - EvmSyncEventsReceived(EvmSyncEventsReceived), - SyncStart(SyncStart), + HistoricalEvmSyncStart(HistoricalEvmSyncStart), + HistoricalNetSyncStart(HistoricalNetSyncStart), SyncEffect(SyncEffect), - SyncEnd(SyncEnd), + SyncEnded(SyncEnded), + EffectsEnabled(EffectsEnabled), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -297,6 +298,10 @@ where (self.payload, self.ctx.ts()) } + pub fn into_components(self) -> (EnclaveEventData, EventContext) { + (self.payload, self.ctx) + } + pub fn get_ctx(&self) -> &EventContext { &self.ctx } @@ -318,6 +323,16 @@ impl EventContextAccessors for EnclaveEvent { fn aggregate_id(&self) -> AggregateId { self.ctx.aggregate_id() } + fn block(&self) -> Option { + self.ctx.block() + } + fn source(&self) -> EventSource { + self.ctx.source() + } + fn with_source(mut self, source: EventSource) -> Self { + self.ctx = self.ctx.with_source(source); + self + } } impl EventContextSeq for EnclaveEvent { @@ -329,8 +344,9 @@ impl EventContextSeq for EnclaveEvent { impl EnclaveEvent { pub fn clone_unsequenced(&self) -> EnclaveEvent { let ts = self.ts(); + let block = self.block(); let data = self.clone().into_data(); - EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts) + EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts, block, self.source()) } pub fn to_typed_event(&self, data: T) -> TypedEvent { @@ -350,9 +366,22 @@ impl EnclaveEvent { #[cfg(feature = "test-helpers")] impl EnclaveEvent { - /// test-helpers only utility function to create a new unsequenced event + /// test-helpers only utility function to create a new sequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, None, time).into_sequenced(seq) + EnclaveEvent::::new_with_timestamp(data, None, time, None, EventSource::Local) + .into_sequenced(seq) + } + + /// test-helpers only utility function to create a new sequenced event + pub fn from_data_ec(data: EnclaveEventData, ec: EventContext) -> Self { + EnclaveEvent::::new_with_timestamp( + data, + Some(ec.clone()), + ec.ts(), + ec.block(), + EventSource::Local, + ) + .into_sequenced(ec.seq()) } /// test-helpers only utility function to remove time information from an event @@ -397,8 +426,12 @@ impl ErrorEvent for EnclaveEvent { let aggregate_id = AggregateId::new(0); // Error events use default aggregate_id let ctx = caused_by - .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) - .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)); + .map(|cause| { + EventContext::from_cause(id, cause, ts, aggregate_id, None, EventSource::Local) + }) + .unwrap_or_else(|| { + EventContext::new_origin(id, ts, aggregate_id, None, EventSource::Local) + }); Ok(EnclaveEvent { payload: payload.into(), @@ -441,6 +474,7 @@ impl EnclaveEventData { EnclaveEventData::TicketSubmitted(ref data) => Some(data.e3_id.clone()), EnclaveEventData::EncryptionKeyCreated(ref data) => Some(data.e3_id.clone()), EnclaveEventData::ComputeResponse(ref data) => Some(data.e3_id.clone()), + EnclaveEventData::TestEvent(ref data) => data.e3_id.clone(), EnclaveEventData::SignedProofFailed(ref data) => Some(data.e3_id.clone()), EnclaveEventData::E3Failed(ref data) => Some(data.e3_id.clone()), EnclaveEventData::E3StageChanged(ref data) => Some(data.e3_id.clone()), @@ -451,12 +485,8 @@ impl EnclaveEventData { impl WithAggregateId for EnclaveEventData { fn get_aggregate_id(&self) -> AggregateId { - let maybe_e3_id = self.get_e3_id(); - if let Some(e3_id) = maybe_e3_id { - AggregateId::new(e3_id.chain_id() as usize) - } else { - AggregateId::new(0) - } + let chain_id = self.get_e3_id().map(|e3_id| e3_id.chain_id()); + AggregateId::from_chain_id(chain_id) } } @@ -512,10 +542,11 @@ impl_event_types!( SignedProofFailed, OutgoingSyncRequested, NetSyncEventsReceived, - EvmSyncEventsReceived, - SyncStart, + HistoricalEvmSyncStart, + HistoricalNetSyncStart, SyncEffect, - SyncEnd + SyncEnded, + EffectsEnabled ); impl TryFrom<&EnclaveEvent> for EnclaveError { @@ -535,6 +566,16 @@ impl TryFrom> for EnclaveError { } } } +impl From> for EventContext { + fn from(value: EnclaveEvent) -> Self { + (&value).into() + } +} +impl From<&EnclaveEvent> for EventContext { + fn from(value: &EnclaveEvent) -> Self { + value.ctx.clone() + } +} // Add convenience method to EnclaveEvent impl EnclaveEvent { @@ -547,7 +588,13 @@ impl EnclaveEvent { impl fmt::Display for EnclaveEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let t = self.event_type(); - f.write_str(&format!("{} {:?}", colorize(t, Color::Cyan), self)) + let colorized_debug = colorize_event_ids(self); + + let s = match t.as_str() { + "EnclaveError" => format!("{} {}", colorize(t, Color::Red), colorized_debug), + _ => format!("{} {}", colorize(t, Color::Cyan), colorized_debug), + }; + f.write_str(&s) } } @@ -556,6 +603,8 @@ impl EventConstructorWithTimestamp for EnclaveEvent { data: Self::Data, caused_by: Option>, ts: u128, + block: Option, + source: EventSource, ) -> Self { let payload: EnclaveEventData = data.into(); let id = EventId::hash(&payload); @@ -563,8 +612,8 @@ impl EventConstructorWithTimestamp for EnclaveEvent { EnclaveEvent { payload, ctx: caused_by - .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) - .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)), + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id, block, source)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id, block, source)), } } } diff --git a/crates/events/src/enclave_event/sync_end.rs b/crates/events/src/enclave_event/sync_end.rs index 400ba9f5e2..6b2118a72b 100644 --- a/crates/events/src/enclave_event/sync_end.rs +++ b/crates/events/src/enclave_event/sync_end.rs @@ -11,15 +11,15 @@ use std::fmt::{self, Display}; /// Dispatched once the sync process is complete and live listening should continue #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct SyncEnd; +pub struct SyncEnded; -impl SyncEnd { +impl SyncEnded { pub fn new() -> Self { Self {} } } -impl Display for SyncEnd { +impl Display for SyncEnded { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs index 3841785133..89993094ed 100644 --- a/crates/events/src/enclave_event/sync_start.rs +++ b/crates/events/src/enclave_event/sync_start.rs @@ -4,94 +4,123 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use super::EnclaveEventData; -use crate::{CorrelationId, SyncEvmEvent}; +use crate::AggregateId; use crate::{EvmEventConfig, EvmEventConfigChain}; use actix::{Message, Recipient}; use anyhow::Context; use anyhow::Result; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use std::fmt::{self, Display}; -/// This is a processed EvmEvent specifically typed for the Sync actor +use super::EnclaveEvent; +use super::Unsequenced; + +/// Dispatched by the Sync actor when initial data is read and the sync process needs to be started #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct EvmEvent { - data: EnclaveEventData, - block: u64, - chain_id: u64, - ts: u128, - id: CorrelationId, +pub struct HistoricalEvmSyncStart { + /// The initial information for reading historical events from chains. This is generated from + /// from persisted information + pub evm_config: EvmEventConfig, + + #[serde(skip)] + /// We include the sender here so that the evm can communicate directly with the sync actor + pub sender: Option>, // Must be Option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented this is fine as this event is never + // shared } -impl EvmEvent { +impl HistoricalEvmSyncStart { pub fn new( - id: CorrelationId, - data: EnclaveEventData, - block: u64, - ts: u128, - chain_id: u64, + sender: impl Into>, + evm_config: EvmEventConfig, ) -> Self { Self { - id, - data, - block, - ts, - chain_id, + sender: Some(sender.into()), + evm_config, } } - pub fn split(self) -> (EnclaveEventData, u128, u64) { - (self.data, self.ts, self.block) - } - - pub fn get_id(&self) -> CorrelationId { - self.id - } - - pub fn chain_id(&self) -> u64 { - self.chain_id + pub fn get_evm_config(&self, chain_id: u64) -> Result { + Ok(self + .evm_config + .get(&chain_id) + .context("No config found for chain")? + .clone()) } +} - pub fn ts(&self) -> u128 { - self.ts +impl Display for HistoricalEvmSyncStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) } } /// Dispatched by the Sync actor when initial data is read and the sync process needs to be started #[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] -pub struct SyncStart { - /// The initial information for reading historical events from chains. This is generated from - /// from persisted information - pub evm_config: EvmEventConfig, - +pub struct HistoricalNetSyncStart { + pub since: BTreeMap, #[serde(skip)] /// We include the sender here so that the evm can communicate directly with the sync actor - pub sender: Option>, // Must be Option to allow serde deserialize on - // EnclaveEvent as Default is required to be - // implemented this is fine as this event is never - // shared + pub sender: Option>, // Must be Option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented this is fine as this event is never + // shared } -impl SyncStart { - pub fn new(sender: impl Into>, evm_config: EvmEventConfig) -> Self { +impl HistoricalNetSyncStart { + pub fn new( + sender: impl Into>, + since: BTreeMap, + ) -> Self { Self { + since, sender: Some(sender.into()), - evm_config, } } +} - pub fn get_evm_config(&self, chain_id: u64) -> Result { - Ok(self - .evm_config - .get(&chain_id) - .context("No config found for chain")? - .clone()) +impl Display for HistoricalNetSyncStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct HistoricalEvmEventsReceived { + pub events: Vec>, + pub chain_id: u64, +} + +impl HistoricalEvmEventsReceived { + pub fn new(events: Vec>, chain_id: u64) -> Self { + Self { events, chain_id } + } +} + +impl Display for HistoricalEvmEventsReceived { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct HistoricalNetEventsReceived { + pub events: Vec>, +} + +impl HistoricalNetEventsReceived { + pub fn new(events: Vec>) -> Self { + Self { events } } } -impl Display for SyncStart { +impl Display for HistoricalNetEventsReceived { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } diff --git a/crates/events/src/enclave_event/test_event.rs b/crates/events/src/enclave_event/test_event.rs index ab26068958..e38603589d 100644 --- a/crates/events/src/enclave_event/test_event.rs +++ b/crates/events/src/enclave_event/test_event.rs @@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize}; pub struct TestEvent { pub msg: String, pub entropy: u64, + pub e3_id: Option, } impl TestEvent { @@ -19,6 +20,15 @@ impl TestEvent { Self { msg: msg.to_owned(), entropy, + e3_id: None, + } + } + + pub fn with_e3_id(self, id: E3id) -> Self { + Self { + msg: self.msg, + entropy: self.entropy, + e3_id: Some(id), } } } @@ -26,6 +36,8 @@ impl TestEvent { #[cfg(test)] use std::fmt::{self, Display}; +use crate::{AggregateId, E3id}; + #[cfg(test)] impl Display for TestEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs index 7f3da4c0ee..86b0ff07f5 100644 --- a/crates/events/src/enclave_event/typed_event.rs +++ b/crates/events/src/enclave_event/typed_event.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::{ event_context::{AggregateId, EventContext}, - EventContextAccessors, EventContextSeq, EventId, + EventContextAccessors, EventContextSeq, EventId, EventSource, }; use super::Sequenced; @@ -24,8 +24,11 @@ pub struct TypedEvent { } impl TypedEvent { - pub fn new(inner: T, ctx: EventContext) -> Self { - Self { inner, ctx } + pub fn new(inner: T, ctx: impl Into>) -> Self { + Self { + inner, + ctx: ctx.into(), + } } pub fn into_inner(self) -> T { @@ -35,6 +38,10 @@ impl TypedEvent { pub fn get_ctx(&self) -> &EventContext { &self.ctx } + + pub fn into_components(self) -> (T, EventContext) { + (self.inner, self.ctx) + } } impl Deref for TypedEvent { @@ -64,6 +71,18 @@ impl EventContextAccessors for TypedEvent { fn aggregate_id(&self) -> AggregateId { self.ctx.aggregate_id() } + + fn block(&self) -> Option { + self.ctx.block() + } + + fn source(&self) -> EventSource { + self.ctx.source() + } + fn with_source(mut self, source: EventSource) -> Self { + self.ctx = self.ctx.with_source(source); + self + } } impl EventContextSeq for TypedEvent { diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs index 6cb5b9bafa..60743649ba 100644 --- a/crates/events/src/event_context.rs +++ b/crates/events/src/event_context.rs @@ -9,10 +9,18 @@ use std::{fmt, ops::Deref}; use serde::{Deserialize, Serialize}; use crate::{ - E3id, EventContextAccessors, EventContextSeq, EventId, SeqState, Sequenced, Unsequenced, + E3id, EnclaveEventData, EventContextAccessors, EventContextSeq, EventId, SeqState, Sequenced, + Unsequenced, }; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum EventSource { + Local, + Net, + Evm, +} + +#[derive(Clone, Copy, Debug, PartialEq, Ord, PartialOrd, Eq, Hash, Serialize, Deserialize)] pub struct AggregateId(usize); impl AggregateId { @@ -23,15 +31,31 @@ impl AggregateId { pub fn to_usize(&self) -> usize { self.0 } + + /// Create AggregateId from Option + /// None → AggregateId(0), Some(chain_id) → AggregateId(chain_id) + pub fn from_chain_id(chain_id: Option) -> Self { + match chain_id { + None => Self::new(0), + Some(id) => Self::new(id.try_into().unwrap_or(0)), + } + } + + /// Convert back to Option + /// AggregateId(0) → None, otherwise → Some(chain_id) + pub fn to_chain_id(&self) -> Option { + if self.0 == 0 { + None + } else { + Some(self.0 as u64) + } + } } impl From> for AggregateId { fn from(value: Option) -> Self { - if let Some(e3_id) = value { - Self::new(e3_id.chain_id() as usize) - } else { - Self::new(0) - } + let chain_id = value.map(|e3_id| e3_id.chain_id()); + Self::from_chain_id(chain_id) } } @@ -57,6 +81,8 @@ pub struct EventContext { seq: S::Seq, ts: u128, aggregate_id: AggregateId, + block: Option, + source: EventSource, } impl EventContext { @@ -66,6 +92,8 @@ impl EventContext { origin_id: EventId, ts: u128, aggregate_id: AggregateId, + block: Option, + source: EventSource, ) -> Self { Self { id, @@ -74,11 +102,19 @@ impl EventContext { seq: (), ts, aggregate_id, + block, + source, } } - pub fn new_origin(id: EventId, ts: u128, aggregate_id: AggregateId) -> Self { - Self::new(id, id, id, ts, aggregate_id) + pub fn new_origin( + id: EventId, + ts: u128, + aggregate_id: AggregateId, + block: Option, + source: EventSource, + ) -> Self { + Self::new(id, id, id, ts, aggregate_id, block, source) } pub fn from_cause( @@ -86,8 +122,28 @@ impl EventContext { cause: EventContext, ts: u128, aggregate_id: AggregateId, + block: Option, + source: EventSource, ) -> Self { - EventContext::new(id, cause.id(), cause.origin_id(), ts, aggregate_id) + EventContext::new( + id, + cause.id(), + cause.origin_id(), + ts, + aggregate_id, + cause.block.max(block), // block watermark + source, + ) + } + + pub fn with_ts(mut self, ts: u128) -> Self { + self.ts = ts; + self + } + + pub fn with_aggregate(mut self, aggregate_id: AggregateId) -> Self { + self.aggregate_id = aggregate_id; + self } pub fn sequence(self, value: u64) -> EventContext { @@ -98,10 +154,25 @@ impl EventContext { origin_id: self.origin_id, ts: self.ts, aggregate_id: self.aggregate_id, + block: self.block, + source: self.source, } } } +impl From for EventContext { + fn from(value: EnclaveEventData) -> Self { + let id = EventId::hash(value); + EventContext::::new_origin( + id, + 0, + AggregateId::new(0), + Some(0), + EventSource::Local, + ) + } +} + impl EventContextAccessors for EventContext { fn id(&self) -> EventId { self.id @@ -122,6 +193,19 @@ impl EventContextAccessors for EventContext { fn aggregate_id(&self) -> AggregateId { self.aggregate_id } + + fn block(&self) -> Option { + self.block + } + + fn source(&self) -> EventSource { + self.source + } + + fn with_source(mut self, source: EventSource) -> Self { + self.source = source; + self + } } impl EventContextSeq for EventContext { @@ -134,7 +218,7 @@ impl EventContextSeq for EventContext { mod tests { use crate::{ event_context::{AggregateId, EventContext}, - EventId, + EventId, EventSource, }; #[test] @@ -147,16 +231,32 @@ mod tests { EventId::hash(1), 1, AggregateId::new(1), + None, + EventSource::Local, ) .sequence(1); events.push(one.clone()); - let two = - EventContext::from_cause(EventId::hash(2), one, 2, AggregateId::new(1)).sequence(2); + let two = EventContext::from_cause( + EventId::hash(2), + one, + 2, + AggregateId::new(1), + None, + EventSource::Local, + ) + .sequence(2); events.push(two.clone()); - let three = - EventContext::from_cause(EventId::hash(3), two, 3, AggregateId::new(1)).sequence(3); + let three = EventContext::from_cause( + EventId::hash(3), + two, + 3, + AggregateId::new(1), + None, + EventSource::Local, + ) + .sequence(3); events.push(three.clone()); assert_eq!( @@ -169,6 +269,8 @@ mod tests { causation_id: EventId::hash(1), ts: 1, aggregate_id: AggregateId::new(1), + block: None, + source: EventSource::Local }, EventContext { seq: 2, @@ -177,6 +279,8 @@ mod tests { causation_id: EventId::hash(1), ts: 2, aggregate_id: AggregateId::new(1), + block: None, + source: EventSource::Local }, EventContext { seq: 3, @@ -185,6 +289,8 @@ mod tests { causation_id: EventId::hash(2), ts: 3, aggregate_id: AggregateId::new(1), + block: None, + source: EventSource::Local }, ] ) diff --git a/crates/events/src/event_extractor.rs b/crates/events/src/event_extractor.rs new file mode 100644 index 0000000000..34278dc83a --- /dev/null +++ b/crates/events/src/event_extractor.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::{EffectsEnabled, EnclaveEvent, EnclaveEventData, Event, HistoricalEvmSyncStart}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use e3_utils::{actix::oneshot_runner::OneShotRunner, MAILBOX_LIMIT}; + +/// Trait for events that can be extracted from EnclaveEventData +pub trait ExtractableEvent: Message + Send + 'static { + fn extract_from(data: EnclaveEventData) -> Option + where + Self: Sized; +} + +/// Generic event extractor that can handle any ExtractableEvent +pub struct EventExtractor { + dest: Recipient, +} + +impl EventExtractor { + pub fn new(dest: impl Into>) -> Self { + Self { dest: dest.into() } + } + + pub fn setup(dest: impl Into>) -> Addr { + Self::new(dest).start() + } +} + +impl Actor for EventExtractor { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} + +impl Handler for EventExtractor { + type Result = (); + + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let Some(evt) = T::extract_from(msg.into_data()) { + self.dest.do_send(evt) + } + } +} + +impl ExtractableEvent for EffectsEnabled { + fn extract_from(data: EnclaveEventData) -> Option { + if let EnclaveEventData::EffectsEnabled(evt) = data { + Some(evt) + } else { + None + } + } +} + +impl ExtractableEvent for HistoricalEvmSyncStart { + fn extract_from(data: EnclaveEventData) -> Option { + if let EnclaveEventData::HistoricalEvmSyncStart(evt) = data { + Some(evt) + } else { + None + } + } +} + +/// Helper function to set up a one-shot event extractor +pub fn run_once( + f: impl FnOnce(T) -> anyhow::Result<()> + Unpin + Send + 'static, +) -> Addr> { + // EventExtractor filters the given event and sends to the receiver + EventExtractor::::setup( + // OneShotRunner runs the closure once when it first receives the event + OneShotRunner::setup(f), + ) +} diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index f899f5b1aa..6760976f8e 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -8,6 +8,7 @@ use crate::traits::{ErrorEvent, Event}; use crate::EventType; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; +use e3_utils::{colorize, Color, MAILBOX_LIMIT, MAILBOX_LIMIT_LARGE}; use std::collections::{HashMap, VecDeque}; use std::marker::PhantomData; use tracing::info; @@ -49,6 +50,9 @@ pub struct EventBus { impl Actor for EventBus { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT_LARGE) + } } impl EventBus { @@ -131,7 +135,7 @@ impl Handler for EventBus { } // TODO: workshop to work out best display format - tracing::info!(">>> {}", event); + tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); self.track(event); } } @@ -218,6 +222,9 @@ impl EventFilter { impl Actor for EventFilter { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT_LARGE) + } } impl Handler for EventFilter { @@ -400,6 +407,9 @@ impl Handler> for HistoryCollector { impl Actor for HistoryCollector { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for HistoryCollector { diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 5fd9ae190d..3ff819c7ea 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -4,44 +4,24 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::collections::HashMap; + use actix::{Message, Recipient}; use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; -/// Direct event received by the snapshot buffer in order to save snapshot to disk -#[derive(Message, Debug)] -#[rtype("()")] -pub struct CommitSnapshot { - seq: u64, - aggregate_id: AggregateId, -} - -impl CommitSnapshot { - pub fn new(seq: u64, aggregate_id: AggregateId) -> Self { - Self { seq, aggregate_id } - } - - pub fn seq(&self) -> u64 { - self.seq - } - - pub fn aggregate_id(&self) -> AggregateId { - self.aggregate_id - } -} - /// Direct event received by the EventStore to store an event #[derive(Message, Debug)] #[rtype("()")] pub struct StoreEventRequested { pub event: EnclaveEvent, - pub sender: Recipient, + pub sender: Recipient, } impl StoreEventRequested { pub fn new( event: EnclaveEvent, - sender: impl Into>, + sender: impl Into>, ) -> Self { Self { event, @@ -50,67 +30,154 @@ impl StoreEventRequested { } } -/// Get events after timestamp in EventStore +/// The response of a request to get all EventStore events by either sequence or timestamp #[derive(Message, Debug)] #[rtype("()")] -pub struct GetEventsAfter { +pub struct EventStoreQueryResponse { + id: CorrelationId, + events: Vec>, +} + +impl EventStoreQueryResponse { + pub fn new(id: CorrelationId, events: Vec) -> Self { + Self { id, events } + } + + pub fn into_events(self) -> Vec { + self.events + } + + pub fn id(&self) -> CorrelationId { + self.id + } +} + +/// Direct event received by the Sequencer once an event has been stored +#[derive(Message, Debug)] +#[rtype("()")] +pub struct StoreEventResponse(pub EnclaveEvent); + +impl StoreEventResponse { + pub fn into_event(self) -> EnclaveEvent { + self.0 + } +} + +/// Trait for various EventStore query types +pub trait QueryKind { + type Shape: Send; +} + +/// Query by aggregated sequence +pub struct SeqAgg; +impl QueryKind for SeqAgg { + type Shape = HashMap; +} + +/// Query by aggregated timestamp +pub struct TsAgg; +impl QueryKind for TsAgg { + type Shape = HashMap; +} + +/// Query by timestamp +pub struct Ts; +impl QueryKind for Ts { + type Shape = u128; +} + +/// Query by seq +pub struct Seq; +impl QueryKind for Seq { + type Shape = u64; +} + +#[derive(Message, Debug)] +#[rtype("()")] +pub struct EventStoreQueryBy { correlation_id: CorrelationId, - ts: u128, - sender: Recipient, + query: Q::Shape, + sender: Recipient, } -impl GetEventsAfter { +impl EventStoreQueryBy { pub fn new( correlation_id: CorrelationId, - ts: u128, - sender: impl Into>, + query: HashMap, + sender: impl Into>, ) -> Self { Self { correlation_id, - ts, + query, sender: sender.into(), } } - pub fn ts(&self) -> u128 { - self.ts + pub fn query(&self) -> &HashMap { + &self.query } +} - pub fn id(&self) -> CorrelationId { - self.correlation_id +impl EventStoreQueryBy { + pub fn new( + correlation_id: CorrelationId, + query: HashMap, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + query, + sender: sender.into(), + } } - pub fn sender(&self) -> &Recipient { - &self.sender + pub fn query(&self) -> &HashMap { + &self.query } } -#[derive(Message, Debug)] -#[rtype("()")] -pub struct ReceiveEvents { - id: CorrelationId, - events: Vec>, -} +impl EventStoreQueryBy { + pub fn new( + correlation_id: CorrelationId, + query: u128, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + query, + sender: sender.into(), + } + } -impl ReceiveEvents { - pub fn new(id: CorrelationId, events: Vec) -> Self { - Self { id, events } + pub fn query(&self) -> u128 { + self.query } - pub fn events(&self) -> &Vec { - &self.events +} + +impl EventStoreQueryBy { + pub fn new( + correlation_id: CorrelationId, + query: u64, + sender: impl Into>, + ) -> Self { + Self { + correlation_id, + query, + sender: sender.into(), + } } - pub fn id(&self) -> CorrelationId { - self.id + + pub fn query(&self) -> u64 { + self.query } } -/// Direct event received by the Sequencer once an event has been stored -#[derive(Message, Debug)] -#[rtype("()")] -pub struct EventStored(pub EnclaveEvent); +impl EventStoreQueryBy { + pub fn id(&self) -> CorrelationId { + self.correlation_id + } -impl EventStored { - pub fn into_event(self) -> EnclaveEvent { - self.0 + pub fn sender(self) -> Recipient { + self.sender } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 239bd68cc0..41fb44fbf2 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -5,16 +5,21 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - events::{EventStored, StoreEventRequested}, - EventContextAccessors, EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, + events::{StoreEventRequested, StoreEventResponse}, + EventContextAccessors, EventLog, EventStoreQueryBy, EventStoreQueryResponse, Seq, + SequenceIndex, Ts, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; -use tracing::error; +use e3_utils::{major_issue, MAILBOX_LIMIT}; +use tracing::{error, warn}; + +const MAX_STORAGE_ERRORS: u64 = 10; pub struct EventStore { index: I, log: L, + storage_errors: u64, } impl EventStore { @@ -23,19 +28,28 @@ impl EventStore { let sender = msg.sender; let ts = event.ts(); if let Some(_) = self.index.get(ts)? { - bail!("Event already stored at timestamp {ts}!"); + warn!("Event already stored at timestamp {ts}! This might happen when recovering from a snapshot. Skipping storage"); + self.storage_errors += 1; + if self.storage_errors > MAX_STORAGE_ERRORS { + bail!( + "The eventstore had too many storage errors! {}", + self.storage_errors + ); + } + return Ok(()); } let seq = self.log.append(&event)?; self.index.insert(ts, seq)?; - sender.try_send(EventStored(event.into_sequenced(seq)))?; + sender.try_send(StoreEventResponse(event.into_sequenced(seq)))?; Ok(()) } - pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { + pub fn handle_event_store_query_ts(&mut self, msg: EventStoreQueryBy) -> Result<()> { // if there are no events after the timestamp return an empty vector - let Some(seq) = self.index.seek(msg.ts())? else { + let id = msg.id(); + let Some(seq) = self.index.seek(msg.query())? else { msg.sender() - .try_send(ReceiveEvents::new(msg.id(), vec![]))?; + .try_send(EventStoreQueryResponse::new(id, vec![]))?; return Ok(()); }; // read and return the events @@ -45,38 +59,70 @@ impl EventStore { .map(|(s, e)| e.into_sequenced(s)) .collect::>(); - msg.sender().try_send(ReceiveEvents::new(msg.id(), evts))?; + msg.sender() + .try_send(EventStoreQueryResponse::new(id, evts))?; + Ok(()) + } + + pub fn handle_event_store_query_seq(&mut self, msg: EventStoreQueryBy) -> Result<()> { + // if there are no events after the timestamp return an empty vector + let id = msg.id(); + + // read and return the events + let evts = self + .log + .read_from(msg.query()) + .map(|(s, e)| e.into_sequenced(s)) + .collect::>(); + + msg.sender() + .try_send(EventStoreQueryResponse::new(id, evts))?; Ok(()) } } impl EventStore { pub fn new(index: I, log: L) -> Self { - Self { index, log } + Self { + index, + log, + storage_errors: 0, + } } } impl Actor for EventStore { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for EventStore { type Result = (); fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { - match self.handle_store_event_requested(msg) { - Ok(_) => (), - Err(e) => panic!("{e}"), // panic here because when event storage fails we really need - // to just give up + if let Err(e) = self.handle_store_event_requested(msg) { + panic!("{}", major_issue("Could not store event in eventstore.", e)) + // panic here because when event storage fails we really need + // to just give up + } + } +} + +impl Handler> for EventStore { + type Result = (); + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_ts(msg) { + error!("{e}"); } } } -impl Handler for EventStore { +impl Handler> for EventStore { type Result = (); - fn handle(&mut self, msg: GetEventsAfter, _: &mut Self::Context) -> Self::Result { - match self.handle_get_events_after(msg) { - Ok(_) => (), - Err(e) => error!("{e}"), + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_seq(msg) { + error!("{e}"); } } } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 4d9a6b3321..ba214df7ff 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -3,24 +3,94 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. - use crate::eventstore::EventStore; use crate::{ - events::{GetEventsAfter, StoreEventRequested}, + events::{EventStoreQueryResponse, StoreEventRequested}, AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; -use crate::{CorrelationId, ReceiveEvents}; -use actix::{Actor, Addr, Handler, Message, Recipient}; +use crate::{CorrelationId, Die, EnclaveEvent, EventStoreQueryBy, Seq, SeqAgg, Ts, TsAgg}; +use actix::{Actor, ActorContext, Addr, AsyncContext, Context, Handler, Recipient}; use anyhow::Result; +use e3_utils::{major_issue, MAILBOX_LIMIT}; use std::collections::HashMap; -use tracing::error; +use tracing::{debug, error, info, warn}; + +/// QueryAggregator - handles a single query's lifecycle +struct QueryAggregator { + parent_id: CorrelationId, + sender: Recipient, + pending: HashMap, + collected_events: Vec, +} + +impl QueryAggregator { + fn new(parent_id: CorrelationId, sender: Recipient) -> Self { + Self { + parent_id, + sender, + pending: HashMap::new(), + collected_events: Vec::new(), + } + } + + fn add_pending(&mut self, sub_query_id: CorrelationId, aggregate_id: AggregateId) { + self.pending.insert(sub_query_id, aggregate_id); + } + + fn pending_aggregates(&self) -> Vec<&AggregateId> { + self.pending.values().collect() + } +} + +impl Actor for QueryAggregator { + type Context = Context; +} + +impl Handler for QueryAggregator { + type Result = (); + + fn handle(&mut self, msg: EventStoreQueryResponse, ctx: &mut Self::Context) -> Self::Result { + let sub_query_id = msg.id(); + + if let Some(aggregate_id) = self.pending.remove(&sub_query_id) { + info!( + "Received response for aggregate {:?}, {} pending", + aggregate_id, + self.pending.len() + ); + self.collected_events.extend(msg.into_events()); + + if self.pending.is_empty() { + info!("All aggregates fulfilled, sending response"); + let response = EventStoreQueryResponse::new( + self.parent_id, + std::mem::take(&mut self.collected_events), + ); + self.sender.do_send(response); + ctx.notify(Die) + } + } else { + warn!("Received response for unknown sub-query: {}", sub_query_id); + } + } +} + +impl Handler for QueryAggregator { + type Result = (); + fn handle(&mut self, msg: Die, ctx: &mut Self::Context) -> Self::Result { + ctx.stop() + } +} + +/// EventStoreRouter - routes events and spawns query aggregators to handle eventstore queries pub struct EventStoreRouter { stores: HashMap>>, } impl EventStoreRouter { pub fn new(stores: HashMap>>) -> Self { + debug!("Making eventstore router..."); let stores = stores .into_iter() .map(|(index, addr)| (AggregateId::new(index), addr)) @@ -29,36 +99,114 @@ impl EventStoreRouter { } pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { + debug!("Handling store event requested...."); let aggregate_id = msg.event.aggregate_id(); - let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { self.stores .get(&AggregateId::new(0)) .expect("Default EventStore for AggregateId(0) not found") }); - let event = msg.event; let sender = msg.sender; - let forwarded_msg = StoreEventRequested::new(event, sender); - store_addr.do_send(forwarded_msg); + store_addr.try_send(forwarded_msg)?; Ok(()) } - pub fn handle_get_events_after(&mut self, msg: GetAggregateEventsAfter) -> Result<()> { - for (aggregate_id, ts) in msg.ts() { - if let Some(store_addr) = self.stores.get(&aggregate_id) { - let get_events_msg = - GetEventsAfter::new(msg.id(), ts.to_owned(), msg.sender.clone()); - store_addr.do_send(get_events_msg); - } + pub fn handle_event_store_query_ts( + &mut self, + msg: EventStoreQueryBy, + _ctx: &mut Context, + ) -> Result<()> { + debug!("Received request for timestamp query."); + let parent_id = msg.id(); + let query = msg.query().clone(); + let sender = msg.sender(); + + let sub_queries: Vec<_> = query + .into_iter() + .filter_map(|(aggregate_id, ts)| { + self.stores + .get(&aggregate_id) + .map(|store_addr| (aggregate_id, ts, CorrelationId::new(), store_addr.clone())) + }) + .collect(); + + if sub_queries.is_empty() { + debug!("No valid stores to query, sending empty response immediately"); + let response = EventStoreQueryResponse::new(parent_id, Vec::new()); + sender.do_send(response); + return Ok(()); + } + + let mut aggregator = QueryAggregator::new(parent_id, sender); + for (aggregate_id, _, sub_query_id, _) in &sub_queries { + aggregator.add_pending(*sub_query_id, aggregate_id.clone()); + } + let aggregator_addr = aggregator.start(); + + for (aggregate_id, ts, sub_query_id, store_addr) in sub_queries { + let get_events_msg = + EventStoreQueryBy::::new(sub_query_id, ts, aggregator_addr.clone().recipient()); + debug!("Sending query for aggregate {:?}", aggregate_id); + store_addr.do_send(get_events_msg); + } + + Ok(()) + } + + pub fn handle_event_store_query_seq( + &mut self, + msg: EventStoreQueryBy, + _ctx: &mut Context, + ) -> Result<()> { + debug!("Received request for sequence query."); + let parent_id = msg.id(); + let query = msg.query().clone(); + let sender = msg.sender(); + + let sub_queries: Vec<_> = query + .into_iter() + .filter_map(|(aggregate_id, seq)| { + self.stores + .get(&aggregate_id) + .map(|store_addr| (aggregate_id, seq, CorrelationId::new(), store_addr.clone())) + }) + .collect(); + + if sub_queries.is_empty() { + debug!("No valid stores to query, sending empty response immediately"); + let response = EventStoreQueryResponse::new(parent_id, Vec::new()); + sender.do_send(response); + return Ok(()); } + + let mut aggregator = QueryAggregator::new(parent_id, sender); + for (aggregate_id, _, sub_query_id, _) in &sub_queries { + aggregator.add_pending(*sub_query_id, aggregate_id.clone()); + } + let aggregator_addr = aggregator.start(); + + for (aggregate_id, seq, sub_query_id, store_addr) in sub_queries { + let get_events_msg = EventStoreQueryBy::::new( + sub_query_id, + seq, + aggregator_addr.clone().recipient(), + ); + debug!("Sending query for aggregate {:?}", aggregate_id); + store_addr.do_send(get_events_msg); + } + Ok(()) } } impl Actor for EventStoreRouter { - type Context = actix::Context; + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for EventStoreRouter { @@ -66,47 +214,27 @@ impl Handler for EventStoreR fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { if let Err(e) = self.handle_store_event_requested(msg) { - error!("Failed to route store event request: {}", e); + panic!("{}", major_issue("Could not store event in eventstore.", e)) } } } -impl Handler for EventStoreRouter { +impl Handler> for EventStoreRouter { type Result = (); - fn handle(&mut self, msg: GetAggregateEventsAfter, _: &mut Self::Context) -> Self::Result { - if let Err(e) = self.handle_get_events_after(msg) { + fn handle(&mut self, msg: EventStoreQueryBy, ctx: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_ts(msg, ctx) { error!("Failed to route get events after request: {}", e); } } } -#[derive(Message, Debug)] -#[rtype("()")] -pub struct GetAggregateEventsAfter { - pub correlation_id: CorrelationId, - pub ts: HashMap, - pub sender: Recipient, -} +impl Handler> for EventStoreRouter { + type Result = (); -impl GetAggregateEventsAfter { - pub fn new( - correlation_id: CorrelationId, - ts: HashMap, - sender: Recipient, - ) -> Self { - Self { - correlation_id, - ts, - sender, + fn handle(&mut self, msg: EventStoreQueryBy, ctx: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_event_store_query_seq(msg, ctx) { + error!("Failed to route get events after request: {}", e); } } - - pub fn id(&self) -> CorrelationId { - self.correlation_id - } - - pub fn ts(&self) -> &HashMap { - &self.ts - } } diff --git a/crates/data/src/into_key.rs b/crates/events/src/into_key.rs similarity index 100% rename from crates/data/src/into_key.rs rename to crates/events/src/into_key.rs diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index b9221447d7..d772e652bd 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -6,34 +6,44 @@ mod bus_handle; mod correlation_id; +mod data_events; mod e3id; mod enclave_event; mod event_context; +mod event_extractor; mod event_id; mod eventbus; mod events; mod eventstore; mod eventstore_router; pub mod hlc; +mod into_key; mod ordered_set; pub mod prelude; mod seed; mod sequencer; +mod snapshot_buffer; +mod store_keys; mod sync; mod traits; pub use bus_handle::*; pub use correlation_id::*; +pub use data_events::*; pub use e3id::*; pub use enclave_event::*; pub use event_context::*; +pub use event_extractor::*; pub use event_id::*; pub use eventbus::*; pub use events::*; pub use eventstore::*; pub use eventstore_router::*; +pub use into_key::*; pub use ordered_set::*; pub use seed::*; pub use sequencer::*; +pub use snapshot_buffer::*; +pub use store_keys::*; pub use sync::*; pub use traits::*; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 552f502833..cb8a929243 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -4,54 +4,66 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; - use crate::{ - events::{CommitSnapshot, EventStored, StoreEventRequested}, - EnclaveEvent, EventBus, EventContextAccessors, EventContextSeq, Sequenced, Unsequenced, + events::{StoreEventRequested, StoreEventResponse}, + EnclaveEvent, EventBus, Sequenced, Unsequenced, }; +use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; +use anyhow::Result; +use e3_utils::{major_issue, MAILBOX_LIMIT_LARGE}; /// Component to sequence the storage of events pub struct Sequencer { bus: Addr>>, eventstore: Recipient, - buffer: Recipient, } impl Sequencer { pub fn new( bus: &Addr>>, eventstore: impl Into>, - buffer: impl Into>, ) -> Self { Self { bus: bus.clone(), eventstore: eventstore.into(), - buffer: buffer.into(), } } + + fn handle_store_event_response(&self, msg: StoreEventResponse) -> Result<()> { + let event = msg.into_event(); + self.bus.try_send(event)?; + Ok(()) + } } impl Actor for Sequencer { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT_LARGE) + } } impl Handler> for Sequencer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - self.eventstore - .do_send(StoreEventRequested::new(msg, ctx.address())) + if let Err(e) = self + .eventstore + .try_send(StoreEventRequested::new(msg, ctx.address())) + { + panic!("{}", major_issue("Could not store event in eventstore.", e)) + } } } -impl Handler for Sequencer { +impl Handler for Sequencer { type Result = (); - fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { - let event = msg.into_event(); - let seq = event.seq(); - self.buffer - .do_send(CommitSnapshot::new(seq, event.aggregate_id())); - self.bus.do_send(event) + fn handle(&mut self, msg: StoreEventResponse, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_store_event_response(msg) { + panic!( + "{}", + major_issue("Could not send event to snapshot_buffer or bus.", e) + ) + } } } @@ -73,7 +85,7 @@ mod tests { ]; for d in event_data.clone() { - bus.publish(d)?; + bus.publish_without_context(d)?; } let expected = event_data diff --git a/crates/events/src/snapshot_buffer/aggregate_config.rs b/crates/events/src/snapshot_buffer/aggregate_config.rs new file mode 100644 index 0000000000..20a9e6cf98 --- /dev/null +++ b/crates/events/src/snapshot_buffer/aggregate_config.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +use crate::AggregateId; +use std::{collections::HashMap, time::Duration}; + +/// Central configuration for aggregates in the WriteBuffer +#[derive(Debug, Clone)] +pub struct AggregateConfig { + pub delays: HashMap, +} + +impl AggregateConfig { + pub fn get_delay(&self, id: &AggregateId) -> Duration { + self.delays + .get(id) + .cloned() + .unwrap_or(Duration::from_micros(0)) + } +} + +impl AggregateConfig { + /// Create a new AggregateConfig with the specified delays + pub fn new(mut delays: HashMap) -> Self { + // Always handle AggregatId of 0 with a delay of 0 + if let None = delays.get(&AggregateId::new(0)) { + delays.insert(AggregateId::new(0), Duration::from_micros(0)); + } + Self { delays } + } + + /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured + pub fn indexed_ids(&self) -> Vec { + self.delays.keys().map(|id| id.to_usize()).collect() + } + + pub fn aggregates(&self) -> Vec { + self.delays.keys().cloned().collect() + } +} diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs new file mode 100644 index 0000000000..acbb517557 --- /dev/null +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +use std::mem::replace; + +use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient}; +use e3_utils::MAILBOX_LIMIT; +use tracing::debug; + +use crate::{trap, Die, EType, Insert, InsertBatch, PanicDispatcher}; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct Flush; + +pub struct Batch { + inserts: Vec, + db: Recipient, +} + +impl Batch { + pub fn new(db: impl Into>, inserts: Vec) -> Self { + Self { + inserts, + db: db.into(), + } + } + pub fn spawn(db: impl Into>, inserts: Vec) -> Addr { + Self::new(db, inserts).start() + } +} + +impl Actor for Batch { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +impl Handler for Batch { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + self.inserts.push(msg) + } +} + +impl Handler for Batch { + type Result = (); + fn handle(&mut self, _: Flush, ctx: &mut Self::Context) -> Self::Result { + let inserts = replace(&mut self.inserts, Vec::new()); + trap(EType::IO, &PanicDispatcher::new(), || { + if inserts.len() > 0 { + self.db.try_send(InsertBatch::new(inserts))?; + } + ctx.notify(Die); + Ok(()) + }) + } +} + +impl Handler for Batch { + type Result = (); + fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + } +} diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs new file mode 100644 index 0000000000..71b88f3147 --- /dev/null +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +use super::{ + batch::{Batch, Flush}, + timelock_queue::{Clock, StartTimelock}, + AggregateConfig, UpdateDestination, +}; +use crate::{ + trap, AggregateId, EType, EnclaveEvent, EventContextAccessors, EventContextSeq, Insert, + InsertBatch, PanicDispatcher, Sequenced, StoreKeys, +}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use anyhow::Context; +use e3_utils::MAILBOX_LIMIT; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tracing::{debug, info, trace, warn}; + +type Seq = u64; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct FlushSeq(pub Seq); + +impl FlushSeq { + pub fn seq(&self) -> u64 { + self.0 + } +} + +pub struct BatchRouter { + config: AggregateConfig, + aggregates: HashMap, + batches: HashMap>, + block_height_seen: HashMap, + timelock_queue: Recipient, + db: Recipient, + clock: Arc, +} + +impl Actor for BatchRouter { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +impl BatchRouter { + pub fn new( + config: &AggregateConfig, + timelock_queue: impl Into>, + db: impl Into>, + ) -> Self { + Self::with_clock( + config, + timelock_queue, + db, + Arc::new(super::timelock_queue::SystemClock), + ) + } + + pub fn with_clock( + config: &AggregateConfig, + timelock_queue: impl Into>, + db: impl Into>, + clock: Arc, + ) -> Self { + Self { + batches: HashMap::new(), + aggregates: HashMap::new(), + config: config.clone(), + timelock_queue: timelock_queue.into(), + block_height_seen: HashMap::new(), + db: db.into(), + clock, + } + } + + fn get_highest_block(&mut self, agg: AggregateId, block: Option) -> u64 { + let highest = block + .into_iter() + .chain(self.block_height_seen.get(&agg).copied()) + .max() + .unwrap_or(0); + + self.block_height_seen.insert(agg, highest); + highest + } +} + +impl Handler for BatchRouter { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + // Messages without context go straight to disk + // This is probably direct datastore manipulation + let Some(ctx) = msg.ctx() else { + debug!("Message without context. Flushing straight to disk."); + self.db.try_send(InsertBatch::new(vec![msg]))?; + return Ok(()); + }; + + // Route to existing batch, or fall back to disk + match self.batches.get(&ctx.seq()) { + Some(batch) => { + debug!("Forwarding to batch actor for seq={}", ctx.seq()); + batch.try_send(msg)?; + } + // This must mean that this insert is late + None => { + debug!( + "No batch available for seq={} assuming this is late. Flushing to disk.", + ctx.seq() + ); + self.db.try_send(InsertBatch::new(vec![msg]))?; + } + } + Ok(()) + }) + } +} + +impl Handler> for BatchRouter { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + let ec = msg.get_ctx(); + trap(EType::IO, &PanicDispatcher::new(), || { + let prev_seq = ec.seq() - 1; + if self.batches.contains_key(&prev_seq) { + let prev_agg = self + .aggregates + .get(&prev_seq) + .context("invariant: prev_agg MUST exist if batches has a batch")?; + + debug!( + "Preparing timelock to clear batch for seq={}, agg={}", + prev_seq, prev_agg + ); + let delay = self.config.get_delay(prev_agg); + + let now = Duration::from_micros(self.clock.now_micros()); + + self.timelock_queue + .try_send(StartTimelock::new(prev_seq, now, delay))?; + } + + debug!("Creating batch for {}", ec.seq()); + let agg_id = ec.aggregate_id(); + let highest_block = self.get_highest_block(agg_id, ec.block()); + let batch = Batch::spawn( + self.db.clone(), + vec![ + Insert::new_with_context( + &StoreKeys::aggregate_seq(agg_id), + encode_u64(ec.seq()), + ec.clone(), + ), + Insert::new_with_context( + &StoreKeys::aggregate_block(agg_id), + encode_u64(highest_block), + ec.clone(), + ), + Insert::new_with_context( + &StoreKeys::aggregate_ts(agg_id), + encode_u128(ec.ts()), + ec.clone(), + ), + ], + ); + + self.batches.insert(ec.seq(), batch); + self.aggregates.insert(ec.seq(), ec.aggregate_id()); + + Ok(()) + }) + } +} + +impl Handler for BatchRouter { + type Result = (); + fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + debug!("Flushing sequence... {}", msg.seq()); + if let Some(batch) = self.batches.get(&msg.seq()) { + batch.try_send(Flush)?; + self.batches.remove(&msg.seq()); + self.aggregates.remove(&msg.seq()); + } + Ok(()) + }) + } +} + +impl Handler for BatchRouter { + type Result = (); + fn handle(&mut self, msg: UpdateDestination, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + self.db = msg.0; + Ok(()) + }) + } +} + +/// Encode the same as bincode without using a result +fn encode_u64(value: u64) -> Vec { + value.to_le_bytes().to_vec() +} + +/// Encode the same as bincode without using a result +fn encode_u128(value: u128) -> Vec { + value.to_le_bytes().to_vec() +} diff --git a/crates/events/src/snapshot_buffer/mod.rs b/crates/events/src/snapshot_buffer/mod.rs new file mode 100644 index 0000000000..84a4905094 --- /dev/null +++ b/crates/events/src/snapshot_buffer/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +mod aggregate_config; +mod batch; +mod batch_router; +mod snapshot_buffer; +mod timelock_queue; + +pub use aggregate_config::*; +pub use snapshot_buffer::*; +pub use timelock_queue::Tick; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs new file mode 100644 index 0000000000..bf73313580 --- /dev/null +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +use super::{ + batch_router::{BatchRouter, FlushSeq}, + timelock_queue::{Clock, StartTimelock, SystemClock, Tick, TimelockQueue}, + AggregateConfig, +}; +use crate::{trap, EType, EnclaveEvent, Insert, InsertBatch, PanicDispatcher}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use anyhow::Result; +use e3_utils::MAILBOX_LIMIT; +use std::sync::Arc; +use tracing::{info, trace}; + +#[derive(Message)] +#[rtype(result = "()")] +struct SetDependencies { + router: Addr, + timelock: Addr, +} + +impl SetDependencies { + pub fn new(router: Addr, timelock: Addr) -> Self { + Self { + router: router.into(), + timelock, + } + } +} + +#[derive(Message)] +#[rtype(result = "()")] +pub struct Start; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct UpdateDestination(pub Recipient); +impl UpdateDestination { + pub fn new(base: impl Into>) -> Self { + Self(base.into()) + } +} + +pub struct SnapshotBuffer { + router: Option>, + timelock: Option>, + tickable: Option>, +} + +impl SnapshotBuffer { + pub fn new() -> Self { + SnapshotBuffer { + router: None, + timelock: None, + tickable: None, + } + } + + pub fn spawn( + config: &AggregateConfig, + store: impl Into>, + ) -> Result> { + info!("spawning SnapshotBuffer..."); + let (addr, _) = Self::with_clock(config, store, Arc::new(SystemClock), Some(1))?; + Ok(addr) + } + + pub fn with_clock( + config: &AggregateConfig, + store: impl Into>, + clock: Arc, + interval: Option, + ) -> Result<(Addr, Addr)> { + let addr = Self::new().start(); + let store = store.into(); + let router = + BatchRouter::with_clock(config, addr.clone(), store.clone(), clock.clone()).start(); + let timelock = TimelockQueue::with_clock(addr.clone(), clock, interval).start(); + addr.try_send(SetDependencies::new(router, timelock.clone()))?; + Ok((addr, timelock)) + } +} + +impl Actor for SnapshotBuffer { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref timelock) = self.timelock { + timelock.try_send(msg)?; + } + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: SetDependencies, _: &mut Self::Context) -> Self::Result { + let SetDependencies { timelock, router } = msg; + self.timelock = Some(timelock.clone().into()); + self.tickable = Some(timelock.into()); + self.router = Some(router); + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + trace!("Forwarding Insert message to batch router..."); + router.try_send(msg)?; + }; + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: Tick, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref tickable) = self.tickable { + tickable.try_send(msg)?; + } + Ok(()) + }) + } +} + +impl Handler for SnapshotBuffer { + type Result = (); + fn handle(&mut self, msg: UpdateDestination, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + if let Some(ref router) = self.router { + router.try_send(msg)?; + } + Ok(()) + }) + } +} + +#[cfg(test)] +mod mock_store { + use std::mem::replace; + + use crate::InsertBatch; + use actix::{Actor, Handler, Message}; + + #[derive(Message)] + #[rtype(result = "Vec")] + pub struct GetEvts; + + #[derive(Default)] + pub struct MockStore { + evts: Vec, + } + + impl Actor for MockStore { + type Context = actix::Context; + } + + impl Handler for MockStore { + type Result = (); + + fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { + self.evts.push(msg); + } + } + + impl Handler for MockStore { + type Result = Vec; + fn handle(&mut self, _: GetEvts, _: &mut Self::Context) -> Self::Result { + replace(&mut self.evts, Vec::new()) + } + } +} + +#[cfg(test)] +mod tests { + use super::super::timelock_queue::mock_clock::MockClock; + use super::mock_store::GetEvts; + use super::{mock_store, SnapshotBuffer}; + use crate::snapshot_buffer::timelock_queue::Tick; + use crate::{ + AggregateConfig, AggregateId, E3id, EnclaveEvent, EventContext, EventContextAccessors, + EventContextSeq, EventId, EventSource, Insert, InsertBatch, Sequenced, SyncEnded, + TestEvent, + }; + use actix::Actor; + use anyhow::Result; + use e3_test_helpers::with_tracing; + use std::collections::HashMap; + use std::sync::Arc; + use std::time::Duration; + use tracing::info; + + fn create_ec(ag: usize, seq: u64) -> EventContext { + EventContext::new_origin( + EventId::hash(1), + 1000, + AggregateId::new(ag), + None, + EventSource::Local, + ) + .sequence(seq) + } + + fn create_event(ec: &EventContext) -> EnclaveEvent { + EnclaveEvent::::from_data_ec( + TestEvent::new("hello", ec.seq()) + .with_e3_id(E3id::new("1", *ec.aggregate_id() as u64)) + .into(), + ec.clone(), + ) + } + + #[actix::test] + async fn test_snapshot_buffer() -> Result<()> { + let _guard = with_tracing("debug"); + let mut delays = HashMap::new(); + delays.insert(AggregateId::new(0), Duration::from_micros(0)); + delays.insert(AggregateId::new(23), Duration::from_micros(30)); + delays.insert(AggregateId::new(1), Duration::from_micros(60)); + + let config = &AggregateConfig::new(delays); + let store = mock_store::MockStore::default().start(); + + let clock = Arc::new(MockClock::new(1000)); + let (buffer, timelock) = + SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; + + buffer + .send(EnclaveEvent::from_data_ec( + SyncEnded::new().into(), + create_ec(0, 9), + )) + .await?; + + info!("TimelockQueue should be empty"); + timelock.send(Tick).await?; + + let ec = create_ec(23, 10); + let enclave_10 = create_event(&ec); + + let mut inserts_10 = vec![]; + inserts_10.push(Insert::new_with_context("one", b"one".to_vec(), ec.clone())); + inserts_10.push(Insert::new_with_context("two", b"two".to_vec(), ec.clone())); + + let ec = create_ec(1, 11); + let enclave_11 = create_event(&ec); + + let mut inserts_11 = vec![]; + inserts_11.push(Insert::new_with_context("one", b"one".to_vec(), ec.clone())); + inserts_11.push(Insert::new_with_context("two", b"two".to_vec(), ec.clone())); + + let ec = create_ec(0, 12); + let enclave_12 = create_event(&ec); + + // send event 10 + buffer.send(enclave_10).await?; + + info!("TimelockQueue should hold all seq=9 inserts"); + timelock.send(Tick).await?; + + // send the first insert for seq 10 + buffer.send(inserts_10[0].clone()).await?; + + // send event 11 + info!("Sending event seq=11 this should start the timelock for all the seq=10 inserts"); + buffer.send(enclave_11).await?; + + // send a late insert for 10 + buffer.send(inserts_10[1].clone()).await?; + + // send the other inserts for 11 + buffer.send(inserts_11[0].clone()).await?; + buffer.send(inserts_11[1].clone()).await?; + + // send event 12 + info!("Sending event seq=12 this should start the timelock for all the seq=11 inserts"); + buffer.send(enclave_12).await?; + + // Nothing happens as there has not been enough delay + info!("Clock=1020 : Checking for events but there should be nothing that has flushed..."); + clock.set(Duration::from_micros(1020)); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + // assert_eq!(0, batches.len()); + assert_eq!(1, batches.len()); + let InsertBatch(inserts) = batches.first().unwrap(); + assert_eq!(3, inserts.len()); // Have sequence,block and ts written as inserts + + // Time is up so lets flush aggregate 23 (but not aggregate 1) + info!("Clock=1030 : Checking for events Tick should flush batch 10..."); + clock.set(Duration::from_micros(1030)); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(1, batches.len()); + let InsertBatch(inserts) = batches.first().unwrap(); + assert_eq!(5, inserts.len()); // Have 5 inserts as sequence,block and ts get written + + // Not ready yet + info!("Clock=1050 : Not ready yet..."); + clock.set(Duration::from_micros(1050)); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(0, batches.len()); + + // Time is up so lets flush aggregate 1 + info!("Clock=1060 : should have all aggregate 1 changes in batch 11..."); + clock.set(Duration::from_micros(1060)); + timelock.send(Tick).await?; + let batches = store.send(GetEvts).await?; + assert_eq!(1, batches.len()); + let InsertBatch(inserts) = batches.first().unwrap(); + assert_eq!(5, inserts.len()); // Have 5 inserts as sequence,block and ts get written + + Ok(()) + } +} diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs new file mode 100644 index 0000000000..0b04cc577a --- /dev/null +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. +use crate::{trap, EType, PanicDispatcher}; +use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient}; +use e3_utils::MAILBOX_LIMIT; +use std::{ + cmp::{Ordering, Reverse}, + collections::BinaryHeap, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tracing::debug; + +use super::batch_router::FlushSeq; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct StartTimelock { + seq: u64, + now: Duration, + delay: Duration, +} + +impl StartTimelock { + pub fn new(seq: u64, now: Duration, delay: Duration) -> Self { + Self { seq, now, delay } + } + pub fn new_micros(seq: u64, now: u64, delay: u64) -> Self { + Self::new( + seq, + Duration::from_micros(now), + Duration::from_micros(delay), + ) + } +} + +pub trait Clock: Send + Sync + 'static { + fn now_micros(&self) -> u64; +} + +// Production implementation +#[derive(Clone, Default)] +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_micros(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_micros() as u64 + } +} + +#[derive(Message)] +#[rtype(result = "()")] +pub struct Tick; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct Timelock { + expiry: Duration, + seq: u64, +} + +impl Timelock { + pub fn new(expiry: Duration, seq: u64) -> Self { + Self { expiry, seq } + } +} + +impl Ord for Timelock { + fn cmp(&self, other: &Self) -> Ordering { + self.expiry.cmp(&other.expiry) + } +} + +impl PartialOrd for Timelock { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +pub struct TimelockQueue { + timelocks: BinaryHeap>, + batch_router: Recipient, + clock: Arc, + interval: Option, // Seconds +} + +impl TimelockQueue { + pub fn new(batch_router: impl Into>) -> Self { + Self::with_clock(batch_router, Arc::new(SystemClock), Some(1)) + } + + pub fn spawn(batch_router: impl Into>) -> Addr { + Self::new(batch_router).start() + } + + pub fn with_clock( + batch_router: impl Into>, + clock: Arc, + interval: Option, + ) -> Self { + Self { + batch_router: batch_router.into(), + timelocks: BinaryHeap::new(), + clock, + interval, + } + } + + fn next_timelock_lt(&mut self, now: Duration) -> bool { + if let Some(peek) = self.timelocks.peek() { + peek.0.expiry <= now + } else { + false + } + } +} + +impl Actor for TimelockQueue { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + let Some(interval) = self.interval else { + debug!("TimelockQueue in manual mode - will tick when requested."); + return; + }; + + // Send Tick to self every second + debug!("TimelockQueue is ticking every {}s", interval); + ctx.run_interval(Duration::from_secs(interval), |_, ctx| { + ctx.address().do_send(Tick); + }); + } +} + +impl Handler for TimelockQueue { + type Result = (); + fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { + debug!("Start timelock: {:?}", msg.delay); + let expiry = msg.now + msg.delay; + self.timelocks.push(Reverse(Timelock::new(expiry, msg.seq))); + } +} + +impl Handler for TimelockQueue { + type Result = (); + fn handle(&mut self, _: Tick, _: &mut Self::Context) -> Self::Result { + trap(EType::IO, &PanicDispatcher::new(), || { + let now_time = Duration::from_micros(self.clock.now_micros()); + debug!( + "Running timelock tick. waiting times: {:?}.", + self.timelocks + .iter() + .map(|t| t.0.expiry.saturating_sub(now_time)) + .collect::>(), + ); + + while self.timelocks.len() > 0 && self.next_timelock_lt(now_time) { + if let Some(tl) = self.timelocks.pop() { + let seq = tl.0.seq; + debug!("Flushing seq {}", seq); + self.batch_router.try_send(FlushSeq(seq))?; + } + } + Ok(()) + }) + } +} + +#[cfg(test)] +pub mod mock_clock { + + use std::{ + sync::{ + atomic::{AtomicU64, Ordering as AtomicOrdering}, + Arc, + }, + time::Duration, + }; + + use super::Clock; + + #[derive(Clone)] + pub struct MockClock { + current_time: Arc, + } + + impl MockClock { + pub fn new(initial_time: u64) -> Self { + Self { + current_time: Arc::new(AtomicU64::new(initial_time)), + } + } + + pub fn set(&self, time: Duration) { + self.current_time + .store(time.as_micros() as u64, AtomicOrdering::SeqCst); + } + + pub fn advance(&self, micros: Duration) { + self.current_time + .fetch_add(micros.as_micros() as u64, AtomicOrdering::SeqCst); + } + } + + impl Clock for MockClock { + fn now_micros(&self) -> u64 { + self.current_time.load(AtomicOrdering::SeqCst) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use actix::prelude::*; + use mock_clock::MockClock; + use std::sync::{Arc, Mutex}; + use tokio::time::sleep; + + // ==================== Mock Router ==================== + + struct MockBatchRouter { + received_seqs: Arc>>, + } + + impl MockBatchRouter { + fn new(received_seqs: Arc>>) -> Self { + Self { received_seqs } + } + } + + impl Actor for MockBatchRouter { + type Context = Context; + } + + impl Handler for MockBatchRouter { + type Result = (); + + fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { + self.received_seqs.lock().unwrap().push(msg.0); + } + } + + // ==================== Tests ==================== + + #[actix::test] + async fn test_tick_with_mock_clock_no_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); // Start at t=1000 + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); + + // Add timelock expiring at t=2000 + queue + .send(StartTimelock::new_micros(42, 1000, 1000)) + .await + .unwrap(); + + // Tick at t=1000 - nothing should expire + queue.send(Tick).await.unwrap(); + sleep(Duration::from_millis(10)).await; + + assert!(received_seqs.lock().unwrap().is_empty()); + } + + #[actix::test] + async fn test_tick_with_mock_clock_after_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); + + // Add timelock expiring at t=2000 + queue + .send(StartTimelock::new_micros(42, 1000, 1000)) + .await + .unwrap(); + + // Advance clock past expiry + clock.set(Duration::from_millis(2500)); + + // Now tick should flush + queue.send(Tick).await.unwrap(); + sleep(Duration::from_millis(10)).await; + + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 1); + assert_eq!(seqs[0], 42); + } + + #[actix::test] + async fn test_tick_exact_expiry_boundary() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); + + // Add timelock expiring at exactly t=2000 + queue + .send(StartTimelock::new_micros(42, 1000, 1000)) + .await + .unwrap(); + + // Set clock to exact expiry time + clock.set(Duration::from_millis(2000)); + + queue.send(Tick).await.unwrap(); + sleep(Duration::from_millis(10)).await; + + // Should flush at exact expiry (<=) + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 1); + assert_eq!(seqs[0], 42); + } + + #[actix::test] + async fn test_tick_one_microsecond_before_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); + + // Add timelock expiring at t=2000 + queue + .send(StartTimelock::new_micros(42, 1000, 1000)) + .await + .unwrap(); + + // Set clock to 1 microsecond before expiry + clock.set(Duration::from_micros(1999)); + + queue.send(Tick).await.unwrap(); + sleep(Duration::from_micros(10)).await; + + // Should NOT flush + assert!(received_seqs.lock().unwrap().is_empty()); + } + + #[actix::test] + async fn test_multiple_timelocks_partial_expiry() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(0); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); + + // Add timelocks with different expiries + queue + .send(StartTimelock::new_micros(1, 0, 1000)) + .await + .unwrap(); // expires at 1000 + queue + .send(StartTimelock::new_micros(2, 0, 2000)) + .await + .unwrap(); // expires at 2000 + queue + .send(StartTimelock::new_micros(3, 0, 3000)) + .await + .unwrap(); // expires at 3000 + + // Advance to 1500 - only first should expire + clock.set(Duration::from_micros(1500)); + queue.send(Tick).await.unwrap(); + sleep(Duration::from_micros(10)).await; + + { + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 1); + assert_eq!(seqs[0], 1); + } + + // Advance to 2500 - second should expire + clock.set(Duration::from_micros(2500)); + queue.send(Tick).await.unwrap(); + sleep(Duration::from_micros(10)).await; + + { + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 2); + assert_eq!(seqs[1], 2); + } + + // Advance to 5000 - third should expire + clock.set(Duration::from_micros(5000)); + queue.send(Tick).await.unwrap(); + sleep(Duration::from_micros(10)).await; + + { + let seqs = received_seqs.lock().unwrap(); + assert_eq!(seqs.len(), 3); + assert_eq!(seqs[2], 3); + } + } + + #[actix::test] + async fn test_clock_advance_helper() { + let received_seqs = Arc::new(Mutex::new(Vec::new())); + let mock_router = MockBatchRouter::new(received_seqs.clone()).start(); + + let clock = MockClock::new(1000); + let queue = + TimelockQueue::with_clock(mock_router.recipient(), Arc::new(clock.clone()), None) + .start(); + + queue + .send(StartTimelock::new_micros(42, 1000, 500)) + .await + .unwrap(); + + // Use advance instead of set + clock.advance(Duration::from_millis(600)); // Now at 1600, expiry is 1500 + + queue.send(Tick).await.unwrap(); + sleep(Duration::from_millis(10)).await; + + assert_eq!(received_seqs.lock().unwrap().len(), 1); + } +} diff --git a/crates/config/src/store_keys.rs b/crates/events/src/store_keys.rs similarity index 82% rename from crates/config/src/store_keys.rs rename to crates/events/src/store_keys.rs index 6b0eae21ae..d0b89d22c6 100644 --- a/crates/config/src/store_keys.rs +++ b/crates/events/src/store_keys.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_events::E3id; +use crate::{AggregateId, E3id}; pub struct StoreKeys; @@ -76,4 +76,16 @@ impl StoreKeys { pub fn ciphernode_selector() -> String { String::from("//ciphernode_selector") } + + pub fn aggregate_seq(aggregate_id: AggregateId) -> String { + format!("//aggregate_seq/{}", aggregate_id) + } + + pub fn aggregate_block(aggregate_id: AggregateId) -> String { + format!("//aggregate_block/{}", aggregate_id) + } + + pub fn aggregate_ts(aggregate_id: AggregateId) -> String { + format!("//aggregate_ts/{}", aggregate_id) + } } diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs index bc13534344..55ff422853 100644 --- a/crates/events/src/sync.rs +++ b/crates/events/src/sync.rs @@ -4,26 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::{BTreeMap, HashSet}; - -use crate::EvmEvent; -use actix::Message; use serde::{Deserialize, Serialize}; -type Chainid = u64; -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[rtype(result = "()")] -pub enum SyncEvmEvent { - /// Signal that this reader has completed historical sync - HistoricalSyncComplete(ChainId), - /// An actual event from the blockchain - Event(EvmEvent), -} - -impl From for SyncEvmEvent { - fn from(event: EvmEvent) -> SyncEvmEvent { - SyncEvmEvent::Event(event) - } -} +use std::collections::{BTreeMap, HashSet}; type ChainId = u64; type DeployBlock = u64; @@ -55,6 +37,13 @@ impl EvmEventConfig { config: BTreeMap::new(), } } + + pub fn from_config(config: impl Into>) -> Self { + EvmEventConfig { + config: config.into(), + } + } + pub fn get(&self, chain_id: &ChainId) -> Option<&EvmEventConfigChain> { self.config.get(&chain_id) } @@ -66,4 +55,8 @@ impl EvmEventConfig { pub fn chains(&self) -> HashSet { self.config.keys().cloned().collect() } + + pub fn deploy_block(&self, chain_id: u64) -> Option { + self.get(&chain_id).map(|c| c.deploy_block()) + } } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 3feba8f434..5ac78dd3ed 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -11,7 +11,7 @@ use std::hash::Hash; use crate::{ event_context::{AggregateId, EventContext}, - EnclaveEvent, EventId, EventType, Sequenced, Unsequenced, + EnclaveEvent, EventId, EventSource, EventType, Sequenced, Unsequenced, }; /// Trait that must be implemented by events used with EventBus @@ -56,13 +56,16 @@ pub trait EventFactory { /// event ordering. /// /// This method should be used for events that originated from remote sources. + /// + /// The Option `caused_by` is for correlation when we send a remote request and receive a response. + /// Block should be provided when the event is from the blockchain fn event_from_remote_source( &self, data: impl Into, - // NOTE: `caused_by` makes sense here as we could be sending out requests and receiving - // responses that relate to the request caused_by: Option>, ts: u128, + block: Option, + source: EventSource, ) -> Result; } @@ -83,12 +86,42 @@ pub trait EventPublisher { /// to the event bus. /// /// This method should be used for events that have originated locally. - fn publish(&self, data: impl Into) -> Result<()>; + /// + /// The ctx parameter is to pass on the current context to the local event. + fn publish( + &self, + data: impl Into, + caused_by: impl Into>, + ) -> Result<()>; + /// This creates a context based on the given data. This should only be used when an event is + /// the origin event and does not originate remotely. This is also useful in tests. + fn publish_without_context(&self, data: impl Into) -> Result<()>; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering and publish it. /// /// This method should be used for events that originated from remote sources. - fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()>; + fn publish_from_remote( + &self, + data: impl Into, + remote_ts: u128, + block: Option, + source: EventSource, + ) -> Result<()>; + /// Create a new event from the given event data, apply the given remote HLC time to ensure correct + /// event ordering and publish it. + /// + /// This method should be used for events that originated from remote sources as a response to + /// a request we have sent + /// + /// The `caused_by` parameter is for correlation when we send a remote request and receive a response. + fn publish_from_remote_as_response( + &self, + data: impl Into, + remote_ts: u128, + caused_by: impl Into>, + block: Option, + source: EventSource, + ) -> Result<()>; /// Dispatch the given event without applying any HLC transformation. fn naked_dispatch(&self, event: E); } @@ -116,6 +149,8 @@ pub trait EventConstructorWithTimestamp: Event + Sized { data: Self::Data, caused_by: Option>, ts: u128, + block: Option, + source: EventSource, ) -> Self; } @@ -153,6 +188,12 @@ pub trait EventContextAccessors { fn ts(&self) -> u128; /// The aggregate id for this event fn aggregate_id(&self) -> AggregateId; + /// The highest block watermark we have seen + fn block(&self) -> Option; + /// The event source + fn source(&self) -> EventSource; + /// Apply a new source fluently + fn with_source(self, source: EventSource) -> Self; } pub trait EventContextSeq { @@ -168,6 +209,8 @@ pub trait WithAggregateId { /// An EventContextManager hold the current event context for use in event publishing and /// persistence management pub trait EventContextManager { - fn set_ctx(&mut self, value: &EventContext); + fn set_ctx(&mut self, value: C) + where + C: Into>; fn get_ctx(&self) -> Option>; } diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index b3e77c6bda..f41e5e035f 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -35,5 +35,6 @@ zeroize = { workspace = true } e3-evm = { workspace = true } e3-entrypoint = { workspace = true } e3-ciphernode-builder = { workspace = true } +e3-test-helpers = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } tracing-subscriber = { workspace = true } diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 456b63fd96..d56a9e6c4f 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -19,11 +19,11 @@ use alloy::{ }; use anyhow::Result; use e3_events::{ - prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, - EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, PublicKeyAggregated, - Seed, Shutdown, TicketGenerated, TicketId, + prelude::*, run_once, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, + EffectsEnabled, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, + PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; -use e3_utils::NotifySync; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use tracing::{error, info, trace}; sol!( @@ -231,7 +231,7 @@ pub struct CiphernodeRegistrySolWriter

{ } impl CiphernodeRegistrySolWriter

{ - pub async fn new( + pub fn new( bus: &BusHandle, provider: EthProvider

, contract_address: Address, @@ -243,42 +243,52 @@ impl CiphernodeRegistrySolWriter }) } - pub async fn attach( + pub fn attach( bus: &BusHandle, provider: EthProvider

, contract_address: Address, is_aggregator: bool, - ) -> Result>> { - let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address) - .await? - .start(); - - if is_aggregator { - bus.subscribe_all( - &[ - EventType::PublicKeyAggregated, - EventType::CommitteeFinalizeRequested, - ], - addr.clone().into(), - ) - } + ) { + let runner = run_once::({ + let bus = bus.clone(); + move |_| { + let addr = + CiphernodeRegistrySolWriter::new(&bus, provider, contract_address)?.start(); + + if is_aggregator { + bus.subscribe_all( + &[ + EventType::PublicKeyAggregated, + EventType::CommitteeFinalizeRequested, + ], + addr.clone().into(), + ) + } + + bus.subscribe_all( + &[ + // Subscribe to TicketGenerated for ticket submission + EventType::TicketGenerated, + // Stop gracefully on shutdown + EventType::Shutdown, + ], + addr.clone().into(), + ); - bus.subscribe_all( - &[ - // Subscribe to TicketGenerated for ticket submission - EventType::TicketGenerated, - // Stop gracefully on shutdown - EventType::Shutdown, - ], - addr.clone().into(), - ); - - Ok(addr) + Ok(()) + } + }); + + bus.subscribe(EventType::EffectsEnabled, runner.recipient()); } } impl Actor for CiphernodeRegistrySolWriter

{ type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler @@ -539,18 +549,14 @@ impl CiphernodeRegistrySol { CiphernodeRegistrySolReader::setup(processor) } - pub async fn attach_writer

( + pub fn attach_writer

( bus: &BusHandle, provider: EthProvider

, contract_address: Address, is_aggregator: bool, - ) -> Result>> - where + ) where P: Provider + WalletProvider + Clone + 'static, { - let writer = - CiphernodeRegistrySolWriter::attach(bus, provider, contract_address, is_aggregator) - .await?; - Ok(writer) + CiphernodeRegistrySolWriter::attach(bus, provider, contract_address, is_aggregator); } } diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs deleted file mode 100644 index 69df39b45c..0000000000 --- a/crates/evm/src/enclave_sol.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::{ - enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - events::EvmEventProcessor, evm_parser::EvmParser, helpers::EthProvider, -}; -use actix::Addr; -use alloy::providers::{Provider, WalletProvider}; -use alloy_primitives::Address; -use anyhow::Result; -use e3_events::BusHandle; - -pub struct EnclaveSol; - -impl EnclaveSol { - pub async fn attach( - processor: &EvmEventProcessor, - bus: &BusHandle, - write_provider: EthProvider, - contract_address: Address, - ) -> Result> - where - W: Provider + WalletProvider + Clone + 'static, - { - let addr = EnclaveSolReader::setup(processor); - - EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; - - Ok(addr) - } -} diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 68a1f6adc9..a95416dce0 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -7,7 +7,6 @@ use crate::helpers::EthProvider; use crate::send_tx_with_retry; use actix::prelude::*; -use actix::Addr; use alloy::{ primitives::Address, providers::{Provider, WalletProvider}, @@ -18,14 +17,15 @@ use alloy::{ rpc::types::TransactionReceipt, }; use anyhow::Result; -use e3_events::prelude::*; use e3_events::BusHandle; -use e3_events::EnclaveEvent; use e3_events::EnclaveEventData; use e3_events::EventType; use e3_events::Shutdown; +use e3_events::{prelude::*, EffectsEnabled}; +use e3_events::{run_once, EnclaveEvent}; use e3_events::{E3id, EType, PlaintextAggregated}; use e3_utils::NotifySync; +use e3_utils::MAILBOX_LIMIT; use tracing::info; sol!( @@ -54,22 +54,28 @@ impl EnclaveSolWriter

{ }) } - pub async fn attach( - bus: &BusHandle, - provider: EthProvider

, - contract_address: Address, - ) -> Result>> { - let addr = EnclaveSolWriter::new(bus, provider, contract_address)?.start(); - bus.subscribe_all( - &[EventType::PlaintextAggregated, EventType::Shutdown], - addr.clone().into(), - ); - Ok(addr) + pub fn attach(bus: &BusHandle, provider: EthProvider

, contract_address: Address) { + let addr = run_once::({ + let bus = bus.clone(); + move |_| { + let addr = EnclaveSolWriter::new(&bus, provider, contract_address)?.start(); + bus.subscribe_all( + &[EventType::PlaintextAggregated, EventType::Shutdown], + addr.clone().into(), + ); + Ok(()) + } + }); + + bus.subscribe(EventType::EffectsEnabled, addr.recipient()); } } impl Actor for EnclaveSolWriter

{ type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EnclaveSolWriter

{ diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs index 4b59deaaeb..b6f2bcaff1 100644 --- a/crates/evm/src/events.rs +++ b/crates/evm/src/events.rs @@ -6,7 +6,11 @@ use actix::{Message, Recipient}; use alloy::rpc::types::Log; -use e3_events::{CorrelationId, EvmEvent}; +use anyhow::Result; +use e3_events::{ + BusHandle, CorrelationId, EnclaveEvent, EnclaveEventData, EventFactory, EventSource, + Unsequenced, +}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -31,6 +35,57 @@ impl HistoricalSyncComplete { } } +/// This is a processed EvmEvent specifically typed for the Sync actor +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct EvmEvent { + data: EnclaveEventData, + block: u64, + chain_id: u64, + ts: u128, + id: CorrelationId, +} + +impl EvmEvent { + pub fn new( + id: CorrelationId, + data: EnclaveEventData, + block: u64, + ts: u128, + chain_id: u64, + ) -> Self { + Self { + id, + data, + block, + ts, + chain_id, + } + } + + pub fn split(self) -> (EnclaveEventData, u128, u64) { + (self.data, self.ts, self.block) + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } + + pub fn chain_id(&self) -> u64 { + self.chain_id + } + + pub fn ts(&self) -> u128 { + self.ts + } + + pub fn into_enclave_event(self, bus: &BusHandle) -> Result> { + let data = self.data; + let ts = self.ts; + bus.event_from_remote_source(data, None, ts, Some(self.block), EventSource::Evm) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub enum EnclaveEvmEvent { diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs index fa8eadf72c..c0ba917892 100644 --- a/crates/evm/src/evm_chain_gateway.rs +++ b/crates/evm/src/evm_chain_gateway.rs @@ -4,19 +4,21 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use std::mem::take; + use crate::events::EnclaveEvmEvent; use crate::HistoricalSyncComplete; use actix::{Actor, Handler}; use actix::{Addr, Recipient}; use anyhow::Result; use anyhow::{bail, Context}; +use e3_events::EType; use e3_events::{ - trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, SyncEnd, - SyncEvmEvent, SyncStart, + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, + HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnded, Unsequenced, }; -use e3_events::{EType, EvmEvent}; use e3_events::{Event, EventPublisher}; -use tracing::info; +use e3_utils::MAILBOX_LIMIT; /// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. /// It coordinates event flow between these components. @@ -25,16 +27,28 @@ pub struct EvmChainGateway { status: SyncStatus, } +#[derive(Clone, Default, Debug)] +struct ForwardToSyncActorData { + pub sender: Option>, + pub buffer: Vec>, +} + +impl ForwardToSyncActorData { + pub fn add_event(&mut self, event: EnclaveEvent) { + self.buffer.push(event); + } +} + /// This state machine coordinates the function of the EvmChainGateway #[derive(Clone, Debug)] enum SyncStatus { /// Intial State - Init(Vec), // Include a buffer to hold events that arrive too early - /// After SyncStart we forward all events to SyncActor - ForwardToSyncActor(Option>), + Init(Vec>), // Include a buffer to hold events that arrive too early + /// After HistoricalEvmSyncStart we forward all events to SyncActor + ForwardToSyncActor(ForwardToSyncActorData), /// Once the chain has completed historical sync then we buffer all "live" events until sync is /// complete - BufferUntilLive(Vec), + BufferUntilLive(Vec>), /// Forward all events directly to the bus Live, } @@ -48,8 +62,8 @@ impl Default for SyncStatus { impl SyncStatus { pub fn forward_to_sync_actor( &mut self, - sender: Recipient, - ) -> Result> { + sender: Recipient, + ) -> Result>> { let Self::Init(buffer) = self else { bail!( "Cannot change state to ForwardToSyncActor when state is {:?}", @@ -58,23 +72,27 @@ impl SyncStatus { }; let buffer = std::mem::take(buffer); - *self = SyncStatus::ForwardToSyncActor(Some(sender)); + *self = SyncStatus::ForwardToSyncActor(ForwardToSyncActorData { + sender: Some(sender), + buffer: Vec::new(), + }); Ok(buffer) } - pub fn buffer_until_live(&mut self) -> Result> { + pub fn buffer_until_live(&mut self) -> Result { let Self::ForwardToSyncActor(sender) = self else { bail!( "Cannot change state to BufferUntilLive when state is {:?}", self ); }; - let sender = std::mem::take(sender).context("Cannot call buffer_until_live twice")?; + + let state_data = take(sender); *self = SyncStatus::BufferUntilLive(vec![]); - Ok(sender) + Ok(state_data) } - pub fn live(&mut self) -> Result> { + pub fn live(&mut self) -> Result>> { let Self::BufferUntilLive(buffer) = self else { bail!("Cannot change state to Live when state is {:?}", self); }; @@ -95,17 +113,18 @@ impl EvmChainGateway { pub fn setup(bus: &BusHandle) -> Addr { let addr = Self::new(bus).start(); bus.subscribe_all( - &[EventType::SyncStart, EventType::SyncEnd], + &[EventType::HistoricalEvmSyncStart, EventType::SyncEnded], addr.clone().recipient(), ); addr } - fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { - info!("Processing SyncStart message"); - // Received a SyncStart event from the event bus. Get the sender within that event and forward + fn handle_sync_start(&mut self, msg: HistoricalEvmSyncStart) -> Result<()> { + // Received a HistoricalEvmSyncStart event from the event bus. Get the sender within that event and forward // all events to that actor - let sender = msg.sender.context("No sender on SyncStart Message")?; + let sender = msg + .sender + .context("No sender on HistoricalEvmSyncStart Message")?; let mut buffer = self.status.forward_to_sync_actor(sender)?; // Drain any events that were buffered early for evt in buffer.drain(..) { @@ -114,8 +133,7 @@ impl EvmChainGateway { Ok(()) } - fn handle_sync_end(&mut self, _: SyncEnd) -> Result<()> { - info!("Processing SyncEnd message"); + fn handle_sync_ended(&mut self, _: SyncEnded) -> Result<()> { let buffer = self.status.live()?; for evt in buffer { self.publish_evm_event(evt)?; @@ -123,9 +141,8 @@ impl EvmChainGateway { Ok(()) } - fn publish_evm_event(&mut self, msg: EvmEvent) -> Result<()> { - let (data, ts, _) = msg.split(); - self.bus.publish_from_remote(data, ts)?; + fn publish_evm_event(&mut self, msg: EnclaveEvent) -> Result<()> { + self.bus.naked_dispatch(msg); Ok(()) } @@ -136,7 +153,7 @@ impl EvmChainGateway { Ok(()) } EnclaveEvmEvent::Event(event) => { - self.process_evm_event(event)?; + self.process_evm_event(event.into_enclave_event(&self.bus)?)?; Ok(()) } _ => panic!("EvmChainGateway is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), @@ -144,35 +161,21 @@ impl EvmChainGateway { } fn forward_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { - info!( - "handling historical sync complete for chain_id({})", - event.chain_id - ); - let sender = self.status.buffer_until_live()?; - info!("Sending historical sync complete event to sender."); - sender.try_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id))?; + let state = self.status.buffer_until_live()?; + let sender = state + .sender + .context("ForwardToSyncActor state must hold a sender")?; + let event = HistoricalEvmEventsReceived::new(state.buffer, event.chain_id); + sender.try_send(event)?; Ok(()) } - fn process_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + fn process_evm_event(&mut self, msg: EnclaveEvent) -> Result<()> { match &mut self.status { - SyncStatus::BufferUntilLive(buffer) => { - info!("saving evm event({}) to pre-live buffer", msg.get_id()); - buffer.push(msg) - } - SyncStatus::ForwardToSyncActor(Some(sync_actor)) => { - info!("forwarding evm event({}) to SyncActor", msg.get_id()); - sync_actor.do_send(msg.into()); - } - SyncStatus::Live => { - info!("publishing evm event({})", msg.get_id()); - self.publish_evm_event(msg)? - } - SyncStatus::Init(buffer) => { - info!("saving evm event({}) to pre-sync buffer", msg.get_id()); - buffer.push(msg) - } - _ => (), + SyncStatus::Init(buffer) => buffer.push(msg), + SyncStatus::BufferUntilLive(buffer) => buffer.push(msg), + SyncStatus::ForwardToSyncActor(state) => state.add_event(msg), + SyncStatus::Live => self.publish_evm_event(msg)?, }; Ok(()) } @@ -180,15 +183,18 @@ impl EvmChainGateway { impl Actor for EvmChainGateway { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EvmChainGateway { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.clone(), || { + trap(EType::Evm, &self.bus.with_ec(msg.get_ctx()), || { match msg.into_data() { - EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, - EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e)?, + EnclaveEventData::HistoricalEvmSyncStart(e) => self.handle_sync_start(e)?, + EnclaveEventData::SyncEnded(e) => self.handle_sync_ended(e)?, _ => (), } Ok(()) @@ -198,38 +204,50 @@ impl Handler for EvmChainGateway { impl Handler for EvmChainGateway { type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) } } #[cfg(test)] mod tests { + use crate::EvmEvent; + use super::*; use e3_ciphernode_builder::EventSystem; - use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TestEvent}; + use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TakeEvents, TestEvent}; use tokio::sync::mpsc; + use tracing_subscriber::{fmt, EnvFilter}; struct SyncEventCollector { - tx: mpsc::UnboundedSender, + tx: mpsc::UnboundedSender, } impl Actor for SyncEventCollector { type Context = actix::Context; } - impl Handler for SyncEventCollector { + impl Handler for SyncEventCollector { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) { + fn handle(&mut self, msg: HistoricalEvmEventsReceived, _: &mut Self::Context) { let _ = self.tx.send(msg); } } #[actix::test] - async fn test_evm_chain_gateway() { + async fn test_evm_chain_gateway() -> Result<()> { + let _foo = tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new("info")) + .with_test_writer() + .finish(), + ); + let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle().unwrap(); + let bus: BusHandle = system.handle()?; + + let history_collector = bus.history(); let (tx, mut rx) = mpsc::unbounded_channel(); let collector = SyncEventCollector { tx }.start(); @@ -238,59 +256,91 @@ mod tests { let chain_id = 1u64; - // SyncStart: Init -> ForwardToSyncActor + // HistoricalEvmSyncStart: Init -> ForwardToSyncActor let mut evm_config = EvmEventConfig::new(); evm_config.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish(SyncStart::new(collector.clone(), evm_config)) + bus.publish_without_context(HistoricalEvmSyncStart::new(collector.clone(), evm_config)) .unwrap(); // Send EVM event while forwarding - should reach collector let evm_event = EvmEvent::new( CorrelationId::new(), - TestEvent { - msg: "test".to_string(), - entropy: 1, - } - .into(), + TestEvent::new("Before Complete", 1).into(), 100, 12345, chain_id, ); - // This will actually arrive earlier than SyncStart but aught to be buffered - addr.do_send(EnclaveEvmEvent::Event(evm_event)); - let received = rx.recv().await.unwrap(); - assert!(matches!(received, SyncEvmEvent::Event(_))); + // This will actually arrive earlier than HistoricalEvmSyncStart but aught to be buffered + addr.send(EnclaveEvmEvent::Event(evm_event)).await?; // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive - addr.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + addr.send(EnclaveEvmEvent::HistoricalSyncComplete( HistoricalSyncComplete::new(chain_id, None), - )); + )) + .await?; + // Normal Synchronizer will take this and wait for other events before flushing events to + // the bus here we simulate it let received = rx.recv().await.unwrap(); - assert!(matches!(received, SyncEvmEvent::HistoricalSyncComplete(_))); + for event in received.events { + bus.naked_dispatch(event); + } // Send EVM event while buffering - should be buffered (not received) let buffered_event = EvmEvent::new( CorrelationId::new(), - TestEvent { - msg: "buffered".to_string(), - entropy: 2, - } - .into(), + TestEvent::new("Before SyncEnded", 2).into(), + 101, + 12346, + chain_id, + ); + addr.send(EnclaveEvmEvent::Event(buffered_event)).await?; + + // The Synchronizer will publish the SyncEnded event when it has all the information it needs + // and has published everything to the bus + bus.publish_without_context(SyncEnded::new())?; + + let after_event = EvmEvent::new( + CorrelationId::new(), + TestEvent::new("After SyncEnded", 2).into(), 101, 12346, chain_id, ); - addr.do_send(EnclaveEvmEvent::Event(buffered_event)); - // SyncEnd: BufferUntilLive -> Live (publishes buffered events to bus) - bus.publish(SyncEnd::new()).unwrap(); + addr.send(EnclaveEvmEvent::Event(after_event)).await?; + + let full = history_collector.send(TakeEvents::new(5)).await?; - // Allow time for async message processing - tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let test_events: Vec = full + .iter() + .filter_map(|e| { + if let EnclaveEventData::TestEvent(TestEvent { msg, .. }) = e.get_data() { + Some(msg.to_string()) + } else { + None + } + }) + .collect(); - // Verify no more messages were sent to collector (buffered events go to bus, not collector) - assert!(rx.try_recv().is_err()); + assert_eq!( + test_events, + vec!["Before Complete", "Before SyncEnded", "After SyncEnded"] + ); + + let event_types: Vec = full.iter().map(|e| e.event_type()).collect(); + + assert_eq!( + event_types, + vec![ + "HistoricalEvmSyncStart", + "TestEvent", + "SyncEnded", + "TestEvent", + "TestEvent" + ] + ); + Ok(()) } } diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs index 301b383f59..2bc3aad605 100644 --- a/crates/evm/src/evm_hub.rs +++ b/crates/evm/src/evm_hub.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Addr, Handler}; +use e3_utils::MAILBOX_LIMIT; use crate::events::{EnclaveEvmEvent, EvmEventProcessor}; @@ -25,11 +26,14 @@ impl EvmHub { impl Actor for EvmHub { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EvmHub { type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { let EnclaveEvmEvent::Log { .. } = msg.clone() else { return; }; diff --git a/crates/evm/src/evm_parser.rs b/crates/evm/src/evm_parser.rs index 2710fd10a9..e4549fd8ff 100644 --- a/crates/evm/src/evm_parser.rs +++ b/crates/evm/src/evm_parser.rs @@ -5,12 +5,13 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler}; -use e3_events::{hlc::HlcTimestamp, EnclaveEventData, EvmEvent}; -use tracing::info; +use e3_events::{hlc::HlcTimestamp, EnclaveEventData}; +use e3_utils::MAILBOX_LIMIT; +use tracing::{debug, info}; use crate::{ events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, - ExtractorFn, + EvmEvent, ExtractorFn, }; pub struct EvmParser { @@ -20,6 +21,9 @@ pub struct EvmParser { impl Actor for EvmParser { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl EvmParser { @@ -41,7 +45,7 @@ impl Handler for EvmParser { id, timestamp, }) => { - info!("processing event({})", msg.get_id()); + debug!("processing event({})", msg.get_id()); let extractor = self.extractor; if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs index fd37247026..54fbab66f3 100644 --- a/crates/evm/src/evm_read_interface.rs +++ b/crates/evm/src/evm_read_interface.rs @@ -17,17 +17,18 @@ use alloy_primitives::Address; use anyhow::anyhow; use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscriber, EventType}; use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; +use e3_utils::MAILBOX_LIMIT; use futures_util::stream::StreamExt; use std::collections::{HashMap, HashSet}; use tokio::select; use tokio::sync::oneshot; -use tracing::{error, info, instrument, warn}; +use tracing::{debug, error, info, instrument, warn}; pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; pub struct EvmReadInterfaceParams

{ provider: EthProvider

, - processor: Recipient, + next: Recipient, bus: BusHandle, filters: Filters, } @@ -74,8 +75,8 @@ pub struct EvmReadInterface

{ shutdown_rx: Option>, /// The sender for the shutdown signal this is only used internally shutdown_tx: Option>, - /// Processor to forward events an actor - processor: EvmEventProcessor, + /// Processor to forward events + next: EvmEventProcessor, /// Event bus for error propagation only bus: BusHandle, /// Filters to configure when to seek from @@ -89,7 +90,7 @@ impl EvmReadInterface

{ provider: Some(params.provider), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), - processor: params.processor, + next: params.next, bus: params.bus, filters: params.filters, } @@ -97,13 +98,13 @@ impl EvmReadInterface

{ pub fn setup( provider: &EthProvider

, - next: &EvmEventProcessor, + next: impl Into, bus: &BusHandle, filters: Filters, ) -> Addr { let params = EvmReadInterfaceParams { provider: provider.clone(), - processor: next.clone(), + next: next.into(), bus: bus.clone(), filters, }; @@ -119,9 +120,11 @@ impl Actor for EvmReadInterface

{ type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + // let reader_addr = ctx.address(); let bus = self.bus.clone(); - let processor = self.processor.clone(); + let next = self.next.clone(); let filters = self.filters.clone(); let Some(provider) = self.provider.take() else { @@ -136,7 +139,7 @@ impl Actor for EvmReadInterface

{ }; ctx.spawn( - async move { stream_from_evm(provider, processor, shutdown, &bus, filters).await } + async move { stream_from_evm(provider, next, shutdown, &bus, filters).await } .into_actor(self), ); } @@ -148,7 +151,7 @@ impl Actor for EvmReadInterface

{ #[instrument(name = "evm_interface", skip_all)] async fn stream_from_evm( provider: EthProvider

, - processor: EvmEventProcessor, + next: EvmEventProcessor, mut shutdown: oneshot::Receiver<()>, bus: &BusHandle, filters: Filters, @@ -165,8 +168,8 @@ async fn stream_from_evm( let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); last_id = Some(evt.get_id()); - info!("Sending event({})", evt.get_id()); - processor.do_send(evt) + debug!("Sending event({})", evt.get_id()); + next.do_send(evt) } } Err(e) => { @@ -176,11 +179,11 @@ async fn stream_from_evm( } } let historical_sync_event = HistoricalSyncComplete::new(chain_id, last_id); - warn!( + info!( "Historical Sync Complete event({})", historical_sync_event.get_id() ); - processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + next.do_send(EnclaveEvmEvent::HistoricalSyncComplete( historical_sync_event, )); @@ -196,13 +199,13 @@ async fn stream_from_evm( match maybe_log { Some(log) => { let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; - processor.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp))) + next.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp))) } None => break, // Stream ended } } _ = &mut shutdown => { - info!("Received shutdown signal, stopping EVM stream"); + warn!("Received shutdown signal, stopping EVM stream"); match provider_ref.unsubscribe(id).await { Ok(_) => info!("Unsubscribed successfully from EVM event stream"), Err(err) => error!("Cannot unsubscribe from EVM event stream: {}", err), diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs index 01c963de5a..953dbce34e 100644 --- a/crates/evm/src/evm_router.rs +++ b/crates/evm/src/evm_router.rs @@ -7,6 +7,7 @@ use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; use actix::{Actor, Handler}; use alloy_primitives::Address; +use e3_utils::MAILBOX_LIMIT; use std::collections::HashMap; use tracing::{debug, error, info}; @@ -42,11 +43,14 @@ impl EvmRouter { impl Actor for EvmRouter { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for EvmRouter { type Result = (); - fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { match msg.clone() { // Take all log events and route them EnclaveEvmEvent::Log(EvmLog { log, .. }) => { diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs index 1faf833d21..075bb81977 100644 --- a/crates/evm/src/fix_historical_order.rs +++ b/crates/evm/src/fix_historical_order.rs @@ -8,7 +8,8 @@ use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; use actix::{Actor, Addr, Handler}; use bloom::{BloomFilter, ASMS}; use e3_events::CorrelationId; -use tracing::info; +use e3_utils::MAILBOX_LIMIT; +use tracing::{debug, info}; pub struct FixHistoricalOrder { dest: EvmEventProcessor, @@ -36,7 +37,7 @@ impl FixHistoricalOrder { })) = self.pending_sync_complete { if self.seen_ids.contains(id) { - info!("Forwarding historical send complete event"); + debug!("Forwarding historical send complete event"); self.dest .do_send(self.pending_sync_complete.take().unwrap()); } @@ -50,6 +51,9 @@ impl FixHistoricalOrder { impl Actor for FixHistoricalOrder { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for FixHistoricalOrder { @@ -57,13 +61,13 @@ impl Handler for FixHistoricalOrder { fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { let id = msg.get_id(); - info!("Receiving EnclaveEvmEvent event({})", msg.get_id()); + debug!("Receiving EnclaveEvmEvent event({})", msg.get_id()); match msg { none_hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { prev_event: None, .. }) => { - info!( + debug!( "Historical order event({}) has no previous event. Forwarding...", id ); @@ -73,7 +77,7 @@ impl Handler for FixHistoricalOrder { prev_event: Some(prev), .. }) => { - info!( + debug!( "Historical order event({}) has previous event({}). Buffering...", id, prev ); @@ -82,7 +86,7 @@ impl Handler for FixHistoricalOrder { } EnclaveEvmEvent::Processed(id) => self.track_id(id), other => { - info!("Forwarding event({})", other.get_id()); + debug!("Forwarding event({})", other.get_id()); self.track_id(other.get_id()); self.dest.do_send(other); } diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 47f8b45f31..93045c9804 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -6,7 +6,6 @@ mod bonding_registry_sol; mod ciphernode_registry_sol; -mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; mod events; @@ -17,7 +16,6 @@ mod evm_read_interface; mod evm_router; mod fix_historical_order; pub mod helpers; -mod one_shot_runnner; mod repo; mod sync_start_extractor; @@ -25,7 +23,6 @@ pub use bonding_registry_sol::BondingRegistrySolReader; pub use ciphernode_registry_sol::{ CiphernodeRegistrySol, CiphernodeRegistrySolReader, CiphernodeRegistrySolWriter, }; -pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; pub use events::*; @@ -36,6 +33,5 @@ pub use evm_read_interface::*; pub use evm_router::*; pub use fix_historical_order::*; pub use helpers::*; -pub use one_shot_runnner::*; pub use repo::*; pub use sync_start_extractor::*; diff --git a/crates/evm/src/repo.rs b/crates/evm/src/repo.rs index 1fc79ae011..c9b37b92a9 100644 --- a/crates/evm/src/repo.rs +++ b/crates/evm/src/repo.rs @@ -4,8 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; +use e3_events::StoreKeys; use crate::EvmReadInterfaceState; diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs index 7aa25b20bd..9bcebdc53b 100644 --- a/crates/evm/src/sync_start_extractor.rs +++ b/crates/evm/src/sync_start_extractor.rs @@ -5,29 +5,33 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Addr, Handler, Recipient}; -use e3_events::{EnclaveEvent, EnclaveEventData, Event, SyncStart}; +use e3_events::{EnclaveEvent, EnclaveEventData, Event, HistoricalEvmSyncStart}; +use e3_utils::MAILBOX_LIMIT; pub struct SyncStartExtractor { - dest: Recipient, + dest: Recipient, } impl SyncStartExtractor { - pub fn new(dest: impl Into>) -> Self { + pub fn new(dest: impl Into>) -> Self { Self { dest: dest.into() } } - pub fn setup(dest: impl Into>) -> Addr { + pub fn setup(dest: impl Into>) -> Addr { Self::new(dest).start() } } impl Actor for SyncStartExtractor { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for SyncStartExtractor { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _ctx: &mut Self::Context) -> Self::Result { - if let EnclaveEventData::SyncStart(evt) = msg.into_data() { + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::HistoricalEvmSyncStart(evt) = msg.into_data() { self.dest.do_send(evt) } } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index b87029282d..a2b0736d6b 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -16,14 +16,13 @@ use alloy::{ use anyhow::Result; use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ - prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, EvmEventConfig, - EvmEventConfigChain, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, + prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEventConfig, + EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, SyncEnded, + TestEvent, }; use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; use std::{sync::Arc, time::Duration}; use tokio::time::sleep; -use tracing::subscriber::DefaultGuard; -use tracing_subscriber::{fmt, EnvFilter}; sol!( #[sol(rpc)] @@ -42,10 +41,10 @@ fn test_event_extractor( return None; }; Some( - TestEvent { - msg: event.value, - entropy: event.count.try_into().unwrap(), // This prevents de-duplication in tests - } + TestEvent::new( + &event.value, + event.count.try_into().unwrap(), // This prevents de-duplication in tests + ) .into(), ) } @@ -61,24 +60,8 @@ impl TestEventParser { } } -async fn get_msgs(history_collector: &Addr>) -> Result> { - let history = history_collector - .send(GetEvents::::new()) - .await?; - let msgs: Vec = history - .into_iter() - .filter_map(|evt| match evt.into_data() { - EnclaveEventData::TestEvent(data) => Some(data.msg), - _ => None, - }) - .collect(); - - Ok(msgs) -} - struct FakeSyncActor { bus: BusHandle, - buffer: Vec, } impl Actor for FakeSyncActor { @@ -87,47 +70,30 @@ impl Actor for FakeSyncActor { impl FakeSyncActor { pub fn setup(bus: &BusHandle) -> Addr { - Self { - bus: bus.clone(), - buffer: Vec::new(), - } - .start() + Self { bus: bus.clone() }.start() } } -impl Handler for FakeSyncActor { +impl Handler for FakeSyncActor { type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + mut msg: HistoricalEvmEventsReceived, + _: &mut Self::Context, + ) -> Self::Result { trap(EType::Sync, &self.bus.clone(), || { - match msg { - // Buffer events as the sync actor receives them - SyncEvmEvent::Event(event) => self.buffer.push(event), - // When we hear that sync is complete send all events on chain then publish SyncEnd - SyncEvmEvent::HistoricalSyncComplete(_) => { - for evt in self.buffer.drain(..) { - let (data, ts, _) = evt.split(); - self.bus.publish_from_remote(data, ts)?; - } - self.bus.publish(SyncEnd::new())?; - } - }; + for evt in msg.events.drain(..) { + self.bus.naked_dispatch(evt); + } + self.bus.publish_without_context(SyncEnded::new())?; Ok(()) }) } } -fn add_tracing() -> DefaultGuard { - tracing::subscriber::set_default( - fmt() - .with_env_filter(EnvFilter::new("info")) - .with_test_writer() - .finish(), - ) -} - #[actix::test] async fn evm_reader() -> Result<()> { - let _guard = add_tracing(); + let _guard = e3_test_helpers::with_tracing("info"); // Create a WS provider // NOTE: Anvil must be available on $PATH @@ -156,11 +122,11 @@ async fn evm_reader() -> Result<()> { }) .build(); - // SyncStart holds initialization information such as start block and earliest event + // HistoricalEvmSyncStart holds initialization information such as start block and earliest event // This should trigger all chains to start to sync let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish(SyncStart::new(sync, evm_info))?; + bus.publish_without_context(HistoricalEvmSyncStart::new(sync, evm_info))?; sleep(Duration::from_secs(1)).await; contract @@ -197,7 +163,7 @@ async fn evm_reader() -> Result<()> { } #[actix::test] async fn ensure_historical_events() -> Result<()> { - let _guard = add_tracing(); + let _guard = e3_test_helpers::with_tracing("info"); // Create a WS provider // NOTE: Anvil must be available on $PATH @@ -238,7 +204,7 @@ async fn ensure_historical_events() -> Result<()> { .build(); let mut evm_info = EvmEventConfig::new(); evm_info.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish(SyncStart::new(sync, evm_info))?; + bus.publish_without_context(HistoricalEvmSyncStart::new(sync, evm_info))?; for msg in live_events.clone() { contract diff --git a/crates/fhe/src/repo.rs b/crates/fhe/src/repo.rs index 3c17fbeec6..289d272fad 100644 --- a/crates/fhe/src/repo.rs +++ b/crates/fhe/src/repo.rs @@ -4,9 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use crate::FheSnapshot; diff --git a/crates/indexer/tests/integration.rs b/crates/indexer/tests/integration.rs index 014bf2f582..dcb0641d25 100644 --- a/crates/indexer/tests/integration.rs +++ b/crates/indexer/tests/integration.rs @@ -43,7 +43,7 @@ sol!( async fn test_indexer() -> Result<()> { const E3_ID: u64 = 10; const THRESHOLD: u64 = 10; - const INDEXER_DELAY_MS: u64 = 10; + const INDEXER_DELAY_MS: u64 = 30; let param_set = DEFAULT_BFV_PRESET.into(); let params = build_bfv_params_from_set_arc(param_set); diff --git a/crates/keyshare/src/encryption_key_collector.rs b/crates/keyshare/src/encryption_key_collector.rs index b9bc2db99b..15d5ef5d81 100644 --- a/crates/keyshare/src/encryption_key_collector.rs +++ b/crates/keyshare/src/encryption_key_collector.rs @@ -11,8 +11,11 @@ use std::{ }; use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, SpawnHandle}; -use e3_events::{E3id, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated}; +use e3_events::{ + E3id, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, TypedEvent, +}; use e3_trbfv::PartyId; +use e3_utils::MAILBOX_LIMIT; use tracing::{info, warn}; const DEFAULT_COLLECTION_TIMEOUT: Duration = Duration::from_secs(60); @@ -85,6 +88,7 @@ impl Actor for EncryptionKeyCollector { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); info!( e3_id = %self.e3_id, "EncryptionKeyCollector started, scheduling timeout in {:?}", @@ -96,9 +100,14 @@ impl Actor for EncryptionKeyCollector { } } -impl Handler for EncryptionKeyCollector { +impl Handler> for EncryptionKeyCollector { type Result = (); - fn handle(&mut self, msg: EncryptionKeyCreated, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); let start = Instant::now(); info!("EncryptionKeyCollector: EncryptionKeyCreated received"); @@ -141,7 +150,8 @@ impl Handler for EncryptionKeyCollector { ctx.cancel_future(handle); } - let event: AllEncryptionKeysCollected = self.keys.clone().into(); + let event: TypedEvent = + TypedEvent::new(self.keys.clone().into(), ec); self.parent.do_send(event); } diff --git a/crates/keyshare/src/repo.rs b/crates/keyshare/src/repo.rs index e7d17eb618..90fe4864cb 100644 --- a/crates/keyshare/src/repo.rs +++ b/crates/keyshare/src/repo.rs @@ -4,9 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use crate::ThresholdKeyshareState; diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index 44819d10c4..cca0bc00e3 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -12,8 +12,9 @@ use e3_events::{ prelude::*, trap, BusHandle, CiphernodeSelected, CiphertextOutputPublished, ComputeRequest, ComputeResponse, ComputeResponseKind, CorrelationId, DecryptionshareCreated, Die, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, EncryptionKey, - EncryptionKeyCollectionFailed, EncryptionKeyCreated, EncryptionKeyPending, KeyshareCreated, - PartyId, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated, TypedEvent, + EncryptionKeyCollectionFailed, EncryptionKeyCreated, EncryptionKeyPending, EventContext, + KeyshareCreated, PartyId, Sequenced, ThresholdShare, ThresholdShareCollectionFailed, + ThresholdShareCreated, TypedEvent, }; use e3_fhe::create_crp; use e3_fhe_params::{BfvParamSet, BfvPreset}; @@ -28,8 +29,8 @@ use e3_trbfv::{ shares::{BfvEncryptedShares, EncryptableVec, Encrypted, ShamirShare, SharedSecret}, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; -use e3_utils::NotifySync; use e3_utils::{to_ordered_vec, utility_types::ArcBytes}; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use fhe::bfv::{PublicKey, SecretKey}; use fhe_traits::{DeserializeParametrized, Serialize}; use rand::{rngs::OsRng, SeedableRng}; @@ -39,7 +40,7 @@ use std::{ mem, sync::{Arc, Mutex}, }; -use tracing::{error, info, trace, warn}; +use tracing::{info, trace, warn}; use crate::encryption_key_collector::{AllEncryptionKeysCollected, EncryptionKeyCollector}; use crate::threshold_share_collector::ThresholdShareCollector; @@ -322,6 +323,9 @@ impl ThresholdKeyshare { impl Actor for ThresholdKeyshare { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl ThresholdKeyshare { @@ -367,7 +371,7 @@ impl ThresholdKeyshare { pub fn handle_threshold_share_created( &mut self, - msg: ThresholdShareCreated, + msg: TypedEvent, self_addr: Addr, ) -> Result<()> { let state = self.state.try_get()?; @@ -390,7 +394,7 @@ impl ThresholdKeyshare { pub fn handle_encryption_key_created( &mut self, - msg: EncryptionKeyCreated, + msg: TypedEvent, self_addr: Addr, ) -> Result<()> { info!("Received EncryptionKeyCreated forwarding to encryption key collector!"); @@ -424,6 +428,7 @@ impl ThresholdKeyshare { msg: TypedEvent, address: Addr, ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("CiphernodeSelected received."); // Ensure the collectors are created let _ = self.ensure_collector(address.clone()); @@ -441,21 +446,36 @@ impl ThresholdKeyshare { let state = self.state.try_get()?; let e3_id = state.e3_id.clone(); - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { s.new_state(KeyshareState::CollectingEncryptionKeys( CollectingEncryptionKeysData { sk_bfv: sk_bfv_encrypted.clone(), pk_bfv: pk_bfv_bytes.clone(), - ciphernode_selected: msg.into_inner(), + ciphernode_selected: msg, }, )) })?; - self.bus.publish(EncryptionKeyPending { - e3_id, - key: Arc::new(EncryptionKey::new(state.party_id, pk_bfv_bytes)), - params_preset: self.share_enc_preset, - })?; + // let state = self.state.try_get()?; + // self.bus.publish( + // EncryptionKeyCreated { + // e3_id: state.e3_id.clone(), + // key: Arc::new(EncryptionKey { + // party_id: state.party_id, + // pk_bfv: pk_bfv_bytes, + // }), + // external: false, + // }, + // ec, + // )?; + self.bus.publish( + EncryptionKeyPending { + e3_id, + key: Arc::new(EncryptionKey::new(state.party_id, pk_bfv_bytes)), + params_preset: self.share_enc_preset, + }, + ec, + )?; Ok(()) } @@ -463,8 +483,9 @@ impl ThresholdKeyshare { /// 1a. AllEncryptionKeysCollected - All BFV keys received, start share generation pub fn handle_all_encryption_keys_collected( &mut self, - msg: AllEncryptionKeysCollected, + msg: TypedEvent, ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!( "AllEncryptionKeysCollected - {} keys received", msg.keys.len() @@ -472,7 +493,7 @@ impl ThresholdKeyshare { let current: CollectingEncryptionKeysData = self.state.try_get()?.try_into()?; - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { s.new_state(KeyshareState::GeneratingThresholdShare( GeneratingThresholdShareData { sk_sss: None, @@ -484,16 +505,21 @@ impl ThresholdKeyshare { }, )) })?; - self.handle_gen_esi_sss_requested(GenEsiSss(current.ciphernode_selected.clone()))?; - self.handle_gen_pk_share_and_sk_sss_requested(GenPkShareAndSkSss( - current.ciphernode_selected, + self.handle_gen_esi_sss_requested(TypedEvent::new( + GenEsiSss(current.ciphernode_selected.clone()), + ec.clone(), + ))?; + self.handle_gen_pk_share_and_sk_sss_requested(TypedEvent::new( + GenPkShareAndSkSss(current.ciphernode_selected), + ec, ))?; Ok(()) } /// 2. GenEsiSss - pub fn handle_gen_esi_sss_requested(&self, msg: GenEsiSss) -> Result<()> { + pub fn handle_gen_esi_sss_requested(&self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("GenEsiSss on ThresholdKeyshare"); let evt = msg.0; @@ -526,17 +552,18 @@ impl ThresholdKeyshare { e3_id, ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } /// 2a. GenEsiSss result pub fn handle_gen_esi_sss_response(&mut self, res: TypedEvent) -> Result<()> { - let output: GenEsiSssResponse = res.into_inner().try_into()?; + let (res, ec) = res.into_components(); + let output: GenEsiSssResponse = res.try_into()?; let esi_sss = output.esi_sss; - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; info!("try_store_esi_sss"); @@ -574,13 +601,17 @@ impl ThresholdKeyshare { .. }) = self.state.get() { - self.handle_shares_generated()?; + self.handle_shares_generated(ec)?; } Ok(()) } /// 3. GenPkShareAndSkSss - pub fn handle_gen_pk_share_and_sk_sss_requested(&self, msg: GenPkShareAndSkSss) -> Result<()> { + pub fn handle_gen_pk_share_and_sk_sss_requested( + &self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("GenPkShareAndSkSss on ThresholdKeyshare"); let CiphernodeSelected { seed, e3_id, .. } = msg.0; let state = self @@ -605,7 +636,7 @@ impl ThresholdKeyshare { e3_id, ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } @@ -614,14 +645,15 @@ impl ThresholdKeyshare { &mut self, res: TypedEvent, ) -> Result<()> { + let (res, ec) = res.into_components(); + let output: GenPkShareAndSkSssResponse = res - .into_inner() .try_into() .context("Error extracting data from compute process")?; let (pk_share, sk_sss) = (output.pk_share, output.sk_sss); - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { info!("try_store_pk_share_and_sk_sss"); let current: GeneratingThresholdShareData = s.clone().try_into()?; let next = match current.esi_sss { @@ -653,13 +685,13 @@ impl ThresholdKeyshare { .. }) = self.state.get() { - self.handle_shares_generated()?; + self.handle_shares_generated(ec)?; } Ok(()) } /// 4. SharesGenerated - Encrypt shares with BFV and publish - pub fn handle_shares_generated(&mut self) -> Result<()> { + pub fn handle_shares_generated(&mut self, ec: EventContext) -> Result<()> { let Some(ThresholdKeyshareState { state: KeyshareState::AggregatingDecryptionKey(AggregatingDecryptionKey { @@ -730,12 +762,15 @@ impl ThresholdKeyshare { anyhow!("Failed to extract share for party {}", recipient_party_id) })?; - self.bus.publish(ThresholdShareCreated { - e3_id: e3_id.clone(), - share: Arc::new(party_share), - target_party_id: recipient_party_id as u64, - external: false, - })?; + self.bus.publish( + ThresholdShareCreated { + e3_id: e3_id.clone(), + share: Arc::new(party_share), + target_party_id: recipient_party_id as u64, + external: false, + }, + ec.clone(), + )?; } Ok(()) } @@ -745,8 +780,9 @@ impl ThresholdKeyshare { /// 5. AllThresholdSharesCollected - Decrypt received shares using BFV and aggregate pub fn handle_all_threshold_shares_collected( &self, - msg: AllThresholdSharesCollected, + msg: TypedEvent, ) -> Result<()> { + let (msg, ec) = msg.into_components(); info!("AllThresholdSharesCollected"); let cipher = self.cipher.clone(); let state = self.state.try_get()?; @@ -809,7 +845,7 @@ impl ThresholdKeyshare { e3_id.clone(), ); - self.bus.publish(event)?; + self.bus.publish(event, ec)?; Ok(()) } @@ -818,14 +854,14 @@ impl ThresholdKeyshare { &mut self, res: TypedEvent, ) -> Result<()> { + let (res, ec) = res.into_components(); let output: CalculateDecryptionKeyResponse = res - .into_inner() .try_into() .context("Error extracting data from compute process")?; let (sk_poly_sum, es_poly_sum) = (output.sk_poly_sum, output.es_poly_sum); - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; info!("Try store decryption key"); @@ -847,11 +883,14 @@ impl ThresholdKeyshare { let address = state.get_address().to_owned(); let current: ReadyForDecryption = state.clone().try_into()?; - self.bus.publish(KeyshareCreated { - pubkey: current.pk_share, - e3_id: e3_id.clone(), - node: address, - })?; + self.bus.publish( + KeyshareCreated { + pubkey: current.pk_share, + e3_id: e3_id.clone(), + node: address, + }, + ec, + )?; Ok(()) } @@ -859,10 +898,11 @@ impl ThresholdKeyshare { /// CiphertextOutputPublished pub fn handle_ciphertext_output_published( &mut self, - msg: CiphertextOutputPublished, + msg: TypedEvent, ) -> Result<()> { + let (msg, ec) = msg.into_components(); // Set state to decrypting - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; let current: ReadyForDecryption = s.clone().try_into()?; @@ -895,7 +935,7 @@ impl ThresholdKeyshare { CorrelationId::new(), e3_id.clone(), ); - self.bus.publish(event)?; // CalculateDecryptionShareRequest + self.bus.publish(event, ec)?; // CalculateDecryptionShareRequest Ok(()) } @@ -904,7 +944,8 @@ impl ThresholdKeyshare { &mut self, res: TypedEvent, ) -> Result<()> { - let msg: CalculateDecryptionShareResponse = res.into_inner().try_into()?; + let (res, ec) = res.into_components(); + let msg: CalculateDecryptionShareResponse = res.try_into()?; let state = self.state.try_get()?; let party_id = state.party_id; let node = state.address; @@ -919,10 +960,10 @@ impl ThresholdKeyshare { }; // send the decryption share - self.bus.publish(event)?; + self.bus.publish(event, ec.clone())?; // mark as complete - self.state.try_mutate(|s| { + self.state.try_mutate(&ec, |s| { use KeyshareState as K; info!("Decryption share sending process is complete"); @@ -937,16 +978,21 @@ impl ThresholdKeyshare { impl Handler for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.clone().into_data() { + let (msg, ec) = msg.into_components(); + match msg { EnclaveEventData::CiphernodeSelected(data) => { - self.notify_sync(ctx, msg.to_typed_event(data)) + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CiphertextOutputPublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) } - EnclaveEventData::CiphertextOutputPublished(data) => self.notify_sync(ctx, data), EnclaveEventData::ThresholdShareCreated(data) => { - let _ = self.handle_threshold_share_created(data, ctx.address()); + let _ = + self.handle_threshold_share_created(TypedEvent::new(data, ec), ctx.address()); } EnclaveEventData::EncryptionKeyCreated(data) => { - let _ = self.handle_encryption_key_created(data, ctx.address()); + let _ = + self.handle_encryption_key_created(TypedEvent::new(data, ec), ctx.address()); } EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), EnclaveEventData::E3Failed(data) => { @@ -973,7 +1019,7 @@ impl Handler for ThresholdKeyshare { } } EnclaveEventData::ComputeResponse(data) => { - self.notify_sync(ctx, msg.to_typed_event(data)) + self.notify_sync(ctx, TypedEvent::new(data, ec)) } _ => (), } @@ -983,9 +1029,11 @@ impl Handler for ThresholdKeyshare { impl Handler> for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_compute_response(msg) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg), + ) } } @@ -996,36 +1044,56 @@ impl Handler> for ThresholdKeyshare { msg: TypedEvent, ctx: &mut Self::Context, ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_ciphernode_selected(msg, ctx.address()) - }) + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_ciphernode_selected(msg, ctx.address()), + ) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: AllEncryptionKeysCollected, _: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_all_encryption_keys_collected(msg) - }) + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_all_encryption_keys_collected(msg), + ) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: AllThresholdSharesCollected, _: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_all_threshold_shares_collected(msg) - }) + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_all_threshold_shares_collected(msg), + ) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: CiphertextOutputPublished, _: &mut Self::Context) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - self.handle_ciphertext_output_published(msg) - }) + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_ciphertext_output_published(msg), + ) } } @@ -1036,23 +1104,24 @@ impl Handler for ThresholdKeyshare { msg: EncryptionKeyCollectionFailed, ctx: &mut Self::Context, ) -> Self::Result { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "Encryption key collection failed: {}", - msg.reason - ); - - // Clear the collector reference since it's stopped - self.encryption_key_collector = None; - - // Publish failure event to event bus for sync tracking - if let Err(e) = self.bus.publish(msg) { - error!("Failed to publish EncryptionKeyCollectionFailed: {}", e); - } - - // Stop this actor since we can't proceed without all encryption keys - ctx.stop(); + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "Encryption key collection failed: {}", + msg.reason + ); + + // Clear the collector reference since it's stopped + self.encryption_key_collector = None; + + // Publish failure event to event bus for sync tracking + self.bus.publish_without_context(msg)?; + + // Stop this actor since we can't proceed without all encryption keys + ctx.stop(); + Ok(()) + }) } } @@ -1063,22 +1132,23 @@ impl Handler for ThresholdKeyshare { msg: ThresholdShareCollectionFailed, ctx: &mut Self::Context, ) -> Self::Result { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "Threshold share collection failed: {}", - msg.reason - ); + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "Threshold share collection failed: {}", + msg.reason + ); - // Clear the collector reference since it's stopped - self.decryption_key_collector = None; + // Clear the collector reference since it's stopped + self.decryption_key_collector = None; - // Publish failure event to event bus for sync tracking - if let Err(e) = self.bus.publish(msg) { - error!("Failed to publish ThresholdShareCollectionFailed: {}", e); - } + // Publish failure event to event bus for sync tracking + self.bus.publish_without_context(msg)?; - ctx.stop(); + ctx.stop(); + Ok(()) + }) } } diff --git a/crates/keyshare/src/threshold_share_collector.rs b/crates/keyshare/src/threshold_share_collector.rs index d0393c34cf..ae1167352f 100644 --- a/crates/keyshare/src/threshold_share_collector.rs +++ b/crates/keyshare/src/threshold_share_collector.rs @@ -11,8 +11,11 @@ use std::{ }; use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, SpawnHandle}; -use e3_events::{E3id, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated}; +use e3_events::{ + E3id, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated, TypedEvent, +}; use e3_trbfv::PartyId; +use e3_utils::MAILBOX_LIMIT; use tracing::{info, warn}; use crate::{AllThresholdSharesCollected, ThresholdKeyshare}; @@ -31,11 +34,17 @@ pub(crate) enum CollectorState { pub struct ThresholdShareCollectionTimeout; pub struct ThresholdShareCollector { + /// The E3id for the round e3_id: E3id, + /// The partys the collector expects to receive from todo: HashSet, + /// The parent actor that has requested collection parent: Addr, + /// The current state of the collector state: CollectorState, + /// The shares collected shares: HashMap>, + /// A timeout handle for when this collector will report failure timeout_handle: Option, } @@ -57,6 +66,7 @@ impl Actor for ThresholdShareCollector { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); info!( e3_id = %self.e3_id, "ThresholdShareCollector started, scheduling timeout in {:?}", @@ -68,9 +78,14 @@ impl Actor for ThresholdShareCollector { } } -impl Handler for ThresholdShareCollector { +impl Handler> for ThresholdShareCollector { type Result = (); - fn handle(&mut self, msg: ThresholdShareCreated, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); let start = Instant::now(); info!("ThresholdShareCollector: ThresholdShareCreated received by collector"); @@ -108,7 +123,8 @@ impl Handler for ThresholdShareCollector { ctx.cancel_future(handle); } - let event: AllThresholdSharesCollected = self.shares.clone().into(); + let event: TypedEvent = + TypedEvent::new(self.shares.clone().into(), ec); self.parent.do_send(event); } info!( diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index 4cbc792f78..97f985bdcf 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -17,11 +17,21 @@ use actix::prelude::*; use actix::{Actor, Handler}; use anyhow::Result; use e3_crypto::Cipher; +use e3_events::run_once; +use e3_events::trap_fut; +use e3_events::BusHandle; +use e3_events::ComputeRequestErrorKind; +use e3_events::EType; +use e3_events::EffectsEnabled; +use e3_events::EnclaveEvent; +use e3_events::EnclaveEventData; +use e3_events::EventPublisher; +use e3_events::EventSubscriber; +use e3_events::TypedEvent; +use e3_events::{ComputeRequest, ComputeRequestError, ComputeResponse, EventType}; use e3_events::{ - BusHandle, ComputeRequest, ComputeRequestError, ComputeRequestErrorKind, ComputeRequestKind, - ComputeResponse, EType, EnclaveEvent, EnclaveEventData, ErrorDispatcher, Event, EventPublisher, - EventSubscriber, EventType, PkBfvProofRequest, PkBfvProofResponse, ZkError as ZkEventError, - ZkRequest, ZkResponse, + ComputeRequestKind, PkBfvProofRequest, PkBfvProofResponse, ZkError as ZkEventError, ZkRequest, + ZkResponse, }; use e3_fhe_params::{BfvParamSet, BfvPreset}; use e3_trbfv::calculate_decryption_key::calculate_decryption_key; @@ -31,6 +41,7 @@ use e3_trbfv::gen_esi_sss::gen_esi_sss; use e3_trbfv::gen_pk_share_and_sk_sss::gen_pk_share_and_sk_sss; use e3_trbfv::{TrBFVError, TrBFVRequest, TrBFVResponse}; use e3_utils::SharedRng; +use e3_utils::MAILBOX_LIMIT; use e3_zk_helpers::circuits::dkg::pk::circuit::{PkCircuit, PkCircuitData}; use e3_zk_prover::{Provable, ZkBackend, ZkProver}; use fhe::bfv::PublicKey; @@ -90,7 +101,21 @@ impl Multithread { report: Option>, ) -> Addr { let addr = Self::new(bus.clone(), rng.clone(), cipher.clone(), task_pool, report).start(); - bus.subscribe(EventType::ComputeRequest, addr.clone().recipient()); + + bus.subscribe( + EventType::EffectsEnabled, + run_once::({ + let bus = bus.clone(); + let addr = addr.clone(); + move |_| { + bus.subscribe(EventType::ComputeRequest, addr.clone().recipient()); + info!("Multithread actor listening for events."); + Ok(()) + } + }) + .recipient(), + ); + addr } @@ -117,41 +142,42 @@ impl Multithread { impl Actor for Multithread { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for Multithread { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { info!("Multithread received EnclaveEvent!"); - match msg.get_data() { - EnclaveEventData::ComputeRequest(data) => ctx.notify(data.clone()), + let (data, ec) = msg.into_components(); + match data { + EnclaveEventData::ComputeRequest(data) => ctx.notify(TypedEvent::new(data, ec)), _ => (), } } } -impl Handler for Multithread { +impl Handler> for Multithread { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: ComputeRequest, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { let cipher = self.cipher.clone(); let rng = self.rng.clone(); let bus = self.bus.clone(); let pool = self.task_pool.clone(); let report = self.report.clone(); let zk_prover = self.zk_prover.clone(); - - Box::pin(async move { - match handle_compute_request_event(msg, bus, cipher, rng, pool, report, zk_prover).await - { - Ok(_) => (), - Err(e) => error!("{e}"), - } - }) + trap_fut( + EType::Computation, + &self.bus.clone(), + handle_compute_request_event(msg, bus, cipher, rng, pool, report, zk_prover), + ) } } async fn handle_compute_request_event( - msg: ComputeRequest, + msg: TypedEvent, bus: BusHandle, cipher: Arc, rng: SharedRng, @@ -161,6 +187,8 @@ async fn handle_compute_request_event( ) -> anyhow::Result<()> { let msg_string = msg.to_string(); let job_name = msg_string.clone(); + let (msg, ctx) = msg.into_components(); + // We spawn a thread on rayon moving to "sync"-land let (result, duration) = pool .spawn(job_name, TaskTimeouts::default(), move || { @@ -173,11 +201,11 @@ async fn handle_compute_request_event( }; match result { - Ok(val) => bus.publish(val)?, + Ok(val) => bus.publish(val, ctx)?, Err(e) => { // Publish ComputeRequestError so ProofRequestActor can handle it // and continue without proof if needed - bus.publish(e)? + bus.publish(e, ctx)? } }; Ok(()) diff --git a/crates/multithread/src/report.rs b/crates/multithread/src/report.rs index 9eedc23dab..da9a05defc 100644 --- a/crates/multithread/src/report.rs +++ b/crates/multithread/src/report.rs @@ -7,6 +7,7 @@ use std::{collections::HashMap, thread, time::Duration}; use actix::{Actor, Handler, Message, MessageResponse}; +use e3_utils::MAILBOX_LIMIT; #[derive(Message)] #[rtype(result = "FlattenedReport")] @@ -34,6 +35,9 @@ pub struct MultithreadReport { impl Actor for MultithreadReport { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl MultithreadReport { diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml index 46f00d41fc..37c2540de3 100644 --- a/crates/net/Cargo.toml +++ b/crates/net/Cargo.toml @@ -13,6 +13,7 @@ async-std = { workspace = true } async-trait = { workspace = true } bincode = { workspace = true } chrono = { workspace = true } +derivative = { workspace = true } futures = { workspace = true } e3-crypto = { workspace = true } e3-config = { workspace = true } diff --git a/crates/net/src/cid.rs b/crates/net/src/cid.rs index e9ee58df05..86678b39ca 100644 --- a/crates/net/src/cid.rs +++ b/crates/net/src/cid.rs @@ -4,13 +4,14 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::ops::{Deref, DerefMut}; - +use derivative::Derivative; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use std::ops::{Deref, DerefMut}; -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct Cid(pub Vec); +#[derive(Derivative, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derivative(Debug)] +pub struct Cid(#[derivative(Debug(format_with = "e3_utils::formatters::hexf"))] pub Vec); impl Cid { pub fn from_content(content: &[u8]) -> Self { diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 44e29556cf..51089fbde3 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -11,17 +11,21 @@ use crate::{ Cid, }; use actix::prelude::*; +use anyhow::Context; use anyhow::Result; use chrono::{DateTime, Utc}; use e3_events::{ - prelude::*, BusHandle, CiphernodeSelected, CorrelationId, DocumentKind, DocumentMeta, - DocumentReceived, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, - EncryptionKeyCreated, EncryptionKeyReceived, Event, EventType, Filter, PartyId, - PublishDocumentRequested, ThresholdShareCreated, + prelude::*, trap, trap_fut, BusHandle, CiphernodeSelected, CorrelationId, DocumentKind, + DocumentMeta, DocumentReceived, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, + EncryptionKeyCreated, EncryptionKeyReceived, Event, EventContext, EventSource, EventType, + Filter, PartyId, PublishDocumentRequested, Sequenced, ThresholdShareCreated, TypedEvent, }; -use e3_utils::retry::{retry_with_backoff, to_retry}; use e3_utils::ArcBytes; use e3_utils::NotifySync; +use e3_utils::{ + retry::{retry_with_backoff, to_retry}, + MAILBOX_LIMIT, +}; use futures::TryFutureExt; use serde::{Deserialize, Serialize}; use std::{ @@ -30,7 +34,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::sync::{broadcast, mpsc}; -use tracing::{debug, error, info}; +use tracing::{debug, info}; const KADEMLIA_PUT_TIMEOUT: Duration = Duration::from_secs(30); const KADEMLIA_GET_TIMEOUT: Duration = Duration::from_secs(30); @@ -55,7 +59,7 @@ pub struct DocumentPublisher { } impl DocumentPublisher { - /// Create a new NetEventTranslator actor + /// Create a new DocumentPublisher actor pub fn new( bus: &BusHandle, tx: &mpsc::Sender, @@ -131,64 +135,79 @@ impl DocumentPublisher { impl Actor for DocumentPublisher { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for DocumentPublisher { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::PublishDocumentRequested(data) => ctx.notify(data), - EnclaveEventData::CiphernodeSelected(data) => self.notify_sync(ctx, data), - EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::PublishDocumentRequested(data) => { + ctx.notify(TypedEvent::new(data, ec)) + } + EnclaveEventData::CiphernodeSelected(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for DocumentPublisher { +impl Handler> for DocumentPublisher { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: PublishDocumentRequested, _: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { let tx = self.tx.clone(); - let msg = msg.clone(); + let (msg, ec) = msg.into_components(); let rx = self.rx.clone(); let bus = self.bus.clone(); let topic = self.topic.clone(); - Box::pin(async move { - match handle_publish_document_requested(tx, rx, msg, topic).await { - Ok(_) => (), - Err(e) => { - error!(error=?e, "Could not handle publish document requested"); - bus.err(EType::IO, e) - } - } - }) + trap_fut( + EType::IO, + &bus.with_ec(&ec), + handle_publish_document_requested(tx, rx, msg, topic, bus), + ) } } -impl Handler for DocumentPublisher { +impl Handler> for DocumentPublisher { type Result = (); - fn handle(&mut self, msg: CiphernodeSelected, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_ciphernode_selected(msg) { - Ok(_) => (), - Err(e) => { - error!("{e}") - } - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { + self.handle_ciphernode_selected(msg) + }) } } -impl Handler for DocumentPublisher { +impl Handler> for DocumentPublisher { type Result = (); - fn handle(&mut self, msg: E3RequestComplete, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_e3_request_complete(msg) { - Ok(_) => (), - Err(e) => { - error!("{e}") - } - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { + self.handle_e3_request_complete(msg) + }) } } +/// Receiving DocumentPublishedNotification from libp2p impl Handler for DocumentPublisher { type Result = ResponseFuture<()>; fn handle( @@ -199,18 +218,13 @@ impl Handler for DocumentPublisher { let ids = self.ids.clone(); let bus = self.bus.clone(); let tx = self.tx.clone(); - let msg = msg.clone(); let rx = self.rx.clone(); - - Box::pin(async move { - match handle_document_published_notification(tx, rx, bus.clone(), ids, msg).await { - Ok(_) => (), - Err(e) => { - error!(error=?e, "Could not handle document published notification"); - bus.err(EType::IO, e); - } - } - }) + let msg = msg.clone(); + trap_fut( + EType::IO, + &bus, + handle_document_published_notification(tx, rx, bus.clone(), ids, msg), + ) } } @@ -233,6 +247,7 @@ pub async fn handle_publish_document_requested( rx: Arc>, event: PublishDocumentRequested, topic: impl Into, + bus: BusHandle, ) -> Result<()> { let value = event.value; let key = Cid::from_content(&value); @@ -247,7 +262,7 @@ pub async fn handle_publish_document_requested( 1000, ) .await?; - let notification = DocumentPublishedNotification::new(event.meta, key); + let notification = DocumentPublishedNotification::new(event.meta, key, bus.ts()?); broadcast_document_published_notification(tx, rx, notification, topic).await?; Ok(()) } @@ -283,10 +298,15 @@ pub async fn handle_document_published_notification( .await?; debug!("Sending received event..."); - bus.publish(DocumentReceived { - meta: event.meta, - value, - })?; + bus.publish_from_remote( + DocumentReceived { + meta: event.meta, + value, + }, + event.ts, + None, + EventSource::Net, + )?; Ok(()) } @@ -421,6 +441,7 @@ impl EventConverter { receivable: ReceivableDocument, e3_id: &E3id, party_id: u64, + ctx: EventContext, ) -> Result<()> { let value = ArcBytes::from_bytes(&receivable.to_bytes()?); let meta = DocumentMeta::new( @@ -430,13 +451,17 @@ impl EventConverter { None, ); self.bus - .publish(PublishDocumentRequested::new(meta, value))?; + .publish(PublishDocumentRequested::new(meta, value), ctx)?; Ok(()) } /// Local node created a threshold share (already split per-party by ThresholdKeyshare). /// Publishes the single-party document with appropriate filter. - pub fn handle_threshold_share_created(&self, msg: ThresholdShareCreated) -> Result<()> { + pub fn handle_threshold_share_created( + &self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ctx) = msg.into_components(); if msg.external { return Ok(()); } @@ -447,53 +472,65 @@ impl EventConverter { msg.share.party_id, target_party_id, msg.e3_id ); + let e3_id = msg.e3_id.clone(); + self.publish_filtered( - ReceivableDocument::ThresholdShareCreated(msg.clone()), - &msg.e3_id, + ReceivableDocument::ThresholdShareCreated(msg), + &e3_id, target_party_id, + ctx, )?; Ok(()) } - fn handle_encryption_key_created(&self, msg: EncryptionKeyCreated) -> Result<()> { + fn handle_encryption_key_created(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); if msg.external { return Ok(()); } - let receivable = ReceivableDocument::EncryptionKeyCreated(msg.clone()); + + let meta = DocumentMeta::new(msg.e3_id.clone(), DocumentKind::TrBFV, vec![], None); + let receivable = ReceivableDocument::EncryptionKeyCreated(msg); let value = ArcBytes::from_bytes(&receivable.to_bytes()?); - let meta = DocumentMeta::new(msg.e3_id, DocumentKind::TrBFV, vec![], None); self.bus - .publish(PublishDocumentRequested::new(meta, value))?; + .publish(PublishDocumentRequested::new(meta, value), ctx)?; Ok(()) } /// Convert received document to internal events. /// Note: Filtering already happened in DocumentPublisher before DHT fetch. - fn handle_document_received(&self, msg: DocumentReceived) -> Result<()> { - let receivable = ReceivableDocument::from_bytes(&msg.value.extract_bytes())?; - + fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); + let receivable = ReceivableDocument::from_bytes(&msg.value.extract_bytes()) + .context("Could not deserialize document bytes")?; match receivable { ReceivableDocument::ThresholdShareCreated(evt) => { debug!( "Received ThresholdShareCreated from party {} for target party {}", evt.share.party_id, evt.target_party_id ); - self.bus.publish(ThresholdShareCreated { - external: true, - e3_id: evt.e3_id, - share: evt.share, - target_party_id: evt.target_party_id, - })?; + self.bus.publish( + ThresholdShareCreated { + external: true, + e3_id: evt.e3_id, + share: evt.share, + target_party_id: evt.target_party_id, + }, + ctx.clone(), + )?; } ReceivableDocument::EncryptionKeyCreated(evt) => { debug!( "Received EncryptionKeyCreated from party {}", evt.key.party_id ); - self.bus.publish(EncryptionKeyReceived { - e3_id: evt.e3_id, - key: evt.key, - })?; + self.bus.publish( + EncryptionKeyReceived { + e3_id: evt.e3_id, + key: evt.key, + }, + ctx, + )?; } } Ok(()) @@ -507,42 +544,64 @@ impl Actor for EventConverter { impl Handler for EventConverter { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::ThresholdShareCreated(data) => self.notify_sync(ctx, data), - EnclaveEventData::EncryptionKeyCreated(data) => self.notify_sync(ctx, data), - EnclaveEventData::DocumentReceived(data) => self.notify_sync(ctx, data), + let (data, ec) = msg.into_components(); + match data { + EnclaveEventData::ThresholdShareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::EncryptionKeyCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::DocumentReceived(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for EventConverter { +impl Handler> for EventConverter { type Result = (); - fn handle(&mut self, msg: ThresholdShareCreated, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_threshold_share_created(msg) { - Ok(_) => (), - Err(err) => error!("{err}"), - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_threshold_share_created(msg), + ) } } -impl Handler for EventConverter { +impl Handler> for EventConverter { type Result = (); - fn handle(&mut self, msg: EncryptionKeyCreated, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_encryption_key_created(msg) { - Ok(_) => (), - Err(err) => error!("{err}"), - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_encryption_key_created(msg), + ) } } -impl Handler for EventConverter { +impl Handler> for EventConverter { type Result = (); - fn handle(&mut self, msg: DocumentReceived, _ctx: &mut Self::Context) -> Self::Result { - match self.handle_document_received(msg) { - Ok(_) => (), - Err(err) => error!("{err}"), - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_document_received(msg), + ) } } @@ -611,7 +670,7 @@ mod tests { let e3_id = E3id::new("1243", 1); // 1. Send a request to publish - bus.publish(PublishDocumentRequested { + bus.publish_without_context(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), })?; @@ -685,7 +744,7 @@ mod tests { let cid = Cid::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish(CiphernodeSelected { + bus.publish_without_context(CiphernodeSelected { e3_id: e3_id.clone(), threshold_m: 3, threshold_n: 5, @@ -696,6 +755,7 @@ mod tests { GossipData::DocumentPublishedNotification(DocumentPublishedNotification { key: cid.clone(), meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), + ts: 123, }), ))?; @@ -747,7 +807,7 @@ mod tests { let e3_id = E3id::new("1243", 1); // Send a request to publish - bus.publish(PublishDocumentRequested { + bus.publish_without_context(PublishDocumentRequested { meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), value: value.clone(), })?; @@ -797,7 +857,7 @@ mod tests { let cid = Cid::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish(CiphernodeSelected { + bus.publish_without_context(CiphernodeSelected { e3_id: e3_id.clone(), threshold_m: 3, threshold_n: 5, @@ -814,6 +874,7 @@ mod tests { vec![], Some(expires_at), ), + ts: 123, }), ))?; @@ -827,6 +888,7 @@ mod tests { GossipData::DocumentPublishedNotification(DocumentPublishedNotification { key: cid.clone(), meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], Some(expires_at)), + ts: 100, }), ))?; @@ -847,7 +909,7 @@ mod tests { net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { key: cid, correlation_id, - value, + value, // This will error because this is not a ReceivableDocument which is fine })?; // wait for events to settle @@ -855,8 +917,12 @@ mod tests { // Check event was dispatched let events = history.send(GetEvents::new()).await?; - let Some(EnclaveEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = - events.last().map(|e| e.get_data()) + let Some(EnclaveEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = events + .iter() + // Filter out the error + .filter(|e| e.event_type() != "EnclaveError") + .last() + .map(|e| e.get_data()) else { bail!("No event sent"); }; diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index d394c80448..4be60f6987 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -7,7 +7,10 @@ use crate::Cid; use actix::Message; use anyhow::{bail, Context, Result}; -use e3_events::{AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; +use e3_events::{ + AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, EventContextAccessors, EventSource, + Sequenced, Unsequenced, +}; use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, @@ -23,6 +26,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::sync::{broadcast, mpsc}; +use tracing::{error, info}; /// Incoming/Outgoing GossipData. We disambiguate on concerns relative to the net package. #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] @@ -59,7 +63,7 @@ impl TryFrom for EnclaveEvent { bail!("GossipData was not the GossipBytes variant"); }; - Ok(EnclaveEvent::from_bytes(&bytes)?) + Ok(EnclaveEvent::from_bytes(&bytes)?.with_source(EventSource::Net)) } } @@ -71,6 +75,7 @@ pub struct SyncRequestValue { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct SyncResponseValue { pub events: Vec, + pub ts: u128, } #[derive(Message, Clone, Debug)] @@ -85,10 +90,14 @@ pub struct SyncRequestReceived { #[rtype("()")] pub struct OutgoingSyncRequestSucceeded { pub value: SyncResponseValue, + pub correlation_id: CorrelationId, } #[derive(Debug, Clone)] -pub struct OutgoingSyncRequestFailed; +pub struct OutgoingSyncRequestFailed { + pub correlation_id: CorrelationId, + pub error: String, +} /// NetInterface Commands are sent to the network peer over a mspc channel #[derive(Debug)] @@ -117,7 +126,10 @@ pub enum NetCommand { Shutdown, /// Called from the syning node to request libp2p events from a random peer node starting /// from the given timestamp. - OutgoingSyncRequest { value: SyncRequestValue }, + OutgoingSyncRequest { + correlation_id: CorrelationId, + value: SyncRequestValue, + }, /// Send libp2p events back to a peer that requested a sync. SyncResponse { value: SyncResponseValue, @@ -132,6 +144,7 @@ impl NetCommand { N::DhtPutRecord { correlation_id, .. } => Some(*correlation_id), N::DhtGetRecord { correlation_id, .. } => Some(*correlation_id), N::GossipPublish { correlation_id, .. } => Some(*correlation_id), + N::OutgoingSyncRequest { correlation_id, .. } => Some(*correlation_id), _ => None, } } @@ -198,6 +211,7 @@ pub enum NetEvent { /// Received gossipsub events from a peer in response to a `SyncRequest`. OutgoingSyncRequestSucceeded(OutgoingSyncRequestSucceeded), OutgoingSyncRequestFailed(OutgoingSyncRequestFailed), + AllPeersDialed, } #[derive(Clone, Debug)] @@ -216,6 +230,8 @@ impl NetEvent { N::DhtGetRecordSucceeded { correlation_id, .. } => Some(*correlation_id), N::DhtPutRecordError { correlation_id, .. } => Some(*correlation_id), N::DhtPutRecordSucceeded { correlation_id, .. } => Some(*correlation_id), + N::OutgoingSyncRequestSucceeded(msg) => Some(msg.correlation_id), + N::OutgoingSyncRequestFailed(msg) => Some(msg.correlation_id), _ => None, } } @@ -228,11 +244,12 @@ impl NetEvent { pub struct DocumentPublishedNotification { pub meta: DocumentMeta, pub key: Cid, + pub ts: u128, } impl DocumentPublishedNotification { - pub fn new(meta: DocumentMeta, key: Cid) -> Self { - Self { meta, key } + pub fn new(meta: DocumentMeta, key: Cid, ts: u128) -> Self { + Self { meta, key, ts } } pub fn to_bytes(&self) -> Result> { @@ -270,21 +287,36 @@ where let debug_cmd = format!("{:?}", command); // Send the command to NetInterface + info!( + "call_and_await_response: SENDING command {:?} with timeout {:?}", + command, timeout + ); net_cmds.send(command).await?; let result = tokio::time::timeout(timeout, async { loop { match rx.recv().await { Ok(event) => { + info!("RECEIVED and event {:?}", event); // Only process events matching our correlation ID if event.correlation_id() == Some(id) { if let Some(result) = matcher(&event) { return result; } // None means unexpected event type, keep waiting - } // Ignore events with non-matching IDs + info!("matcher failed for event {:?}! skipping...", event); + } else { + // Ignore events with non-matching IDs + info!( + "correlation failed for event {:?} against correlation={:?}! skipping...", + event, id + ); + } } Err(broadcast::error::RecvError::Lagged(_)) => continue, - Err(e) => return Err(e.into()), + Err(e) => { + error!("RECEIVED an error from rx: {:?} returning error", e); + return Err(e.into()); + } } } }) @@ -293,10 +325,46 @@ where result } +pub async fn await_event( + net_events: &Arc>, + matcher: F, + timeout: Duration, +) -> Result +where + F: Fn(&NetEvent) -> Option, +{ + let mut rx = net_events.resubscribe(); + + let result = tokio::time::timeout(timeout, async { + loop { + match rx.recv().await { + Ok(event) => { + if let Some(result) = matcher(&event) { + return Ok(result); + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(e) => return Err(e.into()), + } + } + }) + .await + .map_err(|_| anyhow::anyhow!(format!("Timed out waiting for event")))?; + result +} + +pub fn estimate_hashmap_size(map: &HashMap) -> usize { + let entry_size = size_of::() + size_of::(); + let capacity = map.capacity(); + + // HashMap uses ~1 byte of overhead per slot for metadata + capacity * (entry_size + 1) + size_of::>() +} + #[cfg(test)] mod tests { use e3_events::{ - EnclaveEvent, EventConstructorWithTimestamp, Sequenced, TestEvent, Unsequenced, + EnclaveEvent, EventConstructorWithTimestamp, EventSource, Sequenced, TestEvent, Unsequenced, }; use super::GossipData; @@ -304,8 +372,13 @@ mod tests { #[test] fn test_enclave_event_gossip_lifecycle() -> anyhow::Result<()> { // event is created locally - let event: EnclaveEvent = - EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), None, 31415); + let event: EnclaveEvent = EnclaveEvent::new_with_timestamp( + TestEvent::new("fish", 42).into(), + None, + 31415, + None, + EventSource::Local, + ); // event is sequenced after bus.publish() adds a sequence number let event: EnclaveEvent = event.into_sequenced(90210); diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index eb7c49065a..4b12a34985 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -8,13 +8,86 @@ mod cid; mod dialer; mod document_publisher; pub mod events; +mod net_event_buffer; mod net_event_translator; mod net_interface; mod net_sync_manager; mod repo; +use std::sync::Arc; + +use actix::{Addr, Recipient}; +use anyhow::bail; pub use cid::Cid; pub use document_publisher::*; +use e3_crypto::Cipher; +use e3_data::Repository; +use e3_events::{run_once, BusHandle, EffectsEnabled, EventStoreQueryBy, EventSubscriber, TsAgg}; +use libp2p::identity::ed25519; +use net_event_buffer::NetEventBuffer; pub use net_event_translator::*; pub use net_interface::*; +use net_sync_manager::NetSyncManager; pub use repo::*; +use tracing::{info, instrument}; + +/// Spawn a Libp2p interface and hook it up to this actor +#[instrument(name = "libp2p", skip_all)] +pub async fn setup_net( + bus: BusHandle, + peers: Vec, + cipher: &Arc, + quic_port: u16, + repository: Repository>, + eventstore: impl Into>>, +) -> anyhow::Result<(tokio::task::JoinHandle>, String)> { + let topic = "tmp-enclave-gossip-topic"; + + // Get existing keypair or generate a new one + let mut bytes = match repository.read().await? { + Some(bytes) => { + info!("Found keypair in repository"); + cipher.decrypt_data(&bytes)? + } + None => bail!("No network keypair found in repository, please generate a new one using `enclave net generate-key`"), + }; + + // Create peer from keypair + let keypair: libp2p::identity::Keypair = + ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; + + // Generate a new interface to read and write peer events to + let mut interface = NetInterface::new(&keypair, peers, Some(quic_port), topic)?; + + // NOTE: Pass the unbuffered rx to SyncManager as it must operate before live events are + // processed + let _net_sync = NetSyncManager::setup( + &bus, + &interface.tx(), + &Arc::new(interface.rx()), + eventstore.into(), + ); + + // Buffer all incoming events until SyncEnded + let rx = Arc::new(NetEventBuffer::setup(&bus, &interface.rx())); + let tx = interface.tx(); + + let runner = run_once::({ + let bus = bus.clone(); + let rx = rx.clone(); + let topic = topic.to_owned(); + let tx = tx.clone(); + move |_| { + NetEventTranslator::setup(&bus, &tx, &rx, &topic); + DocumentPublisher::setup(&bus, &tx, &rx, &topic); + Ok(()) + } + }); + + bus.subscribe(e3_events::EventType::EffectsEnabled, runner.recipient()); + + // TODO: actix::spawn might avoid all the cleanup code + let handle = tokio::spawn(async move { Ok(interface.start().await?) }); + + Ok((handle, keypair.public().to_peer_id().to_string())) +} diff --git a/crates/net/src/net_event_buffer.rs b/crates/net/src/net_event_buffer.rs new file mode 100644 index 0000000000..77f830eece --- /dev/null +++ b/crates/net/src/net_event_buffer.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, AsyncContext, Handler, Message}; +use anyhow::{anyhow, bail, Result}; +use e3_events::{ + trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, Event, EventSubscriber, EventType, + SyncEnded, +}; +use tokio::sync::broadcast::{self, error::RecvError}; + +use crate::events::NetEvent; + +#[derive(Debug)] +enum NetState { + Running, + Syncing(Vec), +} + +impl NetState { + pub fn run(&mut self) -> Result> { + let Self::Syncing(buffer) = self else { + bail!("Cannot change state to Running when state is {:?}", self); + }; + let buffer = std::mem::take(buffer); + *self = Self::Running; + Ok(buffer) + } +} + +/// Actor that controls a broadcast channel which will Buffer NetEvents until it receives a +/// EnclaveEvent::SyncEnded at which time it will release al events to the broadcast channel +pub struct NetEventBuffer { + state: NetState, + input_rx: Option>, + output_tx: broadcast::Sender, + bus: BusHandle, +} + +impl NetEventBuffer { + pub fn setup( + bus: &BusHandle, + input_rx: &broadcast::Receiver, + ) -> broadcast::Receiver { + let input_rx = input_rx.resubscribe(); + let (output_tx, output_rx) = broadcast::channel(1024); + + let actor = Self { + state: NetState::Syncing(Vec::new()), + input_rx: Some(input_rx), + output_tx: output_tx.clone(), + bus: bus.clone(), + }; + + let addr = actor.start(); + + // Subscribe to EnclaveEvent on the bus + bus.subscribe(EventType::SyncEnded, addr.clone().recipient()); + + output_rx + } + + fn handle_enclave_event(&mut self, msg: EnclaveEvent) -> Result<()> { + if let EnclaveEventData::SyncEnded(m) = msg.get_data() { + return self.process_sync_ended(m.clone()); + } + Ok(()) + } + + fn process_sync_ended(&mut self, _: SyncEnded) -> Result<()> { + let pending = self.state.run()?; + for event in pending { + self.forward_event(event)?; + } + Ok(()) + } + + fn forward_event(&mut self, event: NetEvent) -> Result<()> { + self.output_tx + .send(event) + .map_err(|e| anyhow!("Failed to forward event: {}", e))?; + Ok(()) + } +} + +impl Actor for NetEventBuffer { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + // Spawn task to read from broadcast channel + let addr = ctx.address(); + let mut input_rx = self.input_rx.take().expect("input_rx should be present"); + + actix::spawn(async move { + loop { + match input_rx.recv().await { + Ok(event) => addr.do_send(IncomingNetEvent(event)), + Err(RecvError::Lagged(_)) => continue, + Err(RecvError::Closed) => break, + } + } + }); + } +} + +#[derive(Message)] +#[rtype(result = "()")] +struct IncomingNetEvent(NetEvent); + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: IncomingNetEvent, _: &mut Self::Context) { + trap(EType::Net, &self.bus.clone(), || { + match &mut self.state { + NetState::Syncing(buffer) => buffer.push(msg.0), + NetState::Running => { + self.forward_event(msg.0)?; + } + } + Ok(()) + }) + } +} + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + self.handle_enclave_event(msg) + }) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::events::{GossipData, NetEvent}; + use e3_ciphernode_builder::EventSystem; + use e3_events::EventPublisher; + use tokio::{ + sync::broadcast, + time::{sleep, timeout}, + }; + + #[actix::test] + async fn test_buffers_until_sync_ended() -> Result<()> { + // Setup + let system = EventSystem::new("test").with_fresh_bus(); + let bus = system.handle()?; + let (input_tx, input_rx) = broadcast::channel(16); + let mut output_rx = NetEventBuffer::setup(&bus, &input_rx); + + // Send events while syncing - should be buffered + let event1 = NetEvent::GossipData(GossipData::GossipBytes(vec![1, 2, 3])); + let event2 = NetEvent::GossipData(GossipData::GossipBytes(vec![4, 5, 6])); + input_tx.send(event1.clone()).unwrap(); + input_tx.send(event2.clone()).unwrap(); + + // Give actor time to process + sleep(Duration::from_millis(10)).await; + + // Verify no events forwarded yet (should timeout) + assert!( + timeout(Duration::from_millis(50), output_rx.recv()) + .await + .is_err(), + "Events should be buffered, not forwarded during sync" + ); + + // Send SyncEnded event + bus.publish_without_context(SyncEnded::new()).unwrap(); + + // Now buffered events should be forwarded + let received1 = output_rx.recv().await.unwrap(); + let received2 = output_rx.recv().await.unwrap(); + + assert!( + matches!(received1, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![1, 2, 3]) + ); + assert!( + matches!(received2, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![4, 5, 6]) + ); + + // Send new event after sync - should forward immediately + let event3 = NetEvent::GossipData(GossipData::GossipBytes(vec![7, 8, 9])); + input_tx.send(event3.clone()).unwrap(); + + let received3 = + tokio::time::timeout(tokio::time::Duration::from_millis(100), output_rx.recv()) + .await + .expect("Event should be forwarded immediately after sync") + .unwrap(); + + assert!( + matches!(received3, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![7, 8, 9]) + ); + + Ok(()) + } +} diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 962827d5bf..e5f641ec76 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -1,5 +1,4 @@ // SPDX-License-Identifier: LGPL-3.0-only -// // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. @@ -7,6 +6,8 @@ use crate::events::GossipData; use crate::events::NetCommand; use crate::events::NetEvent; +use crate::net_event_buffer::NetEventBuffer; +use crate::net_sync_manager::NetSyncManager; use crate::DocumentPublisher; use crate::NetInterface; /// Actor for connecting to an libp2p client via it's mpsc channel interface @@ -22,9 +23,13 @@ use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::Event; use e3_events::EventContextAccessors; +use e3_events::EventSource; +use e3_events::EventStoreQueryBy; use e3_events::EventType; +use e3_events::TsAgg; use e3_events::Unsequenced; use e3_events::{CorrelationId, EnclaveEvent, EventId}; +use e3_utils::MAILBOX_LIMIT; use libp2p::identity::ed25519; use std::collections::HashSet; use std::sync::Arc; @@ -47,11 +52,14 @@ pub struct NetEventTranslator { impl Actor for NetEventTranslator { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } /// Libp2pEvent is used to send data to the NetInterface from the NetEventTranslator #[derive(Message, Clone, Debug, PartialEq, Eq)] -#[rtype(result = "anyhow::Result<()>")] +#[rtype(result = "()")] struct LibP2pEvent(pub GossipData); impl NetEventTranslator { @@ -110,96 +118,71 @@ impl NetEventTranslator { } } - /// Spawn a Libp2p interface and hook it up to this actor - #[instrument(name = "libp2p", skip_all)] - pub async fn setup_with_interface( - bus: BusHandle, - peers: Vec, - cipher: &Arc, - quic_port: u16, - repository: Repository>, - ) -> Result<( - Addr, - Option>, - tokio::task::JoinHandle>, - String, - )> { - // TODO: We should separate NetInterface from NetEventTranslator - let topic = "tmp-enclave-gossip-topic"; - // Get existing keypair or generate a new one - let mut bytes = match repository.read().await? { - Some(bytes) => { - info!("Found keypair in repository"); - cipher.decrypt_data(&bytes)? - } - None => bail!("No network keypair found in repository, please generate a new one using `enclave net generate-key`"), - }; - - // Create peer from keypair - let keypair: libp2p::identity::Keypair = - ed25519::Keypair::try_from_bytes(&mut bytes)?.try_into()?; - let mut interface = NetInterface::new(&keypair, peers, Some(quic_port), topic)?; - - // Setup and start net event translator - let rx = &Arc::new(interface.rx()); - let addr = NetEventTranslator::setup(&bus, &interface.tx(), rx, topic); - let maybe_publisher = Some(DocumentPublisher::setup(&bus, &interface.tx(), rx, topic)); - - let handle = tokio::spawn(async move { Ok(interface.start().await?) }); - - Ok(( - addr, - maybe_publisher, - handle, - keypair.public().to_peer_id().to_string(), - )) + fn process_gossip_event(&mut self, msg: EnclaveEvent) -> Result<()> { + // if we have seen this event before dont rebroadcast + let id = msg.event_id(); + if self.sent_events.contains(&id) { + trace!(evt_id=%id,"Have seen event before not rebroadcasting!"); + return Ok(()); + } + + warn!("GossipPublish event: {}", msg.event_type()); + let topic = self.topic.clone(); + let data: GossipData = msg.try_into()?; + + self.tx.try_send(NetCommand::GossipPublish { + topic, + data, + correlation_id: CorrelationId::new(), + })?; + + Ok(()) } -} -impl Handler for NetEventTranslator { - type Result = anyhow::Result<()>; - fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { - let LibP2pEvent(data) = msg; + fn handle_enclave_event(&mut self, msg: EnclaveEvent) -> Result<()> { + // Ignore events that should be considered local + if !Self::is_forwardable_event(&msg) { + let id = msg.event_id(); + trace!(evt_id=%id,"Local events should not be rebroadcast so ignoring"); + return Ok(()); + } + + self.process_gossip_event(msg)?; + + Ok(()) + } + + fn publish_event(&mut self, event: EnclaveEvent) -> Result<()> { + let id = event.id(); + let (data, ec) = event.into_components(); + self.bus + .publish_from_remote(data, ec.ts(), None, EventSource::Net)?; + self.sent_events.insert(id); + Ok(()) + } + + fn handle_remote_event(&mut self, msg: LibP2pEvent) -> Result<()> { + let data = msg.0; let event: EnclaveEvent = data.try_into()?; - self.sent_events.insert(event.id()); - let (data, ts) = event.split(); - self.bus.publish_from_remote(data, ts)?; + self.publish_event(event)?; Ok(()) } } -impl Handler for NetEventTranslator { +impl Handler for NetEventTranslator { type Result = (); - fn handle(&mut self, event: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { - let sent_events = self.sent_events.clone(); - let tx = self.tx.clone(); - let evt = event.clone(); - let topic = self.topic.clone(); - let id: EventId = evt.clone().into(); - - // Ignore events that should be considered local - if !Self::is_forwardable_event(&evt) { - trace!(evt_id=%id,"Local events should not be rebroadcast so ignoring"); - return Ok(()); - } - - // if we have seen this event before dont rebroadcast - if sent_events.contains(&id) { - trace!(evt_id=%id,"Have seen event before not rebroadcasting!"); - return Ok(()); - } - - warn!("GossipPublish event: {}", event.event_type()); - let data: GossipData = evt.try_into()?; - - tx.try_send(NetCommand::GossipPublish { - topic, - data, - correlation_id: CorrelationId::new(), - })?; + self.handle_remote_event(msg) + }) + } +} - Ok(()) +impl Handler for NetEventTranslator { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + self.handle_enclave_event(msg) }) } } diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index e32fba0f59..3f5ee0053f 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -27,12 +27,13 @@ use libp2p::{ StreamProtocol, Swarm, }; use rand::prelude::IteratorRandom; -use std::sync::atomic::AtomicBool; use std::{ collections::HashMap, + fmt::Display, sync::{atomic::Ordering, Arc}, time::Instant, }; +use std::{hash::Hash, sync::atomic::AtomicBool}; use std::{io::Error, time::Duration}; use tokio::{select, sync::broadcast, sync::mpsc}; @@ -43,8 +44,8 @@ const MAX_KADEMLIA_PAYLOAD_MB: usize = 10; const MAX_GOSSIP_MSG_SIZE_KB: usize = 700; use crate::events::{ - GossipData, NetCommand, OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, - SyncResponseValue, + estimate_hashmap_size, GossipData, NetCommand, OutgoingSyncRequestFailed, + OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, SyncResponseValue, }; use crate::events::{NetEvent, PutOrStoreError}; use crate::{dialer::dial_peers, Cid}; @@ -145,7 +146,7 @@ impl NetInterface { let peers = self.peers.clone(); async move { dial_peers(&cmd_tx, &event_tx, &peers).await?; - + event_tx.send(NetEvent::AllPeersDialed)?; return anyhow::Ok(()); } }); @@ -193,12 +194,15 @@ fn create_behaviour( gossipsub::MessageAuthenticity::Signed(key.clone()), gossipsub_config, )?; + let request_response_config = + request_response::Config::default().with_request_timeout(Duration::from_secs(30)); + let sync = CborRequestResponse::::new( [( StreamProtocol::new("/enclave/sync/0.0.1"), ProtocolSupport::Full, )], - request_response::Config::default(), + request_response_config, ); let mut config = KademliaConfig::new(PROTOCOL_NAME); config @@ -389,6 +393,8 @@ async fn process_swarm_event( }, .. })) => { + info!("***INCOMING REQUEST RECEIVED {}!!***", request_id); + // received a request for events event_tx.send(NetEvent::SyncRequestReceived(SyncRequestReceived { request_id, @@ -398,15 +404,69 @@ async fn process_swarm_event( } SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { - message: RequestResponseMessage::Response { response, .. }, + message: + RequestResponseMessage::Response { + request_id, + response, + .. + }, .. })) => { + info!("***OUTGOING SYNC REQUEST RESPONSE***{}", request_id); // received a response to a request for events + let correlation_id = correlator.expire(request_id)?; + info!("correlation_id = {correlation_id}"); event_tx.send(NetEvent::OutgoingSyncRequestSucceeded( - OutgoingSyncRequestSucceeded { value: response }, + OutgoingSyncRequestSucceeded { + value: response, + correlation_id, + }, ))?; } + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync( + RequestResponseEvent::OutboundFailure { + peer, + request_id, + error, + }, + )) => { + info!( + "***OUTBOUND SYNC REQUEST FAILURE*** peer={}, request_id={}, error={:?}", + peer, request_id, error + ); + let correlation_id = correlator.expire(request_id)?; + event_tx.send(NetEvent::OutgoingSyncRequestFailed( + OutgoingSyncRequestFailed { + correlation_id, + error: format!("Outbound sync request failed: {:?}", error), + }, + ))?; + } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::InboundFailure { + peer, + request_id, + error, + })) => { + info!( + "***INBOUND SYNC REQUEST FAILURE*** peer={}, request_id={}, error={:?}", + peer, request_id, error + ); + // Handle inbound failures - log for now, could add more sophisticated error handling if needed + } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::ResponseSent { + peer, + request_id, + })) => { + info!( + "***SYNC RESPONSE SENT*** peer={}, request_id={}", + peer, request_id + ); + // Successfully sent response to inbound request + } + unknown => { trace!("Unknown event: {:?}", unknown); } @@ -452,7 +512,10 @@ async fn process_swarm_command( key, } => handle_get_record(swarm, correlator, correlation_id, key), NetCommand::Shutdown => handle_shutdown(swarm, shutdown_flag), - NetCommand::OutgoingSyncRequest { value } => handle_outgoing_sync_request(swarm, value), + NetCommand::OutgoingSyncRequest { + correlation_id, + value, + } => handle_outgoing_sync_request(swarm, correlator, correlation_id, value), NetCommand::SyncResponse { value, channel } => handle_sync_response(swarm, channel, value), } } @@ -581,8 +644,11 @@ fn handle_shutdown( fn handle_outgoing_sync_request( swarm: &mut Swarm, + correlator: &mut Correlator, + correlation_id: CorrelationId, value: SyncRequestValue, ) -> Result<()> { + info!("***OUTGOING SYNC REQUEST!! {} ***", correlation_id); // TODO: // This is a first pass. // Lots of stuff to work through here: @@ -600,8 +666,15 @@ fn handle_outgoing_sync_request( bail!("No peer found on swarm!") }; + info!("VALUE SIZE: {:?}", estimate_hashmap_size(&value.since)); + // Request events - swarm.behaviour_mut().sync.send_request(&peer, value); + let query_id = swarm.behaviour_mut().sync.send_request(&peer, value); + info!( + "QueryId={} correlated with correlation_id={}", + query_id, correlation_id + ); + correlator.track(query_id, correlation_id); Ok(()) } @@ -610,11 +683,13 @@ fn handle_sync_response( channel: OnceTake>, value: SyncResponseValue, ) -> Result<()> { + info!("Sending response..."); let channel = channel.try_take()?; match swarm.behaviour_mut().sync.send_response(channel, value) { Ok(_) => (), - Err(_res) => { + Err(value) => { // TODO: report failure + error!("sync sending response failure! {:?}", value); } } Ok(()) @@ -623,7 +698,7 @@ fn handle_sync_response( /// This correlates query_id and correlation_id. #[derive(Clone)] struct Correlator { - inner: HashMap, + inner: HashMap, } impl Correlator { @@ -633,13 +708,13 @@ impl Correlator { } } /// Add a pairing between query_id and correlation_id - pub fn track(&mut self, query_id: QueryId, correlation_id: CorrelationId) { - self.inner.insert(query_id, correlation_id); + pub fn track(&mut self, query_id: impl Display, correlation_id: CorrelationId) { + self.inner.insert(format!("{query_id}"), correlation_id); } /// Remove the pairing and return the correlation_id - pub fn expire(&mut self, query_id: &QueryId) -> Result { + pub fn expire(&mut self, query_id: impl Display) -> Result { self.inner - .remove(query_id) + .remove(&format!("{query_id}")) .ok_or_else(|| anyhow::anyhow!("Failed to correlate query_id={}", query_id)) } } diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs index 7d6c9a5a21..3ac9d8beec 100644 --- a/crates/net/src/net_sync_manager.rs +++ b/crates/net/src/net_sync_manager.rs @@ -4,22 +4,22 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; +use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; use anyhow::{anyhow, bail, Result}; use e3_events::{ prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, - EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, - ReceiveEvents, Unsequenced, + EnclaveEventData, EventSource, EventStoreQueryBy, EventStoreQueryResponse, EventType, + HistoricalNetSyncStart, NetSyncEventsReceived, TsAgg, TypedEvent, Unsequenced, }; -use e3_utils::{retry_with_backoff, to_retry, OnceTake}; +use e3_utils::{retry_with_backoff, to_retry, OnceTake, MAILBOX_LIMIT}; use futures::TryFutureExt; use libp2p::request_response::ResponseChannel; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::sync::{broadcast, mpsc}; -use tracing::debug; +use tracing::{debug, info}; use crate::events::{ - call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, + await_event, call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, SyncResponseValue, }; @@ -28,11 +28,11 @@ pub struct NetSyncManager { bus: BusHandle, /// NetCommand sender to forward commands to the NetInterface tx: mpsc::Sender, - /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so - /// that we do not do excessive resubscribes without actually listening for events. + /// NetEvents receiver to receive events rx: Arc>, - eventstore: Recipient, + eventstore: Recipient>, requests: HashMap>>, + peers_ready: bool, } impl NetSyncManager { @@ -40,15 +40,15 @@ impl NetSyncManager { bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, - eventstore: Recipient, + eventstore: Recipient>, ) -> Self { Self { bus: bus.clone(), tx: tx.clone(), - rx: rx.clone(), - + rx: Arc::clone(rx), eventstore, requests: HashMap::new(), + peers_ready: false, } } @@ -56,11 +56,13 @@ impl NetSyncManager { bus: &BusHandle, tx: &mpsc::Sender, rx: &Arc>, - eventstore: Recipient, + eventstore: Recipient>, ) -> Addr { let mut events = rx.resubscribe(); let addr = Self::new(bus, tx, rx, eventstore).start(); + bus.subscribe(EventType::HistoricalNetSyncStart, addr.clone().recipient()); + // Forward from NetEvent tokio::spawn({ debug!("Spawning event receive loop!"); @@ -69,8 +71,9 @@ impl NetSyncManager { while let Ok(event) = events.recv().await { debug!("Received event {:?}", event); match event { - NetEvent::OutgoingSyncRequestSucceeded(value) => addr.do_send(value), + // Someone is asking for our sync NetEvent::SyncRequestReceived(value) => addr.do_send(value), + NetEvent::AllPeersDialed => addr.do_send(AllPeersDialed), _ => (), } } @@ -83,45 +86,72 @@ impl NetSyncManager { impl Actor for NetSyncManager { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } /// Event broadcast from event bus impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::OutgoingSyncRequested(data) => ctx.notify(data), + let (msg, ec) = msg.into_components(); + match msg { + // We are making a sync request of another node + EnclaveEventData::HistoricalNetSyncStart(data) => ctx.notify(TypedEvent::new(data, ec)), _ => (), } } } /// SyncRequest is called on start up to fetch remote events -impl Handler for NetSyncManager { +impl Handler> for NetSyncManager { type Result = ResponseFuture<()>; - fn handle(&mut self, msg: OutgoingSyncRequested, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + info!("HISTORICAL_NET_SYNC_START"); trap_fut( EType::Net, - &self.bus.clone(), - handle_sync_request_event(self.tx.clone(), self.rx.clone(), msg, ctx.address()), + &self.bus.with_ec(msg.get_ctx()), + handle_sync_request_event( + self.tx.clone(), + self.rx.clone(), + msg, + ctx.address(), + !self.peers_ready, + ), ) } } /// We have received the sync response from the remote peer -impl Handler for NetSyncManager { +impl Handler> for NetSyncManager { type Result = (); - fn handle(&mut self, msg: OutgoingSyncRequestSucceeded, _: &mut Self::Context) -> Self::Result { - trap(EType::Net, &self.bus.clone(), || { - self.bus.publish(NetSyncEventsReceived { - events: msg - .value - .events - .iter() - .cloned() - .map(|data| data.try_into()) - .collect::>>>()?, - })?; + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + let (msg, ctx) = msg.into_components(); + self.bus.publish_from_remote_as_response( + NetSyncEventsReceived { + events: msg + .value + .events + .iter() + .cloned() + .map(|data| data.try_into()) + .collect::>>>()?, + }, + msg.value.ts, + ctx, + None, + EventSource::Net, + )?; Ok(()) }); @@ -133,9 +163,12 @@ impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: SyncRequestReceived, ctx: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus, || { + info!("GOT SyncRequestReceived"); let id = CorrelationId::new(); + info!("STORING channel in requests map..."); self.requests.insert(id, msg.channel); - self.eventstore.try_send(GetAggregateEventsAfter::new( + info!("QUERYING eventstore..."); + self.eventstore.try_send(EventStoreQueryBy::::new( id, msg.value.since, ctx.address().recipient(), @@ -146,22 +179,27 @@ impl Handler for NetSyncManager { } /// Receive Events from EventStore -impl Handler for NetSyncManager { +impl Handler for NetSyncManager { type Result = (); - fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { trap(EType::Net, &self.bus.clone(), || { + info!("RECEIVED response from eventstore."); let Some(channel) = self.requests.get(&msg.id()) else { bail!("request not found with {}", msg.id()); }; - + info!( + "GOT CHANNEL: Got channel from request store. NOW sending SyncResponse with channel={:?}", + channel + ); self.tx.try_send(NetCommand::SyncResponse { value: SyncResponseValue { events: msg - .events() + .into_events() .into_iter() - .cloned() + .filter(|e| e.source() == EventSource::Net) .map(|ev| ev.try_into()) .collect::>()?, + ts: self.bus.ts()?, // NOTE: We are storing a local timestamp on this response }, channel: channel.to_owned(), })?; @@ -171,6 +209,18 @@ impl Handler for NetSyncManager { } } +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, _: AllPeersDialed, _: &mut Self::Context) -> Self::Result { + info!("Received handler: All peers dialed"); + self.peers_ready = true; + } +} + +#[derive(Message)] +#[rtype(result = "()")] +struct AllPeersDialed; + const SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); async fn sync_request( @@ -178,10 +228,13 @@ async fn sync_request( net_events: Arc>, since: HashMap, ) -> Result { + info!("RUNNING sync request..."); + let id = CorrelationId::new(); call_and_await_response( net_cmds, net_events, NetCommand::OutgoingSyncRequest { + correlation_id: id, value: SyncRequestValue { since }, }, |e| match e.clone() { @@ -199,11 +252,35 @@ async fn sync_request( async fn handle_sync_request_event( net_cmds: mpsc::Sender, net_events: Arc>, - event: OutgoingSyncRequested, - address: impl Into>, + event: TypedEvent, + address: impl Into>>, + wait_for_event: bool, ) -> Result<()> { + info!("Sync request event received"); + let (event, ctx) = event.into_components(); + info!("Waiting for peers to have been contacted..."); + if wait_for_event { + await_event( + &net_events, + |e| { + if matches!(e, &NetEvent::AllPeersDialed) { + info!("AllPeersDialed matched!"); + Some(e.clone()) + } else { + None + } + }, + Duration::from_secs(30), + ) + .await?; + } + info!("handle_sync_request_event: All peers have been dialed."); + + // Make the sync request + // value returned includes the timestamp from the remote peer let value = retry_with_backoff( || { + info!("Running SYNC REQUEST!!"); sync_request( net_cmds.clone(), net_events.clone(), @@ -212,10 +289,11 @@ async fn handle_sync_request_event( .map_err(to_retry) }, 4, - 1000, + 5000, ) .await?; - address.into().try_send(value)?; + // send the sync request succeeded to ourselves + address.into().try_send(TypedEvent::new(value, ctx))?; Ok(()) } diff --git a/crates/net/src/repo.rs b/crates/net/src/repo.rs index a4f2c518a4..43840164da 100644 --- a/crates/net/src/repo.rs +++ b/crates/net/src/repo.rs @@ -4,8 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; +use e3_events::StoreKeys; pub trait NetRepositoryFactory { fn libp2p_keypair(&self) -> Repository>; diff --git a/crates/request/src/repo.rs b/crates/request/src/repo.rs index 455c6de9b0..e076348895 100644 --- a/crates/request/src/repo.rs +++ b/crates/request/src/repo.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; use e3_events::E3id; +use e3_events::StoreKeys; use crate::{E3ContextSnapshot, E3Meta, E3RouterSnapshot}; diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index e52a316dba..d7a0a52b7e 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -27,6 +27,7 @@ use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::EventType; use e3_events::{E3id, EnclaveEvent, Event}; +use e3_utils::MAILBOX_LIMIT; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; @@ -145,12 +146,15 @@ impl E3Router { impl Actor for E3Router { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for E3Router { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Event, &self.bus.clone(), || { + trap(EType::Event, &self.bus.with_ec(msg.get_ctx()), || { // If we are shutting down then bail on anything else if let EnclaveEventData::Shutdown(_) = msg.get_data() { for (_, ctx) in self.contexts.iter() { @@ -184,8 +188,8 @@ impl Handler for E3Router { } context.forward_message(&msg, &mut self.buffer); - - match msg.into_data() { + let (data, ctx) = msg.into_components(); + match data { EnclaveEventData::PlaintextAggregated(_) => { // Here we are detemining that by receiving the PlaintextAggregated event our request is // complete and we can notify everyone. This might change as we consider other factors @@ -195,7 +199,7 @@ impl Handler for E3Router { }; // Send to bus so all other actors can react to a request being complete. - self.bus.publish(event)?; + self.bus.publish(event, ctx)?; } EnclaveEventData::E3RequestComplete(_) => { // Note this will be sent above to the children who can kill themselves based on diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 7245c10b69..dd3eed23a8 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -4,18 +4,20 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::WithSortitionPartyTicket; +use crate::WithSortitionTicket; use actix::prelude::*; use anyhow::bail; use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::E3RequestComplete; +use e3_events::TypedEvent; use e3_events::{ prelude::*, trap, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, E3id, EType, EnclaveEvent, EnclaveEventData, EventType, Shutdown, TicketGenerated, TicketId, }; use e3_request::E3Meta; use e3_utils::NotifySync; +use e3_utils::MAILBOX_LIMIT; use std::collections::HashMap; use tracing::info; @@ -29,6 +31,9 @@ pub struct CiphernodeSelector { impl Actor for CiphernodeSelector { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl CiphernodeSelector { @@ -64,27 +69,30 @@ impl CiphernodeSelector { impl Handler for CiphernodeSelector { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), - EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CommitteeFinalized(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } } -impl Handler> for CiphernodeSelector { +impl Handler>> for CiphernodeSelector { type Result = (); fn handle( &mut self, - data: WithSortitionPartyTicket, - _ctx: &mut Self::Context, + data: WithSortitionTicket>, + _: &mut Self::Context, ) -> Self::Result { - let bus = self.bus.clone(); - - trap(EType::Sortition, &bus.clone(), || { - self.e3_cache.try_mutate(|mut cache| { + trap(EType::Sortition, &self.bus.with_ec(data.get_ctx()), || { + self.e3_cache.try_mutate(data.get_ctx(), |mut cache| { info!( "Mutating e3_cache: appending data: {:?}", data.e3_id.clone() @@ -107,18 +115,20 @@ impl Handler> for CiphernodeSelector { info!(node = &data.address(), "Ciphernode was not selected"); return Ok(()); } - if let Some(tid) = data.ticket_id() { info!( node = &data.address(), ticket_id = tid, "Ticket generated for score sortition" ); - bus.publish(TicketGenerated { - e3_id: data.e3_id.clone(), - ticket_id: TicketId::Score(tid), - node: data.address().to_owned(), - })?; + self.bus.publish( + TicketGenerated { + e3_id: data.e3_id.clone(), + ticket_id: TicketId::Score(tid), + node: data.address().to_owned(), + }, + data.get_ctx().to_owned(), + )?; } Ok(()) @@ -126,72 +136,93 @@ impl Handler> for CiphernodeSelector { } } -impl Handler for CiphernodeSelector { +impl Handler> for CiphernodeSelector { type Result = (); - fn handle(&mut self, msg: E3RequestComplete, _: &mut Self::Context) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), move || { - self.e3_cache.try_mutate(|mut cache| { - cache.remove(&msg.e3_id); - Ok(cache) - }) - }) + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::Sortition, + &self.bus.with_ec(msg.get_ctx()), + move || { + self.e3_cache.try_mutate(msg.get_ctx(), |mut cache| { + cache.remove(&msg.e3_id); + Ok(cache) + }) + }, + ) } } -impl Handler for CiphernodeSelector { +impl Handler> for CiphernodeSelector { type Result = (); - fn handle(&mut self, msg: CommitteeFinalized, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), move || { - info!("CiphernodeSelector received CommitteeFinalized."); - let bus = self.bus.clone(); - info!("Getting e3_cache..."); - let Some(e3_cache) = self.e3_cache.get() else { - bail!("Could not get cache"); - }; - - info!("Getting e3_meta..."); - let Some(e3_meta) = e3_cache.get(&msg.e3_id) else { - bail!( - "Could not find E3Meta on CiphernodeSelector for {}", - msg.e3_id - ); - }; - - // Check if this node is in the finalized committee - if !msg.committee.contains(&self.address) { - info!(node = self.address, "Node not in finalized committee"); - return Ok(()); - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::Sortition, + &self.bus.with_ec(msg.get_ctx()), + move || { + let (msg, ec) = msg.into_components(); + info!("CiphernodeSelector received CommitteeFinalized."); + let bus = self.bus.clone(); + info!("Getting e3_cache..."); + let Some(e3_cache) = self.e3_cache.get() else { + bail!("Could not get cache"); + }; + + info!("Getting e3_meta..."); + let Some(e3_meta) = e3_cache.get(&msg.e3_id) else { + bail!( + "Could not find E3Meta on CiphernodeSelector for {}", + msg.e3_id + ); + }; + + // Check if this node is in the finalized committee + if !msg.committee.contains(&self.address) { + info!(node = self.address, "Node not in finalized committee"); + return Ok(()); + } + + // Retrieve E3 metadata from repository + let Some(party_id) = msg.committee.iter().position(|addr| addr == &self.address) + else { + info!( + node = self.address, + "Node address not found in committee list (should not happen)" + ); + return Ok(()); + }; - // Retrieve E3 metadata from repository - let Some(party_id) = msg.committee.iter().position(|addr| addr == &self.address) else { info!( node = self.address, - "Node address not found in committee list (should not happen)" + party_id = party_id, + "Node is in finalized committee, emitting CiphernodeSelected" ); - return Ok(()); - }; - - info!( - node = self.address, - party_id = party_id, - "Node is in finalized committee, emitting CiphernodeSelected" - ); - - bus.publish(CiphernodeSelected { - party_id: party_id as u64, - e3_id: msg.e3_id, - threshold_m: e3_meta.threshold_m, - threshold_n: e3_meta.threshold_n, - esi_per_ct: e3_meta.esi_per_ct, - error_size: e3_meta.error_size.clone(), - params: e3_meta.params.clone(), - seed: e3_meta.seed, - })?; - Ok(()) - }) + bus.publish( + CiphernodeSelected { + party_id: party_id as u64, + e3_id: msg.e3_id, + threshold_m: e3_meta.threshold_m, + threshold_n: e3_meta.threshold_n, + esi_per_ct: e3_meta.esi_per_ct, + error_size: e3_meta.error_size.clone(), + params: e3_meta.params.clone(), + seed: e3_meta.seed, + }, + ec, + )?; + + Ok(()) + }, + ) } } diff --git a/crates/sortition/src/repo.rs b/crates/sortition/src/repo.rs index 75a92d0ccb..ffa3b84cc9 100644 --- a/crates/sortition/src/repo.rs +++ b/crates/sortition/src/repo.rs @@ -6,9 +6,8 @@ use crate::backends::SortitionBackend; use crate::sortition::NodeStateStore; -use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use e3_events::E3id; +use e3_events::{E3id, StoreKeys}; use e3_request::E3Meta; use std::collections::HashMap; diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index adfab6c8ac..56063df067 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: LGPL-3.0-only +// SPDX-License-Identifier: LGPL-4.0-only // // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY @@ -9,15 +9,16 @@ use crate::ticket_sortition; use crate::CiphernodeSelector; use actix::prelude::*; use alloy::primitives::U256; -use anyhow::Result; +use anyhow::{anyhow, Result}; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ - prelude::*, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, + prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, ConfigurationUpdated, E3Failed, E3Requested, E3Stage, E3StageChanged, EType, EnclaveEvent, - EventType, OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, + EventContext, EventType, OperatorActivationChanged, PlaintextOutputPublished, Seed, Sequenced, + TicketBalanceUpdated, TypedEvent, }; use e3_events::{BusHandle, E3id, EnclaveEventData}; -use e3_utils::NotifySync; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::ops::Deref; @@ -96,23 +97,15 @@ impl NodeStateStore { } } -/// Message: request the current set of registered node addresses for `chain_id`. -#[derive(Message, Clone, Debug)] -#[rtype(result = "Vec")] -pub struct GetNodes { - /// Target chain. - pub chain_id: u64, -} - #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "()")] -pub struct WithSortitionPartyTicket { +pub struct WithSortitionTicket { inner: T, party_ticket_id: Option<(u64, Option)>, address: String, } -impl WithSortitionPartyTicket { +impl WithSortitionTicket { pub fn new(inner: T, party_ticket_id: Option<(u64, Option)>, address: &str) -> Self { Self { inner, @@ -138,27 +131,77 @@ impl WithSortitionPartyTicket { } } -impl Deref for WithSortitionPartyTicket { +impl Deref for WithSortitionTicket { type Target = T; fn deref(&self) -> &Self::Target { &self.inner } } -/// Message to get the finalized committee nodes for a specific E3. -#[derive(Message, Clone, Debug)] -#[rtype(result = "Vec")] -pub struct GetNodesForE3 { - /// E3 ID to get nodes for. - pub e3_id: e3_events::E3id, - /// Chain ID - pub chain_id: u64, +#[derive(Message, Clone, Debug, PartialEq, Eq)] +#[rtype(result = "()")] +pub struct E3CommitteeContainsRequest +where + T: Send + Sync, +{ + inner: T, + e3_id: E3id, + node: String, + sender: Recipient>, } -/// Message to get the current node state. -#[derive(Message, Clone, Debug)] -#[rtype(result = "Option>")] -pub struct GetNodeState; +impl E3CommitteeContainsRequest +where + T: Send + Sync, +{ + pub fn new( + e3_id: E3id, + node: String, + inner: T, + sender: impl Into>>, + ) -> Self { + Self { + inner, + e3_id, + node, + sender: sender.into(), + } + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq)] +#[rtype(result = "()")] +pub struct E3CommitteeContainsResponse { + inner: T, + is_found_in_committee: bool, +} + +impl E3CommitteeContainsResponse +where + T: Send + Sync, +{ + pub fn new(inner: T, is_found_in_committee: bool) -> Self { + Self { + inner, + is_found_in_committee, + } + } + + pub fn is_found_in_committee(&self) -> bool { + self.is_found_in_committee + } + + pub fn into_inner(self) -> T { + self.inner + } +} + +impl Deref for E3CommitteeContainsResponse { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.inner + } +} /// Sortition actor that manages the sortition algorithm and the node state. pub struct Sortition { @@ -219,7 +262,7 @@ impl Sortition { let node_state = node_state_store.load_or_default(HashMap::new()).await?; let finalized_committees = committees_store.load_or_default(HashMap::new()).await?; - backends.try_mutate(|mut list| { + backends.try_mutate_without_context(|mut list| { list.insert(u64::MAX, default_backend); Ok(list) })?; @@ -260,10 +303,10 @@ impl Sortition { let map = self .backends .get() - .ok_or_else(|| anyhow::anyhow!("Could not get backends cache"))?; + .ok_or_else(|| anyhow!("Could not get backends cache"))?; let backend = map .get(&chain_id) - .ok_or_else(|| anyhow::anyhow!("No backend for chain_id {}", chain_id))?; + .ok_or_else(|| anyhow!("No backend for chain_id {}", chain_id))?; Ok(backend.nodes()) } @@ -288,9 +331,34 @@ impl Sortition { }) } + fn get_committe(&self, e3_id: &E3id) -> Vec { + self.finalized_committees + .get() + .and_then(|committees| committees.get(e3_id).cloned()) + .unwrap_or_else(|| Vec::new()) + } + + fn committee_contains(&mut self, e3_id: E3id, node: String) -> bool { + let committee = self.get_committe(&e3_id); + + if committee.len() == 0 { + // Non blocking error + self.bus.err( + EType::Sortition, + anyhow!("No finalized committee found for E3 {}", e3_id), + ); + } + + committee.contains(&node) + } /// Helper method to decrement active jobs for an E3's committee - fn decrement_jobs_for_e3(&mut self, e3_id: &E3id, reason: &str) { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { + fn decrement_jobs_for_e3( + &mut self, + e3_id: &E3id, + reason: &str, + ec: EventContext, + ) -> Result<()> { + self.node_state.try_mutate(&ec, |mut state_map| { let chain_id = e3_id.chain_id(); let e3_id_str = format!("{}:{}", chain_id, e3_id.e3_id()); @@ -328,46 +396,61 @@ impl Sortition { } Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } + }) } } impl Actor for Sortition { type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for Sortition { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CiphernodeAdded(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CiphernodeRemoved(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::TicketBalanceUpdated(data) => self.notify_sync(ctx, data.clone()), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), + EnclaveEventData::CiphernodeAdded(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CiphernodeRemoved(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::TicketBalanceUpdated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } EnclaveEventData::OperatorActivationChanged(data) => { - self.notify_sync(ctx, data.clone()) + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::ConfigurationUpdated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CommitteePublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::PlaintextOutputPublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::CommitteeFinalized(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) } - EnclaveEventData::ConfigurationUpdated(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CommitteePublished(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::PlaintextOutputPublished(data) => self.notify_sync(ctx, data.clone()), - EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data.clone()), _ => (), } } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: E3Requested, _ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { let e3_id = msg.e3_id.clone(); let chain_id = msg.e3_id.chain_id(); let seed = msg.seed; let threshold_m = msg.threshold_m; let threshold_n = msg.threshold_n; - let buffer = ticket_sortition::calculate_buffer_size(threshold_m, threshold_n); let total_selection_size = threshold_n + buffer; @@ -380,290 +463,312 @@ impl Handler for Sortition { "Performing Sortition with buffer" ); - self.ciphernode_selector - .do_send(WithSortitionPartyTicket::new( - msg, - self.get_node_index(e3_id, seed, total_selection_size, chain_id), - &self.address, - )) - } -} - -impl Handler for Sortition { - type Result = (); - - fn handle(&mut self, msg: CiphernodeAdded, _ctx: &mut Self::Context) -> Self::Result { - let chain_id = msg.chain_id; - let addr = msg.address.clone(); - - if let Err(err) = self.node_state.try_mutate(|mut state_map| { - let chain_state = state_map - .entry(chain_id) - .or_insert_with(NodeStateStore::default); - chain_state - .nodes - .entry(addr.clone()) - .or_insert_with(NodeState::default); - Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } - - if let Err(err) = self.backends.try_mutate(move |mut list_map| { - let default_backend = list_map - .get(&u64::MAX) - .cloned() - .unwrap_or_else(|| SortitionBackend::score()); - - list_map - .entry(chain_id) - .or_insert_with(|| default_backend) - .add(addr); - Ok(list_map) - }) { - self.bus.err(EType::Sortition, err); - } - - info!(address = %msg.address, chain_id = chain_id, "Node added to sortition state"); + self.ciphernode_selector.do_send(WithSortitionTicket::new( + msg, + self.get_node_index(e3_id, seed, total_selection_size, chain_id), + &self.address, + )) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CiphernodeRemoved, _ctx: &mut Self::Context) -> Self::Result { - let chain_id = msg.chain_id; - let addr = msg.address.clone(); + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + let chain_id = msg.chain_id; + let addr = msg.address.clone(); - if let Err(err) = self.node_state.try_mutate(|mut state_map| { - if let Some(chain_state) = state_map.get_mut(&chain_id) { - chain_state.nodes.remove(&addr); - } - Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } - - if let Err(err) = self.backends.try_mutate(move |mut list_map| { - if let Some(backend) = list_map.get_mut(&chain_id) { - backend.remove(addr); - } - Ok(list_map) - }) { - self.bus.err(EType::Sortition, err); - } - - info!(address = %msg.address, chain_id = chain_id, "Node removed from sortition state"); + self.node_state.try_mutate(&ec, |mut state_map| { + let chain_state = state_map + .entry(chain_id) + .or_insert_with(NodeStateStore::default); + chain_state + .nodes + .entry(addr.clone()) + .or_insert_with(NodeState::default); + Ok(state_map) + })?; + self.backends.try_mutate(&ec, move |mut list_map| { + let default_backend = list_map + .get(&u64::MAX) + .cloned() + .unwrap_or_else(|| SortitionBackend::score()); + + list_map + .entry(chain_id) + .or_insert_with(|| default_backend) + .add(addr); + Ok(list_map) + })?; + info!(address = %msg.address, chain_id = chain_id, "Node added to sortition state"); + Ok(()) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: TicketBalanceUpdated, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { - let chain_state = state_map - .entry(msg.chain_id) - .or_insert_with(NodeStateStore::default); - let node = chain_state - .nodes - .entry(msg.operator.clone()) - .or_insert_with(NodeState::default); - node.ticket_balance = msg.new_balance; - - info!( - operator = %msg.operator, - chain_id = msg.chain_id, - new_balance = ?msg.new_balance, - "Updated ticket balance" - ); - - Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + let chain_id = msg.chain_id; + let addr = msg.address.clone(); + + self.node_state.try_mutate(&ec, |mut state_map| { + if let Some(chain_state) = state_map.get_mut(&chain_id) { + chain_state.nodes.remove(&addr); + } + Ok(state_map) + })?; + self.backends.try_mutate(&ec, move |mut list_map| { + if let Some(backend) = list_map.get_mut(&chain_id) { + backend.remove(addr); + } + Ok(list_map) + })?; + info!(address = %msg.address, chain_id = chain_id, "Node removed from sortition state"); + Ok(()) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: OperatorActivationChanged, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { - // Update all entries for this operator across all chains - for (_, chain_state) in state_map.iter_mut() { + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.node_state.try_mutate(&ec, |mut state_map| { + let chain_state = state_map + .entry(msg.chain_id) + .or_insert_with(NodeStateStore::default); let node = chain_state .nodes .entry(msg.operator.clone()) .or_insert_with(NodeState::default); - - node.active = msg.active; + node.ticket_balance = msg.new_balance; info!( operator = %msg.operator, - active = msg.active, - "Updated operator active status" + chain_id = msg.chain_id, + new_balance = ?msg.new_balance, + "Updated ticket balance" ); - } - Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } + + Ok(state_map) + }) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: ConfigurationUpdated, _ctx: &mut Self::Context) -> Self::Result { - if msg.parameter == "ticketPrice" { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { - let chain_state = state_map - .entry(msg.chain_id) - .or_insert_with(NodeStateStore::default); - chain_state.ticket_price = msg.new_value; - info!( - chain_id = msg.chain_id, - old_ticket_price = ?msg.old_value, - new_ticket_price = ?msg.new_value, - "ConfigurationUpdated - ticket price updated" - ); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.node_state.try_mutate(&ec, |mut state_map| { + // Update all entries for this operator across all chains + for (_, chain_state) in state_map.iter_mut() { + let node = chain_state + .nodes + .entry(msg.operator.clone()) + .or_insert_with(NodeState::default); + + node.active = msg.active; + + info!( + operator = %msg.operator, + active = msg.active, + "Updated operator active status" + ); + } Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } - } + }) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CommitteePublished, _ctx: &mut Self::Context) -> Self::Result { - if let Err(err) = self.node_state.try_mutate(|mut state_map| { - let chain_id = msg.e3_id.chain_id(); - let e3_id_str = format!("{}:{}", chain_id, msg.e3_id.e3_id()); - let chain_state = state_map - .entry(chain_id) - .or_insert_with(NodeStateStore::default); - - chain_state - .e3_committees - .insert(e3_id_str.clone(), msg.nodes.clone()); - - for node_addr in &msg.nodes { - let node = chain_state - .nodes - .entry(node_addr.clone()) - .or_insert_with(NodeState::default); - node.active_jobs += 1; - - info!( - node = %node_addr, - chain_id = chain_id, - e3_id = ?msg.e3_id, - active_jobs = node.active_jobs, - "Incremented active jobs for node in committee" - ); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + if msg.parameter == "ticketPrice" { + self.node_state.try_mutate(&ec, |mut state_map| { + let chain_state = state_map + .entry(msg.chain_id) + .or_insert_with(NodeStateStore::default); + chain_state.ticket_price = msg.new_value; + info!( + chain_id = msg.chain_id, + old_ticket_price = ?msg.old_value, + new_ticket_price = ?msg.new_value, + "ConfigurationUpdated - ticket price updated" + ); + Ok(state_map) + })?; } - - Ok(state_map) - }) { - self.bus.err(EType::Sortition, err); - } + Ok(()) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: PlaintextOutputPublished, _ctx: &mut Self::Context) -> Self::Result { - self.decrement_jobs_for_e3(&msg.e3_id, "PlaintextOutputPublished"); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.node_state.try_mutate(&ec, |mut state_map| { + let chain_id = msg.e3_id.chain_id(); + let e3_id_str = format!("{}:{}", chain_id, msg.e3_id.e3_id()); + let chain_state = state_map + .entry(chain_id) + .or_insert_with(NodeStateStore::default); + + chain_state + .e3_committees + .insert(e3_id_str.clone(), msg.nodes.clone()); + + for node_addr in &msg.nodes { + let node = chain_state + .nodes + .entry(node_addr.clone()) + .or_insert_with(NodeState::default); + node.active_jobs += 1; + + info!( + node = %node_addr, + chain_id = chain_id, + e3_id = ?msg.e3_id, + active_jobs = node.active_jobs, + "Incremented active jobs for node in committee" + ); + } + + Ok(state_map) + }) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition +where + T: Clone + Send + Sync + 'static, +{ type Result = (); - - fn handle(&mut self, msg: E3Failed, _ctx: &mut Self::Context) -> Self::Result { - let reason = format!("E3Failed: {:?}", msg.reason); - self.decrement_jobs_for_e3(&msg.e3_id, &reason); + fn handle( + &mut self, + msg: E3CommitteeContainsRequest, + _: &mut Self::Context, + ) -> Self::Result { + trap(EType::Sortition, &self.bus.clone(), || { + let response = E3CommitteeContainsResponse::new( + msg.inner, + self.committee_contains(msg.e3_id, msg.node), + ); + msg.sender.try_send(response)?; + Ok(()) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: E3StageChanged, _ctx: &mut Self::Context) -> Self::Result { - match msg.new_stage { - E3Stage::Complete | E3Stage::Failed => { - let reason = format!("E3StageChanged to {:?}", msg.new_stage); - self.decrement_jobs_for_e3(&msg.e3_id, &reason); - } - _ => { - // Non-terminal stages, no action needed - } - } + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.decrement_jobs_for_e3(&msg.e3_id, "PlaintextOutputPublished", ec) + }) } } -impl Handler for Sortition { +impl Handler> for Sortition { type Result = (); - fn handle(&mut self, msg: CommitteeFinalized, _ctx: &mut Self::Context) -> Self::Result { - info!( - e3_id = %msg.e3_id, - committee_size = msg.committee.len(), - "Storing finalized committee" - ); - - if let Err(err) = self.finalized_committees.try_mutate(|mut committees| { - committees.insert(msg.e3_id.clone(), msg.committee.clone()); - Ok(committees) - }) { - self.bus.err(EType::Sortition, err); - } + fn handle(&mut self, msg: TypedEvent, _ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + let reason = format!("E3Failed: {:?}", msg.reason); + self.decrement_jobs_for_e3(&msg.e3_id, &reason, ec) + }) } } -impl Handler for Sortition { - type Result = Vec; +impl Handler> for Sortition { + type Result = (); - fn handle(&mut self, msg: GetNodes, _ctx: &mut Self::Context) -> Self::Result { - self.get_nodes(msg.chain_id).unwrap_or_else(|err| { - tracing::warn!("Failed to get nodes for chain {}: {}", msg.chain_id, err); - Vec::new() + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + match msg.new_stage { + E3Stage::Complete | E3Stage::Failed => { + let reason = format!("E3StageChanged to {:?}", msg.new_stage); + self.decrement_jobs_for_e3(&msg.e3_id, &reason, ec)?; + } + _ => { + // Non-terminal stages, no action needed + } + } + Ok(()) }) } } -impl Handler for Sortition { - type Result = Vec; +impl Handler> for Sortition { + type Result = (); - fn handle(&mut self, msg: GetNodesForE3, _ctx: &mut Self::Context) -> Self::Result { - if msg.e3_id.chain_id() != msg.chain_id { - tracing::warn!( - "Chain ID mismatch: e3_id has chain_id {}, but requested chain_id {}", - msg.e3_id.chain_id(), - msg.chain_id + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + info!( + e3_id = %msg.e3_id, + committee_size = msg.committee.len(), + "Storing finalized committee" ); - return Vec::new(); - } - self.finalized_committees - .get() - .and_then(|committees| committees.get(&msg.e3_id).cloned()) - .unwrap_or_else(|| { - tracing::warn!("No finalized committee found for E3 {}", msg.e3_id); - Vec::new() + self.finalized_committees.try_mutate(&ec, |mut committees| { + committees.insert(msg.e3_id.clone(), msg.committee.clone()); + Ok(committees) }) - } -} - -impl Handler for Sortition { - type Result = Option>; - - fn handle(&mut self, _msg: GetNodeState, _: &mut Self::Context) -> Self::Result { - self.node_state.get() + }) } } diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index 147d026610..268d7b1aa3 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -10,8 +10,12 @@ repository = "https://github.com/gnosisguild/enclave/crates/sync" actix.workspace = true anyhow.workspace = true e3-events.workspace = true +e3-data.workspace = true +e3-config.workspace = true +e3-utils.workspace = true tokio.workspace = true tracing.workspace = true [dev-dependencies] -e3-ciphernode-builder.workspace = true \ No newline at end of file +e3-ciphernode-builder.workspace = true +e3-test-helpers.workspace = true diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index 07b21af2e0..6d0d970c05 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +mod repo; mod sync; +pub use repo::*; pub use sync::*; diff --git a/crates/sync/src/repo.rs b/crates/sync/src/repo.rs new file mode 100644 index 0000000000..a8f763a122 --- /dev/null +++ b/crates/sync/src/repo.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use e3_data::{Repositories, Repository}; +use e3_events::AggregateId; +use e3_events::StoreKeys; + +pub trait SyncRepositoryFactory { + fn aggregate_seq(&self, aggregate_id: AggregateId) -> Repository; + fn aggregate_block(&self, aggregate_id: AggregateId) -> Repository; + fn aggregate_ts(&self, aggregate_id: AggregateId) -> Repository; +} + +impl SyncRepositoryFactory for Repositories { + fn aggregate_seq(&self, aggregate_id: AggregateId) -> Repository { + Repository::new(self.store.scope(StoreKeys::aggregate_seq(aggregate_id))) + } + + fn aggregate_block(&self, aggregate_id: AggregateId) -> Repository { + Repository::new(self.store.scope(StoreKeys::aggregate_block(aggregate_id))) + } + + fn aggregate_ts(&self, aggregate_id: AggregateId) -> Repository { + Repository::new(self.store.scope(StoreKeys::aggregate_ts(aggregate_id))) + } +} diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs index dd2736ae6d..a81eed2737 100644 --- a/crates/sync/src/sync.rs +++ b/crates/sync/src/sync.rs @@ -4,246 +4,249 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::HashSet; - -use actix::{Actor, Addr, AsyncContext, Handler, Message}; -use anyhow::{Context, Result}; +use crate::SyncRepositoryFactory; +use actix::{Message, Recipient}; +use anyhow::Result; +use e3_data::Repositories; use e3_events::{ - trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, - SyncStart, + AggregateConfig, AggregateId, BusHandle, CorrelationId, EffectsEnabled, EnclaveEvent, + EventContextAccessors, EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, + EvmEventConfig, EvmEventConfigChain, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, + HistoricalNetEventsReceived, HistoricalNetSyncStart, SeqAgg, SyncEnded, Unsequenced, +}; +use e3_utils::actix::channel as actix_toolbox; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + time::Duration, }; +use tokio::{sync::mpsc::Receiver, time::timeout}; use tracing::info; -// NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p -type ChainId = u64; - -/// Manage the synchronization of events across. -pub struct Synchronizer { - bus: BusHandle, - evm_config: Option, - evm_buffer: Vec, - evm_to_sync: HashSet, - // net_config: NetEventConfig, -} - -impl Synchronizer { - pub fn new(bus: &BusHandle, evm_config: EvmEventConfig) -> Self { - let evm_to_sync = evm_config.chains(); - Self { - evm_config: Some(evm_config), - bus: bus.clone(), - evm_buffer: Vec::new(), - evm_to_sync, - } +pub async fn sync( + bus: &BusHandle, + default_config: &EvmEventConfig, + repositories: &Repositories, + aggregate_config: &AggregateConfig, + eventstore: &Recipient>, +) -> Result<()> { + // 1. Load snapsshot metadata + info!("Loading snapshot metadata..."); + let snapshot = + SnapshotMeta::read_from_disk(aggregate_config.aggregates(), default_config, repositories) + .await?; + info!( + "Snapshot metadata loaded for {} aggregates.", + snapshot.aggregates().len() + ); + + // 2. Determine the evm blocks to read from based on the SnapshotMeta + let evm_config = snapshot.to_evm_config(); + let _net_config = snapshot.to_net_config(); + + // 3. Load EventStore events since the sequence number found in the snapshot into memory. + info!("Loading EventStore events..."); + let (addr, rx) = actix_toolbox::oneshot::(); + eventstore.try_send(EventStoreQueryBy::::new( + CorrelationId::new(), + snapshot.to_sequence_map(), + addr, + ))?; + let events = rx.await?.into_events(); + info!("{} EventStore events loaded.", events.len()); + + info!("Replaying events to actors..."); + // 4. Replay the EventStore events to all listeners (except effects) + for event in events { + bus.event_bus().try_send(event)?; } - - pub fn setup(bus: &BusHandle, evm_config: EvmEventConfig) -> Addr { - Self::new(bus, evm_config).start() + info!("Events replayed."); + + // TODO: Detect open loops - incase we crashed in the middle of a request we need to play the + // request event again once effects are on + + // 5. Load the historical evm events to memory from all chains + info!("Loading historical blockchain events..."); + let (addr, rx) = actix_toolbox::mpsc::(256); + bus.publish_without_context(HistoricalEvmSyncStart::new(addr, evm_config.clone()))?; + let historical_evm_events = + collect_historical_evm_events(rx, &evm_config, Duration::from_secs(30)).await; + info!( + "{} historical blockchain events loaded.", + historical_evm_events.len() + ); + + // XXX: Skipping as we have bugs in libp2p netevent requests + // 6. Load the historical libp2p events to memory + // info!("Loading historical libp2p events..."); + // let (addr, rx) = actix_toolbox::oneshot::(); + // bus.publish_without_context(HistoricalNetSyncStart::new(addr, net_config.clone()))?; + // let historical_net_events = rx.await?.events; + // info!( + // "{} historical libp2p events loaded.", + // historical_net_events.len() + // ); + + // 7. Sort both the evm and libp2p events together by HLC timestamp + let mut historical = historical_evm_events + .into_iter() + // .chain(historical_net_events) // Commenting out to skip + .collect::>(); + + historical.sort_by_key(|event| event.ts()); + info!("Historical events sorted."); + + // 8. Enable effects + bus.publish_without_context(EffectsEnabled::new())?; + info!("Effects enabled"); + + // 9. Publish the new sorted events to the eventstore + info!("Publishing historical events to actors..."); + for event in historical { + bus.naked_dispatch(event); } + info!("Historical events published."); - fn buffer_evm_event(&mut self, event: EvmEvent) { - info!("buffer evm event({})", event.get_id()); - self.evm_buffer.push(event); - } + bus.publish_without_context(SyncEnded::new())?; + info!("Sync finished."); + // normal live operations - fn handle_sync_complete(&mut self, chain_id: u64) -> Result<()> { - info!("handle sync complete for chain({})", chain_id); - self.evm_to_sync.remove(&chain_id); - info!("{} chains left to sync...", self.evm_to_sync.len()); - if self.evm_to_sync.is_empty() { - self.handle_sync_end()?; - } - Ok(()) - } + Ok(()) +} - fn handle_sync_end(&mut self) -> Result<()> { - info!("all chains synced draining to bus and running sync end"); - // Order all events (theoretically) - self.evm_buffer.sort_by_key(|i| i.ts()); +pub async fn collect_historical_evm_events( + mut receiver: Receiver, + config: &EvmEventConfig, + max_dur: Duration, +) -> Vec> { + // Get expected chain IDs from config + let expected = config.chains(); + let mut received = HashSet::new(); + let mut results = Vec::new(); + + let fut = async { + while received.len() < expected.len() { + if let Some(mut msg) = receiver.recv().await { + // Only accept messages for expected chains we haven't received yet + if expected.contains(&msg.chain_id) && !received.contains(&msg.chain_id) { + received.insert(msg.chain_id); + results.append(&mut msg.events); + } + } else { + break; + } + } + }; - // publish them in order - for evt in self.evm_buffer.drain(..) { - let (data, _, _) = evt.split(); - self.bus.publish(data)?; // Use publish here as historical events will be correctly - // ordered as part of the preparatory process + if let Err(_) = timeout(max_dur, fut).await { + for chain_id in expected.difference(&received) { + eprintln!( + "Error: Timeout waiting for historical events from chain {}", + chain_id + ); } - self.bus.publish(SyncEnd::new())?; - Ok(()) } -} -impl Actor for Synchronizer { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.notify(Bootstrap); - } + results } -impl Handler for Synchronizer { - type Result = (); - fn handle(&mut self, msg: SyncEvmEvent, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::Sync, &self.bus.clone(), || { - match msg { - // Buffer events as the sync actor receives them - SyncEvmEvent::Event(event) => self.buffer_evm_event(event), - // When we hear that sync is complete send all events on chain then publish SyncEnd - SyncEvmEvent::HistoricalSyncComplete(chain_id) => { - self.handle_sync_complete(chain_id)? - } - }; - Ok(()) - }) - } +/// Latest event information in store +#[derive(Clone)] +pub struct AggregateState { + ts: u128, + aggregate_id: AggregateId, + seq: u64, + block: u64, } -impl Handler for Synchronizer { - type Result = (); - fn handle(&mut self, _: Bootstrap, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Sync, &self.bus.clone(), || { - let evm_config = self.evm_config.take().context( - "EvmEventConfig was not set likely Bootstrap was called more than once.", - )?; - - // TODO: Get information about what has and has not been synced then fire SyncStart - self.bus.publish(SyncStart::new(ctx.address(), evm_config)) - }) - } +#[derive(Clone)] +pub struct SnapshotMeta { + aggregate_state: Vec, } -#[derive(Message)] -#[rtype("()")] -pub struct Bootstrap; - -#[cfg(test)] -mod tests { - use super::*; - use e3_ciphernode_builder::EventSystem; - use e3_events::EnclaveEvent; - use e3_events::{ - CorrelationId, EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, - TestEvent, - }; - use std::time::Duration; - use tokio::time::sleep; - - fn hlc_faucet(bus: &BusHandle, num: usize) -> Result> { - let mut queue = Vec::new(); - for _ in 0..num { - queue.push(bus.ts()?) +impl SnapshotMeta { + /// Load the SnapshotMeta from the Snapshot on disk + pub async fn read_from_disk( + ids: Vec, + initial_evm_config: &EvmEventConfig, + repositories: &Repositories, + ) -> Result { + let mut aggregate_state = Vec::new(); + for aggregate_id in ids { + let deploy_block = aggregate_id + .to_chain_id() + .and_then(|chain_id| initial_evm_config.deploy_block(chain_id)) + .unwrap_or(0); + let seq_repo = repositories.aggregate_seq(aggregate_id); + let block_repo = repositories.aggregate_block(aggregate_id); + let ts_repo = repositories.aggregate_ts(aggregate_id); + let seq = seq_repo.read().await?.unwrap_or(0); + let block = block_repo.read().await?.unwrap_or(deploy_block); + let ts = ts_repo.read().await?.unwrap_or(0); + let agg_state = AggregateState { + aggregate_id, + seq, + block, + ts, + }; + aggregate_state.push(agg_state); } - Ok(queue.into_iter()) + Ok(Self { aggregate_state }) } - async fn settle() { - sleep(Duration::from_millis(100)).await; - } - - #[actix::test] - async fn test_synchronizer_full_flow() -> Result<()> { - // Setup event system and synchronizer - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - - // Configure test chains - let mut evm_config = EvmEventConfig::new(); - evm_config.insert(1, EvmEventConfigChain::new(0)); - evm_config.insert(2, EvmEventConfigChain::new(0)); - - // Start synchronizer - let sync_addr = Synchronizer::setup(&bus, evm_config); - settle().await; - - // Verify SyncStart was published - let history = history_collector - .send(GetEvents::::new()) - .await?; - let sync_start_count = history - .into_iter() - .filter(|e| matches!(e.get_data(), EnclaveEventData::SyncStart(_))) - .count(); - assert!(sync_start_count > 0, "SyncStart should be dispatched"); - - // Create test events with timestamps - let mut timelord = hlc_faucet(&bus, 100)?; - let (chain_1, chain_2) = (1, 2); - let (block_1, block_2) = (1, 2); - - // Test events - timestamps generated in order - let h_2_1 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), - EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), - block_1, - timelord.next().unwrap(), - chain_2, - )); - - let h_1_1 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), - EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), - block_1, - timelord.next().unwrap(), - chain_1, - )); - - let h_1_2 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), - EnclaveEventData::TestEvent(TestEvent::new("1-second", 1)), - block_2, - timelord.next().unwrap(), - chain_1, - )); - - let h_2_2 = SyncEvmEvent::Event(EvmEvent::new( - CorrelationId::new(), - EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), - block_2, - timelord.next().unwrap(), - chain_2, - )); - - // Chain completion signals - let hc_1 = SyncEvmEvent::HistoricalSyncComplete(chain_1); - let hc_2 = SyncEvmEvent::HistoricalSyncComplete(chain_2); - - // Send events in mixed order to test sorting - sync_addr.send(h_2_2).await?; - sync_addr.send(h_2_1).await?; - sync_addr.send(hc_2).await?; - sync_addr.send(h_1_1).await?; - sync_addr.send(h_1_2).await?; - sync_addr.send(hc_1).await?; - - settle().await; - - // Get final event history and verify ordering - let history = history_collector - .send(GetEvents::::new()) - .await?; - - let events: Vec = history - .into_iter() - .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) - .collect(); - - let event_strings: Vec = events - .into_iter() - .filter_map(|e| { - if let EnclaveEventData::TestEvent(data) = e.into_data() { - Some(data.msg) + /// Return an EvmEventConfig based on the SnapshotMeta + pub fn to_evm_config(&self) -> EvmEventConfig { + let map: BTreeMap = self + .aggregate_state + .iter() + .map(|s| (s.aggregate_id.to_chain_id(), s.block)) + .filter_map(|s| { + if let Some(chain) = s.0 { + Some((chain, EvmEventConfigChain::new(s.1))) } else { None } }) .collect(); + EvmEventConfig::from_config(map) + } - // Events should be published in timestamp order - assert_eq!( - event_strings, - vec!["2-first", "1-first", "1-second", "2-second"] - ); + pub fn to_net_config(&self) -> BTreeMap { + self.aggregate_state + .iter() + .map(|s| (s.aggregate_id, s.ts)) + .collect() + } - Ok(()) + /// Return a map between AggregateIds and Sequence + pub fn to_sequence_map(&self) -> HashMap { + self.aggregate_state + .iter() + .fold(HashMap::new(), |mut acc, item| { + acc.insert(item.aggregate_id, item.seq); + acc + }) + } + + pub fn aggregates(&self) -> Vec { + self.aggregate_state + .iter() + .map(|s| s.aggregate_id) + .collect() + } +} + +#[derive(Message)] +#[rtype("()")] +pub struct Bootstrap; + +#[derive(Message)] +#[rtype("()")] +pub struct SnapshotLoaded { + pub snapshot: SnapshotMeta, +} +impl SnapshotLoaded { + pub fn new(snapshot: SnapshotMeta) -> Self { + Self { snapshot } } } diff --git a/crates/test-helpers/Cargo.toml b/crates/test-helpers/Cargo.toml index 10c882f257..6cf492e796 100644 --- a/crates/test-helpers/Cargo.toml +++ b/crates/test-helpers/Cargo.toml @@ -35,3 +35,4 @@ rand = { workspace = true } rand_chacha = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 8d38c7b8f4..16ef1612f8 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -32,6 +32,7 @@ use rand::Rng; use rand_chacha::rand_core::SeedableRng; use rand_chacha::ChaCha20Rng; use std::sync::Arc; +use tracing::trace; pub use utils::*; pub fn create_shared_rng_from_u64(value: u64) -> Arc> { @@ -142,7 +143,7 @@ pub fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { || DocumentPublisher::is_document_publisher_event(e) }); } else { - println!("not piping bus to itself"); + trace!("Source = Dest! Not piping bus to itself"); } } } @@ -182,7 +183,7 @@ impl AddToCommittee { self.count += 1; - self.bus.publish(evt.clone())?; + self.bus.publish_without_context(evt.clone())?; Ok(evt.into()) } diff --git a/crates/test-helpers/src/plaintext_writer.rs b/crates/test-helpers/src/plaintext_writer.rs index 943d0a7961..3dafe48959 100644 --- a/crates/test-helpers/src/plaintext_writer.rs +++ b/crates/test-helpers/src/plaintext_writer.rs @@ -10,6 +10,7 @@ use super::write_file_with_dirs; use actix::{Actor, Addr, Context, Handler}; use e3_bfv_client::decode_bytes_to_vec_u64; use e3_events::{prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventType}; +use e3_utils::MAILBOX_LIMIT; use tracing::{error, info}; pub struct PlaintextWriter { @@ -29,6 +30,9 @@ impl PlaintextWriter { impl Actor for PlaintextWriter { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for PlaintextWriter { diff --git a/crates/test-helpers/src/public_key_writer.rs b/crates/test-helpers/src/public_key_writer.rs index e10655d3b5..69298a3eb8 100644 --- a/crates/test-helpers/src/public_key_writer.rs +++ b/crates/test-helpers/src/public_key_writer.rs @@ -11,6 +11,7 @@ use actix::{Actor, Addr, Context, Handler}; use e3_events::{ prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, }; +use e3_utils::MAILBOX_LIMIT; use tracing::info; pub struct PublicKeyWriter { @@ -30,6 +31,9 @@ impl PublicKeyWriter { impl Actor for PublicKeyWriter { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler for PublicKeyWriter { diff --git a/crates/test-helpers/src/utils.rs b/crates/test-helpers/src/utils.rs index ffa6a5a132..2f1f532b8b 100644 --- a/crates/test-helpers/src/utils.rs +++ b/crates/test-helpers/src/utils.rs @@ -6,7 +6,18 @@ use std::{fmt::Debug, fs, io::Write, path::PathBuf}; -use tracing::{error, trace}; +use tracing::{error, subscriber::DefaultGuard, trace}; +use tracing_subscriber::{fmt, EnvFilter}; + +/// Use this at the top of a test to include tracing +pub fn with_tracing(level: &str) -> DefaultGuard { + tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new(level)) + .with_test_writer() + .finish(), + ) +} pub fn write_file_with_dirs(path: &PathBuf, content: &[u8]) -> std::io::Result<()> { let abs_path = if path.is_absolute() { diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index cfdd240155..b8b299a806 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -23,7 +23,9 @@ use e3_net::events::{GossipData, NetEvent}; use e3_net::NetEventTranslator; use e3_sortition::{calculate_buffer_size, RegisteredNode, ScoreSortition, Ticket}; use e3_test_helpers::ciphernode_system::CiphernodeSystemBuilder; -use e3_test_helpers::{create_seed_from_u64, create_shared_rng_from_u64, AddToCommittee}; +use e3_test_helpers::{ + create_seed_from_u64, create_shared_rng_from_u64, with_tracing, AddToCommittee, +}; use e3_trbfv::helpers::calculate_error_size; use e3_utils::rand_eth_addr; use e3_utils::utility_types::ArcBytes; @@ -207,7 +209,7 @@ async fn setup_score_sortition_environment( eth_addrs: &Vec, chain_id: u64, ) -> Result<()> { - bus.publish(ConfigurationUpdated { + bus.publish_without_context(ConfigurationUpdated { parameter: "ticketPrice".to_string(), old_value: U256::ZERO, new_value: U256::from(10_000_000u64), @@ -218,7 +220,7 @@ async fn setup_score_sortition_environment( for addr in eth_addrs { adder.add(addr).await?; - bus.publish(TicketBalanceUpdated { + bus.publish_without_context(TicketBalanceUpdated { operator: addr.clone(), delta: I256::try_from(1_000_000_000u64).unwrap(), new_balance: U256::from(1_000_000_000u64), @@ -226,7 +228,7 @@ async fn setup_score_sortition_environment( chain_id, })?; - bus.publish(OperatorActivationChanged { + bus.publish_without_context(OperatorActivationChanged { operator: addr.clone(), active: true, chain_id, @@ -260,14 +262,8 @@ async fn test_trbfv_actor() -> Result<()> { println!("Running test_trbfv_actor..."); let mut report: Vec<(&str, Duration)> = vec![]; let whole_test = Instant::now(); - use tracing_subscriber::{fmt, EnvFilter}; - - let subscriber = fmt() - .with_env_filter(EnvFilter::new("info")) - .with_test_writer() - .finish(); - let _guard = tracing::subscriber::set_default(subscriber); + let _guard = with_tracing("info"); // NOTE: Here we are trying to make it as clear as possible as to what is going on so attempting to // avoid over abstracting test helpers and favouring straight forward single descriptive @@ -347,6 +343,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_pubkey_aggregation() .with_sortition_score() .with_threshold_plaintext_aggregation() + .testmode_start_buffer_immediately() .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() @@ -364,6 +361,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_zkproof(zk_backend.clone()) .testmode_with_signer(PrivateKeySigner::random()) .with_sortition_score() + .testmode_start_buffer_immediately() .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() @@ -426,11 +424,7 @@ async fn test_trbfv_actor() -> Result<()> { params, }; - println!( - "Publishing E3Requested: e3_id={}, threshold={}/{}", - e3_id, threshold_m, threshold_n - ); - bus.publish(e3_requested)?; + bus.publish_without_context(e3_requested)?; sleep(Duration::from_millis(500)).await; @@ -454,7 +448,7 @@ async fn test_trbfv_actor() -> Result<()> { .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; - bus.publish(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: e3_id.clone(), committee: committee.clone(), chain_id, @@ -572,7 +566,7 @@ async fn test_trbfv_actor() -> Result<()> { e3_id: e3_id.clone(), }; - bus.publish(ciphertext_published_event.clone())?; + bus.publish_without_context(ciphertext_published_event.clone())?; println!("CiphertextOutputPublished event has been dispatched!"); @@ -709,9 +703,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { ..CiphernodeSelected::default() }; - bus.publish(evt_1.clone())?; - bus.publish(evt_2.clone())?; - bus.publish(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network + bus.publish_without_context(evt_1.clone())?; + bus.publish_without_context(evt_2.clone())?; + bus.publish_without_context(local_evt_3.clone())?; // This is a local event which should not be broadcast to the network // check the history of the event bus let history = history_collector @@ -863,7 +857,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { }; // Send e3request - bus.publish(E3Requested { + bus.publish_without_context(E3Requested { e3_id: e3_id.clone(), threshold_m: 2, threshold_n: 2, @@ -872,7 +866,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { ..E3Requested::default() })?; - bus.publish(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: e3_id.clone(), committee: eth_addrs.clone(), chain_id: 1, @@ -888,7 +882,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { assert_eq!(errors.len(), 0); // SEND SHUTDOWN! - bus.publish(Shutdown)?; + bus.publish_without_context(Shutdown)?; // This is probably overkill but required to ensure that all the data is written sleep(Duration::from_secs(1)).await; @@ -953,7 +947,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { use e3_test_helpers::encrypt_ciphertext; let raw_plaintext = vec![vec![4, 5]]; let (ciphertext, expected) = encrypt_ciphertext(¶ms, pubkey, raw_plaintext)?; - bus.publish(CiphertextOutputPublished { + bus.publish_without_context(CiphertextOutputPublished { ciphertext_output: ciphertext .iter() .map(|ct| ArcBytes::from_bytes(&ct.to_bytes())) @@ -1100,7 +1094,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { setup_score_sortition_environment(&bus, ð_addrs, 2).await?; // Send the computation requested event - bus.publish(E3Requested { + bus.publish_without_context(E3Requested { e3_id: E3id::new("1234", 1), threshold_m: 2, threshold_n: 5, @@ -1109,7 +1103,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ..E3Requested::default() })?; - bus.publish(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: E3id::new("1234", 1), committee: eth_addrs.clone(), chain_id: 1, @@ -1144,7 +1138,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ); // Send the computation requested event - bus.publish(E3Requested { + bus.publish_without_context(E3Requested { e3_id: E3id::new("1234", 2), threshold_m: 2, threshold_n: 5, @@ -1153,7 +1147,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { ..E3Requested::default() })?; - bus.publish(CommitteeFinalized { + bus.publish_without_context(CommitteeFinalized { e3_id: E3id::new("1234", 2), committee: eth_addrs.clone(), chain_id: 2, diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index bc2563d2eb..b031a29e86 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -12,6 +12,7 @@ alloy.workspace = true anyhow.workspace = true derivative.workspace = true rand.workspace = true +regex.workspace = true rand_chacha.workspace = true serde.workspace = true tokio.workspace = true diff --git a/crates/utils/src/actix/channel.rs b/crates/utils/src/actix/channel.rs new file mode 100644 index 0000000000..e1b93c538d --- /dev/null +++ b/crates/utils/src/actix/channel.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Context, Handler, Message}; +use tokio::sync::{ + mpsc, + oneshot::{self}, +}; + +// Oneshot == +pub struct Oneshot(Option>); + +impl Actor for Oneshot +where + M: Message + Send + 'static, +{ + type Context = actix::Context; +} + +impl Handler for Oneshot +where + M: Message + Send + 'static, +{ + type Result = (); + + fn handle(&mut self, m: M, _: &mut Context) -> Self::Result { + self.0.take().map(|s| s.send(m)); + } +} + +/// Return a oneshot channel where instead of a oneshot::Sender we use an actix::Recipient. +/// After the oneshot has completed sending messages to the address will noop +pub fn oneshot() -> (actix::Recipient, oneshot::Receiver) +where + M: Message + Send + 'static, +{ + let (tx, rx) = oneshot::channel(); + (Oneshot(Some(tx)).start().recipient(), rx) +} + +// Mpsc == + +pub struct Mpsc(mpsc::Sender); + +impl Actor for Mpsc +where + M: Message + Send + 'static, +{ + type Context = actix::Context; +} + +impl Handler for Mpsc +where + M: Message + Send + 'static, +{ + type Result = (); + fn handle(&mut self, m: M, _: &mut Context) -> Self::Result { + let s = self.0.clone(); + actix::spawn(async move { + if let Err(e) = s.send(m).await { + eprintln!("Failed to send message: {}", e); + } + }); + } +} + +/// Return a mpsc channel where instead of a mpsc::Sender we use an actix::Recipient. +pub fn mpsc(buffer: usize) -> (actix::Recipient, mpsc::Receiver) +where + M: Message + Send + 'static, +{ + let (tx, rx) = mpsc::channel(buffer); + (Mpsc(tx).start().recipient(), rx) +} diff --git a/crates/utils/src/actix.rs b/crates/utils/src/actix/mod.rs similarity index 96% rename from crates/utils/src/actix.rs rename to crates/utils/src/actix/mod.rs index 63cd1dbc02..8da657a2d4 100644 --- a/crates/utils/src/actix.rs +++ b/crates/utils/src/actix/mod.rs @@ -4,8 +4,10 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; +pub mod channel; +pub mod oneshot_runner; +use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; // Helper to allow for bail behaviour in actor model async handlers diff --git a/crates/evm/src/one_shot_runnner.rs b/crates/utils/src/actix/oneshot_runner.rs similarity index 94% rename from crates/evm/src/one_shot_runnner.rs rename to crates/utils/src/actix/oneshot_runner.rs index 6a6576774d..bae7c875d9 100644 --- a/crates/evm/src/one_shot_runnner.rs +++ b/crates/utils/src/actix/oneshot_runner.rs @@ -4,6 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use crate::MAILBOX_LIMIT; use actix::prelude::*; use anyhow::Result; use std::marker::PhantomData; @@ -40,6 +41,9 @@ where M: Message + 'static + Unpin, { type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } } impl Handler for OneShotRunner diff --git a/crates/utils/src/constants.rs b/crates/utils/src/constants.rs new file mode 100644 index 0000000000..0800376848 --- /dev/null +++ b/crates/utils/src/constants.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +// NOTE: Currently this is here as everything depends on utils. We could consider moving this +// closer to the configuration if we need to make this dynamic or create a create just for this. + +// Max message +pub const MAILBOX_LIMIT: usize = 256; +pub const MAILBOX_LIMIT_LARGE: usize = 256 * 10; diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs new file mode 100644 index 0000000000..8ffd8c62b1 --- /dev/null +++ b/crates/utils/src/error.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use tracing::error; + +/// Formats panic errors so they are seen in logs clearly +pub fn major_issue(msg: &str, e: impl Into) -> String { + error!("\n\n\nMAJOR ISSUE: {msg}.\n\nThe error supplied was: {:?}\n\n As a precaution we are crashing the system.\n\n\n", e.into()); + format!("System has crashed. Nothing personal. Goodbye.") +} diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index b8094d900b..77eaa3c9c6 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -5,6 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use core::fmt; +use std::sync::LazyLock; + +use regex::Regex; // Custom formatter function for hex display pub fn hexf(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result { @@ -36,6 +39,7 @@ pub fn truncate(s: String) -> String { } } +#[derive(Clone, Copy)] pub enum Color { Black = 30, Red = 31, @@ -58,3 +62,74 @@ pub enum Color { pub fn colorize(s: T, color: Color) -> String { format!("\x1b[{}m{}\x1b[0m", color as u8, s) } + +// Pre-compile the regex for efficiency +static EVENT_ID_RE: LazyLock = + LazyLock::new(|| Regex::new(r"EventId\(0x([a-fA-F0-9]+)\)").unwrap()); + +/// Hashes a string to an ANSI 256 color within `[hue_min, hue_max]` degrees. +/// +/// # Examples +/// ```ignore +/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 300.0); // orange to purple +/// hash_str_to_ansi_color_in_hue_range(s, 30.0, 330.0); // full spectrum, no red +/// hash_str_to_ansi_color_in_hue_range(s, 0.0, 360.0); // full spectrum +/// ``` +fn hash_str_to_ansi_color_in_hue_range(s: &str, hue_min: f32, hue_max: f32) -> u8 { + let hash: u32 = s + .bytes() + .fold(0u32, |acc, b| acc.wrapping_mul(31).wrapping_add(b as u32)); + + let hue = hue_min + (hash as f32 % (hue_max - hue_min)); + + let (r, g, b) = hsv_to_rgb(hue, 1.0, 1.0); + + rgb_to_ansi256(r, g, b) +} + +fn hsv_to_rgb(h: f32, s: f32, v: f32) -> (f32, f32, f32) { + let c = v * s; + let x = c * (1.0 - ((h / 60.0) % 2.0 - 1.0).abs()); + let m = v - c; + + let (r, g, b) = match (h / 60.0) as u32 { + 0 => (c, x, 0.0), + 1 => (x, c, 0.0), + 2 => (0.0, c, x), + 3 => (0.0, x, c), + 4 => (x, 0.0, c), + _ => (c, 0.0, x), + }; + + (r + m, g + m, b + m) +} + +fn rgb_to_ansi256(r: f32, g: f32, b: f32) -> u8 { + let r6 = (r * 5.0).round() as u8; + let g6 = (g * 5.0).round() as u8; + let b6 = (b * 5.0).round() as u8; + + 16 + 36 * r6 + 6 * g6 + b6 +} + +pub fn colorize_event_ids(value: &T) -> String { + let s = format!("{:?}", value); + let mut result = String::with_capacity(s.len() + 100); + let mut last_end = 0; + for cap in EVENT_ID_RE.captures_iter(&s) { + let full_match = cap.get(0).unwrap(); + let hex_str = cap.get(1).unwrap().as_str(); + result.push_str(&s[last_end..full_match.start()]); + let color = hash_str_to_ansi_color_in_hue_range(hex_str, 30.0, 330.0); // Avoiding red so + // it does not look + // like errors + result.push_str(&format!( + "\x1b[38;5;{}m{}\x1b[0m", + color, + full_match.as_str() + )); + last_end = full_match.end(); + } + result.push_str(&s[last_end..]); + result +} diff --git a/crates/utils/src/helpers.rs b/crates/utils/src/helpers.rs index 9380d01abf..fa24bd7c4d 100644 --- a/crates/utils/src/helpers.rs +++ b/crates/utils/src/helpers.rs @@ -8,6 +8,8 @@ use std::{ sync::{Arc, Mutex}, }; +use tracing::info; + pub fn to_ordered_vec(source: HashMap) -> Vec where K: Ord + Copy, @@ -47,6 +49,7 @@ impl OnceTake { /// Takes the item, returning `None` if already taken. pub fn take(&self) -> Option { + info!("take has been called!"); self.0.lock().unwrap().take() } diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index ec17ce7a9c..857daf3a6d 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -6,13 +6,17 @@ pub mod actix; pub mod alloy; +pub mod constants; +pub mod error; pub mod formatters; pub mod helpers; pub mod path; pub mod retry; pub mod utility_types; -pub use actix::*; +pub use actix::NotifySync; pub use alloy::*; +pub use constants::*; +pub use error::*; pub use formatters::*; pub use helpers::*; pub use path::*; diff --git a/crates/utils/src/retry.rs b/crates/utils/src/retry.rs index 700442a1e1..6eb82724e8 100644 --- a/crates/utils/src/retry.rs +++ b/crates/utils/src/retry.rs @@ -9,6 +9,7 @@ use std::{future::Future, time::Duration}; use tokio::time::sleep; use tracing::{error, warn}; +#[derive(Debug)] pub enum RetryError { Failure(anyhow::Error), Retry(anyhow::Error), @@ -46,6 +47,7 @@ where match operation().await { Ok(value) => return Ok(value), Err(re) => { + tracing::error!("RETRY FAILED {:?}", re); match re { RetryError::Retry(e) => { if current_attempt >= max_attempts { diff --git a/crates/zk-prover/src/actors/proof_request.rs b/crates/zk-prover/src/actors/proof_request.rs index 2a7ffe7408..af815b23b8 100644 --- a/crates/zk-prover/src/actors/proof_request.rs +++ b/crates/zk-prover/src/actors/proof_request.rs @@ -12,8 +12,9 @@ use alloy::signers::local::PrivateKeySigner; use e3_events::{ BusHandle, ComputeRequest, ComputeRequestError, ComputeRequestErrorKind, ComputeResponse, ComputeResponseKind, CorrelationId, E3id, EnclaveEvent, EnclaveEventData, EncryptionKey, - EncryptionKeyCreated, EncryptionKeyPending, Event, EventPublisher, EventSubscriber, EventType, - PkBfvProofRequest, ProofPayload, ProofType, SignedProofPayload, ZkRequest, ZkResponse, + EncryptionKeyCreated, EncryptionKeyPending, EventPublisher, EventSubscriber, EventType, + PkBfvProofRequest, ProofPayload, ProofType, SignedProofPayload, TypedEvent, ZkRequest, + ZkResponse, }; use e3_utils::NotifySync; use tracing::{error, info}; @@ -52,7 +53,8 @@ impl ProofRequestActor { addr } - fn handle_encryption_key_pending(&mut self, msg: EncryptionKeyPending) { + fn handle_encryption_key_pending(&mut self, msg: TypedEvent) { + let (msg, ec) = msg.into_components(); let correlation_id = CorrelationId::new(); self.pending.insert( correlation_id, @@ -72,13 +74,14 @@ impl ProofRequestActor { ); info!("Requesting T0 proof generation"); - if let Err(err) = self.bus.publish(request) { + if let Err(err) = self.bus.publish(request, ec) { error!("Failed to publish ZK proof request: {err}"); self.pending.remove(&correlation_id); } } - fn handle_compute_response(&mut self, msg: ComputeResponse) { + fn handle_compute_response(&mut self, msg: TypedEvent) { + let (msg, ec) = msg.into_components(); let ComputeResponseKind::Zk(ZkResponse::PkBfv(resp)) = msg.response else { return; }; @@ -112,16 +115,19 @@ impl ProofRequestActor { } } - if let Err(err) = self.bus.publish(EncryptionKeyCreated { - e3_id: pending.e3_id, - key: Arc::new(key), - external: false, - }) { + if let Err(err) = self.bus.publish( + EncryptionKeyCreated { + e3_id: pending.e3_id, + key: Arc::new(key), + external: false, + }, + ec, + ) { error!("Failed to publish EncryptionKeyCreated: {err}"); } } - fn handle_compute_request_error(&mut self, msg: ComputeRequestError) { + fn handle_compute_request_error(&mut self, msg: TypedEvent) { let ComputeRequestErrorKind::Zk(err) = msg.get_err() else { return; }; @@ -143,35 +149,54 @@ impl Handler for ProofRequestActor { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::EncryptionKeyPending(data) => self.notify_sync(ctx, data), - EnclaveEventData::ComputeResponse(data) => self.notify_sync(ctx, data), - EnclaveEventData::ComputeRequestError(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::EncryptionKeyPending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + EnclaveEventData::ComputeRequestError(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for ProofRequestActor { +impl Handler> for ProofRequestActor { type Result = (); - fn handle(&mut self, msg: EncryptionKeyPending, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { self.handle_encryption_key_pending(msg) } } -impl Handler for ProofRequestActor { +impl Handler> for ProofRequestActor { type Result = (); - fn handle(&mut self, msg: ComputeResponse, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { self.handle_compute_response(msg) } } -impl Handler for ProofRequestActor { +impl Handler> for ProofRequestActor { type Result = (); - fn handle(&mut self, msg: ComputeRequestError, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { self.handle_compute_request_error(msg) } } diff --git a/crates/zk-prover/src/actors/proof_verification.rs b/crates/zk-prover/src/actors/proof_verification.rs index 5b209d3784..c0fe9ae3c4 100644 --- a/crates/zk-prover/src/actors/proof_verification.rs +++ b/crates/zk-prover/src/actors/proof_verification.rs @@ -26,8 +26,8 @@ use actix::{Actor, Addr, AsyncContext, Context, Handler, Message, Recipient}; use alloy::primitives::Address; use e3_events::{ BusHandle, E3Failed, E3Stage, E3id, EnclaveEvent, EnclaveEventData, EncryptionKey, - EncryptionKeyCreated, EncryptionKeyReceived, Event, EventPublisher, EventSubscriber, EventType, - FailureReason, Proof, SignedProofFailed, SignedProofPayload, + EncryptionKeyCreated, EncryptionKeyReceived, EventContext, EventPublisher, EventSubscriber, + EventType, FailureReason, Proof, Sequenced, SignedProofFailed, SignedProofPayload, TypedEvent, }; use e3_utils::NotifySync; use tracing::{error, info, warn}; @@ -39,7 +39,7 @@ pub struct ZkVerificationRequest { pub proof: Proof, pub e3_id: E3id, pub key: Arc, - pub sender: Recipient, + pub sender: Recipient>, } /// Response from ZK proof verification with context. @@ -66,14 +66,14 @@ struct PendingVerification { /// attribution) and [`E3Failed`] (to stop the E3 computation). pub struct ProofVerificationActor { bus: BusHandle, - verifier: Recipient, + verifier: Recipient>, /// Tracks signed payloads for keys currently being verified, /// keyed by `(e3_id, party_id)`. pending: HashMap<(E3id, u64), PendingVerification>, } impl ProofVerificationActor { - pub fn new(bus: &BusHandle, verifier: Recipient) -> Self { + pub fn new(bus: &BusHandle, verifier: Recipient>) -> Self { Self { bus: bus.clone(), verifier, @@ -81,13 +81,21 @@ impl ProofVerificationActor { } } - pub fn setup(bus: &BusHandle, verifier: Recipient) -> Addr { + pub fn setup( + bus: &BusHandle, + verifier: Recipient>, + ) -> Addr { let addr = Self::new(bus, verifier).start(); bus.subscribe(EventType::EncryptionKeyReceived, addr.clone().into()); addr } - fn handle_encryption_key_received(&mut self, msg: EncryptionKeyReceived, ctx: &Context) { + fn handle_encryption_key_received( + &mut self, + msg: TypedEvent, + ctx: &Context, + ) { + let (msg, ec) = msg.into_components(); let Some(ref proof) = msg.key.proof else { error!( "External key from party {} is missing T0 proof - rejecting", @@ -135,22 +143,33 @@ impl ProofVerificationActor { }, ); - let request = ZkVerificationRequest { - proof: proof.clone(), - e3_id: msg.e3_id, - key: msg.key, - sender: ctx.address().recipient(), - }; + let request = TypedEvent::new( + ZkVerificationRequest { + proof: proof.clone(), + e3_id: msg.e3_id, + key: msg.key, + sender: ctx.address().recipient(), + }, + ec, + ); self.verifier.do_send(request); } - fn publish_key_created(&self, e3_id: E3id, key: Arc) { - if let Err(err) = self.bus.publish(EncryptionKeyCreated { - e3_id, - key, - external: true, - }) { + fn publish_key_created( + &self, + e3_id: E3id, + key: Arc, + ec: EventContext, + ) { + if let Err(err) = self.bus.publish( + EncryptionKeyCreated { + e3_id, + key, + external: true, + }, + ec, + ) { error!("Failed to publish EncryptionKeyCreated: {err}"); } } @@ -164,25 +183,37 @@ impl Handler for ProofVerificationActor { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::EncryptionKeyReceived(data) => self.notify_sync(ctx, data), + let (msg, ec) = msg.into_components(); + match msg { + EnclaveEventData::EncryptionKeyReceived(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } } -impl Handler for ProofVerificationActor { +impl Handler> for ProofVerificationActor { type Result = (); - fn handle(&mut self, msg: EncryptionKeyReceived, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { self.handle_encryption_key_received(msg, ctx) } } -impl Handler for ProofVerificationActor { +impl Handler> for ProofVerificationActor { type Result = (); - fn handle(&mut self, msg: ZkVerificationResponse, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); let pending_key = (msg.e3_id.clone(), msg.key.party_id); let pending = self.pending.remove(&pending_key); @@ -191,7 +222,7 @@ impl Handler for ProofVerificationActor { "T0 proof verified for party {} - accepting key", msg.key.party_id ); - self.publish_key_created(msg.e3_id, msg.key); + self.publish_key_created(msg.e3_id, msg.key, ec.clone()); } else { let error_msg = msg.error.unwrap_or_else(|| "unknown error".to_string()); error!( @@ -209,22 +240,28 @@ impl Handler for ProofVerificationActor { "Emitting SignedProofFailed for party {} (address: {recovered_signer})", msg.key.party_id ); - if let Err(err) = self.bus.publish(SignedProofFailed { - e3_id: msg.e3_id.clone(), - faulting_node: recovered_signer, - proof_type: signed_payload.payload.proof_type, - signed_payload, - }) { + if let Err(err) = self.bus.publish( + SignedProofFailed { + e3_id: msg.e3_id.clone(), + faulting_node: recovered_signer, + proof_type: signed_payload.payload.proof_type, + signed_payload, + }, + ec.clone(), + ) { error!("Failed to publish SignedProofFailed: {err}"); } } // Stop the E3 computation — proof verification failure is fatal - if let Err(err) = self.bus.publish(E3Failed { - e3_id: msg.e3_id, - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::VerificationFailed, - }) { + if let Err(err) = self.bus.publish( + E3Failed { + e3_id: msg.e3_id, + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::VerificationFailed, + }, + ec, + ) { error!("Failed to publish E3Failed: {err}"); } } diff --git a/crates/zk-prover/src/actors/zk_actor.rs b/crates/zk-prover/src/actors/zk_actor.rs index a3c3f02c3e..e8a04bee3c 100644 --- a/crates/zk-prover/src/actors/zk_actor.rs +++ b/crates/zk-prover/src/actors/zk_actor.rs @@ -10,6 +10,7 @@ //! This is an IO actor - it performs file system operations. use actix::{Actor, Context, Handler}; +use e3_events::TypedEvent; use tracing::{debug, error}; use crate::{ZkBackend, ZkProver}; @@ -33,10 +34,15 @@ impl Actor for ZkActor { type Context = Context; } -impl Handler for ZkActor { +impl Handler> for ZkActor { type Result = (); - fn handle(&mut self, msg: ZkVerificationRequest, _ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); debug!( "Verifying proof for circuit: {} (party {})", msg.proof.circuit, msg.key.party_id @@ -51,35 +57,38 @@ impl Handler for ZkActor { msg.key.party_id, ); - let response = match result { - Ok(true) => { - debug!("Proof verification successful"); - ZkVerificationResponse { - verified: true, - error: None, - e3_id: msg.e3_id, - key: msg.key, + let response = TypedEvent::new( + match result { + Ok(true) => { + debug!("Proof verification successful"); + ZkVerificationResponse { + verified: true, + error: None, + e3_id: msg.e3_id, + key: msg.key, + } } - } - Ok(false) => { - error!("Proof verification failed"); - ZkVerificationResponse { - verified: false, - error: Some("Verification returned false".to_string()), - e3_id: msg.e3_id, - key: msg.key, + Ok(false) => { + error!("Proof verification failed"); + ZkVerificationResponse { + verified: false, + error: Some("Verification returned false".to_string()), + e3_id: msg.e3_id, + key: msg.key, + } } - } - Err(e) => { - error!("Proof verification error: {}", e); - ZkVerificationResponse { - verified: false, - error: Some(e.to_string()), - e3_id: msg.e3_id, - key: msg.key, + Err(e) => { + error!("Proof verification error: {}", e); + ZkVerificationResponse { + verified: false, + error: Some(e.to_string()), + e3_id: msg.e3_id, + key: msg.key, + } } - } - }; + }, + ec, + ); // Send response back to the sender msg.sender.do_send(response); diff --git a/examples/CRISP/Cargo.lock b/examples/CRISP/Cargo.lock index 55062840c2..ce0b8cd7f7 100644 --- a/examples/CRISP/Cargo.lock +++ b/examples/CRISP/Cargo.lock @@ -2511,6 +2511,7 @@ dependencies = [ "derivative", "rand 0.8.5", "rand_chacha 0.3.1", + "regex", "serde", "tokio", "tracing", @@ -4813,9 +4814,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr",