diff --git a/.husky/pre-commit b/.husky/pre-commit deleted file mode 100644 index cb2c84d5c3..0000000000 --- a/.husky/pre-commit +++ /dev/null @@ -1 +0,0 @@ -pnpm lint-staged diff --git a/Cargo.lock b/Cargo.lock index 657ab2acbf..c542b28c4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2059,6 +2059,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbor4ii" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "472931dd4dfcc785075b09be910147f9c6258883fc4591d0dac6116392b2daa6" +dependencies = [ + "serde", +] + [[package]] name = "cc" version = "1.2.54" @@ -2854,8 +2863,10 @@ dependencies = [ "e3-fhe", "e3-keyshare", "e3-multithread", + "e3-net", "e3-request", "e3-sortition", + "e3-sync", "e3-trbfv", "e3-utils", "once_cell", @@ -3055,12 +3066,14 @@ dependencies = [ "anyhow", "async-trait", "base64", + "bloom", "e3-ciphernode-builder", "e3-config", "e3-crypto", "e3-data", "e3-entrypoint", "e3-events", + "e3-evm", "e3-fhe-params", "e3-sortition", "e3-trbfv", @@ -3070,6 +3083,7 @@ dependencies = [ "serde", "tokio", "tracing", + "tracing-subscriber", "url", "zeroize", ] @@ -3269,6 +3283,7 @@ dependencies = [ "futures", "hex", "libp2p", + "rand 0.8.5", "serde", "sha2", "tokio", @@ -3416,6 +3431,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "e3-sync" +version = "0.1.7" +dependencies = [ + "actix", + "anyhow", + "e3-ciphernode-builder", + "e3-events", + "tokio", + "tracing", +] + [[package]] name = "e3-test-helpers" version = "0.1.7" @@ -5149,6 +5176,7 @@ dependencies = [ "libp2p-metrics", "libp2p-ping", "libp2p-quic", + "libp2p-request-response", "libp2p-swarm", "libp2p-tcp", "libp2p-upnp", @@ -5409,6 +5437,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "libp2p-request-response" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1356c9e376a94a75ae830c42cdaea3d4fe1290ba409a22c809033d1b7dcab0a6" +dependencies = [ + "async-trait", + "cbor4ii", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.8.5", + "serde", + "smallvec", + "tracing", + "void", + "web-time", +] + [[package]] name = "libp2p-swarm" version = "0.45.1" diff --git a/Cargo.toml b/Cargo.toml index 87908abada..d1f9dac7eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "crates/sdk", "crates/sortition", "crates/support-scripts", + "crates/sync", "crates/test-helpers", "crates/tests", "crates/trbfv", @@ -51,6 +52,8 @@ exclude = [ ] resolver = "3" msrv = "1.86.0" + +[workspace.metadata.release] shared-version = true pre-release-commit-message = "chore: Release {{crate_name}} v{{version}}" pre-release-replacements = [ @@ -98,6 +101,7 @@ e3-sortition = { version = "0.1.7", path = "./crates/sortition" } e3-program-server = { version = "0.1.7", path = "./crates/program-server" } e3-polynomial = { version = "0.1.7", path = "./crates/polynomial" } e3-support-scripts = { version = "0.1.7", path = "./crates/support-scripts" } +e3-sync = { version = "0.1.7", path = "./crates/sync" } e3-test-helpers = { version = "0.1.7", path = "./crates/test-helpers" } e3-tests = { version = "0.1.7", path = "./crates/tests" } e3-trbfv = { version = "0.1.7", path = "./crates/trbfv" } @@ -195,6 +199,8 @@ libp2p = { version = "=0.54.1", features = [ "ping", "quic", "tokio", + "request-response", + "cbor" ]} zeroize = "=1.8.1" diff --git a/crates/Dockerfile b/crates/Dockerfile index f09cb97f62..10a6c49ad5 100644 --- a/crates/Dockerfile +++ b/crates/Dockerfile @@ -74,6 +74,7 @@ COPY crates/safe/Cargo.toml ./safe/Cargo.toml COPY crates/sdk/Cargo.toml ./sdk/Cargo.toml COPY crates/sortition/Cargo.toml ./sortition/Cargo.toml COPY crates/support-scripts/Cargo.toml ./support-scripts/Cargo.toml +COPY crates/sync/Cargo.toml ./sync/Cargo.toml COPY crates/test-helpers/Cargo.toml ./test-helpers/Cargo.toml COPY crates/tests/Cargo.toml ./tests/Cargo.toml COPY crates/trbfv/Cargo.toml ./trbfv/Cargo.toml diff --git a/crates/aggregator/src/committee_finalizer.rs b/crates/aggregator/src/committee_finalizer.rs index 9c2b531214..de319534d8 100644 --- a/crates/aggregator/src/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalizer.rs @@ -9,6 +9,7 @@ use e3_events::{ prelude::*, trap, BusHandle, CommitteeFinalizeRequested, CommitteeRequested, EType, EnclaveEvent, EnclaveEventData, EventType, Shutdown, }; +use e3_utils::NotifySync; use std::collections::HashMap; use std::time::Duration; use tracing::{error, info}; @@ -48,8 +49,8 @@ impl Handler for CommitteeFinalizer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::CommitteeRequested(data) => ctx.notify(data), - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::CommitteeRequested(data) => self.notify_sync(ctx, data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/aggregator/src/publickey_aggregator.rs b/crates/aggregator/src/publickey_aggregator.rs index a3ea9d3d04..bfdf3b0195 100644 --- a/crates/aggregator/src/publickey_aggregator.rs +++ b/crates/aggregator/src/publickey_aggregator.rs @@ -12,8 +12,10 @@ use e3_events::{ prelude::*, BusHandle, Die, E3id, EnclaveEvent, EnclaveEventData, KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, }; +use e3_events::{trap, EType}; use e3_fhe::{Fhe, GetAggregatePublicKey}; use e3_utils::ArcBytes; +use e3_utils::NotifySync; use std::sync::Arc; use tracing::{error, info}; @@ -139,11 +141,14 @@ impl Actor for PublicKeyAggregator { impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::KeyshareCreated(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(_) => ctx.notify(Die), - _ => (), - } + trap(EType::KeyGeneration, &self.bus.clone(), || { + match msg.into_data() { + EnclaveEventData::KeyshareCreated(data) => self.notify_sync(ctx, data)?, + EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + _ => (), + }; + Ok(()) + }); } } @@ -163,10 +168,13 @@ impl Handler for PublicKeyAggregator { self.add_keyshare(pubkey, node)?; if let Some(PublicKeyAggregatorState::Computing { keyshares, .. }) = &self.state.get() { - ctx.notify(ComputeAggregate { - keyshares: keyshares.clone(), - e3_id, - }) + self.notify_sync( + ctx, + ComputeAggregate { + keyshares: keyshares.clone(), + e3_id, + }, + )? } Ok(()) @@ -215,6 +223,6 @@ impl Handler for PublicKeyAggregator { impl Handler for PublicKeyAggregator { type Result = (); fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { - ctx.stop() + ctx.stop(); } } diff --git a/crates/aggregator/src/threshold_plaintext_aggregator.rs b/crates/aggregator/src/threshold_plaintext_aggregator.rs index ca861ff876..2345773ea9 100644 --- a/crates/aggregator/src/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/threshold_plaintext_aggregator.rs @@ -20,6 +20,7 @@ use e3_trbfv::{ TrBFVResponse, }; use e3_utils::utility_types::ArcBytes; +use e3_utils::NotifySync; use tracing::{debug, error, info, trace}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -263,8 +264,8 @@ impl Handler for ThresholdPlaintextAggregator { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { EnclaveEventData::DecryptionshareCreated(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(_) => ctx.notify(Die), - EnclaveEventData::ComputeResponse(data) => ctx.notify(data), + EnclaveEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + EnclaveEventData::ComputeResponse(data) => self.notify_sync(ctx, data), _ => (), } } @@ -318,12 +319,15 @@ impl Handler for ThresholdPlaintextAggregator { .. })) = act.state.get() { - ctx.notify(ComputeAggregate { - shares: shares.clone(), - ciphertext_output: ciphertext_output.clone(), - threshold_m, - threshold_n, - }) + act.notify_sync( + ctx, + ComputeAggregate { + shares: shares.clone(), + ciphertext_output: ciphertext_output.clone(), + threshold_m, + threshold_n, + }, + ) } Ok(()) diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 07b940c81e..495eda85f4 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -21,8 +21,10 @@ e3-evm.workspace = true e3-fhe.workspace = true e3-keyshare.workspace = true e3-multithread.workspace = true +e3-net.workspace = true e3-request.workspace = true e3-sortition.workspace = true +e3-sync.workspace = true e3-trbfv.workspace = true e3-utils.workspace = true rayon.workspace = true diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 2e67d36187..c4af8deaca 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -5,16 +5,22 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::Addr; +use anyhow::Result; use e3_data::{DataStore, InMemStore, StoreAddr}; use e3_events::{BusHandle, EnclaveEvent, HistoryCollector}; +use tokio::task::JoinHandle; -#[derive(Clone, Debug)] +/// A Sharable handle to a Ciphernode. NOTE: clones are available for use in the CiphernodeSystem +/// but they cannot await the task. +#[derive(Debug)] pub struct CiphernodeHandle { pub address: String, pub store: DataStore, pub bus: BusHandle, pub history: Option>>, pub errors: Option>>, + pub peer_id: String, + pub join_handle: JoinHandle>, } impl CiphernodeHandle { @@ -24,6 +30,8 @@ impl CiphernodeHandle { bus: BusHandle, history: Option>>, errors: Option>>, + peer_id: String, + join_handle: JoinHandle>, ) -> Self { Self { address, @@ -31,6 +39,8 @@ impl CiphernodeHandle { bus, history, errors, + peer_id, + join_handle, } } @@ -54,6 +64,10 @@ impl CiphernodeHandle { &self.store } + pub fn split(self) -> (BusHandle, JoinHandle>) { + (self.bus, self.join_handle) + } + pub fn in_mem_store(&self) -> Option<&Addr> { let addr = self.store.get_addr(); if let StoreAddr::InMem(ref store) = addr { diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 9f1d522c21..775666a877 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -4,34 +4,29 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{CiphernodeHandle, EventSystem}; +use crate::event_system::AggregateConfig; +use crate::{CiphernodeHandle, EventSystem, EvmSystemChainBuilder, ProviderCache, WriteEnabled}; use actix::{Actor, Addr}; -use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; use anyhow::Result; use derivative::Derivative; use e3_aggregator::ext::{PublicKeyAggregatorExtension, ThresholdPlaintextAggregatorExtension}; +use e3_aggregator::CommitteeFinalizer; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; -use e3_data::{InMemStore, Repositories, RepositoriesFactory}; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig}; -use e3_evm::{ - helpers::{ - load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, - ProviderConfig, - }, - BondingRegistryReaderRepositoryFactory, BondingRegistrySol, - CiphernodeRegistryReaderRepositoryFactory, CiphernodeRegistrySol, CoordinatorStart, EnclaveSol, - EnclaveSolReader, EnclaveSolReaderRepositoryFactory, EthPrivateKeyRepositoryFactory, - HistoricalEventCoordinator, -}; +use e3_data::{InMemStore, RepositoriesFactory}; +use e3_events::{AggregateId, BusHandle, EnclaveEvent, EventBus, EventBusConfig, EvmEventConfig}; +use e3_evm::{BondingRegistrySolReader, CiphernodeRegistrySolReader, EnclaveSolWriter}; +use e3_evm::{CiphernodeRegistrySol, EnclaveSolReader}; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::ThresholdKeyshareExtension; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; +use e3_net::{NetEventTranslator, NetRepositoryFactory}; use e3_request::E3Router; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; +use e3_sync::Synchronizer; use e3_utils::{rand_eth_addr, SharedRng}; use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; @@ -71,6 +66,20 @@ pub struct CiphernodeBuilder { task_pool: Option, threads: Option, threshold_plaintext_agg: bool, + net_config: Option, +} + +// Simple Net Configuration +#[derive(Debug)] +struct NetConfig { + pub peers: Vec, + pub quic_port: u16, +} + +impl NetConfig { + pub fn new(peers: Vec, quic_port: u16) -> Self { + Self { peers, quic_port } + } } #[derive(Default, Debug)] @@ -121,6 +130,7 @@ impl CiphernodeBuilder { task_pool: None, threads: None, threshold_plaintext_agg: false, + net_config: None, } } @@ -270,10 +280,31 @@ impl CiphernodeBuilder { self } + /// Setup net package components. + pub fn with_net(mut self, peers: Vec, quic_port: u16) -> Self { + self.net_config = Some(NetConfig::new(peers, quic_port)); + self + } + fn create_local_bus() -> Addr> { EventBus::::new(EventBusConfig { deduplicate: true }).start() } + /// Create aggregate configuration from configured chains + async fn create_aggregate_config( + &self, + provider_cache: &mut ProviderCache, + ) -> Result { + let mut chain_providers = Vec::new(); + for chain in &self.chains { + let provider = provider_cache.ensure_read_provider(chain).await?; + chain_providers.push((chain.clone(), provider.chain_id())); + } + + let delays = create_aggregate_delays(&chain_providers)?; + Ok(AggregateConfig::new(delays)) + } + pub async fn build(mut self) -> anyhow::Result { // Local bus for ciphernode events can either be forked from a bus or it can be directly // attached to a source bus @@ -316,139 +347,65 @@ impl CiphernodeBuilder { rand_eth_addr(&self.rng) }; + // Create provider cache early to use for chain validation + let mut provider_cache = ProviderCache::new(); + let aggregate_config = self.create_aggregate_config(&mut provider_cache).await?; + // Get an event system instance. let event_system = if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { - EventSystem::persisted(&addr, log_path, kv_path).with_event_bus(local_bus) + EventSystem::persisted(&addr, log_path, kv_path) + .with_event_bus(local_bus) + .with_aggregate_config(aggregate_config) } else { if let Some(ref store) = self.in_mem_store { - EventSystem::in_mem_from_store(&addr, store).with_event_bus(local_bus) + EventSystem::in_mem_from_store(&addr, store) + .with_event_bus(local_bus) + .with_aggregate_config(aggregate_config.clone()) } else { - EventSystem::in_mem(&addr).with_event_bus(local_bus) + EventSystem::in_mem(&addr) + .with_event_bus(local_bus) + .with_aggregate_config(aggregate_config.clone()) } }; let bus = event_system.handle()?; let store = event_system.store()?; + let cipher = &self.cipher; + let repositories = Arc::new(store.repositories()); - let repositories = store.repositories(); + // Now we add write support as store depends on event system + let mut provider_cache = + provider_cache.with_write_support(Arc::clone(cipher), Arc::clone(&repositories)); // Use the configured backend directly let default_backend = self.sortition_backend.clone(); + let ciphernode_selector = + CiphernodeSelector::attach(&bus, repositories.ciphernode_selector(), &addr).await?; + let sortition = Sortition::attach( &bus, repositories.sortition(), repositories.node_state(), repositories.finalized_committees(), default_backend, + ciphernode_selector, + &addr, ) .await?; - CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) - .await?; - - let mut provider_cache = ProviderCaches::new(); - let cipher = &self.cipher; - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - // TODO: gather an async handle from the event readers that closes when they shutdown and - // join it with the network manager joinhandle below - for chain in self - .chains - .iter() - .filter(|chain| chain.enabled.unwrap_or(true)) - { - if self.contract_components.enclave { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - let write_provider = provider_cache - .ensure_write_provider(&repositories, chain, cipher) - .await?; - EnclaveSol::attach( - &processor, - &bus, - read_provider.clone(), - write_provider.clone(), - &chain.contracts.enclave.address(), - &repositories.enclave_sol_reader(read_provider.chain_id()), - chain.contracts.enclave.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; - } - - if self.contract_components.enclave_reader { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - EnclaveSolReader::attach( - &processor, - &bus, - read_provider.clone(), - &chain.contracts.enclave.address(), - &repositories.enclave_sol_reader(read_provider.chain_id()), - chain.contracts.enclave.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; - } - - if self.contract_components.bonding_registry { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - BondingRegistrySol::attach( - &processor, - &bus, - read_provider.clone(), - &chain.contracts.bonding_registry.address(), - &repositories.bonding_registry_reader(read_provider.chain_id()), - chain.contracts.bonding_registry.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; - } - - if self.contract_components.ciphernode_registry { - let read_provider = provider_cache.ensure_read_provider(chain).await?; - CiphernodeRegistrySol::attach( - &processor, - &bus, - read_provider.clone(), - &chain.contracts.ciphernode_registry.address(), - &repositories.ciphernode_registry_reader(read_provider.chain_id()), - chain.contracts.ciphernode_registry.deploy_block(), - chain.rpc_url.clone(), - ) - .await?; - - match provider_cache - .ensure_write_provider(&repositories, chain, cipher) - .await - { - Ok(write_provider) => { - let _writer = CiphernodeRegistrySol::attach_writer( - &bus, - write_provider.clone(), - &chain.contracts.ciphernode_registry.address(), - self.pubkey_agg, - ) - .await?; - info!("CiphernodeRegistrySolWriter attached for publishing committees"); - - if self.pubkey_agg && matches!(self.sortition_backend, SortitionBackend::Score(_)) { - info!("Attaching CommitteeFinalizer for score sortition"); - e3_aggregator::CommitteeFinalizer::attach(&bus); - } - } - Err(e) => error!( - "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", - e - ), - } - } - } - - // We start after all readers have registered - coordinator.do_send(CoordinatorStart); + // Setup evm system + // TODO: gather an async handle from the event readers in thre following function + // that closes when they shutdown and join it with the network manager joinhandle externally + let evm_config = setup_evm_system( + &self.chains, + &mut provider_cache, + &bus, + &self.contract_components, + self.pubkey_agg, + ) + .await?; // E3 specific setup let mut e3_builder = E3Router::builder(&bus, store.clone()); @@ -482,15 +439,39 @@ impl CiphernodeBuilder { &bus, &sortition, )) } + info!("building..."); + e3_builder.build().await?; + let (join_handle, peer_id) = if let Some(net_config) = self.net_config { + let repositories = store.repositories(); + let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( + bus.clone(), + net_config.peers, + &self.cipher, + net_config.quic_port, + repositories.libp2p_keypair(), + ) + .await?; + (join_handle, peer_id) + } else { + ( + tokio::spawn(std::future::ready(Ok(()))), + "-not set-".to_string(), + ) + }; + + Synchronizer::setup(&bus, evm_config); // TODO: add net config if required + Ok(CiphernodeHandle::new( addr.to_owned(), store, bus, history, errors, + peer_id, + join_handle, )) } @@ -527,68 +508,119 @@ impl CiphernodeBuilder { } } -/// Struct to cache modules required during the ciphernode construction so that providers are only -/// constructed once. -struct ProviderCaches { - signer_cache: Option>, - read_provider_cache: HashMap>, - write_provider_cache: HashMap>, +/// Validate chain ID matches expected configuration +fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { + if let Some(expected_chain_id) = chain.chain_id { + if actual_chain_id != expected_chain_id { + return Err(anyhow::anyhow!( + "Chain '{}' validation failed: expected chain_id {}, but provider returned chain_id {}", + chain.name, expected_chain_id, actual_chain_id + )); + } + } + Ok(()) +} + +/// Build delay configuration for a specific chain +fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, u64) { + let aggregate_id = AggregateId::new(actual_chain_id as usize); + let finalization_ms = chain.finalization_ms.unwrap_or(0); + let delay_us = finalization_ms * 1000; // ms → microseconds + (aggregate_id, delay_us) } -impl ProviderCaches { - pub fn new() -> Self { - ProviderCaches { - signer_cache: None, - read_provider_cache: HashMap::new(), - write_provider_cache: HashMap::new(), - } +/// Build delays configuration from chain providers +fn create_aggregate_delays( + chain_providers: &[(ChainConfig, u64)], +) -> Result> { + let mut delays = HashMap::new(); + + for (chain, actual_chain_id) in chain_providers.into_iter().cloned() { + // Validate chain_id if specified in configuration + validate_chain_id(&chain, actual_chain_id)?; + + // Add delay if configured + let (aggregate_id, delay_us) = create_aggregate_delay(&chain, actual_chain_id); + delays.insert(aggregate_id, delay_us); } - pub async fn ensure_signer( - &mut self, - cipher: &Cipher, - repositories: &Repositories, - ) -> Result> { - if let Some(ref cache) = self.signer_cache { - return Ok(cache.clone()); + Ok(delays) +} + +async fn setup_evm_system( + chains: &Vec, + provider_cache: &mut ProviderCache, + bus: &BusHandle, + contract_components: &ContractComponents, + pubkey_agg: bool, +) -> Result { + let mut evm_config = EvmEventConfig::new(); + for chain in chains.iter().filter(|chain| chain.enabled.unwrap_or(true)) { + let provider = provider_cache.ensure_read_provider(chain).await?; + let chain_id = provider.chain_id(); + evm_config.insert(chain_id, chain.try_into()?); + let mut system = EvmSystemChainBuilder::new(&bus, &provider); + + if contract_components.enclave { + let write_provider = provider_cache.ensure_write_provider(chain).await?; + let contract = &chain.contracts.enclave; + EnclaveSolWriter::attach(&bus, write_provider.clone(), contract.address()?).await?; + system.with_contract(contract.address()?, move |next| { + EnclaveSolReader::setup(&next).recipient() + }); } - let signer = load_signer_from_repository(repositories.eth_private_key(), cipher).await?; - self.signer_cache = Some(signer.clone()); - Ok(signer) - } + if contract_components.enclave_reader { + let contract = &chain.contracts.enclave; - pub async fn ensure_read_provider( - &mut self, - chain: &ChainConfig, - ) -> Result> { - if let Some(cache) = self.read_provider_cache.get(chain) { - return Ok(cache.clone()); + system.with_contract(contract.address()?, move |next| { + EnclaveSolReader::setup(&next).recipient() + }); } - let rpc_url = chain.rpc_url()?; - let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); - let read_provider = provider_config.create_readonly_provider().await?; - self.read_provider_cache - .insert(chain.clone(), read_provider.clone()); - Ok(read_provider) - } - - pub async fn ensure_write_provider( - &mut self, - repositories: &Repositories, - chain: &ChainConfig, - cipher: &Cipher, - ) -> Result> { - if let Some(cache) = self.write_provider_cache.get(chain) { - return Ok(cache.clone()); + + if contract_components.bonding_registry { + let contract = &chain.contracts.bonding_registry; + system.with_contract(contract.address()?, move |next| { + BondingRegistrySolReader::setup(&next).recipient() + }); } - let signer = self.ensure_signer(cipher, repositories).await?; - let rpc_url = chain.rpc_url()?; - let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); - let write_provider = provider_config.create_signer_provider(&signer).await?; - self.write_provider_cache - .insert(chain.clone(), write_provider.clone()); - Ok(write_provider) + if contract_components.ciphernode_registry { + let contract = &chain.contracts.ciphernode_registry; + + system.with_contract(contract.address()?, move |next| { + CiphernodeRegistrySolReader::setup(&next).recipient() + }); + + // TODO: Should we not let this pass and just use '?'? + // Above if we include enclave in the config and we don't have a wallet it will fail + match provider_cache + .ensure_write_provider(&chain) + .await + { + Ok(write_provider) => { + CiphernodeRegistrySol::attach_writer( + &bus, + write_provider.clone(), + contract.address()?, + pubkey_agg, + ) + .await?; + info!("CiphernodeRegistrySolWriter attached for publishing committees"); + + if pubkey_agg { + info!("Attaching CommitteeFinalizer for score sortition"); + CommitteeFinalizer::attach(&bus); + } + } + Err(e) => error!( + "Failed to create write provider (likely no wallet configured), skipping writer attachment: {}", + e + ) + } + } + system.build(); } + + Ok(evm_config) } diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 7fb9b83cc3..921f845423 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -13,15 +13,19 @@ use e3_data::{ }; use e3_events::hlc::Hlc; use e3_events::{ - BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventType, Sequencer, + BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, EventStoreRouter, EventType, + Sequencer, StoreEventRequested, }; +use e3_utils::enumerate_path; use once_cell::sync::OnceCell; +use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; -/// Hold the InMem EventStore instance and InMemStore +pub use e3_data::AggregateConfig; + struct InMemBackend { - eventstore: OnceCell>>, + eventstores: OnceCell>>>, store: OnceCell>, } @@ -29,7 +33,7 @@ struct InMemBackend { struct PersistedBackend { log_path: PathBuf, sled_path: PathBuf, - eventstore: OnceCell>>, + eventstores: OnceCell>>>, store: OnceCell>, } @@ -39,22 +43,10 @@ enum EventSystemBackend { Persisted(PersistedBackend), } -pub enum EventStoreAddr { - InMem(Addr>), - Persisted(Addr>), -} - -impl TryFrom for Addr> { - type Error = anyhow::Error; - fn try_from(value: EventStoreAddr) -> std::result::Result { - if let EventStoreAddr::InMem(addr) = value { - Ok(addr) - } else { - Err(anyhow!( - "address was not EventStore" - )) - } - } +#[derive(Clone)] +pub enum EventStoreAddrs { + InMem(HashMap>>), + Persisted(HashMap>>), } /// EventSystem holds interconnected references to the components that manage events and @@ -83,6 +75,10 @@ pub struct EventSystem { wired: OnceCell<()>, /// Hlc override hlc: OnceCell, + /// Central configuration for aggregates, including delays and other settings + aggregate_config: OnceCell, + /// Cached EventStoreAddrs for idempotency + eventstore_addrs: OnceCell, } impl EventSystem { @@ -96,7 +92,7 @@ impl EventSystem { Self { node_id: EventSystem::node_id(node_id), backend: EventSystemBackend::InMem(InMemBackend { - eventstore: OnceCell::new(), + eventstores: OnceCell::new(), store: OnceCell::new(), }), buffer: OnceCell::new(), @@ -105,6 +101,8 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), + aggregate_config: OnceCell::new(), + eventstore_addrs: OnceCell::new(), } } @@ -113,7 +111,7 @@ impl EventSystem { Self { node_id: EventSystem::node_id(node_id), backend: EventSystemBackend::InMem(InMemBackend { - eventstore: OnceCell::new(), + eventstores: OnceCell::new(), store: OnceCell::from(store.to_owned()), }), buffer: OnceCell::new(), @@ -122,6 +120,8 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), + aggregate_config: OnceCell::new(), + eventstore_addrs: OnceCell::new(), } } @@ -132,7 +132,7 @@ impl EventSystem { backend: EventSystemBackend::Persisted(PersistedBackend { log_path, sled_path, - eventstore: OnceCell::new(), + eventstores: OnceCell::new(), store: OnceCell::new(), }), buffer: OnceCell::new(), @@ -141,6 +141,8 @@ impl EventSystem { handle: OnceCell::new(), wired: OnceCell::new(), hlc: OnceCell::new(), + aggregate_config: OnceCell::new(), + eventstore_addrs: OnceCell::new(), } } @@ -164,16 +166,32 @@ impl EventSystem { self } + /// Add aggregate configuration including delays and other settings + pub fn with_aggregate_config(self, config: AggregateConfig) -> Self { + let _ = self.aggregate_config.set(config); + self + } + /// Get the eventbus address pub fn eventbus(&self) -> Addr> { self.eventbus.get_or_init(get_enclave_event_bus).clone() } + /// Get the aggregate configuration + pub fn aggregate_config(&self) -> AggregateConfig { + self.aggregate_config + .get_or_init(|| AggregateConfig::new(HashMap::new())) + .clone() + } + /// Get the buffer address pub fn buffer(&self) -> Addr { let buffer = self .buffer - .get_or_init(|| WriteBuffer::new().start()) + .get_or_init(|| { + let config = self.aggregate_config(); + WriteBuffer::with_config(config).start() + }) .clone(); self.wire_if_ready(); buffer @@ -182,40 +200,101 @@ impl EventSystem { /// Get the sequencer address pub fn sequencer(&self) -> Result> { self.sequencer - .get_or_try_init(|| match self.eventstore()? { - EventStoreAddr::InMem(es) => { - Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) - } - EventStoreAddr::Persisted(es) => { - Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) + .get_or_try_init(|| { + let router = self.eventstore_router()?; + Ok(Sequencer::new(&self.eventbus(), router, self.buffer()).start()) + }) + .cloned() + } + + /// Get the EventStore addresses + pub fn eventstore_addrs(&self) -> Result { + self.eventstore_addrs + .get_or_try_init(|| { + match &self.backend { + EventSystemBackend::InMem(b) => { + let config = self.aggregate_config(); + let indexes = config.indexed_ids(); + + let addrs = b + .eventstores + .get_or_init(|| { + let mut eventstore_map = HashMap::new(); + for &index in &indexes { + eventstore_map.insert( + index, + EventStore::new( + InMemSequenceIndex::new(), + InMemEventLog::new(), + ) + .start(), + ); + } + eventstore_map + }) + .clone(); + Ok(EventStoreAddrs::InMem(addrs)) + } + EventSystemBackend::Persisted(b) => { + let config = self.aggregate_config(); + let indexes = config.indexed_ids(); + + let addrs = b + .eventstores + .get_or_try_init(|| -> Result<_> { + let mut eventstore_map = HashMap::new(); + for &index in &indexes { + // Enumerate the log path for each eventstore + let enumerated_log_path = enumerate_path(&b.log_path, index); + let tree_name = format!("sequence_index.{}", index); + let index_store = + SledSequenceIndex::new(&b.sled_path, &tree_name)?; + let log = CommitLogEventLog::new(&enumerated_log_path)?; + eventstore_map + .insert(index, EventStore::new(index_store, log).start()); + } + Ok(eventstore_map) + })? + .clone(); + Ok(EventStoreAddrs::Persisted(addrs)) + } } }) .cloned() } - /// Get the EventStore address - pub fn eventstore(&self) -> Result { - match &self.backend { - EventSystemBackend::InMem(b) => { - let addr = b - .eventstore - .get_or_init(|| { - EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()).start() - }) - .clone(); - Ok(EventStoreAddr::InMem(addr)) - } - EventSystemBackend::Persisted(b) => { - let addr = b - .eventstore - .get_or_try_init(|| -> Result<_> { - let index = SledSequenceIndex::new(&b.sled_path, "sequence_index")?; - let log = CommitLogEventLog::new(&b.log_path)?; - Ok(EventStore::new(index, log).start()) - })? - .clone(); - Ok(EventStoreAddr::Persisted(addr)) - } + /// Get an EventStoreRouter for InMem backend + pub fn in_mem_eventstore_router( + &self, + ) -> Result>> { + let eventstores = self.eventstore_addrs()?; + if let EventStoreAddrs::InMem(addrs) = eventstores { + let router = EventStoreRouter::new(addrs); + Ok(router.start()) + } else { + Err(anyhow!("Expected InMem backend but got Persisted")) + } + } + + /// Get an EventStoreRouter for Persisted backend + pub fn persisted_eventstore_router( + &self, + ) -> Result>> { + let eventstores = self.eventstore_addrs()?; + if let EventStoreAddrs::Persisted(addrs) = eventstores { + let router = EventStoreRouter::new(addrs); + Ok(router.start()) + } else { + Err(anyhow!("Expected Persisted backend but got InMem")) + } + } + + /// Get an EventStoreRouter Recipient + pub fn eventstore_router(&self) -> Result> { + let eventstores = self.eventstore_addrs()?; + match &eventstores { + EventStoreAddrs::InMem(_) => Ok(self.in_mem_eventstore_router()?.recipient()), + EventStoreAddrs::Persisted(_) => Ok(self.persisted_eventstore_router()?.recipient()), } } @@ -304,8 +383,9 @@ mod tests { use actix::Message; use e3_events::prelude::*; + use e3_events::CorrelationId; use e3_events::EnclaveEventData; - use e3_events::GetEventsAfter; + use e3_events::ReceiveEvents; use e3_events::TestEvent; use tempfile::TempDir; @@ -396,7 +476,6 @@ mod tests { let system = EventSystem::in_mem("cn1").with_fresh_bus(); let handle = system.handle()?; let datastore = system.store()?; - let eventstore = system.eventstore()?; let listener = Listener { logs: Vec::new(), events: Vec::new(), @@ -415,12 +494,6 @@ mod tests { // NOTE: Eventual consistency // Store should not have data set on it until event has been published - // There is an argument we should instead delay reads until the event has been stored but - // this would: - // a. Promote poor patterns of sharing data through persistence - // b. Add a large amount of complexity to batching Get operations - // For now we allow this inconsistency under the assumption that data is written for - // snapshot storage exclusively. // Let's check the eventual consistency all data points should be none... assert_eq!(datastore.scope("/foo/name").read::().await?, None); @@ -458,11 +531,16 @@ mod tests { let logs = listener.send(GetLogs).await?; assert_eq!(logs, vec!["pink", "yellow", "red", "white"]); - // Get the in mem address for the event store - let es: Addr> = eventstore.try_into()?; + // Get the in mem eventstore router + let router = system.in_mem_eventstore_router()?; - // Get all events after the given timestamp and send them to the listener - es.do_send(GetEventsAfter::new(ts, listener.clone())); + // Get all events after the given timestamp using the router + use e3_events::{AggregateId, GetAggregateEventsAfter}; + let mut ts_map = HashMap::new(); + ts_map.insert(AggregateId::new(0), ts); + let get_events_msg = + GetAggregateEventsAfter::new(CorrelationId::new(), ts_map, listener.clone().into()); + router.do_send(get_events_msg); sleep(Duration::from_millis(100)).await; // Pull the events off the listsner since the timestamp @@ -470,4 +548,47 @@ mod tests { assert_eq!(events, vec!["yellow", "red", "white"]); Ok(()) } + + #[actix::test] + async fn test_multiple_eventstores() -> Result<()> { + use e3_events::AggregateId; + + // Create an AggregateConfig with multiple AggregateIds + let mut delays = HashMap::new(); + delays.insert(AggregateId::new(0), 1000); // 1ms delay + delays.insert(AggregateId::new(1), 2000); // 2ms delay + delays.insert(AggregateId::new(2), 3000); // 3ms delay + let aggregate_config = AggregateConfig::new(delays); + + // Test in-memory eventstores + let system = EventSystem::in_mem("test_multi").with_aggregate_config(aggregate_config); + let Ok(EventStoreAddrs::InMem(addrs)) = system.eventstore_addrs() else { + panic!("Expected InMem event store addrs"); + }; + + // Should create 3 eventstores for 3 AggregateIds + assert_eq!(addrs.len(), 3); + // Test that we can access the first eventstore (index 0) + assert!(addrs.contains_key(&0)); + assert!(addrs.contains_key(&1)); + assert!(addrs.contains_key(&2)); + + // Test persistent eventstores + let tmp = TempDir::new().unwrap(); + let persisted_system = EventSystem::persisted( + "test_persisted", + tmp.path().join("log"), + tmp.path().join("sled"), + ) + .with_aggregate_config(AggregateConfig::new(HashMap::new())); + + let Ok(EventStoreAddrs::Persisted(addrs)) = persisted_system.eventstore_addrs() else { + panic!("Expected Persisted event store addrs"); + }; + + assert_eq!(addrs.len(), 1); + assert!(addrs.contains_key(&0)); + + Ok(()) + } } diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs new file mode 100644 index 0000000000..0c44393463 --- /dev/null +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::mem::replace; + +use actix::Actor; +use alloy::{primitives::Address, providers::Provider}; +use e3_events::{BusHandle, EventSubscriber, EventType, SyncStart}; +use e3_evm::{ + EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, + FixHistoricalOrder, OneShotRunner, SyncStartExtractor, +}; + +pub trait RouteFn: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} +impl RouteFn for F where F: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} + +type RouteFactory = Box; + +// Build the event system for a single chain +pub struct EvmSystemChainBuilder

{ + provider: EthProvider

, + bus: BusHandle, + chain_id: u64, + route_factories: Vec<(Address, RouteFactory)>, +} + +impl EvmSystemChainBuilder

{ + pub fn new(bus: &BusHandle, provider: &EthProvider

) -> Self { + let chain_id = provider.chain_id(); + Self { + bus: bus.clone(), + provider: provider.clone(), + chain_id, + route_factories: Vec::new(), + } + } + + pub fn with_contract( + &mut self, + address: Address, + route_fn: F, + ) -> &mut Self { + self.route_factories.push((address, Box::new(route_fn))); + self + } + + pub fn build(&mut self) { + let gateway = FixHistoricalOrder::setup(EvmChainGateway::setup(&self.bus)); + let runner = SyncStartExtractor::setup(OneShotRunner::setup({ + let bus = self.bus.clone(); + let provider = self.provider.clone(); + let gateway = gateway.clone(); + let chain_id = self.chain_id; + // Only gets consumed once so fine to do this + let route_factories = replace(&mut self.route_factories, Vec::new()); + move |msg: SyncStart| { + let config = msg.get_evm_config(chain_id)?; + let gateway = gateway.recipient(); + let mut router = EvmRouter::new(); + + for (address, route_fn) in route_factories { + let processor = route_fn(gateway.clone()); + router = router.add_route(address, &processor); + } + + router = router.add_fallback(&gateway); + let filters = + Filters::from_routing_table(router.get_routing_table(), config.deploy_block()); + let router = router.start(); + EvmReadInterface::setup(&provider, &router.recipient(), &bus, filters); + Ok(()) + } + })); + self.bus.subscribe(EventType::SyncStart, runner.recipient()); + } +} diff --git a/crates/ciphernode-builder/src/lib.rs b/crates/ciphernode-builder/src/lib.rs index c77952b744..70a5a3079d 100644 --- a/crates/ciphernode-builder/src/lib.rs +++ b/crates/ciphernode-builder/src/lib.rs @@ -8,7 +8,11 @@ mod ciphernode; mod ciphernode_builder; mod event_system; mod eventbus_factory; +mod evm_system; +mod provider_caches; pub use ciphernode::*; pub use ciphernode_builder::*; pub use event_system::*; pub use eventbus_factory::*; +pub use evm_system::*; +pub use provider_caches::*; diff --git a/crates/ciphernode-builder/src/provider_caches.rs b/crates/ciphernode-builder/src/provider_caches.rs new file mode 100644 index 0000000000..628a7e525a --- /dev/null +++ b/crates/ciphernode-builder/src/provider_caches.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; +use anyhow::Result; +use e3_config::chain_config::ChainConfig; +use e3_crypto::Cipher; +use e3_data::Repositories; +use e3_evm::helpers::{ + load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, + ProviderConfig, +}; +use e3_evm::EthPrivateKeyRepositoryFactory; +use std::collections::HashMap; +use std::sync::Arc; + +// Typestate marker types +pub struct ReadOnly; + +pub struct WriteEnabled { + cipher: Arc, + repositories: Arc, +} + +/// Struct to cache modules required during the ciphernode construction so that providers are only +/// constructed once. +pub struct ProviderCache { + signer_cache: Option>, + read_provider_cache: HashMap>, + write_provider_cache: HashMap>, + state: State, +} + +impl ProviderCache { + pub fn new() -> Self { + ProviderCache { + signer_cache: None, + read_provider_cache: HashMap::new(), + write_provider_cache: HashMap::new(), + state: ReadOnly, + } + } + + pub fn from_single_read_provider( + chain: ChainConfig, + provider: EthProvider, + ) -> Self { + ProviderCache { + signer_cache: None, + read_provider_cache: HashMap::from([(chain, provider)]), + write_provider_cache: HashMap::new(), + state: ReadOnly, + } + } + + /// Configure the cache with cipher and repositories to enable write provider support. + pub fn with_write_support( + self, + cipher: Arc, + repositories: Arc, + ) -> ProviderCache { + ProviderCache { + signer_cache: self.signer_cache, + read_provider_cache: self.read_provider_cache, + write_provider_cache: self.write_provider_cache, + state: WriteEnabled { + cipher, + repositories, + }, + } + } +} + +impl Default for ProviderCache { + fn default() -> Self { + Self::new() + } +} + +impl ProviderCache { + pub async fn ensure_read_provider( + &mut self, + chain: &ChainConfig, + ) -> Result> { + if let Some(cache) = self.read_provider_cache.get(chain) { + return Ok(cache.clone()); + } + + let rpc_url = chain.rpc_url()?; + let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); + let read_provider = provider_config.create_readonly_provider().await?; + + self.read_provider_cache + .insert(chain.clone(), read_provider.clone()); + + Ok(read_provider) + } +} + +impl ProviderCache { + pub async fn ensure_signer(&mut self) -> Result> { + if let Some(ref cache) = self.signer_cache { + return Ok(cache.clone()); + } + + let signer = load_signer_from_repository( + self.state.repositories.eth_private_key(), + &self.state.cipher, + ) + .await?; + + self.signer_cache = Some(signer.clone()); + Ok(signer) + } + + pub async fn ensure_write_provider( + &mut self, + chain: &ChainConfig, + ) -> Result> { + if let Some(cache) = self.write_provider_cache.get(chain) { + return Ok(cache.clone()); + } + + let signer = self.ensure_signer().await?; + let rpc_url = chain.rpc_url()?; + let provider_config = ProviderConfig::new(rpc_url, chain.rpc_auth.clone()); + let write_provider = provider_config.create_signer_provider(&signer).await?; + + self.write_provider_cache + .insert(chain.clone(), write_provider.clone()); + + Ok(write_provider) + } +} diff --git a/crates/cli/src/ciphernode/context.rs b/crates/cli/src/ciphernode/context.rs index 61843e8cd5..5d01af5bbb 100644 --- a/crates/cli/src/ciphernode/context.rs +++ b/crates/cli/src/ciphernode/context.rs @@ -66,7 +66,7 @@ pub(crate) struct ChainContext { impl ChainContext { pub(crate) async fn new(config: &AppConfig, selection: Option<&str>) -> Result { let chain = select_chain(config, selection)?; - let bonding_registry = parse_address(chain.contracts.bonding_registry.address())?; + let bonding_registry = parse_address(chain.contracts.bonding_registry.address_str())?; let rpc = chain.rpc_url()?; let cipher = Cipher::from_file(config.key_file()).await?; diff --git a/crates/cli/src/print_env.rs b/crates/cli/src/print_env.rs index 0a6ffa25f5..99ccba221d 100644 --- a/crates/cli/src/print_env.rs +++ b/crates/cli/src/print_env.rs @@ -15,18 +15,30 @@ pub fn extract_env_vars_vite(config: &AppConfig, chain: &str) -> String { let enclave_addr = &chain.contracts.enclave; let registry_addr = &chain.contracts.ciphernode_registry; let bonding_registry_addr = &chain.contracts.bonding_registry; - env_vars.push(format!("VITE_ENCLAVE_ADDRESS={}", enclave_addr.address())); - env_vars.push(format!("VITE_REGISTRY_ADDRESS={}", registry_addr.address())); + env_vars.push(format!( + "VITE_ENCLAVE_ADDRESS={}", + enclave_addr.address_str() + )); + env_vars.push(format!( + "VITE_REGISTRY_ADDRESS={}", + registry_addr.address_str() + )); env_vars.push(format!("VITE_RPC_URL={}", chain.rpc_url)); env_vars.push(format!( "VITE_BONDING_REGISTRY_ADDRESS={}", - bonding_registry_addr.address() + bonding_registry_addr.address_str() )); if let Some(e3_program) = &chain.contracts.e3_program { - env_vars.push(format!("VITE_E3_PROGRAM_ADDRESS={}", e3_program.address())); + env_vars.push(format!( + "VITE_E3_PROGRAM_ADDRESS={}", + e3_program.address_str() + )); } if let Some(fee_token) = &chain.contracts.fee_token { - env_vars.push(format!("VITE_FEE_TOKEN_ADDRESS={}", fee_token.address())); + env_vars.push(format!( + "VITE_FEE_TOKEN_ADDRESS={}", + fee_token.address_str() + )); } } @@ -41,18 +53,18 @@ pub fn extract_env_vars(config: &AppConfig, chain: &str) -> String { let enclave_addr = &chain.contracts.enclave; let registry_addr = &chain.contracts.ciphernode_registry; let bonding_registry_addr = &chain.contracts.bonding_registry; - env_vars.push(format!("ENCLAVE_ADDRESS={}", enclave_addr.address())); + env_vars.push(format!("ENCLAVE_ADDRESS={}", enclave_addr.address_str())); env_vars.push(format!("RPC_URL={}", chain.rpc_url)); - env_vars.push(format!("REGISTRY_ADDRESS={}", registry_addr.address())); + env_vars.push(format!("REGISTRY_ADDRESS={}", registry_addr.address_str())); env_vars.push(format!( "BONDING_REGISTRY_ADDRESS={}", - bonding_registry_addr.address() + bonding_registry_addr.address_str() )); if let Some(e3_program) = &chain.contracts.e3_program { - env_vars.push(format!("E3_PROGRAM_ADDRESS={}", e3_program.address())); + env_vars.push(format!("E3_PROGRAM_ADDRESS={}", e3_program.address_str())); } if let Some(fee_token) = &chain.contracts.fee_token { - env_vars.push(format!("FEE_TOKEN_ADDRESS={}", fee_token.address())); + env_vars.push(format!("FEE_TOKEN_ADDRESS={}", fee_token.address_str())); } } diff --git a/crates/cli/src/start.rs b/crates/cli/src/start.rs index 8ab6b20fa1..2150cd4e2c 100644 --- a/crates/cli/src/start.rs +++ b/crates/cli/src/start.rs @@ -21,7 +21,7 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { // add cli peers to the config config.add_peers(peers); - let (bus, handle, peer_id) = match config.role() { + let node = match config.role() { // Launch in aggregator configuration NodeRole::Aggregator { pubkey_write_path, @@ -44,10 +44,10 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { "LAUNCHING CIPHERNODE: ({}/{}/{})", config.name(), address, - peer_id + node.peer_id ); - tokio::spawn(listen_for_shutdown(bus, handle)).await?; + tokio::spawn(listen_for_shutdown(node)).await?; Ok(()) } diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index 8bed2fe5d5..5074b23086 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -577,7 +577,7 @@ nodes: let chain = config.chains().first().unwrap(); assert_eq!(config.quic_port(), 1235); assert_eq!( - chain.contracts.ciphernode_registry.address(), + chain.contracts.ciphernode_registry.address_str(), "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" ); assert_eq!(config.peers(), vec!["one", "two"]); @@ -688,11 +688,11 @@ chains: assert_eq!(chain.name, "hardhat"); assert_eq!(chain.rpc_url, "ws://localhost:8545"); assert_eq!( - chain.contracts.enclave.address(), + chain.contracts.enclave.address_str(), "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" ); assert_eq!( - chain.contracts.ciphernode_registry.address(), + chain.contracts.ciphernode_registry.address_str(), "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" ); assert_eq!( @@ -803,7 +803,7 @@ chains: } ); assert_eq!( - chain.contracts.enclave.address(), + chain.contracts.enclave.address_str(), "0x1234567890123456789012345678901234567890" ); diff --git a/crates/config/src/chain_config.rs b/crates/config/src/chain_config.rs index 6c25192767..63e241eee4 100644 --- a/crates/config/src/chain_config.rs +++ b/crates/config/src/chain_config.rs @@ -11,7 +11,9 @@ use crate::{ rpc::{RpcAuth, RPC}, }; use anyhow::*; +use e3_events::{EvmEventConfig, EvmEventConfigChain}; use serde::{Deserialize, Serialize}; +use tracing::error; #[derive(Debug, Clone, PartialEq, Hash, Eq, Deserialize, Serialize)] pub struct ChainConfig { @@ -21,6 +23,8 @@ pub struct ChainConfig { #[serde(default)] pub rpc_auth: RpcAuth, pub contracts: ContractAddresses, + pub finalization_ms: Option, + pub chain_id: Option, } impl ChainConfig { @@ -29,3 +33,38 @@ impl ChainConfig { .map_err(|e| anyhow!("Failed to parse RPC URL for chain {}: {}", self.name, e))?) } } + +impl TryFrom<&ChainConfig> for EvmEventConfigChain { + type Error = anyhow::Error; + fn try_from(value: &ChainConfig) -> std::result::Result { + let rpc = value.rpc_url()?; + let contracts = value.contracts.contracts(); + let mut lowest_block: Option = None; + for contract in contracts { + let deploy_block = contract.deploy_block(); + if deploy_block.unwrap_or(0) == 0 && !rpc.is_local() { + let rpc_url = rpc.url().to_string(); + let contract_address = contract.address_str(); + error!( + "Querying from block 0 on a non-local node ({}) without a specific deploy_block is not allowed.", + rpc_url + ); + bail!( + "Misconfiguration: Attempted to query historical events from genesis on a non-local node. \ + Please specify a `deploy_block` for contract address {contract_address} on rpc {rpc_url}" + ); + } + lowest_block = [lowest_block, deploy_block].into_iter().flatten().min(); + } + let start_block = lowest_block.unwrap_or(0); + Ok(EvmEventConfigChain::new(start_block)) + } +} + +impl TryFrom for EvmEventConfigChain { + type Error = anyhow::Error; + fn try_from(value: ChainConfig) -> std::result::Result { + let r = &value; + r.try_into() + } +} diff --git a/crates/config/src/contract.rs b/crates/config/src/contract.rs index cf8115d978..420d3c953c 100644 --- a/crates/config/src/contract.rs +++ b/crates/config/src/contract.rs @@ -4,6 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use alloy_primitives::Address; +use anyhow::Result; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Hash, Eq, Deserialize, Serialize, PartialEq)] @@ -17,7 +19,7 @@ pub enum Contract { } impl Contract { - pub fn address(&self) -> &String { + pub fn address_str(&self) -> &str { use Contract::*; match self { Full { address, .. } => address, @@ -25,6 +27,11 @@ impl Contract { } } + pub fn address(&self) -> Result

{ + let addr = self.address_str().parse()?; + Ok(addr) + } + pub fn deploy_block(&self) -> Option { use Contract::*; match self { @@ -42,3 +49,18 @@ pub struct ContractAddresses { pub e3_program: Option, pub fee_token: Option, } + +impl ContractAddresses { + pub fn contracts(&self) -> Vec<&Contract> { + [ + Some(&self.enclave), + Some(&self.ciphernode_registry), + Some(&self.bonding_registry), + self.e3_program.as_ref(), + self.fee_token.as_ref(), + ] + .into_iter() + .flatten() + .collect() + } +} diff --git a/crates/config/src/rpc.rs b/crates/config/src/rpc.rs index f814b55118..729137c939 100644 --- a/crates/config/src/rpc.rs +++ b/crates/config/src/rpc.rs @@ -12,60 +12,131 @@ use serde::Deserialize; use serde::Serialize; use url::Url; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RpcProtocol { + Http, + Https, + Ws, + Wss, +} + +impl RpcProtocol { + pub fn is_websocket(&self) -> bool { + matches!(self, RpcProtocol::Ws | RpcProtocol::Wss) + } + + pub fn is_secure(&self) -> bool { + matches!(self, RpcProtocol::Https | RpcProtocol::Wss) + } + + pub fn as_str(&self) -> &'static str { + match self { + RpcProtocol::Http => "http", + RpcProtocol::Https => "https", + RpcProtocol::Ws => "ws", + RpcProtocol::Wss => "wss", + } + } +} + #[derive(Clone)] -pub enum RPC { - Http(String), - Https(String), - Ws(String), - Wss(String), +pub struct RPC { + protocol: RpcProtocol, + url: Url, } impl RPC { pub fn from_url(url: &str) -> Result { let parsed = Url::parse(url).context("Invalid URL format")?; - match parsed.scheme() { - "http" => Ok(RPC::Http(url.to_string())), - "https" => Ok(RPC::Https(url.to_string())), - "ws" => Ok(RPC::Ws(url.to_string())), - "wss" => Ok(RPC::Wss(url.to_string())), + let protocol = match parsed.scheme() { + "http" => RpcProtocol::Http, + "https" => RpcProtocol::Https, + "ws" => RpcProtocol::Ws, + "wss" => RpcProtocol::Wss, _ => bail!("Invalid protocol. Expected: http://, https://, ws://, wss://"), + }; + + if parsed.host_str().is_none() { + bail!("URL must contain a host"); } + + Ok(RPC { + protocol, + url: parsed, + }) + } + + pub fn protocol(&self) -> RpcProtocol { + self.protocol + } + + pub fn url(&self) -> &Url { + &self.url + } + + pub fn hostname(&self) -> &str { + // Safe: validated in from_url() - http(s)/ws(s) schemes always require a host + self.url.host_str().expect("RPC URL always has a host") + } + + pub fn port(&self) -> u16 { + // Safe: http(s)/ws(s) always have known default ports + self.url + .port_or_known_default() + .expect("RPC URL always has a port") + } + + pub fn host_with_port(&self) -> String { + format!("{}:{}", self.hostname(), self.port()) } pub fn as_http_url(&self) -> Result { - match self { - RPC::Http(url) | RPC::Https(url) => Ok(url.clone()), - RPC::Ws(url) | RPC::Wss(url) => { - let mut parsed = - Url::parse(url).context(format!("Failed to parse URL: {}", url))?; - parsed - .set_scheme(if self.is_secure() { "https" } else { "http" }) - .map_err(|_| anyhow!("http(s) are valid schemes"))?; - Ok(parsed.to_string()) - } + if !self.protocol.is_websocket() { + Ok(self.url.to_string()) + } else { + let mut parsed = self.url.clone(); + let scheme = if self.protocol.is_secure() { + "https" + } else { + "http" + }; + parsed + .set_scheme(scheme) + .map_err(|_| anyhow!("http(s) are valid schemes"))?; + Ok(parsed.to_string()) } } pub fn as_ws_url(&self) -> Result { - match self { - RPC::Ws(url) | RPC::Wss(url) => Ok(url.clone()), - RPC::Http(url) | RPC::Https(url) => { - let mut parsed = - Url::parse(url).context(format!("Failed to parse URL: {}", url))?; - parsed - .set_scheme(if self.is_secure() { "wss" } else { "ws" }) - .map_err(|_| anyhow!("ws(s) are valid schemes"))?; - Ok(parsed.to_string()) - } + if self.protocol.is_websocket() { + Ok(self.url.to_string()) + } else { + let mut parsed = self.url.clone(); + let scheme = if self.protocol.is_secure() { + "wss" + } else { + "ws" + }; + parsed + .set_scheme(scheme) + .map_err(|_| anyhow!("ws(s) are valid schemes"))?; + Ok(parsed.to_string()) } } pub fn is_websocket(&self) -> bool { - matches!(self, RPC::Ws(_) | RPC::Wss(_)) + self.protocol.is_websocket() } pub fn is_secure(&self) -> bool { - matches!(self, RPC::Https(_) | RPC::Wss(_)) + self.protocol.is_secure() + } + + pub fn is_local(&self) -> bool { + match self.hostname() { + "localhost" | "127.0.0.1" | "::1" => true, + host => host.starts_with("127."), // 127.0.0.0/8 is all loopback + } } } diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 5905f1b7dd..46dc04dea9 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -82,7 +82,7 @@ mod tests { use tempfile::tempdir; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), 123) + EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) } #[test] diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index b2df9273cf..d4516f858f 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -99,6 +99,31 @@ impl DataStore { &self.addr } + /// Get a reference to the Recipient + pub fn get_recipient(&self) -> &Recipient { + &self.get + } + + /// Get a reference to the Recipient + pub fn remove_recipient(&self) -> &Recipient { + &self.remove + } + + /// Get a reference to the Recipient + pub fn insert_recipient(&self) -> &Recipient { + &self.insert + } + + /// Get a reference to the Recipient + pub fn insert_sync_recipient(&self) -> &Recipient { + &self.insert_sync + } + + /// Get a clone of the scope bytes + pub fn scope_bytes(&self) -> &[u8] { + &self.scope + } + /// Changes the scope for the data store. /// Note that if the scope does not start with a slash one is appended. /// ``` diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index 6ecfc25d40..6f36b8c53c 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -7,21 +7,47 @@ use crate::IntoKey; use actix::Message; use anyhow::Result; +use e3_events::{EventContext, Sequenced}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] -pub struct Insert(pub Vec, pub Vec); +pub struct Insert { + key: Vec, + value: Vec, + ctx: Option>, +} + impl Insert { pub fn new(key: K, value: Vec) -> Self { - Self(key.into_key(), value) + Self { + key: key.into_key(), + value, + ctx: None, + } + } + + pub fn new_with_context( + key: K, + value: Vec, + ctx: EventContext, + ) -> Self { + Self { + key: key.into_key(), + value, + ctx: Some(ctx), + } } pub fn key(&self) -> &Vec { - &self.0 + &self.key } pub fn value(&self) -> &Vec { - &self.1 + &self.value + } + + pub fn ctx(&self) -> Option<&EventContext> { + self.ctx.as_ref() } } diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 9b95398919..a283ded98d 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -49,7 +49,7 @@ mod tests { use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; fn event_from(data: impl Into) -> EnclaveEvent { - EnclaveEvent::::new_with_timestamp(data.into().into(), 123) + EnclaveEvent::::new_with_timestamp(data.into().into(), None, 123) } #[test] diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index e1a73c587c..7247a252b0 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -3,10 +3,11 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; +use crate::{Get, Insert, Remove, Repository}; +use actix::Recipient; use anyhow::*; use async_trait::async_trait; +use e3_events::{EventContext, EventContextManager, Sequenced}; use serde::{de::DeserializeOwned, Serialize}; pub trait PersistableData: Serialize + DeserializeOwned + Clone + Send + Sync + 'static {} @@ -18,13 +19,13 @@ pub trait AutoPersist where T: PersistableData, { - /// Load the data from the repository into an auto persist container + /// Load the data from the source into an auto persist container async fn load(&self) -> Result>; - /// Create a new auto persist container and set some data on it to send back to the repository + /// Create a new auto persist container and set some data on it to send back to the source fn send(&self, data: Option) -> Persistable; - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the given default data + /// Load the data from the source into an auto persist container. If there is no persisted data then persist the given default data async fn load_or_default(&self, default: T) -> Result>; - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the given default data + /// Load the data from the source into an auto persist container. If there is no persisted data then persist the given default data async fn load_or_else(&self, f: F) -> Result> where F: Send + FnOnce() -> Result; @@ -35,106 +36,185 @@ impl AutoPersist for Repository where T: PersistableData, { - /// Load the data from the repository into an auto persist container async fn load(&self) -> Result> { - Ok(Persistable::load(self).await?) + self.to_connector().load().await + } + + fn send(&self, data: Option) -> Persistable { + self.to_connector().send(data) + } + + async fn load_or_default(&self, default: T) -> Result> { + self.to_connector().load_or_default(default).await + } + + async fn load_or_else(&self, f: F) -> Result> + where + F: Send + FnOnce() -> Result, + { + self.to_connector().load_or_else(f).await + } +} + +/// Connector to connect to store +#[derive(Clone, Debug)] +pub struct StoreConnector { + pub key: Vec, + pub get: Recipient, + pub insert: Recipient, + pub remove: Recipient, +} + +impl StoreConnector { + pub fn new( + key: &[u8], + get: &Recipient, + insert: &Recipient, + remove: &Recipient, + ) -> Self { + Self { + key: key.to_owned(), + get: get.clone(), + insert: insert.clone(), + remove: remove.clone(), + } + } +} + +#[async_trait] +impl AutoPersist for StoreConnector +where + T: PersistableData, +{ + async fn load(&self) -> Result> { + Persistable::load(self.clone()).await } - /// Create a new auto persist container and set some data on it to send back to the repository fn send(&self, data: Option) -> Persistable { - Persistable::new(data, self).save() + Persistable::new(data, self.clone()).save() } - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the given default data async fn load_or_default(&self, default: T) -> Result> { - Ok(Persistable::load_or_default(self, default).await?) + Persistable::load_or_default(self.clone(), default).await } - /// Load the data from the repository into an auto persist container. If there is no persisted data then persist the result of the callback async fn load_or_else(&self, f: F) -> Result> where F: Send + FnOnce() -> Result, { - Ok(Persistable::load_or_else(self, f).await?) + Persistable::load_or_else(self.clone(), f).await } } -/// A container that automatically persists it's content every time it is mutated or changed. +/// A container that automatically persists its content every time it is mutated or changed. #[derive(Debug)] pub struct Persistable { data: Option, - repo: Repository, + connector: StoreConnector, + ctx: Option>, + staging_mode: bool, } impl Persistable where T: PersistableData, { - /// Create a new container with the given option data and repository - pub fn new(data: Option, repo: &Repository) -> Self { + /// Create a new container with the given data and connector + pub fn new(data: Option, connector: StoreConnector) -> Self { Self { data, - repo: repo.clone(), + connector, + ctx: None, + staging_mode: false, } } - /// Load data from the repository to the container - pub async fn load(repo: &Repository) -> Result { - let data = repo.read().await?; - - Ok(Self::new(data, repo)) + /// Load data from the store + pub async fn load(connector: StoreConnector) -> Result { + let data = Self::read_from_store(&connector).await?; + Ok(Self::new(data, connector)) } - /// Load the data from the repo or save and sync the given default value - pub async fn load_or_default(repo: &Repository, default: T) -> Result { - let instance = Self::new(Some(repo.read().await?.unwrap_or(default)), repo); - + /// Load the data or save and sync the given default value + pub async fn load_or_default(connector: StoreConnector, default: T) -> Result { + let data = Self::read_from_store(&connector).await?.unwrap_or(default); + let instance = Self::new(Some(data), connector); Ok(instance.save()) } - /// Load the data from the repo or save and sync the result of the given callback - pub async fn load_or_else(repo: &Repository, f: F) -> Result + /// Load the data or save and sync the result of the given callback + pub async fn load_or_else(connector: StoreConnector, f: F) -> Result where F: FnOnce() -> Result, { - let data = repo - .read() + let data = Self::read_from_store(&connector) .await? .ok_or_else(|| anyhow!("Not found")) .or_else(|_| f())?; - - let instance = Self::new(Some(data), repo); + let instance = Self::new(Some(data), connector); Ok(instance.save()) } - /// Save the data in the container to the database + async fn read_from_store(connector: &StoreConnector) -> Result> { + let Some(bytes) = connector.get.send(Get::new(&connector.key)).await? else { + return Ok(None); + }; + if bytes == [0] { + return Ok(None); + } + Ok(Some(bincode::deserialize(&bytes)?)) + } + + fn write_to_store(&self) { + if self.staging_mode { + return; + } + + let Some(ref data) = self.data else { + return; + }; + let Result::Ok(serialized) = bincode::serialize(data) else { + tracing::error!("Could not serialize value for persistable"); + return; + }; + + let msg = if let Some(ctx) = self.ctx.clone() { + Insert::new_with_context(&self.connector.key, serialized, ctx) + } else { + Insert::new(&self.connector.key, serialized) + }; + self.connector.insert.do_send(msg); + } + + /// Save the data in the container to the store pub fn save(self) -> Self { - self.checkpoint(); + self.write_to_store(); self } - /// Mutate the content if it is available or return an error if either the mutator function - /// fails or if the data has not been set. + /// Mutate the content if available or return an error pub fn try_mutate(&mut self, mutator: F) -> Result<()> where F: FnOnce(T) -> Result, { let content = self.data.clone().ok_or(anyhow!("Data has not been set"))?; self.data = Some(mutator(content)?); - self.checkpoint(); + self.write_to_store(); Ok(()) } - /// Set the data on both the persistable and the repository. + /// Set the data on both the persistable and the store pub fn set(&mut self, data: T) { self.data = Some(data); - self.checkpoint(); + self.write_to_store(); } - /// Clear the data from both the persistable and the repository. + /// Clear the data from both the persistable and the store pub fn clear(&mut self) { self.data = None; - self.clear_checkpoint(); + self.connector + .remove + .do_send(Remove::new(&self.connector.key)); } /// Get the data currently stored on the container as an Option @@ -142,295 +222,141 @@ where self.data.clone() } - /// Get the data from the container or return an error. + /// Get the data from the container or return an error pub fn try_get(&self) -> Result { self.data .clone() .ok_or(anyhow!("Data was not set on container.")) } - /// Returns true if there is data on the container and false if there is not. + /// Returns true if there is data on the container pub fn has(&self) -> bool { self.data.is_some() } - /// Get an immutable reference to the data on the container if the data is not set on the - /// container return an error - pub fn try_with(&self, f: F) -> Result - where - F: FnOnce(&T) -> Result, - { - match &self.data { - Some(data) => f(data), - None => Err(anyhow!("Data was not set on container.")), - } + /// Enter staging mode - changes held in memory only + pub fn stage(&mut self) { + self.staging_mode = true; } -} -impl Snapshot for Persistable -where - T: PersistableData, -{ - type Snapshot = T; - fn snapshot(&self) -> Result { - Ok(self - .data - .clone() - .ok_or(anyhow!("No data stored on container"))?) + /// Commit mode - writes current state and enables persistence + pub fn commit(&mut self) { + self.staging_mode = false; + self.write_to_store(); } } -impl Checkpoint for Persistable -where - T: PersistableData, -{ - fn repository(&self) -> &Repository { - &self.repo +impl EventContextManager for Persistable { + fn get_ctx(&self) -> Option> { + self.ctx.clone() } -} -#[async_trait] -impl FromSnapshotWithParams for Persistable -where - T: PersistableData, -{ - type Params = Repository; - async fn from_snapshot(params: Repository, snapshot: T) -> Result { - Ok(Persistable::new(Some(snapshot), ¶ms)) + fn set_ctx(&mut self, value: &EventContext) { + self.ctx = Some(value.clone()) } } #[cfg(test)] mod tests { - use crate::{AutoPersist, DataStore, GetLog, InMemStore, Repository}; - use actix::{Actor, Addr}; - use anyhow::{anyhow, Result}; - - fn get_repo() -> (Repository, Addr) { - let addr = InMemStore::new(true).start(); - let store = DataStore::from(&addr).scope("/"); - let repo: Repository = Repository::new(store); - (repo, addr) - } - - #[actix::test] - async fn persistable_loads_with_default() -> Result<()> { - let (repo, addr) = get_repo::>(); - let container = repo - .clone() - .load_or_default(vec!["berlin".to_string()]) - .await?; - - assert_eq!(addr.send(GetLog).await?.len(), 1); - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_loads_with_default_override() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["berlin".to_string()]); - let container = repo - .clone() - .load_or_default(vec!["amsterdam".to_string()]) - .await?; - - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_load() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["berlin".to_string()]); - let container = repo.clone().load().await?; - - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_send() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["amsterdam".to_string()]); - let container = repo.clone().send(Some(vec!["berlin".to_string()])); - - assert_eq!(repo.read().await?, Some(vec!["berlin".to_string()])); - assert_eq!(container.get(), Some(vec!["berlin".to_string()])); - Ok(()) - } - - #[actix::test] - async fn persistable_mutate() -> Result<()> { - let (repo, addr) = get_repo::>(); - - let mut container = repo.clone().send(Some(vec!["berlin".to_string()])); - - container.try_mutate(|mut list| { - list.push(String::from("amsterdam")); - Ok(list) - })?; - - assert_eq!( - repo.read().await?, - Some(vec!["berlin".to_string(), "amsterdam".to_string()]) - ); + use actix::{Actor, Addr, Handler, Message}; - assert_eq!(addr.send(GetLog).await?.len(), 2); - - Ok(()) - } - - #[actix::test] - async fn test_clear_persistable() -> Result<()> { - let (repo, _) = get_repo::>(); - let repo_ref = &repo; - let mut container = repo_ref.send(Some(vec!["berlin".to_string()])); - - assert!(container.has()); - container.clear(); - assert!(!container.has()); - assert_eq!(repo_ref.read().await?, None); - Ok(()) - } - - #[actix::test] - async fn test_set_persistable() -> Result<()> { - let (repo, _) = get_repo::>(); - let mut container = repo.clone().send(None); - - container.set(vec!["amsterdam".to_string()]); - - assert!(container.has()); - assert_eq!(repo.read().await?, Some(vec!["amsterdam".to_string()])); - Ok(()) - } - - #[actix::test] - async fn test_try_get_with_data() -> Result<()> { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(Some(vec!["berlin".to_string()])); - - let result = container.try_get()?; - assert_eq!(result, vec!["berlin".to_string()]); - Ok(()) - } + use crate::{Get, Insert, Remove}; - #[actix::test] - async fn test_try_get_without_data() { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(None); + use super::{Persistable, StoreConnector}; - assert!(container.try_get().is_err()); + #[derive(Debug, Clone)] + enum Evts { + Get, + Insert(Insert), + Remove, } - #[actix::test] - async fn test_try_with_success() -> Result<()> { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(Some(vec!["berlin".to_string()])); - - let length = container.try_with(|data| Ok(data.len()))?; - assert_eq!(length, 1); - Ok(()) + struct MockConnector { + key: Vec, + events: Vec, } + #[derive(Message)] + #[rtype("Vec")] + struct GetEvents; - #[actix::test] - async fn test_try_with_failure() { - let (repo, _) = get_repo::>(); - let container = repo.clone().send(None); - - let result = container.try_with(|data| Ok(data.len())); - assert!(result.is_err()); + impl Actor for MockConnector { + type Context = actix::Context; } - #[actix::test] - async fn test_try_mutate_failure() { - let (repo, _) = get_repo::>(); - let mut container = repo.clone().send(None); - - let result = container.try_mutate(|mut list| { - list.push(String::from("amsterdam")); - Ok(list) - }); - assert!(result.is_err()); + impl Handler for MockConnector { + type Result = Vec; + fn handle(&mut self, _msg: GetEvents, _ctx: &mut Self::Context) -> Self::Result { + self.events.clone() + } } - #[actix::test] - async fn test_mutate_with_error() -> Result<()> { - let (repo, _) = get_repo::>(); - let mut container = repo.clone().send(Some(vec!["berlin".to_string()])); - - let result = - container.try_mutate(|_| -> Result> { Err(anyhow!("Mutation failed")) }); - - assert!(result.is_err()); - // Original data should remain unchanged - assert_eq!(container.try_get()?, vec!["berlin".to_string()]); - Ok(()) + impl Handler for MockConnector { + type Result = Option>; + fn handle(&mut self, _msg: Get, _ctx: &mut Self::Context) -> Self::Result { + self.events.push(Evts::Get); + None + } } - #[actix::test] - async fn test_load_or_else_success_with_empty_repo() -> Result<()> { - let (repo, _) = get_repo::>(); - - let container = repo - .clone() - .load_or_else(|| Ok(vec!["amsterdam".to_string()])) - .await?; - - assert_eq!(container.try_get()?, vec!["amsterdam".to_string()]); - assert_eq!(repo.read().await?, Some(vec!["amsterdam".to_string()])); - Ok(()) + impl Handler for MockConnector { + type Result = (); + fn handle(&mut self, msg: Insert, _ctx: &mut Self::Context) -> Self::Result { + self.events.push(Evts::Insert(msg)); + } } - #[actix::test] - async fn test_load_or_else_skips_callback_when_data_exists() -> Result<()> { - let (repo, _) = get_repo::>(); - repo.write(&vec!["berlin".to_string()]); - - let container = repo - .clone() - .load_or_else(|| { - panic!("This callback should not be called!"); - #[allow(unreachable_code)] - Ok(vec!["amsterdam".to_string()]) - }) - .await?; - - assert_eq!(container.try_get()?, vec!["berlin".to_string()]); - Ok(()) + impl Handler for MockConnector { + type Result = (); + fn handle(&mut self, _msg: Remove, _ctx: &mut Self::Context) -> Self::Result { + self.events.push(Evts::Remove); + } } - #[actix::test] - async fn test_load_or_else_propagates_callback_error() -> Result<()> { - let (repo, _) = get_repo::>(); + impl MockConnector { + fn new(key: impl Into>) -> Self { + Self { + key: key.into(), + events: Vec::new(), + } + } - let result = repo - .clone() - .load_or_else(|| Err(anyhow!("Failed to create default data"))) - .await; - - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("Failed to create default data")); - assert_eq!(repo.read().await?, None); - Ok(()) + fn to_store_connector(self) -> (Addr, StoreConnector) { + let key = self.key.clone(); + let addr = self.start(); + ( + addr.clone(), + StoreConnector::new( + &key, + &addr.clone().recipient(), + &addr.clone().recipient(), + &addr.clone().recipient(), + ), + ) + } } #[actix::test] - async fn test_load_or_else_custom_error_message() -> Result<()> { - let (repo, _) = get_repo::>(); - let error_msg = "Custom initialization error"; + async fn test_persistable_staging() { + let (addr, connector) = MockConnector::new(b"loc").to_store_connector(); + let mut p = Persistable::new(Some(42i32), connector); + + p.set(100); + let events = addr.send(GetEvents).await.unwrap(); + assert_eq!(events.len(), 1); + assert!( + matches!(&events[0], Evts::Insert(msg) if msg.value() == &bincode::serialize(&100i32).unwrap()) + ); - let result = repo.load_or_else(|| Err(anyhow!(error_msg))).await; + p.stage(); + p.set(200); + let events = addr.send(GetEvents).await.unwrap(); + assert_eq!(events.len(), 1); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains(error_msg)); - Ok(()) + p.commit(); + let events = addr.send(GetEvents).await.unwrap(); + assert_eq!(events.len(), 2); + assert!( + matches!(&events[1], Evts::Insert(msg) if msg.value() == &bincode::serialize(&200i32).unwrap()) + ); } } diff --git a/crates/data/src/repository.rs b/crates/data/src/repository.rs index 085719ce65..4718a019b6 100644 --- a/crates/data/src/repository.rs +++ b/crates/data/src/repository.rs @@ -8,7 +8,7 @@ use std::marker::PhantomData; use anyhow::Result; -use crate::DataStore; +use crate::{DataStore, StoreConnector}; #[derive(Debug)] pub struct Repository { @@ -24,6 +24,15 @@ impl Repository { _p: PhantomData, } } + + pub fn to_connector(&self) -> StoreConnector { + StoreConnector::new( + self.store.scope_bytes(), + self.store.get_recipient(), + self.store.insert_recipient(), + self.store.remove_recipient(), + ) + } } impl From> for DataStore { diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index c3d39afdb0..c144b33403 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -5,13 +5,55 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::{Actor, Handler, Message, Recipient}; -use e3_events::CommitSnapshot; +use e3_events::hlc::HlcTimestamp; +use e3_events::{AggregateId, CommitSnapshot, EventContextAccessors}; +use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, +}; use crate::{Insert, InsertBatch}; +/// Central configuration for aggregates in the WriteBuffer +#[derive(Debug, Clone)] +pub struct AggregateConfig { + pub delays: HashMap, +} + +impl AggregateConfig { + /// Create a new AggregateConfig with the specified delays + pub fn new(mut delays: HashMap) -> Self { + // Always handle AggregatId of 0 with a delay of 0 + if let None = delays.get(&AggregateId::new(0)) { + delays.insert(AggregateId::new(0), 0); + } + Self { delays } + } + + /// Get the indexed aggregate IDs, defaulting to [0] if no delays are configured + pub fn indexed_ids(&self) -> Vec { + self.delays.keys().map(|id| id.to_usize()).collect() + } +} + +#[derive(Debug)] +struct AggregateBuffer { + buffer: Vec, +} + +impl AggregateBuffer { + fn new() -> Self { + Self { buffer: Vec::new() } + } +} + pub struct WriteBuffer { + /// Destination recipient for batched inserts dest: Option>, - buffer: Vec, + /// Per-aggregate buffers for organizing inserts + aggregate_buffers: HashMap, + /// Per-aggregate wait time configuration + config: AggregateConfig, } impl Actor for WriteBuffer { @@ -22,7 +64,55 @@ impl WriteBuffer { pub fn new() -> Self { Self { dest: None, - buffer: Vec::new(), + aggregate_buffers: HashMap::new(), + config: AggregateConfig::new(HashMap::new()), + } + } + + pub fn with_config(config: AggregateConfig) -> Self { + Self { + dest: None, + aggregate_buffers: HashMap::new(), + config, + } + } + + fn handle_insert(&mut self, msg: Insert) { + let aggregate_id = if let Some(event_ctx) = msg.ctx() { + event_ctx.aggregate_id().clone() + } else { + AggregateId::new(0) + }; + + let agg_buffer = self + .aggregate_buffers + .entry(aggregate_id) + .or_insert_with(|| AggregateBuffer::new()); + agg_buffer.buffer.push(msg.clone()); + } + + fn handle_commit_snapshot(&mut self, msg: CommitSnapshot) { + // Store the sequence number as an Insert message so snapshots hold the most recent event + // they were created against + self.handle_insert(Insert::new( + &format!("//aggregate_seq/{}", msg.aggregate_id()), + msg.seq().to_le_bytes().to_vec(), // Same as bincode avoiding result + )); + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_micros() as u64; + + if let Some(ref dest) = self.dest { + let (updated_buffers, expired_inserts) = + process_expired_inserts(&self.aggregate_buffers, &self.config.delays, now); + if !expired_inserts.is_empty() { + let batch = InsertBatch::new(expired_inserts); + dest.do_send(batch); + } + + self.aggregate_buffers = updated_buffers; } } } @@ -38,22 +128,126 @@ impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - self.buffer.push(msg); + self.handle_insert(msg) + } +} + +fn process_expired_inserts( + aggregate_buffers: &HashMap, + config: &HashMap, + now: u64, +) -> (HashMap, Vec) { + let mut updated_buffers = HashMap::new(); + let mut all_expired_inserts = Vec::new(); + + for (aggregate_id, agg_buffer) in aggregate_buffers { + let delay_micros = config.get(aggregate_id).copied().unwrap_or(0); + let cutoff_time = now.saturating_sub(delay_micros); + let mut expired_inserts = Vec::new(); + let mut remaining_inserts = Vec::new(); + + for insert in &agg_buffer.buffer { + if let Some(ctx) = insert.ctx() { + let event_wall_time = HlcTimestamp::wall_time(ctx.ts()); + if event_wall_time < cutoff_time { + expired_inserts.push(insert.clone()); + } else { + remaining_inserts.push(insert.clone()); + } + } else { + // If there is no context just flush it + expired_inserts.push(insert.clone()); + } + } + + all_expired_inserts.extend(expired_inserts); + + if !remaining_inserts.is_empty() { + let mut new_agg_buffer = AggregateBuffer::new(); + new_agg_buffer.buffer = remaining_inserts; + updated_buffers.insert(aggregate_id.clone(), new_agg_buffer); + } } + + (updated_buffers, all_expired_inserts) } impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { - if let Some(ref dest) = self.dest { - if !self.buffer.is_empty() { - let mut inserts = std::mem::take(&mut self.buffer); - inserts.push(Insert::new("//seq", msg.seq().to_be_bytes().to_vec())); - let batch = InsertBatch::new(inserts); - dest.do_send(batch); - } - } + self.handle_commit_snapshot(msg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::Insert; + use e3_events::{hlc::HlcTimestamp, EventContext, EventId}; + + #[test] + fn test_process_expired_inserts() { + let aggregate_id = AggregateId::new(1); + + // Create test inserts with different timestamps (in microseconds) + // Create proper HlcTimestamps and encode them to u128 + let old_hlc = HlcTimestamp::new(500_000, 0, 1); // 0.5 seconds ago + let new_hlc = HlcTimestamp::new(3_000_000, 0, 2); // 3 seconds from epoch + + let old_ctx = EventContext::new( + EventId::hash(1), + EventId::hash(1), + EventId::hash(1), + old_hlc.into(), + aggregate_id.clone(), + ) + .sequence(1); + + let new_ctx = EventContext::new( + EventId::hash(2), + EventId::hash(2), + EventId::hash(2), + new_hlc.into(), + aggregate_id.clone(), + ) + .sequence(2); + + let old_insert = Insert::new_with_context("old_key", b"old_value".to_vec(), old_ctx); + let new_insert = Insert::new_with_context("new_key", b"new_value".to_vec(), new_ctx); + let insert_no_ctx = Insert::new("no_ctx_key", b"no_ctx_value".to_vec()); + + // Set up aggregate buffer with mixed inserts + let mut agg_buffer = AggregateBuffer::new(); + agg_buffer.buffer.push(old_insert.clone()); + agg_buffer.buffer.push(new_insert.clone()); + agg_buffer.buffer.push(insert_no_ctx.clone()); + + let mut aggregate_buffers = HashMap::new(); + aggregate_buffers.insert(aggregate_id.clone(), agg_buffer); + + // Set config with 1 second delay + let mut delays = HashMap::new(); + delays.insert(aggregate_id.clone(), 1_000_000); // 1 second in microseconds + let config = AggregateConfig::new(delays); + + // Use current time of 2 seconds, so old insert (0.5s) and insert without context should expire, + // new insert (3s) should remain + let now = 2_000_000; // 2 seconds in microseconds + + let (updated_buffers, expired_inserts) = + process_expired_inserts(&aggregate_buffers, &config.delays, now); + + // Verify expired inserts (old insert and insert without context) + assert_eq!(expired_inserts.len(), 2); + assert!(expired_inserts.contains(&old_insert)); + assert!(expired_inserts.contains(&insert_no_ctx)); + + // Verify remaining inserts in buffer + assert_eq!(updated_buffers.len(), 1); + let remaining_buffer = updated_buffers.get(&aggregate_id).unwrap(); + assert_eq!(remaining_buffer.buffer.len(), 1); + assert!(remaining_buffer.buffer.contains(&new_insert)); } } diff --git a/crates/entrypoint/src/helpers/shutdown.rs b/crates/entrypoint/src/helpers/shutdown.rs index 1ff0d6748d..1b8f96e818 100644 --- a/crates/entrypoint/src/helpers/shutdown.rs +++ b/crates/entrypoint/src/helpers/shutdown.rs @@ -4,17 +4,17 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; -use e3_events::{prelude::*, BusHandle, Shutdown}; +use e3_ciphernode_builder::CiphernodeHandle; +use e3_events::{prelude::*, Shutdown}; use std::time::Duration; use tokio::{ select, signal::unix::{signal, SignalKind}, - task::JoinHandle, }; use tracing::{error, info}; -pub async fn listen_for_shutdown(bus: BusHandle, mut handle: JoinHandle>) { +pub async fn listen_for_shutdown(node: CiphernodeHandle) { + let (bus, mut handle) = node.split(); let mut sigterm = signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal stream"); select! { diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index c85e98cee9..3032e43d0c 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -6,12 +6,9 @@ use alloy::primitives::Address; use anyhow::Result; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{CiphernodeBuilder, CiphernodeHandle}; use e3_config::AppConfig; use e3_crypto::Cipher; -use e3_data::RepositoriesFactory; -use e3_events::BusHandle; -use e3_net::{NetEventTranslator, NetRepositoryFactory}; use e3_test_helpers::{PlaintextWriter, PublicKeyWriter}; use rand::SeedableRng; use rand_chacha::{rand_core::OsRng, ChaCha20Rng}; @@ -19,17 +16,16 @@ use std::{ path::PathBuf, sync::{Arc, Mutex}, }; -use tokio::task::JoinHandle; pub async fn execute( config: &AppConfig, address: Address, pubkey_write_path: Option, plaintext_write_path: Option, -) -> Result<(BusHandle, JoinHandle>, String)> { +) -> Result { let rng = Arc::new(Mutex::new(ChaCha20Rng::from_rng(OsRng)?)); let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); - let builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) + let node = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_address(&address.to_string()) .with_persistence(&config.log_file(), &config.db_file()) .with_chains(&config.chains()) @@ -39,30 +35,19 @@ pub async fn execute( .with_contract_ciphernode_registry() .with_max_threads() .with_pubkey_aggregation() - .with_threshold_plaintext_aggregation(); - - // TODO: put net package provisioning in the ciphernode-builder: - let node = builder.build().await?; - let store = node.store(); - let repositories = store.repositories(); - let bus = node.bus.clone(); - let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( - bus.clone(), - config.peers(), - &cipher, - config.quic_port(), - repositories.libp2p_keypair(), - ) - .await?; + .with_threshold_plaintext_aggregation() + .with_net(config.peers(), config.quic_port()) + .build() + .await?; // These are here purely for our integration test so leaving out of the builder if let Some(path) = pubkey_write_path { - PublicKeyWriter::attach(&path, bus.clone()); + PublicKeyWriter::attach(&path, node.bus().clone()); } if let Some(path) = plaintext_write_path { - PlaintextWriter::attach(&path, bus.clone()); + PlaintextWriter::attach(&path, node.bus().clone()); } - Ok((bus, join_handle, peer_id)) + Ok(node) } diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 2a4a42abef..9dae6086f7 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -6,7 +6,7 @@ use alloy::primitives::Address; use anyhow::Result; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{CiphernodeBuilder, CiphernodeHandle}; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; @@ -19,13 +19,10 @@ use tokio::task::JoinHandle; use tracing::instrument; #[instrument(name = "app", skip_all)] -pub async fn execute( - config: &AppConfig, - address: Address, -) -> Result<(BusHandle, JoinHandle>, String)> { +pub async fn execute(config: &AppConfig, address: Address) -> Result { let rng = Arc::new(Mutex::new(rand_chacha::ChaCha20Rng::from_rng(OsRng)?)); let cipher = Arc::new(Cipher::from_file(&config.key_file()).await?); - let builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) + let node = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_address(&address.to_string()) .with_persistence(&config.log_file(), &config.db_file()) .with_sortition_score() @@ -34,19 +31,10 @@ pub async fn execute( .with_contract_bonding_registry() .with_max_threads() .with_contract_ciphernode_registry() - .with_trbfv(); + .with_trbfv() + .with_net(config.peers(), config.quic_port()) + .build() + .await?; - let node = builder.build().await?; - let repositories = node.store().repositories(); - let bus = node.bus.clone(); - let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( - bus.clone(), - config.peers(), - &cipher, - config.quic_port(), - repositories.libp2p_keypair(), - ) - .await?; - - Ok((bus, join_handle, peer_id)) + Ok(node) } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 26985f5b73..1863054021 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -12,55 +12,59 @@ use derivative::Derivative; use tracing::error; use crate::{ + event_context::EventContext, hlc::Hlc, sequencer::Sequencer, traits::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventType, HistoryCollector, - Sequenced, Subscribe, Unsequenced, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, EventContextManager, EventType, + HistoryCollector, Sequenced, Subscribe, Unsequenced, Unsubscribe, }; #[derive(Clone, Derivative)] #[derivative(Debug, PartialEq, Eq)] pub struct BusHandle { /// EventBus that actors can consume sequenced events from - consumer: Addr>>, + event_bus: Addr>>, /// Sequencer that new events should be produced from - producer: Addr, + sequencer: Addr, /// Hlc clock used to time all events created on this BusHandle #[derivative(Debug = "ignore")] hlc: Arc, + /// Temporary context for events the bus publishes + ctx: Option>, } impl BusHandle { /// Create a new BusHandle pub fn new( - consumer: Addr>>, - producer: Addr, + event_bus: Addr>>, + sequencer: Addr, hlc: Hlc, ) -> Self { Self { - consumer, - producer, + event_bus, + sequencer, hlc: Arc::new(hlc), + ctx: None, } } /// Return a HistoryCollector for examining events that have passed through on the events bus pub fn history(&self) -> Addr>> { - EventBus::>::history(&self.consumer) + EventBus::>::history(&self.event_bus) } - /// Access the producer to internally dispatch am event to - pub fn producer(&self) -> &Addr { - &self.producer + /// Access the sequencer to internally dispatch am event to + pub fn sequencer(&self) -> &Addr { + &self.sequencer } - /// Access the consumer to internally subscribe to events - pub fn consumer(&self) -> &Addr>> { - &self.consumer + /// Access the event_bus to internally subscribe to events + pub fn event_bus(&self) -> &Addr>> { + &self.event_bus } /// Get a new timestamp. Note this ticks over the internal Hlc. @@ -81,36 +85,41 @@ impl BusHandle { impl EventPublisher> for BusHandle { fn publish(&self, data: impl Into) -> Result<()> { - let evt = self.event_from(data)?; - self.producer.do_send(evt); + let evt = self.event_from(data, self.get_ctx())?; + self.sequencer.do_send(evt); Ok(()) } fn publish_from_remote(&self, data: impl Into, ts: u128) -> Result<()> { - let evt = self.event_from_remote_source(data, ts)?; - self.producer.do_send(evt); + let evt = self.event_from_remote_source(data, self.get_ctx(), ts)?; + self.sequencer.do_send(evt); Ok(()) } fn naked_dispatch(&self, event: EnclaveEvent) { - self.producer.do_send(event); + self.sequencer.do_send(event); } } impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: impl Into) { - match self.event_from_error(err_type, error) { - Ok(evt) => self.producer.do_send(evt), + match self.event_from_error(err_type, error, self.get_ctx()) { + Ok(evt) => self.sequencer.do_send(evt), Err(e) => error!("{e}"), } } } impl EventFactory> for BusHandle { - fn event_from(&self, data: impl Into) -> Result> { + fn event_from( + &self, + data: impl Into, + caused_by: Option>, + ) -> Result> { let ts = self.hlc.tick()?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), + caused_by, ts.into(), )) } @@ -118,11 +127,13 @@ impl EventFactory> for BusHandle { fn event_from_remote_source( &self, data: impl Into, + caused_by: Option>, ts: u128, ) -> Result> { let ts = self.hlc.receive(&ts.into())?; Ok(EnclaveEvent::::new_with_timestamp( data.into(), + caused_by, ts.into(), )) } @@ -133,15 +144,17 @@ impl ErrorFactory> for BusHandle { &self, err_type: EType, error: impl Into, + caused_by: Option>, ) -> Result> { let ts = self.hlc.tick()?; - EnclaveEvent::::from_error(err_type, error, ts.into()) + EnclaveEvent::::from_error(err_type, error, ts.into(), caused_by) } } impl EventSubscriber> for BusHandle { fn subscribe(&self, event_type: EventType, recipient: Recipient>) { - self.consumer.do_send(Subscribe::new(event_type, recipient)) + self.event_bus + .do_send(Subscribe::new(event_type, recipient)) } fn subscribe_all( @@ -150,10 +163,24 @@ impl EventSubscriber> for BusHandle { recipient: Recipient>, ) { for event_type in event_types.into_iter() { - self.consumer + self.event_bus .do_send(Subscribe::new(*event_type, recipient.clone())); } } + + fn unsubscribe(&self, event_type: &str, recipient: Recipient>) { + self.event_bus + .do_send(Unsubscribe::new(event_type, recipient)); + } +} + +impl EventContextManager for BusHandle { + fn set_ctx(&mut self, value: &EventContext) { + self.ctx = Some(value.clone()); + } + fn get_ctx(&self) -> Option> { + self.ctx.clone() + } } #[cfg(test)] @@ -192,7 +219,7 @@ mod tests { impl Handler for Forwarder { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - let ts = msg.get_ts(); + let ts = msg.ts(); self.dest.publish_from_remote(msg.into_data(), ts).unwrap() } } @@ -263,7 +290,7 @@ mod tests { // Sort by HLC timestamp let mut sorted_events = events.clone(); - sorted_events.sort_by_key(|e| e.get_ts()); + sorted_events.sort_by_key(|e| e.ts()); // Extract the payloads/names in HLC-sorted order let ordered_names: Vec<_> = sorted_events @@ -282,7 +309,7 @@ mod tests { ); // ASSERTION 2: All timestamps are unique (HLC guarantee) - let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); + let timestamps: Vec<_> = sorted_events.iter().map(|e| e.ts()).collect(); let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); assert_eq!( timestamps.len(), diff --git a/crates/events/src/correlation_id.rs b/crates/events/src/correlation_id.rs index b7f808deeb..ee3e3dadc5 100644 --- a/crates/events/src/correlation_id.rs +++ b/crates/events/src/correlation_id.rs @@ -5,7 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use std::{ + fmt, + fmt::Debug, fmt::Display, + fmt::Formatter, sync::atomic::{AtomicUsize, Ordering}, }; @@ -14,7 +17,7 @@ use serde::{Deserialize, Serialize}; static NEXT_CORRELATION_ID: AtomicUsize = AtomicUsize::new(1); /// CorrelationId provides a way to correlate commands and the events they create. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct CorrelationId { id: usize, } @@ -31,3 +34,9 @@ impl Display for CorrelationId { write!(f, "{}", self.id) } } + +impl Debug for CorrelationId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.id) + } +} diff --git a/crates/events/src/enclave_event/enclave_error.rs b/crates/events/src/enclave_event/enclave_error.rs index 241ce8f563..5f9100ded3 100644 --- a/crates/events/src/enclave_event/enclave_error.rs +++ b/crates/events/src/enclave_event/enclave_error.rs @@ -6,7 +6,11 @@ use actix::Message; use serde::{Deserialize, Serialize}; -use std::fmt::{self, Display}; +use std::{ + fmt::{self, Display}, + future::Future, + pin::Pin, +}; use crate::{BusHandle, ErrorDispatcher}; @@ -37,6 +41,7 @@ pub enum EType { PlaintextAggregation, Decryption, Sortition, + Sync, Data, Event, Computation, @@ -71,3 +76,21 @@ where Err(e) => bus.err(err_type, e), } } + +/// Function to accept a future that resolves to a result. If result is an Err variant it is trapped and +/// sent to the bus as an ErrorEvent +pub fn trap_fut( + err_type: EType, + bus: &BusHandle, + fut: F, +) -> Pin + Send>> +where + F: Future> + Send + 'static, +{ + let bus = bus.clone(); + Box::pin(async move { + if let Err(e) = fut.await { + bus.err(err_type, e); + } + }) +} diff --git a/crates/events/src/enclave_event/evm_sync_events_received.rs b/crates/events/src/enclave_event/evm_sync_events_received.rs new file mode 100644 index 0000000000..0fcf1e3347 --- /dev/null +++ b/crates/events/src/enclave_event/evm_sync_events_received.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use super::{EnclaveEvent, Unsequenced}; + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct EvmSyncEventsReceived { + pub events: Vec>, +} + +impl EvmSyncEventsReceived { + pub fn new(events: Vec>) -> Self { + Self { events } + } +} + +impl Display for EvmSyncEventsReceived { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 26765e9eac..514ec33337 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -21,19 +21,26 @@ mod e3_requested; mod enclave_error; mod encryption_key_collection_failed; mod encryption_key_created; +mod evm_sync_events_received; mod keyshare_created; +mod net_sync_events_received; mod operator_activation_changed; +mod outgoing_sync_requested; mod plaintext_aggregated; mod plaintext_output_published; mod publickey_aggregated; mod publish_document; mod shutdown; +mod sync_effect; +mod sync_end; +mod sync_start; mod test_event; mod threshold_share_collection_failed; mod threshold_share_created; mod ticket_balance_updated; mod ticket_generated; mod ticket_submitted; +mod typed_event; pub use ciphernode_added::*; pub use ciphernode_removed::*; @@ -53,24 +60,32 @@ use e3_utils::{colorize, Color}; pub use enclave_error::*; pub use encryption_key_collection_failed::*; pub use encryption_key_created::*; +pub use evm_sync_events_received::*; pub use keyshare_created::*; +pub use net_sync_events_received::*; pub use operator_activation_changed::*; +pub use outgoing_sync_requested::*; pub use plaintext_aggregated::*; pub use plaintext_output_published::*; pub use publickey_aggregated::*; pub use publish_document::*; pub use shutdown::*; use strum::IntoStaticStr; +pub use sync_effect::*; +pub use sync_end::*; +pub use sync_start::*; pub use test_event::*; pub use threshold_share_collection_failed::*; pub use threshold_share_created::*; pub use ticket_balance_updated::*; pub use ticket_generated::*; pub use ticket_submitted::*; +pub use typed_event::*; use crate::{ - traits::{ErrorEvent, Event, EventConstructorWithTimestamp}, - E3id, EventId, + event_context::{AggregateId, EventContext}, + traits::{ErrorEvent, Event, EventConstructorWithTimestamp, EventContextAccessors}, + E3id, EventContextSeq, EventId, WithAggregateId, }; use actix::Message; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -191,9 +206,15 @@ pub enum EnclaveEventData { EncryptionKeyCreated(EncryptionKeyCreated), EncryptionKeyCollectionFailed(EncryptionKeyCollectionFailed), ThresholdShareCollectionFailed(ThresholdShareCollectionFailed), - ComputeRequest(ComputeRequest), - ComputeResponse(ComputeResponse), - ComputeRequestError(ComputeRequestError), + ComputeRequest(ComputeRequest), // ComputeRequested + ComputeResponse(ComputeResponse), // ComputeResponseReceived + ComputeRequestError(ComputeRequestError), // ComputeRequestFailed + OutgoingSyncRequested(OutgoingSyncRequested), + NetSyncEventsReceived(NetSyncEventsReceived), + EvmSyncEventsReceived(EvmSyncEventsReceived), + SyncStart(SyncStart), + SyncEffect(SyncEffect), + SyncEnd(SyncEnd), /// This is a test event to use in testing TestEvent(TestEvent), } @@ -234,11 +255,13 @@ impl SeqState for Sequenced { #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] +#[serde(bound( + serialize = "S: SeqState, S::Seq: Serialize", + deserialize = "S: SeqState, S::Seq: DeserializeOwned" +))] pub struct EnclaveEvent { - id: EventId, payload: EnclaveEventData, - seq: S::Seq, - ts: u128, + ctx: EventContext, } impl EnclaveEvent @@ -253,38 +276,57 @@ where bincode::deserialize(bytes) } - pub fn get_id(&self) -> EventId { - self.into() + pub fn split(self) -> (EnclaveEventData, u128) { + (self.payload, self.ctx.ts()) } - pub fn get_ts(&self) -> u128 { - self.ts + pub fn get_ctx(&self) -> &EventContext { + &self.ctx } +} - pub fn split(self) -> (EnclaveEventData, u128) { - (self.payload, self.ts) +impl EventContextAccessors for EnclaveEvent { + fn causation_id(&self) -> EventId { + self.ctx.causation_id() + } + fn origin_id(&self) -> EventId { + self.ctx.origin_id() + } + fn ts(&self) -> u128 { + self.ctx.ts() + } + fn id(&self) -> EventId { + self.ctx.id() + } + fn aggregate_id(&self) -> AggregateId { + self.ctx.aggregate_id() } } -impl EnclaveEvent { - pub fn get_seq(&self) -> u64 { - self.seq +impl EventContextSeq for EnclaveEvent { + fn seq(&self) -> u64 { + self.ctx.seq() } +} +impl EnclaveEvent { pub fn clone_unsequenced(&self) -> EnclaveEvent { - let ts = self.get_ts(); + let ts = self.ts(); let data = self.clone().into_data(); - EnclaveEvent::new_with_timestamp(data, ts) + EnclaveEvent::new_with_timestamp(data, Some(self.ctx.clone()), ts) + } + + pub fn to_typed_event(&self, data: T) -> TypedEvent { + let ctx: EventContext = self.get_ctx().clone(); + TypedEvent::new(data, ctx) } } impl EnclaveEvent { pub fn into_sequenced(self, seq: u64) -> EnclaveEvent { EnclaveEvent:: { - id: self.id, payload: self.payload, - ts: self.ts, - seq, + ctx: self.ctx.sequence(seq), } } } @@ -293,12 +335,12 @@ impl EnclaveEvent { impl EnclaveEvent { /// test-helpers only utility function to create a new unsequenced event pub fn new_stored_event(data: EnclaveEventData, time: u128, seq: u64) -> Self { - EnclaveEvent::::new_with_timestamp(data, time).into_sequenced(seq) + EnclaveEvent::::new_with_timestamp(data, None, time).into_sequenced(seq) } /// test-helpers only utility function to remove time information from an event pub fn strip_ts(&self) -> EnclaveEvent { - EnclaveEvent::new_stored_event(self.get_data().clone(), 0, self.get_seq()) + EnclaveEvent::new_stored_event(self.get_data().clone(), 0, self.seq()) } } @@ -306,12 +348,12 @@ impl Event for EnclaveEvent { type Id = EventId; type Data = EnclaveEventData; - fn event_type(&self) -> String { - self.payload.event_type() + fn event_id(&self) -> Self::Id { + self.ctx.id() } - fn event_id(&self) -> Self::Id { - self.get_id() + fn event_type(&self) -> String { + self.payload.event_type() } fn get_data(&self) -> &EnclaveEventData { @@ -331,33 +373,38 @@ impl ErrorEvent for EnclaveEvent { err_type: Self::ErrType, msg: impl Into, ts: u128, + caused_by: Option>, ) -> anyhow::Result { let payload = EnclaveError::new(err_type, msg); let id = EventId::hash(&payload); + let aggregate_id = AggregateId::new(0); // Error events use default aggregate_id + + let ctx = caused_by + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)); + Ok(EnclaveEvent { payload: payload.into(), - id, - seq: (), - ts, + ctx, }) } } impl From> for EventId { fn from(value: EnclaveEvent) -> Self { - value.id + value.ctx.id() } } impl From<&EnclaveEvent> for EventId { fn from(value: &EnclaveEvent) -> Self { - value.id.clone() + value.ctx.id() } } -impl EnclaveEvent { +impl EnclaveEventData { pub fn get_e3_id(&self) -> Option { - match self.payload { + match self { EnclaveEventData::KeyshareCreated(ref data) => Some(data.e3_id.clone()), EnclaveEventData::E3Requested(ref data) => Some(data.e3_id.clone()), EnclaveEventData::PublicKeyAggregated(ref data) => Some(data.e3_id.clone()), @@ -380,6 +427,29 @@ impl EnclaveEvent { } } +impl WithAggregateId for EnclaveEventData { + fn get_aggregate_id(&self) -> AggregateId { + let maybe_e3_id = self.get_e3_id(); + if let Some(e3_id) = maybe_e3_id { + AggregateId::new(e3_id.chain_id() as usize) + } else { + AggregateId::new(0) + } + } +} + +impl EnclaveEvent { + pub fn get_e3_id(&self) -> Option { + self.payload.get_e3_id() + } +} + +impl WithAggregateId for EnclaveEvent { + fn get_aggregate_id(&self) -> AggregateId { + self.payload.get_aggregate_id() + } +} + impl_event_types!( KeyshareCreated, E3Requested, @@ -412,7 +482,13 @@ impl_event_types!( ThresholdShareCollectionFailed, ComputeRequest, ComputeResponse, - ComputeRequestError + ComputeRequestError, + OutgoingSyncRequested, + NetSyncEventsReceived, + EvmSyncEventsReceived, + SyncStart, + SyncEffect, + SyncEnd ); impl TryFrom<&EnclaveEvent> for EnclaveError { @@ -449,14 +525,19 @@ impl fmt::Display for EnclaveEvent { } impl EventConstructorWithTimestamp for EnclaveEvent { - fn new_with_timestamp(data: Self::Data, ts: u128) -> Self { - let payload = data.into(); + fn new_with_timestamp( + data: Self::Data, + caused_by: Option>, + ts: u128, + ) -> Self { + let payload: EnclaveEventData = data.into(); let id = EventId::hash(&payload); + let aggregate_id = payload.get_aggregate_id(); EnclaveEvent { - id, payload, - seq: (), - ts, + ctx: caused_by + .map(|cause| EventContext::from_cause(id, cause, ts, aggregate_id)) + .unwrap_or_else(|| EventContext::new_origin(id, ts, aggregate_id)), } } } diff --git a/crates/events/src/enclave_event/net_sync_events_received.rs b/crates/events/src/enclave_event/net_sync_events_received.rs new file mode 100644 index 0000000000..eefe1828fb --- /dev/null +++ b/crates/events/src/enclave_event/net_sync_events_received.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use super::{EnclaveEvent, Unsequenced}; + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct NetSyncEventsReceived { + pub events: Vec>, +} + +impl NetSyncEventsReceived { + pub fn new(events: Vec>) -> Self { + Self { events } + } +} + +impl Display for NetSyncEventsReceived { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/outgoing_sync_requested.rs b/crates/events/src/enclave_event/outgoing_sync_requested.rs new file mode 100644 index 0000000000..d9a89b6996 --- /dev/null +++ b/crates/events/src/enclave_event/outgoing_sync_requested.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +use crate::AggregateId; + +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct OutgoingSyncRequested { + // TODO: this should be the event to trigger evm sync too + pub since: Vec<(AggregateId, u128)>, +} + +impl Display for OutgoingSyncRequested { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/sync_effect.rs b/crates/events/src/enclave_event/sync_effect.rs new file mode 100644 index 0000000000..b503422933 --- /dev/null +++ b/crates/events/src/enclave_event/sync_effect.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +/// Dispatched from the Sync actor once the effect events are to be run but before buffered events are to +/// be dispatched +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncEffect; + +impl SyncEffect { + pub fn new() -> Self { + Self {} + } +} + +impl Display for SyncEffect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/sync_end.rs b/crates/events/src/enclave_event/sync_end.rs new file mode 100644 index 0000000000..400ba9f5e2 --- /dev/null +++ b/crates/events/src/enclave_event/sync_end.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::Message; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +/// Dispatched once the sync process is complete and live listening should continue +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncEnd; + +impl SyncEnd { + pub fn new() -> Self { + Self {} + } +} + +impl Display for SyncEnd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/sync_start.rs b/crates/events/src/enclave_event/sync_start.rs new file mode 100644 index 0000000000..3841785133 --- /dev/null +++ b/crates/events/src/enclave_event/sync_start.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::EnclaveEventData; +use crate::{CorrelationId, SyncEvmEvent}; +use crate::{EvmEventConfig, EvmEventConfigChain}; +use actix::{Message, Recipient}; +use anyhow::Context; +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Display}; + +/// This is a processed EvmEvent specifically typed for the Sync actor +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct EvmEvent { + data: EnclaveEventData, + block: u64, + chain_id: u64, + ts: u128, + id: CorrelationId, +} + +impl EvmEvent { + pub fn new( + id: CorrelationId, + data: EnclaveEventData, + block: u64, + ts: u128, + chain_id: u64, + ) -> Self { + Self { + id, + data, + block, + ts, + chain_id, + } + } + + pub fn split(self) -> (EnclaveEventData, u128, u64) { + (self.data, self.ts, self.block) + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } + + pub fn chain_id(&self) -> u64 { + self.chain_id + } + + pub fn ts(&self) -> u128 { + self.ts + } +} + +/// Dispatched by the Sync actor when initial data is read and the sync process needs to be started +#[derive(Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct SyncStart { + /// The initial information for reading historical events from chains. This is generated from + /// from persisted information + pub evm_config: EvmEventConfig, + + #[serde(skip)] + /// We include the sender here so that the evm can communicate directly with the sync actor + pub sender: Option>, // Must be Option to allow serde deserialize on + // EnclaveEvent as Default is required to be + // implemented this is fine as this event is never + // shared +} + +impl SyncStart { + pub fn new(sender: impl Into>, evm_config: EvmEventConfig) -> Self { + Self { + sender: Some(sender.into()), + evm_config, + } + } + + pub fn get_evm_config(&self, chain_id: u64) -> Result { + Ok(self + .evm_config + .get(&chain_id) + .context("No config found for chain")? + .clone()) + } +} + +impl Display for SyncStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/events/src/enclave_event/typed_event.rs b/crates/events/src/enclave_event/typed_event.rs new file mode 100644 index 0000000000..7f3da4c0ee --- /dev/null +++ b/crates/events/src/enclave_event/typed_event.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::ops::Deref; + +use actix::Message; +use serde::{Deserialize, Serialize}; + +use crate::{ + event_context::{AggregateId, EventContext}, + EventContextAccessors, EventContextSeq, EventId, +}; + +use super::Sequenced; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct TypedEvent { + inner: T, + ctx: EventContext, +} + +impl TypedEvent { + pub fn new(inner: T, ctx: EventContext) -> Self { + Self { inner, ctx } + } + + pub fn into_inner(self) -> T { + self.inner + } + + pub fn get_ctx(&self) -> &EventContext { + &self.ctx + } +} + +impl Deref for TypedEvent { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl EventContextAccessors for TypedEvent { + fn id(&self) -> EventId { + self.ctx.id() + } + + fn ts(&self) -> u128 { + self.ctx.ts() + } + + fn origin_id(&self) -> EventId { + self.ctx.origin_id() + } + + fn causation_id(&self) -> EventId { + self.ctx.causation_id() + } + + fn aggregate_id(&self) -> AggregateId { + self.ctx.aggregate_id() + } +} + +impl EventContextSeq for TypedEvent { + fn seq(&self) -> u64 { + self.ctx.seq() + } +} + +impl From<(T, &EventContext)> for TypedEvent { + fn from(value: (T, &EventContext)) -> Self { + Self { + inner: value.0, + ctx: value.1.clone(), + } + } +} diff --git a/crates/events/src/event_context.rs b/crates/events/src/event_context.rs new file mode 100644 index 0000000000..6cb5b9bafa --- /dev/null +++ b/crates/events/src/event_context.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::{fmt, ops::Deref}; + +use serde::{Deserialize, Serialize}; + +use crate::{ + E3id, EventContextAccessors, EventContextSeq, EventId, SeqState, Sequenced, Unsequenced, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct AggregateId(usize); + +impl AggregateId { + pub fn new(value: usize) -> Self { + Self(value) + } + + pub fn to_usize(&self) -> usize { + self.0 + } +} + +impl From> for AggregateId { + fn from(value: Option) -> Self { + if let Some(e3_id) = value { + Self::new(e3_id.chain_id() as usize) + } else { + Self::new(0) + } + } +} + +impl Deref for AggregateId { + type Target = usize; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Display for AggregateId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", &self.0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EventContext { + id: EventId, + causation_id: EventId, + origin_id: EventId, + seq: S::Seq, + ts: u128, + aggregate_id: AggregateId, +} + +impl EventContext { + pub fn new( + id: EventId, + causation_id: EventId, + origin_id: EventId, + ts: u128, + aggregate_id: AggregateId, + ) -> Self { + Self { + id, + causation_id, + origin_id, + seq: (), + ts, + aggregate_id, + } + } + + pub fn new_origin(id: EventId, ts: u128, aggregate_id: AggregateId) -> Self { + Self::new(id, id, id, ts, aggregate_id) + } + + pub fn from_cause( + id: EventId, + cause: EventContext, + ts: u128, + aggregate_id: AggregateId, + ) -> Self { + EventContext::new(id, cause.id(), cause.origin_id(), ts, aggregate_id) + } + + pub fn sequence(self, value: u64) -> EventContext { + EventContext:: { + seq: value, + id: self.id, + causation_id: self.causation_id, + origin_id: self.origin_id, + ts: self.ts, + aggregate_id: self.aggregate_id, + } + } +} + +impl EventContextAccessors for EventContext { + fn id(&self) -> EventId { + self.id + } + + fn causation_id(&self) -> EventId { + self.causation_id + } + + fn origin_id(&self) -> EventId { + self.origin_id + } + + fn ts(&self) -> u128 { + self.ts + } + + fn aggregate_id(&self) -> AggregateId { + self.aggregate_id + } +} + +impl EventContextSeq for EventContext { + fn seq(&self) -> u64 { + self.seq + } +} + +#[cfg(test)] +mod tests { + use crate::{ + event_context::{AggregateId, EventContext}, + EventId, + }; + + #[test] + fn test_event_context_cycle() { + let mut events = vec![]; + + let one = EventContext::new( + EventId::hash(1), + EventId::hash(1), + EventId::hash(1), + 1, + AggregateId::new(1), + ) + .sequence(1); + events.push(one.clone()); + + let two = + EventContext::from_cause(EventId::hash(2), one, 2, AggregateId::new(1)).sequence(2); + events.push(two.clone()); + + let three = + EventContext::from_cause(EventId::hash(3), two, 3, AggregateId::new(1)).sequence(3); + events.push(three.clone()); + + assert_eq!( + events, + vec![ + EventContext { + seq: 1, + id: EventId::hash(1), + origin_id: EventId::hash(1), + causation_id: EventId::hash(1), + ts: 1, + aggregate_id: AggregateId::new(1), + }, + EventContext { + seq: 2, + id: EventId::hash(2), + origin_id: EventId::hash(1), + causation_id: EventId::hash(1), + ts: 2, + aggregate_id: AggregateId::new(1), + }, + EventContext { + seq: 3, + id: EventId::hash(3), + origin_id: EventId::hash(1), + causation_id: EventId::hash(2), + ts: 3, + aggregate_id: AggregateId::new(1), + }, + ] + ) + } +} diff --git a/crates/events/src/event_id.rs b/crates/events/src/event_id.rs index 60b2fccdcb..cec880d5fc 100644 --- a/crates/events/src/event_id.rs +++ b/crates/events/src/event_id.rs @@ -12,7 +12,7 @@ use std::{ hash::{DefaultHasher, Hash, Hasher}, }; -#[derive(Derivative, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Derivative, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[derivative(Debug)] pub struct EventId(#[derivative(Debug(format_with = "e3_utils::formatters::hexf"))] pub [u8; 32]); diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 96450e4921..5fd9ae190d 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -6,20 +6,27 @@ use actix::{Message, Recipient}; -use crate::{EnclaveEvent, Sequenced, Unsequenced}; +use crate::{AggregateId, CorrelationId, EnclaveEvent, Sequenced, Unsequenced}; /// Direct event received by the snapshot buffer in order to save snapshot to disk #[derive(Message, Debug)] #[rtype("()")] -pub struct CommitSnapshot(u64); +pub struct CommitSnapshot { + seq: u64, + aggregate_id: AggregateId, +} impl CommitSnapshot { - pub fn new(seq: u64) -> Self { - Self(seq) + pub fn new(seq: u64, aggregate_id: AggregateId) -> Self { + Self { seq, aggregate_id } } pub fn seq(&self) -> u64 { - self.0 + self.seq + } + + pub fn aggregate_id(&self) -> AggregateId { + self.aggregate_id } } @@ -47,29 +54,53 @@ impl StoreEventRequested { #[derive(Message, Debug)] #[rtype("()")] pub struct GetEventsAfter { - pub ts: u128, - pub sender: Recipient, + correlation_id: CorrelationId, + ts: u128, + sender: Recipient, } impl GetEventsAfter { - pub fn new(ts: u128, sender: impl Into>) -> Self { + pub fn new( + correlation_id: CorrelationId, + ts: u128, + sender: impl Into>, + ) -> Self { Self { + correlation_id, ts, sender: sender.into(), } } + + pub fn ts(&self) -> u128 { + self.ts + } + + pub fn id(&self) -> CorrelationId { + self.correlation_id + } + + pub fn sender(&self) -> &Recipient { + &self.sender + } } #[derive(Message, Debug)] #[rtype("()")] -pub struct ReceiveEvents(Vec>); +pub struct ReceiveEvents { + id: CorrelationId, + events: Vec>, +} impl ReceiveEvents { - pub fn new(events: Vec) -> Self { - Self(events) + pub fn new(id: CorrelationId, events: Vec) -> Self { + Self { id, events } } pub fn events(&self) -> &Vec { - &self.0 + &self.events + } + pub fn id(&self) -> CorrelationId { + self.id } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 5218459196..239bd68cc0 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -6,7 +6,7 @@ use crate::{ events::{EventStored, StoreEventRequested}, - EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, + EventContextAccessors, EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; @@ -21,7 +21,7 @@ impl EventStore { pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { let event = msg.event; let sender = msg.sender; - let ts = event.get_ts(); + let ts = event.ts(); if let Some(_) = self.index.get(ts)? { bail!("Event already stored at timestamp {ts}!"); } @@ -33,8 +33,9 @@ impl EventStore { pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { // if there are no events after the timestamp return an empty vector - let Some(seq) = self.index.seek(msg.ts)? else { - msg.sender.try_send(ReceiveEvents::new(vec![]))?; + let Some(seq) = self.index.seek(msg.ts())? else { + msg.sender() + .try_send(ReceiveEvents::new(msg.id(), vec![]))?; return Ok(()); }; // read and return the events @@ -43,10 +44,12 @@ impl EventStore { .read_from(seq) .map(|(s, e)| e.into_sequenced(s)) .collect::>(); - msg.sender.try_send(ReceiveEvents::new(evts))?; + + msg.sender().try_send(ReceiveEvents::new(msg.id(), evts))?; Ok(()) } } + impl EventStore { pub fn new(index: I, log: L) -> Self { Self { index, log } diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs new file mode 100644 index 0000000000..4d9a6b3321 --- /dev/null +++ b/crates/events/src/eventstore_router.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::eventstore::EventStore; +use crate::{ + events::{GetEventsAfter, StoreEventRequested}, + AggregateId, EventContextAccessors, EventLog, SequenceIndex, +}; +use crate::{CorrelationId, ReceiveEvents}; +use actix::{Actor, Addr, Handler, Message, Recipient}; +use anyhow::Result; +use std::collections::HashMap; +use tracing::error; + +pub struct EventStoreRouter { + stores: HashMap>>, +} + +impl EventStoreRouter { + pub fn new(stores: HashMap>>) -> Self { + let stores = stores + .into_iter() + .map(|(index, addr)| (AggregateId::new(index), addr)) + .collect(); + Self { stores } + } + + pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { + let aggregate_id = msg.event.aggregate_id(); + + let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { + self.stores + .get(&AggregateId::new(0)) + .expect("Default EventStore for AggregateId(0) not found") + }); + + let event = msg.event; + let sender = msg.sender; + + let forwarded_msg = StoreEventRequested::new(event, sender); + store_addr.do_send(forwarded_msg); + Ok(()) + } + + pub fn handle_get_events_after(&mut self, msg: GetAggregateEventsAfter) -> Result<()> { + for (aggregate_id, ts) in msg.ts() { + if let Some(store_addr) = self.stores.get(&aggregate_id) { + let get_events_msg = + GetEventsAfter::new(msg.id(), ts.to_owned(), msg.sender.clone()); + store_addr.do_send(get_events_msg); + } + } + Ok(()) + } +} + +impl Actor for EventStoreRouter { + type Context = actix::Context; +} + +impl Handler for EventStoreRouter { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_store_event_requested(msg) { + error!("Failed to route store event request: {}", e); + } + } +} + +impl Handler for EventStoreRouter { + type Result = (); + + fn handle(&mut self, msg: GetAggregateEventsAfter, _: &mut Self::Context) -> Self::Result { + if let Err(e) = self.handle_get_events_after(msg) { + error!("Failed to route get events after request: {}", e); + } + } +} + +#[derive(Message, Debug)] +#[rtype("()")] +pub struct GetAggregateEventsAfter { + pub correlation_id: CorrelationId, + pub ts: HashMap, + pub sender: Recipient, +} + +impl GetAggregateEventsAfter { + pub fn new( + correlation_id: CorrelationId, + ts: HashMap, + sender: Recipient, + ) -> Self { + Self { + correlation_id, + ts, + sender, + } + } + + pub fn id(&self) -> CorrelationId { + self.correlation_id + } + + pub fn ts(&self) -> &HashMap { + &self.ts + } +} diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index a49c077e79..c476fdca62 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -33,8 +33,11 @@ pub enum HlcError { /// HLC timestamp #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct HlcTimestamp { + /// Physical timestamp in microseconds since UNIX epoch pub ts: u64, + /// Logical counter for same-timestamp ordering pub counter: u32, + /// Unique node identifier for tie-breaking pub node: u32, } @@ -44,6 +47,11 @@ impl HlcTimestamp { Self { ts, counter, node } } + /// Extract wall time from a u128 timestamp + pub fn wall_time(ts: u128) -> u64 { + Self::from_u128(ts).ts + } + /// Packs the HLC timestamp into a 128bit big-endian representation. /// /// Layout: @@ -173,7 +181,9 @@ pub struct Hlc { #[derive(PartialEq)] struct HlcInner { + /// Current timestamp value ts: u64, + /// Current logical counter counter: u32, } @@ -194,7 +204,7 @@ impl PartialEq for Hlc { } impl Hlc { - const DEFAULT_MAX_DRIFT: u64 = 60_000_000; // 60 sec + const DEFAULT_MAX_DRIFT: u64 = 5 * 60 * 1_000_000; // 5 min pub fn new(node: u32) -> Self { Self { diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index 863ed76b54..b9221447d7 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -8,26 +8,32 @@ mod bus_handle; mod correlation_id; mod e3id; mod enclave_event; +mod event_context; mod event_id; mod eventbus; mod events; mod eventstore; +mod eventstore_router; pub mod hlc; mod ordered_set; pub mod prelude; mod seed; mod sequencer; +mod sync; mod traits; pub use bus_handle::*; pub use correlation_id::*; pub use e3id::*; pub use enclave_event::*; +pub use event_context::*; pub use event_id::*; pub use eventbus::*; pub use events::*; pub use eventstore::*; +pub use eventstore_router::*; pub use ordered_set::*; pub use seed::*; pub use sequencer::*; +pub use sync::*; pub use traits::*; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index a521637def..552f502833 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -8,7 +8,7 @@ use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; use crate::{ events::{CommitSnapshot, EventStored, StoreEventRequested}, - EnclaveEvent, EventBus, Sequenced, Unsequenced, + EnclaveEvent, EventBus, EventContextAccessors, EventContextSeq, Sequenced, Unsequenced, }; /// Component to sequence the storage of events @@ -48,8 +48,9 @@ impl Handler for Sequencer { type Result = (); fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { let event = msg.into_event(); - let seq = event.get_seq(); - self.buffer.do_send(CommitSnapshot::new(seq)); + let seq = event.seq(); + self.buffer + .do_send(CommitSnapshot::new(seq, event.aggregate_id())); self.bus.do_send(event) } } diff --git a/crates/events/src/sync.rs b/crates/events/src/sync.rs new file mode 100644 index 0000000000..bc13534344 --- /dev/null +++ b/crates/events/src/sync.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::collections::{BTreeMap, HashSet}; + +use crate::EvmEvent; +use actix::Message; +use serde::{Deserialize, Serialize}; +type Chainid = u64; +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub enum SyncEvmEvent { + /// Signal that this reader has completed historical sync + HistoricalSyncComplete(ChainId), + /// An actual event from the blockchain + Event(EvmEvent), +} + +impl From for SyncEvmEvent { + fn from(event: EvmEvent) -> SyncEvmEvent { + SyncEvmEvent::Event(event) + } +} + +type ChainId = u64; +type DeployBlock = u64; + +/// Configuration value object for starting the evm reader for a specific chain +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmEventConfigChain { + deploy_block: DeployBlock, +} + +impl EvmEventConfigChain { + pub fn new(deploy_block: DeployBlock) -> Self { + Self { deploy_block } + } + pub fn deploy_block(&self) -> u64 { + self.deploy_block + } +} + +/// Configuration value object for starting the evm reader for all chains +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmEventConfig { + config: BTreeMap, // Need BTreeMap because of Hash +} + +impl EvmEventConfig { + pub fn new() -> Self { + Self { + config: BTreeMap::new(), + } + } + pub fn get(&self, chain_id: &ChainId) -> Option<&EvmEventConfigChain> { + self.config.get(&chain_id) + } + + pub fn insert(&mut self, key: ChainId, value: EvmEventConfigChain) { + self.config.insert(key, value); + } + + pub fn chains(&self) -> HashSet { + self.config.keys().cloned().collect() + } +} diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 1f0ee6ba8e..3feba8f434 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -9,7 +9,10 @@ use anyhow::Result; use std::fmt::Display; use std::hash::Hash; -use crate::{EnclaveEvent, EventType, Unsequenced}; +use crate::{ + event_context::{AggregateId, EventContext}, + EnclaveEvent, EventId, EventType, Sequenced, Unsequenced, +}; /// Trait that must be implemented by events used with EventBus pub trait Event: @@ -18,10 +21,9 @@ pub trait Event: type Id: Hash + Eq + Clone + Unpin + Send + Sync + Display; /// Payload for the Event - type Data; - - fn event_type(&self) -> String; + type Data: WithAggregateId; fn event_id(&self) -> Self::Id; + fn event_type(&self) -> String; fn get_data(&self) -> &Self::Data; fn into_data(self) -> Self::Data; } @@ -36,6 +38,7 @@ pub trait ErrorEvent: Event { err_type: Self::ErrType, error: impl Into, ts: u128, + caused_by: Option>, ) -> Result; } @@ -44,18 +47,34 @@ pub trait EventFactory { /// Create a new event from the given event data, apply a local HLC timestamp. /// /// This method should be used for events that have originated locally. - fn event_from(&self, data: impl Into) -> Result; + fn event_from( + &self, + data: impl Into, + caused_by: Option>, + ) -> Result; /// Create a new event from the given event data, apply the given remote HLC time to ensure correct /// event ordering. /// /// This method should be used for events that originated from remote sources. - fn event_from_remote_source(&self, data: impl Into, ts: u128) -> Result; + fn event_from_remote_source( + &self, + data: impl Into, + // NOTE: `caused_by` makes sense here as we could be sending out requests and receiving + // responses that relate to the request + caused_by: Option>, + ts: u128, + ) -> Result; } /// An ErrorFactory creates errors. pub trait ErrorFactory { /// Create an error event from the given error. - fn event_from_error(&self, err_type: E::ErrType, error: impl Into) -> Result; + fn event_from_error( + &self, + err_type: E::ErrType, + error: impl Into, + caused_by: Option>, + ) -> Result; } /// An EventPublisher publishes events on it's internal EventBus @@ -86,12 +105,18 @@ pub trait EventSubscriber { fn subscribe(&self, event_type: EventType, recipient: Recipient); /// Subscribe the recipient to events matching any of the given event types fn subscribe_all(&self, event_types: &[EventType], recipient: Recipient); + /// Subscribe the recipient to events matching the given event type + fn unsubscribe(&self, event_type: &str, recipient: Recipient); } /// Trait to create an event with a timestamp from its associated type data pub trait EventConstructorWithTimestamp: Event + Sized { /// Create an event passing attaching a specific timestamp. - fn new_with_timestamp(data: Self::Data, ts: u128) -> Self; + fn new_with_timestamp( + data: Self::Data, + caused_by: Option>, + ts: u128, + ) -> Self; } pub trait CompositeEvent: EventConstructorWithTimestamp {} @@ -115,3 +140,34 @@ pub trait EventLog: Unpin + 'static { /// Read all events starting from the given sequence number (inclusive) fn read_from(&self, from: u64) -> Box)>>; } + +/// EventContext allows consumers to extract infrastructure metadata from event objects +pub trait EventContextAccessors { + /// The unique id for this event + fn id(&self) -> EventId; + /// The event that caused this event to occur + fn causation_id(&self) -> EventId; + /// The root event that caused this event to occur + fn origin_id(&self) -> EventId; + /// The timestamp when the event occurred timestamp is encoded HlcTimestamp format + fn ts(&self) -> u128; + /// The aggregate id for this event + fn aggregate_id(&self) -> AggregateId; +} + +pub trait EventContextSeq { + /// The sequence number of the event + fn seq(&self) -> u64; +} + +pub trait WithAggregateId { + /// Extract the aggregate id from the object + fn get_aggregate_id(&self) -> AggregateId; +} + +/// An EventContextManager hold the current event context for use in event publishing and +/// persistence management +pub trait EventContextManager { + fn set_ctx(&mut self, value: &EventContext); + fn get_ctx(&self) -> Option>; +} diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index e7071e3870..b3e77c6bda 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -14,6 +14,7 @@ alloy-primitives = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } +bloom = { workspace = true } e3-fhe-params = { workspace = true } e3-crypto = { workspace = true } e3-config = { workspace = true } @@ -31,6 +32,8 @@ url = { workspace = true } zeroize = { workspace = true } [dev-dependencies] +e3-evm = { workspace = true } e3-entrypoint = { workspace = true } e3-ciphernode-builder = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } +tracing-subscriber = { workspace = true } diff --git a/crates/evm/src/bonding_registry_sol.rs b/crates/evm/src/bonding_registry_sol.rs index 20652f3554..e9dca8572a 100644 --- a/crates/evm/src/bonding_registry_sol.rs +++ b/crates/evm/src/bonding_registry_sol.rs @@ -4,20 +4,15 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{ - event_reader::EvmEventReaderState, helpers::EthProvider, EnclaveEvmEvent, EvmEventReader, -}; -use actix::{Addr, Recipient}; +use crate::{events::EvmEventProcessor, evm_parser::EvmParser}; +use actix::{Actor, Addr}; use alloy::{ primitives::{LogData, B256}, - providers::Provider, sol, sol_types::SolEvent, }; -use anyhow::Result; -use e3_data::Repository; -use e3_events::{BusHandle, EnclaveEventData}; -use tracing::{error, info, trace}; +use e3_events::EnclaveEventData; +use tracing::{error, trace}; sol!( #[sol(rpc)] @@ -141,64 +136,8 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< /// Connects to BondingRegistry.sol converting EVM events to EnclaveEvents pub struct BondingRegistrySolReader; - impl BondingRegistrySolReader { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result>> - where - P: Provider + Clone + 'static, - { - let addr = EvmEventReader::attach( - provider, - extractor, - contract_address, - start_block, - processor, - bus, - repository, - rpc_url, - ) - .await?; - - info!(address=%contract_address, "BondingRegistrySolReader is listening to address"); - - Ok(addr) - } -} - -/// Wrapper for a reader -pub struct BondingRegistrySol; - -impl BondingRegistrySol { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result<()> - where - P: Provider + Clone + 'static, - { - BondingRegistrySolReader::attach( - processor, - bus, - provider, - contract_address, - repository, - start_block, - rpc_url, - ) - .await?; - Ok(()) + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() } } diff --git a/crates/evm/src/ciphernode_registry_sol.rs b/crates/evm/src/ciphernode_registry_sol.rs index 0913bdf0ff..3448590ce9 100644 --- a/crates/evm/src/ciphernode_registry_sol.rs +++ b/crates/evm/src/ciphernode_registry_sol.rs @@ -5,9 +5,9 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - event_reader::EvmEventReaderState, + events::{EnclaveEvmEvent, EvmEventProcessor}, + evm_parser::EvmParser, helpers::{send_tx_with_retry, EthProvider}, - EnclaveEvmEvent, EvmEventReader, }; use actix::prelude::*; use alloy::{ @@ -18,12 +18,12 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; -use e3_data::Repository; use e3_events::{ prelude::*, BusHandle, CommitteeFinalizeRequested, CommitteeFinalized, E3id, EType, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, OrderedSet, PublicKeyAggregated, Seed, Shutdown, TicketGenerated, TicketId, }; +use e3_utils::NotifySync; use tracing::{error, info, trace}; sol!( @@ -218,33 +218,8 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result>> - where - P: Provider + Clone + 'static, - { - let addr = EvmEventReader::attach( - provider, - extractor, - contract_address, - start_block, - processor, - bus, - repository, - rpc_url, - ) - .await?; - - info!(address=%contract_address, "CiphernodeRegistrySolReader is listening to address"); - - Ok(addr) + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() } } @@ -271,10 +246,10 @@ impl CiphernodeRegistrySolWriter pub async fn attach( bus: &BusHandle, provider: EthProvider

, - contract_address: &str, + contract_address: Address, is_aggregator: bool, ) -> Result>> { - let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address.parse()?) + let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address) .await? .start(); @@ -330,7 +305,7 @@ impl Handler ctx.notify(data); } } - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } @@ -555,35 +530,14 @@ pub async fn publish_committee_to_registry( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result<()> - where - P: Provider + Clone + 'static, - { - CiphernodeRegistrySolReader::attach( - processor, - bus, - provider, - contract_address, - repository, - start_block, - rpc_url, - ) - .await?; - Ok(()) + pub fn attach(processor: &Recipient) -> Addr { + CiphernodeRegistrySolReader::setup(processor) } pub async fn attach_writer

( bus: &BusHandle, provider: EthProvider

, - contract_address: &str, + contract_address: Address, is_aggregator: bool, ) -> Result>> where diff --git a/crates/evm/src/enclave_sol.rs b/crates/evm/src/enclave_sol.rs index a8a6bebf30..69df39b45c 100644 --- a/crates/evm/src/enclave_sol.rs +++ b/crates/evm/src/enclave_sol.rs @@ -6,44 +6,30 @@ use crate::{ enclave_sol_reader::EnclaveSolReader, enclave_sol_writer::EnclaveSolWriter, - event_reader::EvmEventReaderState, helpers::EthProvider, EnclaveEvmEvent, + events::EvmEventProcessor, evm_parser::EvmParser, helpers::EthProvider, }; -use actix::Recipient; +use actix::Addr; use alloy::providers::{Provider, WalletProvider}; +use alloy_primitives::Address; use anyhow::Result; -use e3_data::Repository; use e3_events::BusHandle; pub struct EnclaveSol; impl EnclaveSol { - pub async fn attach( - processor: &Recipient, + pub async fn attach( + processor: &EvmEventProcessor, bus: &BusHandle, - read_provider: EthProvider, write_provider: EthProvider, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result<()> + contract_address: Address, + ) -> Result> where - R: Provider + Clone + 'static, W: Provider + WalletProvider + Clone + 'static, { - EnclaveSolReader::attach( - processor, - bus, - read_provider, - contract_address, - repository, - start_block, - rpc_url, - ) - .await?; + let addr = EnclaveSolReader::setup(processor); EnclaveSolWriter::attach(bus, write_provider, contract_address).await?; - Ok(()) + Ok(addr) } } diff --git a/crates/evm/src/enclave_sol_reader.rs b/crates/evm/src/enclave_sol_reader.rs index 11c4343fe8..8d89f9c8bb 100644 --- a/crates/evm/src/enclave_sol_reader.rs +++ b/crates/evm/src/enclave_sol_reader.rs @@ -4,19 +4,16 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::event_reader::EvmEventReaderState; -use crate::helpers::EthProvider; -use crate::{EnclaveEvmEvent, EvmEventReader}; -use actix::{Addr, Recipient}; +use crate::events::EvmEventProcessor; +use crate::evm_parser::EvmParser; +use actix::{Actor, Addr}; use alloy::primitives::{LogData, B256}; -use alloy::providers::Provider; use alloy::{sol, sol_types::SolEvent}; -use anyhow::Result; -use e3_data::Repository; -use e3_events::{BusHandle, E3id, EnclaveEventData}; +use e3_events::E3id; +use e3_events::EnclaveEventData; use e3_fhe_params::decode_bfv_params_arc; use e3_trbfv::helpers::calculate_error_size; -use e3_utils::utility_types::ArcBytes; +use e3_utils::ArcBytes; use num_bigint::BigUint; use tracing::{error, info, trace, warn}; @@ -141,32 +138,7 @@ pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option< pub struct EnclaveSolReader; impl EnclaveSolReader { - pub async fn attach

( - processor: &Recipient, - bus: &BusHandle, - provider: EthProvider

, - contract_address: &str, - repository: &Repository, - start_block: Option, - rpc_url: String, - ) -> Result>> - where - P: Provider + Clone + 'static, - { - let addr = EvmEventReader::attach( - provider, - extractor, - contract_address, - start_block, - processor, - bus, - repository, - rpc_url, - ) - .await?; - - info!(address=%contract_address, "EnclaveSolReader is listening to address"); - - Ok(addr) + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() } } diff --git a/crates/evm/src/enclave_sol_writer.rs b/crates/evm/src/enclave_sol_writer.rs index 0526bdfc2d..68a1f6adc9 100644 --- a/crates/evm/src/enclave_sol_writer.rs +++ b/crates/evm/src/enclave_sol_writer.rs @@ -25,6 +25,7 @@ use e3_events::EnclaveEventData; use e3_events::EventType; use e3_events::Shutdown; use e3_events::{E3id, EType, PlaintextAggregated}; +use e3_utils::NotifySync; use tracing::info; sol!( @@ -56,9 +57,9 @@ impl EnclaveSolWriter

{ pub async fn attach( bus: &BusHandle, provider: EthProvider

, - contract_address: &str, + contract_address: Address, ) -> Result>> { - let addr = EnclaveSolWriter::new(bus, provider, contract_address.parse()?)?.start(); + let addr = EnclaveSolWriter::new(bus, provider, contract_address)?.start(); bus.subscribe_all( &[EventType::PlaintextAggregated, EventType::Shutdown], addr.clone().into(), @@ -82,7 +83,7 @@ impl Handler for E ctx.notify(data); } } - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/evm/src/event_reader.rs b/crates/evm/src/event_reader.rs deleted file mode 100644 index e1b6299512..0000000000 --- a/crates/evm/src/event_reader.rs +++ /dev/null @@ -1,352 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::helpers::EthProvider; -use actix::prelude::*; -use actix::{Addr, Recipient}; -use alloy::eips::BlockNumberOrTag; -use alloy::primitives::Address; -use alloy::primitives::{LogData, B256}; -use alloy::providers::Provider; -use alloy::rpc::types::Filter; -use anyhow::{anyhow, Result}; -use e3_data::{AutoPersist, Persistable, Repository}; -use e3_events::{prelude::*, EType, EnclaveEvent, EnclaveEventData, EventId, EventType}; -use e3_events::{BusHandle, Event}; -use futures_util::stream::StreamExt; -use std::collections::HashSet; -use tokio::select; -use tokio::sync::oneshot; -use tracing::{error, info, instrument, trace, warn}; - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[rtype(result = "()")] -pub enum EnclaveEvmEvent { - /// Register a reader with the coordinator before it starts processing - RegisterReader, - /// Signal that this reader has completed historical sync - HistoricalSyncComplete, - /// An actual event from the blockchain - Event { - event: EnclaveEventData, - block: Option, - }, -} - -impl EnclaveEvmEvent { - pub fn new(event: EnclaveEventData, block: Option) -> Self { - Self::Event { event, block } - } - - pub fn get_id(&self) -> EventId { - EventId::hash(self.clone()) - } -} - -pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; - -pub struct EvmEventReaderParams

{ - provider: EthProvider

, - extractor: ExtractorFn, - contract_address: Address, - start_block: Option, - processor: Recipient, - bus: BusHandle, - state: Persistable, - rpc_url: String, -} - -#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] -pub struct EvmEventReaderState { - pub ids: HashSet, - pub last_block: Option, -} - -/// Connects to Enclave.sol converting EVM events to EnclaveEvents -pub struct EvmEventReader

{ - /// The alloy provider - provider: Option>, - /// The contract address - contract_address: Address, - /// The Extractor function to determine which events to extract and convert to EnclaveEvents - extractor: ExtractorFn, - /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used - /// internally. You should send the Shutdown signal to the reader directly or via the EventBus - shutdown_rx: Option>, - /// The sender for the shutdown signal this is only used internally - shutdown_tx: Option>, - /// The block that processing should start from - start_block: Option, - /// Processor to forward events an actor - processor: Recipient, - /// Event bus for error propagation only - bus: BusHandle, - /// The auto persistable state of the event reader - state: Persistable, - /// The RPC URL for the provider - rpc_url: String, -} - -impl EvmEventReader

{ - pub fn new(params: EvmEventReaderParams

) -> Self { - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - Self { - contract_address: params.contract_address, - provider: Some(params.provider), - extractor: params.extractor, - shutdown_rx: Some(shutdown_rx), - shutdown_tx: Some(shutdown_tx), - start_block: params.start_block, - processor: params.processor, - bus: params.bus, - state: params.state, - rpc_url: params.rpc_url, - } - } - - pub async fn attach( - provider: EthProvider

, - extractor: ExtractorFn, - contract_address: &str, - start_block: Option, - processor: &Recipient, - bus: &BusHandle, - repository: &Repository, - rpc_url: String, - ) -> Result> { - let sync_state = repository - .clone() - .load_or_default(EvmEventReaderState::default()) - .await?; - - let params = EvmEventReaderParams { - provider, - extractor, - contract_address: contract_address.parse()?, - start_block, - processor: processor.clone(), - bus: bus.clone(), - state: sync_state, - rpc_url, - }; - - let addr = EvmEventReader::new(params).start(); - - processor.do_send(EnclaveEvmEvent::RegisterReader); - - bus.subscribe(EventType::Shutdown, addr.clone().into()); - Ok(addr) - } -} - -impl Actor for EvmEventReader

{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - let reader_addr = ctx.address(); - let bus = self.bus.clone(); - - let Some(provider) = self.provider.take() else { - error!("Could not start event reader as provider has already been used."); - return; - }; - - let extractor = self.extractor; - let Some(shutdown) = self.shutdown_rx.take() else { - bus.err(EType::Evm, anyhow!("shutdown already called")); - return; - }; - - let contract_address = self.contract_address; - let start_block = self.start_block; - let rpc_url = self.rpc_url.clone(); - - ctx.spawn( - async move { - stream_from_evm( - provider, - &contract_address, - reader_addr.clone(), - extractor, - shutdown, - start_block, - &bus, - rpc_url, - ) - .await - } - .into_actor(self), - ); - } -} - -#[instrument(name = "evm_event_reader", skip_all)] -async fn stream_from_evm( - provider: EthProvider

, - contract_address: &Address, - reader_addr: Addr>, - extractor: fn(&LogData, Option<&B256>, u64) -> Option, - mut shutdown: oneshot::Receiver<()>, - start_block: Option, - bus: &BusHandle, - rpc_url: String, -) { - let chain_id = provider.chain_id(); - let provider_ref = provider.provider(); - - if start_block.unwrap_or(0) == 0 && !is_local_node(&rpc_url) { - error!( - "Querying from block 0 on a non-local node ({}) without a specific start_block is not allowed.", - rpc_url - ); - bus.err( - EType::Evm, - anyhow!( - "Misconfiguration: Attempted to query historical events from genesis on a non-local node. \ - Please specify a `start_block` for contract address {contract_address} on chain {chain_id} using rpc {rpc_url}" - ) - ); - return; - } - - let historical_filter = Filter::new() - .address(*contract_address) - .from_block(start_block.unwrap_or(0)); - let current_filter = Filter::new() - .address(*contract_address) - .from_block(BlockNumberOrTag::Latest); - - // Historical events - match provider_ref.get_logs(&historical_filter).await { - Ok(historical_logs) => { - info!("Fetched {} historical events", historical_logs.len()); - for log in historical_logs { - let block_number = log.block_number; - if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { - trace!("Processing historical log"); - reader_addr.do_send(EnclaveEvmEvent::new(event, block_number)); - } - } - - reader_addr.do_send(EnclaveEvmEvent::HistoricalSyncComplete); - } - Err(e) => { - error!("Failed to fetch historical events: {}", e); - bus.err(EType::Evm, anyhow!(e)); - return; - } - } - - info!("Subscribing to live events"); - match provider_ref.subscribe_logs(¤t_filter).await { - Ok(subscription) => { - let id: B256 = subscription.local_id().clone(); - let mut stream = subscription.into_stream(); - - loop { - select! { - maybe_log = stream.next() => { - match maybe_log { - Some(log) => { - let block_number = log.block_number; - trace!("Received log from EVM"); - - let Some(event) = extractor(log.data(), log.topic0(), chain_id) else { - trace!("Unknown log from EVM. This will happen from time to time."); - continue; - }; - - trace!("Extracted EVM Event: {:?}", event); - reader_addr.do_send(EnclaveEvmEvent::new(event, block_number)); - } - None => break, // Stream ended - } - } - _ = &mut shutdown => { - info!("Received shutdown signal, stopping EVM stream"); - match provider_ref.unsubscribe(id).await { - Ok(_) => info!("Unsubscribed successfully from EVM event stream"), - Err(err) => error!("Cannot unsubscribe from EVM event stream: {}", err), - }; - break; - } - } - } - } - Err(e) => { - bus.err(EType::Evm, anyhow!("{}", e)); - } - } - - info!("Exiting stream loop"); -} - -fn is_local_node(rpc_url: &str) -> bool { - rpc_url.contains("localhost") || rpc_url.contains("127.0.0.1") -} - -impl Handler for EvmEventReader

{ - type Result = (); - - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - if let EnclaveEventData::Shutdown(_) = msg.into_data() { - if let Some(shutdown) = self.shutdown_tx.take() { - let _ = shutdown.send(()); - } - } - } -} - -impl Handler for EvmEventReader

{ - type Result = (); - - #[instrument(name = "evm_event_reader", skip_all)] - fn handle(&mut self, msg: EnclaveEvmEvent, _: &mut Self::Context) -> Self::Result { - match msg { - EnclaveEvmEvent::RegisterReader | EnclaveEvmEvent::HistoricalSyncComplete => { - self.processor.do_send(msg); - } - - EnclaveEvmEvent::Event { event, block } => { - match self.state.try_mutate(|mut state| { - let temp_wrapped = EnclaveEvmEvent::Event { - event: event.clone(), - block, - }; - let event_id = temp_wrapped.get_id(); - - trace!("Processing event: {}", event_id); - trace!("Cache length: {}", state.ids.len()); - - if state.ids.contains(&event_id) { - warn!( - "Event id {} has already been seen and was not forwarded", - &event_id - ); - return Ok(state); - } - - let event_type = event.event_type(); - - self.processor.do_send(EnclaveEvmEvent::Event { - event: event.clone(), - block, - }); - - // Save processed IDs - trace!("Storing event(EVM) in cache {}({})", event_type, event_id); - state.ids.insert(event_id); - state.last_block = block; - - Ok(state) - }) { - Ok(_) => (), - Err(err) => self.bus.err(EType::Evm, err), - } - } - } - } -} diff --git a/crates/evm/src/events.rs b/crates/evm/src/events.rs new file mode 100644 index 0000000000..4b59deaaeb --- /dev/null +++ b/crates/evm/src/events.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Message, Recipient}; +use alloy::rpc::types::Log; +use e3_events::{CorrelationId, EvmEvent}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct HistoricalSyncComplete { + pub chain_id: u64, + pub prev_event: Option, + pub id: CorrelationId, +} + +impl HistoricalSyncComplete { + pub fn new(chain_id: u64, prev_event: Option) -> Self { + let id = CorrelationId::new(); + Self { + id, + chain_id, + prev_event, + } + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub enum EnclaveEvmEvent { + /// Signal that this reader has completed historical sync + HistoricalSyncComplete(HistoricalSyncComplete), + /// An actual event from the blockchain + Event(EvmEvent), + /// Raw log data from the provider + Log(EvmLog), + /// Dummy event to report that an event was processed. This is required to ensure that the + /// appropriate events are ordered correctly + Processed(CorrelationId), +} + +impl EnclaveEvmEvent { + pub fn get_id(&self) -> CorrelationId { + match self { + EnclaveEvmEvent::HistoricalSyncComplete(e) => e.get_id(), + EnclaveEvmEvent::Log(e) => e.get_id(), + EnclaveEvmEvent::Event(e) => e.get_id(), + EnclaveEvmEvent::Processed(id) => id.to_owned(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmLog { + pub id: CorrelationId, + pub log: Log, + pub timestamp: u64, + pub chain_id: u64, +} + +impl EvmLog { + pub fn new(log: Log, chain_id: u64, timestamp: u64) -> Self { + let id = CorrelationId::new(); + Self { + log, + chain_id, + id, + timestamp, + } + } + + pub fn get_id(&self) -> CorrelationId { + self.id + } +} + +#[cfg(test)] +use alloy_primitives::Address; + +#[cfg(test)] +impl EvmLog { + pub fn test_log(address: Address, chain_id: u64, timestamp: u64) -> EvmLog { + let id = CorrelationId::new(); + EvmLog { + log: Log { + inner: alloy_primitives::Log { + address, + ..Default::default() + }, + ..Default::default() + }, + chain_id, + id, + timestamp, + } + } +} + +pub type EvmEventProcessor = Recipient; diff --git a/crates/evm/src/evm_chain_gateway.rs b/crates/evm/src/evm_chain_gateway.rs new file mode 100644 index 0000000000..d730c1585f --- /dev/null +++ b/crates/evm/src/evm_chain_gateway.rs @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::events::EnclaveEvmEvent; +use crate::HistoricalSyncComplete; +use actix::{Actor, Handler}; +use actix::{Addr, Recipient}; +use anyhow::Result; +use anyhow::{bail, Context}; +use e3_events::{ + trap, BusHandle, EnclaveEvent, EnclaveEventData, EventSubscriber, EventType, SyncEnd, + SyncEvmEvent, SyncStart, +}; +use e3_events::{EType, EvmEvent}; +use e3_events::{Event, EventPublisher}; +use tracing::info; + +/// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. +/// It coordinates event flow between these components. +pub struct EvmChainGateway { + bus: BusHandle, + status: SyncStatus, +} + +/// This state machine coordinates the function of the EvmChainGateway +#[derive(Clone, Debug)] +enum SyncStatus { + /// Intial State + Init(Vec), // Include a buffer to hold events that arrive too early + /// After SyncStart we forward all events to SyncActor + ForwardToSyncActor(Option>), + /// Once the chain has completed historical sync then we buffer all "live" events until sync is + /// complete + BufferUntilLive(Vec), + /// Forward all events directly to the bus + Live, +} + +impl Default for SyncStatus { + fn default() -> Self { + Self::Init(Vec::new()) + } +} + +impl SyncStatus { + pub fn forward_to_sync_actor( + &mut self, + sender: Recipient, + ) -> Result> { + let Self::Init(buffer) = self else { + bail!( + "Cannot change state to ForwardToSyncActor when state is {:?}", + self + ); + }; + + let buffer = std::mem::take(buffer); + *self = SyncStatus::ForwardToSyncActor(Some(sender)); + Ok(buffer) + } + + pub fn buffer_until_live(&mut self) -> Result> { + let Self::ForwardToSyncActor(sender) = self else { + bail!( + "Cannot change state to BufferUntilLive when state is {:?}", + self + ); + }; + let sender = std::mem::take(sender).context("Cannot call buffer_until_live twice")?; + *self = SyncStatus::BufferUntilLive(vec![]); + Ok(sender) + } + + pub fn live(&mut self) -> Result> { + let Self::BufferUntilLive(buffer) = self else { + bail!("Cannot change state to Live when state is {:?}", self); + }; + let buffer = std::mem::take(buffer); + *self = SyncStatus::Live; + Ok(buffer) + } +} + +impl EvmChainGateway { + pub fn new(bus: &BusHandle) -> Self { + Self { + bus: bus.clone(), + status: SyncStatus::default(), + } + } + + pub fn setup(bus: &BusHandle) -> Addr { + let addr = Self::new(bus).start(); + bus.subscribe_all( + &[EventType::SyncStart, EventType::SyncEnd], + addr.clone().recipient(), + ); + addr + } + + fn handle_sync_start(&mut self, msg: SyncStart) -> Result<()> { + info!("Processing SyncStart message"); + // Received a SyncStart event from the event bus. Get the sender within that event and forward + // all events to that actor + let sender = msg.sender.context("No sender on SyncStart Message")?; + let mut buffer = self.status.forward_to_sync_actor(sender)?; + // Drain any events that were buffered early + for evt in buffer.drain(..) { + self.process_evm_event(evt)?; + } + Ok(()) + } + + fn handle_sync_end(&mut self, _: SyncEnd) -> Result<()> { + info!("Processing SyncEnd message"); + let buffer = self.status.live()?; + for evt in buffer { + self.publish_evm_event(evt)?; + } + Ok(()) + } + + fn publish_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + let (data, ts, _) = msg.split(); + self.bus.publish_from_remote(data, ts)?; + Ok(()) + } + + fn handle_evm_event(&mut self, msg: EnclaveEvmEvent) -> Result<()> { + match msg { + EnclaveEvmEvent::HistoricalSyncComplete(e) => { + self.forward_historical_sync_complete(e)?; + Ok(()) + } + EnclaveEvmEvent::Event(event) => { + self.process_evm_event(event)?; + Ok(()) + } + _ => panic!("EvmChainGateway is only designed to receive EnclaveEvmEvent::HistoricalSyncComplete or EnclaveEvmEvent::Event events"), + } + } + + fn forward_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { + info!( + "handling historical sync complete for chain_id({})", + event.chain_id + ); + let sender = self.status.buffer_until_live()?; + info!("Sending historical sync complete event to sender."); + sender.try_send(SyncEvmEvent::HistoricalSyncComplete(event.chain_id))?; + Ok(()) + } + + fn process_evm_event(&mut self, msg: EvmEvent) -> Result<()> { + match &mut self.status { + SyncStatus::BufferUntilLive(buffer) => { + info!("saving evm event({}) to pre-live buffer", msg.get_id()); + buffer.push(msg) + } + SyncStatus::ForwardToSyncActor(Some(sync_actor)) => { + info!("forwarding evm event({}) to SyncActor", msg.get_id()); + sync_actor.do_send(msg.into()); + } + SyncStatus::Live => { + info!("publishing evm event({})", msg.get_id()); + self.publish_evm_event(msg)? + } + SyncStatus::Init(buffer) => { + info!("saving evm event({}) to pre-sync buffer", msg.get_id()); + buffer.push(msg) + } + _ => (), + }; + Ok(()) + } +} + +impl Actor for EvmChainGateway { + type Context = actix::Context; +} + +impl Handler for EvmChainGateway { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || { + match msg.into_data() { + EnclaveEventData::SyncStart(e) => self.handle_sync_start(e)?, + EnclaveEventData::SyncEnd(e) => self.handle_sync_end(e)?, + _ => (), + } + Ok(()) + }) + } +} + +impl Handler for EvmChainGateway { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use e3_ciphernode_builder::EventSystem; + + use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TestEvent}; + use tokio::sync::mpsc; + + struct SyncEventCollector { + tx: mpsc::UnboundedSender, + } + + impl Actor for SyncEventCollector { + type Context = actix::Context; + } + + impl Handler for SyncEventCollector { + type Result = (); + fn handle(&mut self, msg: SyncEvmEvent, _: &mut Self::Context) { + let _ = self.tx.send(msg); + } + } + + #[actix::test] + async fn test_evm_chain_gateway() { + let system = EventSystem::new("test").with_fresh_bus(); + let bus = system.handle().unwrap(); + + let (tx, mut rx) = mpsc::unbounded_channel(); + let collector = SyncEventCollector { tx }.start(); + + let addr = EvmChainGateway::setup(&bus); + + let chain_id = 1u64; + + // SyncStart: Init -> ForwardToSyncActor + let mut evm_config = EvmEventConfig::new(); + evm_config.insert(chain_id, EvmEventConfigChain::new(0)); + bus.publish(SyncStart::new(collector.clone(), evm_config)) + .unwrap(); + + // Send EVM event while forwarding - should reach collector + let evm_event = EvmEvent::new( + CorrelationId::new(), + TestEvent { + msg: "test".to_string(), + entropy: 1, + } + .into(), + 100, + 12345, + chain_id, + ); + // This will actually arrive earlier than SyncStart but aught to be buffered + addr.do_send(EnclaveEvmEvent::Event(evm_event)); + + let received = rx.recv().await.unwrap(); + assert!(matches!(received, SyncEvmEvent::Event(_))); + + // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive + addr.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + HistoricalSyncComplete::new(chain_id, None), + )); + + let received = rx.recv().await.unwrap(); + assert!(matches!(received, SyncEvmEvent::HistoricalSyncComplete(_))); + + // Send EVM event while buffering - should be buffered (not received) + let buffered_event = EvmEvent::new( + CorrelationId::new(), + TestEvent { + msg: "buffered".to_string(), + entropy: 2, + } + .into(), + 101, + 12346, + chain_id, + ); + addr.do_send(EnclaveEvmEvent::Event(buffered_event)); + + // SyncEnd: BufferUntilLive -> Live (publishes buffered events to bus) + bus.publish(SyncEnd::new()).unwrap(); + + // Allow time for async message processing + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // Verify no more messages were sent to collector (buffered events go to bus, not collector) + assert!(rx.try_recv().is_err()); + } +} diff --git a/crates/evm/src/evm_hub.rs b/crates/evm/src/evm_hub.rs new file mode 100644 index 0000000000..3c22437424 --- /dev/null +++ b/crates/evm/src/evm_hub.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Addr, Handler}; + +use crate::events::{EnclaveEvmEvent, EvmEventProcessor}; + +pub struct EvmHub { + nexts: Vec, +} + +impl EvmHub { + pub fn new(nexts: Vec) -> Self { + Self { nexts } + } + + pub fn setup(nexts: Vec) -> Addr { + let addr = Self::new(nexts).start(); + addr + } +} + +impl Actor for EvmHub { + type Context = actix::Context; +} + +impl Handler for EvmHub { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + let EnclaveEvmEvent::Log { .. } = msg.clone() else { + return; + }; + + for next in self.nexts.clone() { + next.do_send(msg.clone()); + } + } +} + +#[cfg(test)] +mod tests { + use crate::events::EvmLog; + + use super::*; + use actix::prelude::*; + use alloy::primitives::address; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + use std::time::Duration; + use tokio::time::sleep; + + #[actix::test] + async fn test_evm_hub_forwards_log_events_to_all_processors() { + // Arrange + let call_count = Arc::new(AtomicUsize::new(0)); + + // Create mock processors that track invocations + let count1 = call_count.clone(); + let count2 = call_count.clone(); + + let processor1 = TestProcessor { call_count: count1 }.start(); + let processor2 = TestProcessor { call_count: count2 }.start(); + + let hub = EvmHub::setup(vec![ + processor1.clone().recipient(), + processor2.clone().recipient(), + ]); + + let log_event = EnclaveEvmEvent::Log(EvmLog::test_log( + address!("0x1111111111111111111111111111111111111111"), + 1, + 0, + )); + + hub.send(log_event).await.unwrap(); + + sleep(Duration::from_millis(10)).await; + // Assert + assert_eq!(call_count.load(Ordering::SeqCst), 2); + } + + // Helper test actor + struct TestProcessor { + call_count: Arc, + } + + impl Actor for TestProcessor { + type Context = Context; + } + + impl Handler for TestProcessor { + type Result = (); + + fn handle(&mut self, _msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + self.call_count.fetch_add(1, Ordering::SeqCst); + } + } +} diff --git a/crates/evm/src/evm_parser.rs b/crates/evm/src/evm_parser.rs new file mode 100644 index 0000000000..2710fd10a9 --- /dev/null +++ b/crates/evm/src/evm_parser.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Handler}; +use e3_events::{hlc::HlcTimestamp, EnclaveEventData, EvmEvent}; +use tracing::info; + +use crate::{ + events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}, + ExtractorFn, +}; + +pub struct EvmParser { + next: EvmEventProcessor, + extractor: ExtractorFn, +} + +impl Actor for EvmParser { + type Context = actix::Context; +} + +impl EvmParser { + pub fn new(next: &EvmEventProcessor, extractor: ExtractorFn) -> Self { + Self { + next: next.clone(), + extractor, + } + } +} + +impl Handler for EvmParser { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { + match msg.clone() { + EnclaveEvmEvent::Log(EvmLog { + log, + chain_id, + id, + timestamp, + }) => { + info!("processing event({})", msg.get_id()); + let extractor = self.extractor; + + if let Some(event) = extractor(log.data(), log.topic0(), chain_id) { + let err = "Log should always have metadata because we listen to non-pending blocks. If you are seeing this it is likely because there is an issue with how we are subscribing to blocks"; + let block = log.block_number.expect(err); + let log_index = log.log_index.expect(err); + let ts = from_log_chain_id_to_ts(timestamp, log_index, chain_id); + self.next.do_send(EnclaveEvmEvent::Event(EvmEvent::new( + // note we use the id from the log event above! + id, event, block, ts, chain_id, + ))) + } else { + self.next.do_send(EnclaveEvmEvent::Processed(id)) + } + } + hist @ EnclaveEvmEvent::HistoricalSyncComplete(..) => self.next.do_send(hist), + _ => (), + } + } +} + +fn from_log_chain_id_to_ts(block_timestamp: u64, log_index: u64, chain_id: u64) -> u128 { + let ts = block_timestamp.saturating_mul(1_000_000); + + // Use log_index as counter (orders logs within same block) + let counter = log_index as u32; + + // Use transaction_index as node (or chain_id if you have it) + let node = chain_id as u32; + + HlcTimestamp::new(ts, counter, node).into() +} diff --git a/crates/evm/src/evm_read_interface.rs b/crates/evm/src/evm_read_interface.rs new file mode 100644 index 0000000000..ece45967f2 --- /dev/null +++ b/crates/evm/src/evm_read_interface.rs @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; +use crate::helpers::EthProvider; +use crate::HistoricalSyncComplete; +use actix::prelude::*; +use actix::{Addr, Recipient}; +use alloy::eips::BlockNumberOrTag; +use alloy::primitives::{LogData, B256}; +use alloy::providers::Provider; +use alloy::rpc::types::Filter; +use alloy_primitives::Address; +use anyhow::anyhow; +use e3_events::{BusHandle, CorrelationId, ErrorDispatcher, Event, EventSubscriber, EventType}; +use e3_events::{EType, EnclaveEvent, EnclaveEventData, EventId}; +use futures_util::stream::StreamExt; +use std::collections::{HashMap, HashSet}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::select; +use tokio::sync::oneshot; +use tracing::{error, info, instrument, warn}; + +pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; + +pub struct EvmReadInterfaceParams

{ + provider: EthProvider

, + processor: Recipient, + bus: BusHandle, + filters: Filters, +} + +#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] +pub struct EvmReadInterfaceState { + pub ids: HashSet, + pub last_block: Option, +} + +#[derive(Clone, Default)] +pub struct Filters { + historical: Filter, + current: Filter, +} + +impl Filters { + pub fn new(addresses: Vec

, start_block: u64) -> Self { + let historical = Filter::new() + .address(addresses.clone()) + .from_block(start_block); + let current = Filter::new() + .address(addresses) + .from_block(BlockNumberOrTag::Latest); + + Self { + historical, + current, + } + } + + pub fn from_routing_table(table: &HashMap, start_block: u64) -> Self { + let addresses: Vec
= table.keys().cloned().collect(); + Self::new(addresses, start_block) + } +} + +/// Connects to Enclave.sol converting EVM events to EnclaveEvents +pub struct EvmReadInterface

{ + /// The alloy provider + provider: Option>, + /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used + /// internally. You should send the Shutdown signal to the reader directly or via the EventBus + shutdown_rx: Option>, + /// The sender for the shutdown signal this is only used internally + shutdown_tx: Option>, + /// Processor to forward events an actor + processor: EvmEventProcessor, + /// Event bus for error propagation only + bus: BusHandle, + /// Filters to configure when to seek from + filters: Filters, +} + +impl EvmReadInterface

{ + pub fn new(params: EvmReadInterfaceParams

) -> Self { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + Self { + provider: Some(params.provider), + shutdown_rx: Some(shutdown_rx), + shutdown_tx: Some(shutdown_tx), + processor: params.processor, + bus: params.bus, + filters: params.filters, + } + } + + pub fn setup( + provider: &EthProvider

, + next: &EvmEventProcessor, + bus: &BusHandle, + filters: Filters, + ) -> Addr { + let params = EvmReadInterfaceParams { + provider: provider.clone(), + processor: next.clone(), + bus: bus.clone(), + filters, + }; + + let addr = EvmReadInterface::new(params).start(); + + bus.subscribe(EventType::Shutdown, addr.clone().into()); + addr + } +} + +impl Actor for EvmReadInterface

{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + // let reader_addr = ctx.address(); + let bus = self.bus.clone(); + let processor = self.processor.clone(); + let filters = self.filters.clone(); + + let Some(provider) = self.provider.take() else { + error!("Could not start event reader as provider has already been used."); + return; + }; + + // let extractor = self.extractor; + let Some(shutdown) = self.shutdown_rx.take() else { + bus.err(EType::Evm, anyhow!("shutdown already called")); + return; + }; + + ctx.spawn( + async move { stream_from_evm(provider, processor, shutdown, &bus, filters).await } + .into_actor(self), + ); + } +} + +// TODO: split this up into: +// 1. historical request (will finish) +// 2. current listener (run indefinitely) +#[instrument(name = "evm_interface", skip_all)] +async fn stream_from_evm( + provider: EthProvider

, + processor: EvmEventProcessor, + mut shutdown: oneshot::Receiver<()>, + bus: &BusHandle, + filters: Filters, +) { + let chain_id = provider.chain_id(); + let provider_ref = provider.provider(); + let mut last_id: Option = None; + let mut timestamp_tracker = TimestampTracker::new(); + // Historical events + match provider_ref.get_logs(&filters.historical).await { + Ok(historical_logs) => { + info!("Fetched {} historical events", historical_logs.len()); + for log in historical_logs { + let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; + let evt = EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); + last_id = Some(evt.get_id()); + info!("Sending event({})", evt.get_id()); + processor.do_send(evt) + } + } + Err(e) => { + error!("Failed to fetch historical events: {}", e); + bus.err(EType::Evm, anyhow!(e)); + return; + } + } + let historical_sync_event = HistoricalSyncComplete::new(chain_id, last_id); + warn!( + "Historical Sync Complete event({})", + historical_sync_event.get_id() + ); + processor.do_send(EnclaveEvmEvent::HistoricalSyncComplete( + historical_sync_event, + )); + + info!("Subscribing to live events"); + match provider_ref.subscribe_logs(&filters.current).await { + Ok(subscription) => { + let id: B256 = subscription.local_id().clone(); + let mut stream = subscription.into_stream(); + + loop { + select! { + maybe_log = stream.next() => { + match maybe_log { + Some(log) => { + let timestamp = timestamp_tracker.get(provider_ref, log.block_number).await; + processor.do_send(EnclaveEvmEvent::Log(EvmLog::new(log, chain_id, timestamp))) + } + None => break, // Stream ended + } + } + _ = &mut shutdown => { + info!("Received shutdown signal, stopping EVM stream"); + match provider_ref.unsubscribe(id).await { + Ok(_) => info!("Unsubscribed successfully from EVM event stream"), + Err(err) => error!("Cannot unsubscribe from EVM event stream: {}", err), + }; + break; + } + } + } + } + Err(e) => { + bus.err(EType::Evm, anyhow!("{}", e)); + } + } + + info!("Exiting stream loop"); +} + +impl Handler for EvmReadInterface

{ + type Result = (); + + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::Shutdown(_) = msg.into_data() { + if let Some(shutdown) = self.shutdown_tx.take() { + let _ = shutdown.send(()); + } + } + } +} + +/// Cache utility to keep track of timestamps +struct TimestampTracker { + current: Option<(u64, u64)>, // (block_number, timestamp) +} + +impl TimestampTracker { + fn new() -> Self { + Self { current: None } + } + + async fn get(&mut self, provider: &P, block_number: Option) -> u64 { + let Some(bn) = block_number else { + error!("BLOCK NUMBER NOT FOUND ON LOG!"); + return 0; + }; + + if let Some((cached_bn, ts)) = self.current { + if bn == cached_bn { + return ts; + } + } + + let ts = provider + .get_block_by_number(bn.into()) + .await + .ok() + .flatten() + .map(|b| b.header.timestamp) + .unwrap_or(0); + + self.current = Some((bn, ts)); + ts + } +} diff --git a/crates/evm/src/evm_router.rs b/crates/evm/src/evm_router.rs new file mode 100644 index 0000000000..790cb78487 --- /dev/null +++ b/crates/evm/src/evm_router.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::events::{EnclaveEvmEvent, EvmEventProcessor, EvmLog}; +use actix::{Actor, Addr, Handler}; +use alloy_primitives::Address; +use std::collections::HashMap; +use tracing::{debug, error, info}; + +/// Directs EnclaveEvmEvent::Log events to the correct upstream processors. Drops all other event +/// types +pub struct EvmRouter { + routing_table: HashMap, + fallback: Option, +} + +impl EvmRouter { + pub fn new() -> Self { + Self { + routing_table: HashMap::new(), + fallback: None, + } + } + + pub fn add_route(mut self, address: Address, dest: &EvmEventProcessor) -> Self { + self.routing_table.insert(address, dest.clone()); + self + } + + pub fn add_fallback(mut self, fallback: &EvmEventProcessor) -> Self { + self.fallback = Some(fallback.clone()); + self + } + + pub fn get_routing_table(&self) -> &HashMap { + &self.routing_table + } +} + +impl Actor for EvmRouter { + type Context = actix::Context; +} + +impl Handler for EvmRouter { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, ctx: &mut Self::Context) -> Self::Result { + match msg.clone() { + // Take all log events and route them + EnclaveEvmEvent::Log(EvmLog { log, .. }) => { + let address = log.address(); + if let Some(dest) = self.routing_table.get(&address) { + debug!("Found address {address} in routing table forwarding to destination."); + dest.do_send(msg); + } else { + error!( + "Could not find a route for log with address = {:?}", + log.address() + ) + } + } + _ => { + if let Some(fallback) = self.fallback.clone() { + info!("Sending event({}) to fallback", msg.get_id()); + fallback.do_send(msg) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use actix::prelude::*; + use alloy_primitives::address; + use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, + }; + use tokio::time::sleep; + + struct TestProcessor(Arc); + + impl Actor for TestProcessor { + type Context = Context; + } + + impl Handler for TestProcessor { + type Result = (); + fn handle(&mut self, _msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + #[actix::test] + async fn test_evm_router_routes_log_to_correct_processor() { + let received_count = Arc::new(AtomicUsize::new(0)); + let processor_addr = TestProcessor(received_count.clone()).start(); + let addr = address!("0x1111111111111111111111111111111111111111"); + let test_log = EvmLog::test_log(addr, 1, 0); + let test_address = test_log.log.address(); + + let router = EvmRouter::new() + .add_route(test_address, &processor_addr.recipient()) + .start(); + + router.do_send(EnclaveEvmEvent::Log(test_log)); + + sleep(Duration::from_millis(10)).await; + + assert_eq!(received_count.load(Ordering::SeqCst), 1); + } + + #[actix::test] + async fn test_evm_router_ignores_log_with_unknown_address() { + let received_count = Arc::new(AtomicUsize::new(0)); + let processor_addr = TestProcessor(received_count.clone()).start(); + + let router_addr = address!("0x1111111111111111111111111111111111111111"); + let log_addr = address!("0x2222222222222222222222222222222222222222"); + + let test_log = EvmLog::test_log(log_addr, 1, 0); + + let router = EvmRouter::new() + .add_route(router_addr, &processor_addr.recipient()) + .start(); + + router.do_send(EnclaveEvmEvent::Log(test_log)); + + sleep(Duration::from_millis(10)).await; + + assert_eq!(received_count.load(Ordering::SeqCst), 0); + } +} diff --git a/crates/evm/src/fix_historical_order.rs b/crates/evm/src/fix_historical_order.rs new file mode 100644 index 0000000000..1faf833d21 --- /dev/null +++ b/crates/evm/src/fix_historical_order.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::{EnclaveEvmEvent, EvmEventProcessor, HistoricalSyncComplete}; +use actix::{Actor, Addr, Handler}; +use bloom::{BloomFilter, ASMS}; +use e3_events::CorrelationId; +use tracing::info; + +pub struct FixHistoricalOrder { + dest: EvmEventProcessor, + pending_sync_complete: Option, + seen_ids: BloomFilter, +} + +impl FixHistoricalOrder { + pub fn new(dest: impl Into) -> Self { + Self { + dest: dest.into(), + pending_sync_complete: None, + seen_ids: BloomFilter::with_rate(0.001, 10_000), + } + } + + pub fn setup(dest: impl Into) -> Addr { + Self::new(dest).start() + } + + fn send_pending(&mut self) { + if let Some(EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { + prev_event: Some(ref id), + .. + })) = self.pending_sync_complete + { + if self.seen_ids.contains(id) { + info!("Forwarding historical send complete event"); + self.dest + .do_send(self.pending_sync_complete.take().unwrap()); + } + } + } + + fn track_id(&mut self, id: CorrelationId) { + self.seen_ids.insert(&id); + } +} + +impl Actor for FixHistoricalOrder { + type Context = actix::Context; +} + +impl Handler for FixHistoricalOrder { + type Result = (); + + fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + let id = msg.get_id(); + info!("Receiving EnclaveEvmEvent event({})", msg.get_id()); + match msg { + none_hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { + prev_event: None, + .. + }) => { + info!( + "Historical order event({}) has no previous event. Forwarding...", + id + ); + self.dest.do_send(none_hist); + } + hist @ EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete { + prev_event: Some(prev), + .. + }) => { + info!( + "Historical order event({}) has previous event({}). Buffering...", + id, prev + ); + + self.pending_sync_complete = Some(hist); + } + EnclaveEvmEvent::Processed(id) => self.track_id(id), + other => { + info!("Forwarding event({})", other.get_id()); + self.track_id(other.get_id()); + self.dest.do_send(other); + } + } + self.send_pending(); + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use crate::EvmLog; + + use super::*; + use actix::prelude::*; + use alloy_primitives::Address; + use tokio::{sync::mpsc, time::sleep}; + + struct Collector(mpsc::UnboundedSender); + + impl Actor for Collector { + type Context = Context; + } + + impl Handler for Collector { + type Result = (); + fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) { + let _ = self.0.send(msg); + } + } + + #[actix::test] + async fn test_reorders_sync_complete_after_referenced_event() { + let (tx, mut rx) = mpsc::unbounded_channel(); + let fix = FixHistoricalOrder::setup(Collector(tx).start()); + + let log_1 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 1, 1)); + let log_2 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 2, 2)); + let log_3 = EnclaveEvmEvent::Log(EvmLog::test_log(Address::ZERO, 3, 3)); + + let sync_complete = EnclaveEvmEvent::HistoricalSyncComplete(HistoricalSyncComplete::new( + 1, + Some(log_3.get_id()), + )); + + // Send logs 1, 2, 3 + fix.send(log_1.clone()).await.unwrap(); + // Send sync complete FIRST (out of order - references log_3 which hasn't been seen) + fix.send(sync_complete.clone()).await.unwrap(); + fix.send(log_2.clone()).await.unwrap(); + fix.send(log_3.clone()).await.unwrap(); + + sleep(Duration::from_secs(1)).await; + + // Collect results + let mut received = vec![]; + while let Ok(msg) = rx.try_recv() { + received.push(msg); + } + + // The sync complete should have been held until log_3 was seen + assert_eq!(received.len(), 4); + assert_eq!(received[0], log_1); + assert_eq!(received[1], log_2); + assert_eq!(received[2], log_3); + assert_eq!(received[3], sync_complete); + } +} diff --git a/crates/evm/src/historical_event_coordinator.rs b/crates/evm/src/historical_event_coordinator.rs deleted file mode 100644 index 5b02e33b89..0000000000 --- a/crates/evm/src/historical_event_coordinator.rs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::EnclaveEvmEvent; -use actix::prelude::*; -use e3_events::{prelude::*, trap, BusHandle, EType, EnclaveEventData}; -use tracing::info; - -#[derive(Clone)] -struct BufferedEvent { - block: u64, - event: EnclaveEventData, -} - -/// Message to start forwarding buffered events after all readers have registered -#[derive(Message)] -#[rtype(result = "()")] -pub struct CoordinatorStart; - -/// Coordinates historical replay across all EvmEventReaders. -/// Buffers historical events, then sorts + publishes once all readers finish. -pub struct HistoricalEventCoordinator { - /// Count of readers that have registered - registered_count: usize, - /// Count of readers that have completed historical sync - completed_count: usize, - /// Buffered events during historical sync - buffered_events: Vec, - /// Target to forward events to (typically EventBus) - target: BusHandle, - /// Whether we've started forwarding (after Start message) - started: bool, -} - -impl HistoricalEventCoordinator { - pub fn new(target: BusHandle) -> Self { - Self { - registered_count: 0, - completed_count: 0, - buffered_events: Vec::new(), - target, - started: false, - } - } - - pub fn setup(target: BusHandle) -> Addr { - Self::new(target).start() - } - - fn all_readers_complete(&self) -> bool { - self.registered_count > 0 && self.registered_count == self.completed_count - } - - fn flush_buffered_events(&mut self) -> anyhow::Result<()> { - // Ordering by block number. But we should also consider the tx_index and log_index. - self.buffered_events.sort_by_key(|e| e.block); - - let count = self.buffered_events.len(); - for BufferedEvent { event, .. } in self.buffered_events.drain(..) { - self.target.publish(event)?; - } - - info!( - "HistoricalEventCoordinator: replay complete, published {} ordered events", - count - ); - Ok(()) - } -} - -impl Actor for HistoricalEventCoordinator { - type Context = Context; - - fn started(&mut self, _ctx: &mut Self::Context) { - info!("HistoricalEventCoordinator started"); - } -} - -impl Handler for HistoricalEventCoordinator { - type Result = (); - - fn handle(&mut self, msg: EnclaveEvmEvent, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.target.clone(), || match msg { - EnclaveEvmEvent::RegisterReader => { - self.registered_count += 1; - info!( - total_registered = self.registered_count, - "Reader registered with coordinator" - ); - Ok(()) - } - - EnclaveEvmEvent::HistoricalSyncComplete => { - self.completed_count += 1; - info!( - completed = self.completed_count, - total_registered = self.registered_count, - "Reader completed historical sync" - ); - - if self.started && self.all_readers_complete() { - info!("All readers completed historical sync, flushing buffered events"); - self.flush_buffered_events()?; - } - Ok(()) - } - - EnclaveEvmEvent::Event { event, block } => { - if !self.started || !self.all_readers_complete() { - if let Some(block) = block { - self.buffered_events.push(BufferedEvent { block, event }); - } - } else { - self.target.publish(event)?; - } - Ok(()) - } - }) - } -} - -impl Handler for HistoricalEventCoordinator { - type Result = (); - - fn handle(&mut self, _msg: CoordinatorStart, _ctx: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.target.clone(), || { - info!( - registered_readers = self.registered_count, - "Starting HistoricalEventCoordinator" - ); - self.started = true; - - if self.all_readers_complete() { - self.flush_buffered_events()?; - } - Ok(()) - }) - } -} diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index f864b89f8c..47f8b45f31 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -9,19 +9,33 @@ mod ciphernode_registry_sol; mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; -mod event_reader; +mod events; +mod evm_chain_gateway; +mod evm_hub; +mod evm_parser; +mod evm_read_interface; +mod evm_router; +mod fix_historical_order; pub mod helpers; -mod historical_event_coordinator; +mod one_shot_runnner; mod repo; +mod sync_start_extractor; -pub use bonding_registry_sol::{BondingRegistrySol, BondingRegistrySolReader}; +pub use bonding_registry_sol::BondingRegistrySolReader; pub use ciphernode_registry_sol::{ CiphernodeRegistrySol, CiphernodeRegistrySolReader, CiphernodeRegistrySolWriter, }; pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use event_reader::{EnclaveEvmEvent, EvmEventReader, EvmEventReaderState, ExtractorFn}; -pub use helpers::send_tx_with_retry; -pub use historical_event_coordinator::{CoordinatorStart, HistoricalEventCoordinator}; +pub use events::*; +pub use evm_chain_gateway::*; +pub use evm_hub::*; +pub use evm_parser::*; +pub use evm_read_interface::*; +pub use evm_router::*; +pub use fix_historical_order::*; +pub use helpers::*; +pub use one_shot_runnner::*; pub use repo::*; +pub use sync_start_extractor::*; diff --git a/crates/evm/src/one_shot_runnner.rs b/crates/evm/src/one_shot_runnner.rs new file mode 100644 index 0000000000..6a6576774d --- /dev/null +++ b/crates/evm/src/one_shot_runnner.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::prelude::*; +use anyhow::Result; +use std::marker::PhantomData; +use tracing::error; + +pub struct OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static, + M: Message + 'static, +{ + task: Option, + _marker: PhantomData, +} + +impl OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static + Unpin, + M: Message + 'static + Unpin, +{ + pub fn new(task: F) -> Self { + Self { + task: Some(task), + _marker: PhantomData, + } + } + pub fn setup(task: F) -> Addr { + Self::new(task).start() + } +} + +impl Actor for OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static + Unpin, + M: Message + 'static + Unpin, +{ + type Context = Context; +} + +impl Handler for OneShotRunner +where + F: FnOnce(M) -> Result<()> + 'static + Unpin, + M: Message + 'static + Unpin, +{ + type Result = (); + + fn handle(&mut self, msg: M, _ctx: &mut Self::Context) -> Self::Result { + if let Some(task) = self.task.take() { + match task(msg) { + Ok(_) => (), + Err(e) => error!("{e}"), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[derive(Message)] + #[rtype(result = "()")] + struct TestMessage(usize); + + #[actix::test] + async fn test_one_shot_runner() { + let call_count = Arc::new(AtomicUsize::new(0)); + let received_value = Arc::new(AtomicUsize::new(0)); + let call_count_clone = call_count.clone(); + let received_value_clone = received_value.clone(); + + let runner = OneShotRunner::new(move |msg: TestMessage| { + call_count_clone.fetch_add(1, Ordering::SeqCst); + received_value_clone.store(msg.0, Ordering::SeqCst); + Ok(()) + }); + + let addr = runner.start(); + addr.send(TestMessage(42)).await.unwrap(); + addr.send(TestMessage(99)).await.unwrap(); + + assert_eq!(received_value.load(Ordering::SeqCst), 42); + assert_eq!(call_count.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/evm/src/repo.rs b/crates/evm/src/repo.rs index 0455c09f04..1fc79ae011 100644 --- a/crates/evm/src/repo.rs +++ b/crates/evm/src/repo.rs @@ -7,7 +7,7 @@ use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; -use crate::EvmEventReaderState; +use crate::EvmReadInterfaceState; pub trait EthPrivateKeyRepositoryFactory { fn eth_private_key(&self) -> Repository>; @@ -20,21 +20,21 @@ impl EthPrivateKeyRepositoryFactory for Repositories { } pub trait EnclaveSolReaderRepositoryFactory { - fn enclave_sol_reader(&self, chain_id: u64) -> Repository; + fn enclave_sol_reader(&self, chain_id: u64) -> Repository; } impl EnclaveSolReaderRepositoryFactory for Repositories { - fn enclave_sol_reader(&self, chain_id: u64) -> Repository { + fn enclave_sol_reader(&self, chain_id: u64) -> Repository { Repository::new(self.store.scope(StoreKeys::enclave_sol_reader(chain_id))) } } pub trait CiphernodeRegistryReaderRepositoryFactory { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; + fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; } impl CiphernodeRegistryReaderRepositoryFactory for Repositories { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { + fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(StoreKeys::ciphernode_registry_reader(chain_id)), @@ -43,11 +43,11 @@ impl CiphernodeRegistryReaderRepositoryFactory for Repositories { } pub trait BondingRegistryReaderRepositoryFactory { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository; + fn bonding_registry_reader(&self, chain_id: u64) -> Repository; } impl BondingRegistryReaderRepositoryFactory for Repositories { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository { + fn bonding_registry_reader(&self, chain_id: u64) -> Repository { Repository::new( self.store .scope(StoreKeys::bonding_registry_reader(chain_id)), diff --git a/crates/evm/src/sync_start_extractor.rs b/crates/evm/src/sync_start_extractor.rs new file mode 100644 index 0000000000..5cd2a52ee3 --- /dev/null +++ b/crates/evm/src/sync_start_extractor.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Addr, Handler, Recipient}; +use e3_events::{EnclaveEvent, EnclaveEventData, Event, SyncStart}; + +pub struct SyncStartExtractor { + dest: Recipient, +} + +impl SyncStartExtractor { + pub fn new(dest: impl Into>) -> Self { + Self { dest: dest.into() } + } + + pub fn setup(dest: impl Into>) -> Addr { + Self::new(dest).start() + } +} +impl Actor for SyncStartExtractor { + type Context = actix::Context; +} + +impl Handler for SyncStartExtractor { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::SyncStart(evt) = msg.into_data() { + self.dest.do_send(evt) + } + } +} diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 385f890781..b87029282d 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::Addr; +use actix::{Actor, Addr, Handler}; use alloy::{ node_bindings::Anvil, primitives::{FixedBytes, LogData}, @@ -14,15 +14,16 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; -use e3_ciphernode_builder::EventSystem; -use e3_data::Repository; -use e3_entrypoint::helpers::datastore::get_in_mem_store; +use e3_ciphernode_builder::{EventSystem, EvmSystemChainBuilder}; use e3_events::{ - prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, HistoryCollector, Shutdown, TestEvent, + prelude::*, trap, BusHandle, EType, EnclaveEvent, EnclaveEventData, EvmEvent, EvmEventConfig, + EvmEventConfigChain, GetEvents, HistoryCollector, SyncEnd, SyncEvmEvent, SyncStart, TestEvent, }; -use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmEventReader, HistoricalEventCoordinator}; -use std::time::Duration; +use e3_evm::{helpers::EthProvider, EvmEventProcessor, EvmParser}; +use std::{sync::Arc, time::Duration}; use tokio::time::sleep; +use tracing::subscriber::DefaultGuard; +use tracing_subscriber::{fmt, EnvFilter}; sol!( #[sol(rpc)] @@ -52,6 +53,14 @@ fn test_event_extractor( } } +struct TestEventParser; + +impl TestEventParser { + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, test_event_extractor).start() + } +} + async fn get_msgs(history_collector: &Addr>) -> Result> { let history = history_collector .send(GetEvents::::new()) @@ -67,42 +76,93 @@ async fn get_msgs(history_collector: &Addr>) -> R Ok(msgs) } +struct FakeSyncActor { + bus: BusHandle, + buffer: Vec, +} + +impl Actor for FakeSyncActor { + type Context = actix::Context; +} + +impl FakeSyncActor { + pub fn setup(bus: &BusHandle) -> Addr { + Self { + bus: bus.clone(), + buffer: Vec::new(), + } + .start() + } +} + +impl Handler for FakeSyncActor { + type Result = (); + fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Sync, &self.bus.clone(), || { + match msg { + // Buffer events as the sync actor receives them + SyncEvmEvent::Event(event) => self.buffer.push(event), + // When we hear that sync is complete send all events on chain then publish SyncEnd + SyncEvmEvent::HistoricalSyncComplete(_) => { + for evt in self.buffer.drain(..) { + let (data, ts, _) = evt.split(); + self.bus.publish_from_remote(data, ts)?; + } + self.bus.publish(SyncEnd::new())?; + } + }; + Ok(()) + }) + } +} + +fn add_tracing() -> DefaultGuard { + tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new("info")) + .with_test_writer() + .finish(), + ) +} + #[actix::test] async fn evm_reader() -> Result<()> { + let _guard = add_tracing(); + // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; let rpc_url = anvil.ws_endpoint(); // Get RPC URL - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) // Use RPC URL - .await?, - ) - .await?; + let provider = Arc::new( + EthProvider::new( + ProviderBuilder::new() + .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) + .connect_ws(WsConnect::new(rpc_url.clone())) // Use RPC URL + .await?, + ) + .await?, + ); let contract = EmitLogs::deploy(provider.provider()).await?; let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - coordinator.do_send(CoordinatorStart); + let chain_id = provider.chain_id(); + let contract_address = contract.address().clone(); + let sync = FakeSyncActor::setup(&bus); + EvmSystemChainBuilder::new(&bus, &provider) + .with_contract(contract_address, move |upstream| { + TestEventParser::setup(&upstream).recipient() + }) + .build(); + + // SyncStart holds initialization information such as start block and earliest event + // This should trigger all chains to start to sync + let mut evm_info = EvmEventConfig::new(); + evm_info.insert(chain_id, EvmEventConfigChain::new(0)); + bus.publish(SyncStart::new(sync, evm_info))?; + sleep(Duration::from_secs(1)).await; contract .setValue("hello".to_string()) .send() @@ -117,14 +177,12 @@ async fn evm_reader() -> Result<()> { .watch() .await?; - sleep(Duration::from_millis(1)).await; + sleep(Duration::from_secs(1)).await; let history = history_collector .send(GetEvents::::new()) .await?; - assert_eq!(history.len(), 2); - let msgs: Vec<_> = history .into_iter() .filter_map(|evt| match evt.into_data() { @@ -137,9 +195,10 @@ async fn evm_reader() -> Result<()> { Ok(()) } - #[actix::test] async fn ensure_historical_events() -> Result<()> { + let _guard = add_tracing(); + // Create a WS provider // NOTE: Anvil must be available on $PATH let anvil = Anvil::new().block_time(1).try_spawn()?; @@ -152,17 +211,14 @@ async fn ensure_historical_events() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; + let contract_address = contract.address().clone(); + let chain_id = provider.chain_id(); let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let historical_msgs = vec!["these", "are", "historical", "events"]; let live_events = vec!["these", "events", "are", "live"]; - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - for msg in historical_msgs.clone() { contract .setValue(msg.to_string()) @@ -172,19 +228,17 @@ async fn ensure_historical_events() -> Result<()> { .await?; } - EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; + sleep(Duration::from_millis(1)).await; - coordinator.do_send(CoordinatorStart); + let sync = FakeSyncActor::setup(&bus); + EvmSystemChainBuilder::new(&bus, &provider) + .with_contract(contract_address, move |upstream| { + TestEventParser::setup(&upstream).recipient() + }) + .build(); + let mut evm_info = EvmEventConfig::new(); + evm_info.insert(chain_id, EvmEventConfigChain::new(0)); + bus.publish(SyncStart::new(sync, evm_info))?; for msg in live_events.clone() { contract @@ -203,8 +257,6 @@ async fn ensure_historical_events() -> Result<()> { .send(GetEvents::::new()) .await?; - assert_eq!(history.len(), 8); - let msgs: Vec<_> = history .into_iter() .filter_map(|evt| match evt.into_data() { @@ -217,328 +269,3 @@ async fn ensure_historical_events() -> Result<()> { Ok(()) } - -#[actix::test] -async fn ensure_resume_after_shutdown() -> Result<()> { - // Create a WS provider - // NOTE: Anvil must be available on $PATH - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); // Get RPC URL - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) // Use RPC URL - .await?, - ) - .await?; - let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - for msg in ["before", "online"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - let addr1 = EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - - for msg in ["live", "events"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - // Ensure shutdown doesn't cause event to be lost. - sleep(Duration::from_millis(10)).await; - addr1 - .send(EnclaveEvent::new_stored_event(Shutdown.into(), 4321, 42)) - .await?; - - for msg in ["these", "are", "not", "lost"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(10)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs, ["before", "online", "live", "events"]); - - let _ = EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - sleep(Duration::from_millis(10)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!( - msgs, - ["before", "online", "live", "events", "these", "are", "not", "lost"] - ); - - for msg in ["resumed", "data"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(10)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!( - msgs, - ["before", "online", "live", "events", "these", "are", "not", "lost", "resumed", "data"] - ); - - Ok(()) -} - -#[actix::test] -async fn coordinator_single_reader() -> Result<()> { - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) - .await?, - ) - .await?; - let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - for msg in ["historical1", "historical2", "historical3"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - sleep(Duration::from_millis(100)).await; - - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs, ["historical1", "historical2", "historical3"]); - - for msg in ["live1", "live2"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(100)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!( - msgs, - [ - "historical1", - "historical2", - "historical3", - "live1", - "live2" - ] - ); - - Ok(()) -} - -#[actix::test] -async fn coordinator_multiple_readers() -> Result<()> { - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) - .await?, - ) - .await?; - - let contract1 = EmitLogs::deploy(provider.provider()).await?; - let contract2 = EmitLogs::deploy(provider.provider()).await?; - - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository1 = Repository::new(get_in_mem_store()); - let repository2 = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - contract1 - .setValue("contract1_msg1".to_string()) - .send() - .await? - .watch() - .await?; - contract2 - .setValue("contract2_msg1".to_string()) - .send() - .await? - .watch() - .await?; - contract1 - .setValue("contract1_msg2".to_string()) - .send() - .await? - .watch() - .await?; - contract2 - .setValue("contract2_msg2".to_string()) - .send() - .await? - .watch() - .await?; - - EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract1.address().to_string(), - None, - &processor, - &bus, - &repository1, - rpc_url.clone(), - ) - .await?; - - EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract2.address().to_string(), - None, - &processor, - &bus, - &repository2, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - - // Wait for historical events to be processed - sleep(Duration::from_millis(200)).await; - - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs.len(), 4); - assert!(msgs.contains(&"contract1_msg1".to_string())); - assert!(msgs.contains(&"contract2_msg1".to_string())); - assert!(msgs.contains(&"contract1_msg2".to_string())); - assert!(msgs.contains(&"contract2_msg2".to_string())); - - Ok(()) -} - -#[actix::test] -async fn coordinator_no_historical_events() -> Result<()> { - let anvil = Anvil::new().block_time(1).try_spawn()?; - let rpc_url = anvil.ws_endpoint(); - let provider = EthProvider::new( - ProviderBuilder::new() - .wallet(PrivateKeySigner::from_slice(&anvil.keys()[0].to_bytes())?) - .connect_ws(WsConnect::new(rpc_url.clone())) - .await?, - ) - .await?; - let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test").with_fresh_bus(); - let bus = system.handle()?; - let history_collector = bus.history(); - let repository = Repository::new(get_in_mem_store()); - - let coordinator = HistoricalEventCoordinator::setup(bus.clone()); - let processor = coordinator.clone().recipient(); - - EvmEventReader::attach( - provider.clone(), - test_event_extractor, - &contract.address().to_string(), - None, - &processor, - &bus, - &repository, - rpc_url.clone(), - ) - .await?; - - coordinator.do_send(CoordinatorStart); - sleep(Duration::from_millis(50)).await; - - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs.len(), 0); - - for msg in ["live1", "live2"] { - contract - .setValue(msg.to_string()) - .send() - .await? - .watch() - .await?; - } - - sleep(Duration::from_millis(100)).await; - let msgs = get_msgs(&history_collector).await?; - assert_eq!(msgs, ["live1", "live2"]); - - Ok(()) -} diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index f437a11dc8..92e4ba1e8d 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -13,7 +13,7 @@ use e3_events::{ ComputeResponse, CorrelationId, DecryptionshareCreated, Die, E3RequestComplete, E3id, EType, EnclaveEvent, EnclaveEventData, EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, KeyshareCreated, PartyId, ThresholdShare, ThresholdShareCollectionFailed, - ThresholdShareCreated, + ThresholdShareCreated, TypedEvent, }; use e3_fhe::create_crp; use e3_trbfv::{ @@ -27,6 +27,7 @@ use e3_trbfv::{ shares::{BfvEncryptedShares, EncryptableVec, Encrypted, ShamirShare, SharedSecret}, TrBFVConfig, TrBFVRequest, TrBFVResponse, }; +use e3_utils::NotifySync; use e3_utils::{to_ordered_vec, utility_types::ArcBytes}; use fhe::bfv::BfvParameters; use fhe::bfv::{PublicKey, SecretKey}; @@ -406,7 +407,7 @@ impl ThresholdKeyshare { Ok(()) } - pub fn handle_compute_response(&mut self, msg: ComputeResponse) -> Result<()> { + pub fn handle_compute_response(&mut self, msg: TypedEvent) -> Result<()> { match &msg.response { TrBFVResponse::GenEsiSss(_) => self.handle_gen_esi_sss_response(msg), TrBFVResponse::GenPkShareAndSkSss(_) => { @@ -425,7 +426,7 @@ impl ThresholdKeyshare { /// 1. CiphernodeSelected - Generate BFV keys and start collecting pub fn handle_ciphernode_selected( &mut self, - msg: CiphernodeSelected, + msg: TypedEvent, address: Addr, ) -> Result<()> { info!("CiphernodeSelected received."); @@ -447,7 +448,7 @@ impl ThresholdKeyshare { CollectingEncryptionKeysData { sk_bfv: sk_bfv_encrypted.clone(), pk_bfv: pk_bfv_bytes.clone(), - ciphernode_selected: msg.clone(), + ciphernode_selected: msg.into_inner(), }, )) })?; @@ -489,7 +490,6 @@ impl ThresholdKeyshare { }, )) })?; - self.handle_gen_esi_sss_requested(GenEsiSss(current.ciphernode_selected.clone()))?; self.handle_gen_pk_share_and_sk_sss_requested(GenPkShareAndSkSss( current.ciphernode_selected, @@ -537,8 +537,8 @@ impl ThresholdKeyshare { } /// 2a. GenEsiSss result - pub fn handle_gen_esi_sss_response(&mut self, res: ComputeResponse) -> Result<()> { - let output: GenEsiSssResponse = res.try_into()?; + pub fn handle_gen_esi_sss_response(&mut self, res: TypedEvent) -> Result<()> { + let output: GenEsiSssResponse = res.into_inner().try_into()?; let esi_sss = output.esi_sss; @@ -616,8 +616,11 @@ impl ThresholdKeyshare { } /// 3a. GenPkShareAndSkSss result - pub fn handle_gen_pk_share_and_sk_sss_response(&mut self, res: ComputeResponse) -> Result<()> { - let TrBFVResponse::GenPkShareAndSkSss(output) = res.response else { + pub fn handle_gen_pk_share_and_sk_sss_response( + &mut self, + res: TypedEvent, + ) -> Result<()> { + let TrBFVResponse::GenPkShareAndSkSss(output) = res.into_inner().response else { bail!("Error extracting data from compute process") }; @@ -661,7 +664,7 @@ impl ThresholdKeyshare { } /// 4. SharesGenerated - Encrypt shares with BFV and publish - pub fn handle_shares_generated(&self) -> Result<()> { + pub fn handle_shares_generated(&mut self) -> Result<()> { let Some(ThresholdKeyshareState { state: KeyshareState::AggregatingDecryptionKey(AggregatingDecryptionKey { @@ -739,7 +742,6 @@ impl ThresholdKeyshare { external: false, })?; } - Ok(()) } @@ -817,8 +819,11 @@ impl ThresholdKeyshare { } /// 5a. CalculateDecryptionKeyResponse -> KeyshareCreated - pub fn handle_calculate_decryption_key_response(&mut self, res: ComputeResponse) -> Result<()> { - let TrBFVResponse::CalculateDecryptionKey(output) = res.response else { + pub fn handle_calculate_decryption_key_response( + &mut self, + res: TypedEvent, + ) -> Result<()> { + let TrBFVResponse::CalculateDecryptionKey(output) = res.into_inner().response else { bail!("Error extracting data from compute process") }; @@ -901,9 +906,9 @@ impl ThresholdKeyshare { /// CalculateDecryptionShareResponse pub fn handle_calculate_decryption_share_response( &mut self, - res: ComputeResponse, + res: TypedEvent, ) -> Result<()> { - let msg: CalculateDecryptionShareResponse = res.try_into()?; + let msg: CalculateDecryptionShareResponse = res.into_inner().try_into()?; let state = self.state.try_get()?; let party_id = state.party_id; let node = state.address; @@ -936,34 +941,42 @@ impl ThresholdKeyshare { impl Handler for ThresholdKeyshare { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - EnclaveEventData::CiphernodeSelected(data) => ctx.notify(data), - EnclaveEventData::CiphertextOutputPublished(data) => ctx.notify(data), + match msg.clone().into_data() { + EnclaveEventData::CiphernodeSelected(data) => { + self.notify_sync(ctx, msg.to_typed_event(data)) + } + EnclaveEventData::CiphertextOutputPublished(data) => self.notify_sync(ctx, data), EnclaveEventData::ThresholdShareCreated(data) => { let _ = self.handle_threshold_share_created(data, ctx.address()); } EnclaveEventData::EncryptionKeyCreated(data) => { let _ = self.handle_encryption_key_created(data, ctx.address()); } - EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), - EnclaveEventData::ComputeResponse(data) => ctx.notify(data), + EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + EnclaveEventData::ComputeResponse(data) => { + self.notify_sync(ctx, msg.to_typed_event(data)) + } _ => (), } } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: ComputeResponse, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_compute_response(msg) }) } } -impl Handler for ThresholdKeyshare { +impl Handler> for ThresholdKeyshare { type Result = (); - fn handle(&mut self, msg: CiphernodeSelected, ctx: &mut Self::Context) -> Self::Result { + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { trap(EType::KeyGeneration, &self.bus.clone(), || { self.handle_ciphernode_selected(msg, ctx.address()) }) @@ -1055,7 +1068,7 @@ impl Handler for ThresholdKeyshare { fn handle(&mut self, _: E3RequestComplete, ctx: &mut Self::Context) -> Self::Result { self.encryption_key_collector = None; self.decryption_key_collector = None; - ctx.notify(Die); + self.notify_sync(ctx, Die); } } diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index 7bf35affec..09bf9e8f29 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -33,6 +33,7 @@ use e3_trbfv::calculate_threshold_decryption::calculate_threshold_decryption; use e3_trbfv::gen_esi_sss::gen_esi_sss; use e3_trbfv::gen_pk_share_and_sk_sss::gen_pk_share_and_sk_sss; use e3_trbfv::{TrBFVError, TrBFVRequest, TrBFVResponse}; +use e3_utils::NotifySync; use e3_utils::SharedRng; use rand::Rng; use tracing::error; diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml index 8fb6f8ae49..46f00d41fc 100644 --- a/crates/net/Cargo.toml +++ b/crates/net/Cargo.toml @@ -20,13 +20,16 @@ e3-data = { workspace = true } e3-utils = { workspace = true } hex = { workspace = true } libp2p = { workspace = true } +rand = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } e3-events = { workspace = true } -e3-ciphernode-builder = { workspace = true } anyhow = { workspace = true } actix = { workspace = true } zeroize = { workspace = true } + +[dev-dependencies] +e3-ciphernode-builder = { workspace = true } diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index 785ec8eb58..7db3f39c29 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -21,6 +21,7 @@ use e3_events::{ }; use e3_utils::retry::{retry_with_backoff, to_retry}; use e3_utils::ArcBytes; +use e3_utils::NotifySync; use futures::TryFutureExt; use serde::{Deserialize, Serialize}; use std::{ @@ -137,8 +138,8 @@ impl Handler for DocumentPublisher { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { EnclaveEventData::PublishDocumentRequested(data) => ctx.notify(data), - EnclaveEventData::CiphernodeSelected(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), + EnclaveEventData::CiphernodeSelected(data) => self.notify_sync(ctx, data), + EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), _ => (), } } @@ -508,9 +509,9 @@ impl Handler for EventConverter { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::ThresholdShareCreated(data) => ctx.notify(data), - EnclaveEventData::EncryptionKeyCreated(data) => ctx.notify(data), - EnclaveEventData::DocumentReceived(data) => ctx.notify(data), + EnclaveEventData::ThresholdShareCreated(data) => self.notify_sync(ctx, data), + EnclaveEventData::EncryptionKeyCreated(data) => self.notify_sync(ctx, data), + EnclaveEventData::DocumentReceived(data) => self.notify_sync(ctx, data), _ => (), } } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index e042eedd1d..d394c80448 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -7,15 +7,17 @@ use crate::Cid; use actix::Message; use anyhow::{bail, Context, Result}; -use e3_events::{CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; -use e3_utils::ArcBytes; +use e3_events::{AggregateId, CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; +use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, kad::{store, GetRecordError, PutRecordError}, + request_response::{InboundRequestId, ResponseChannel}, swarm::{dial_opts::DialOpts, ConnectionId, DialError}, }; use serde::{Deserialize, Serialize}; use std::{ + collections::HashMap, hash::Hash, sync::Arc, time::{Duration, Instant}, @@ -61,6 +63,33 @@ impl TryFrom for EnclaveEvent { } } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SyncRequestValue { + pub since: HashMap, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SyncResponseValue { + pub events: Vec, +} + +#[derive(Message, Clone, Debug)] +#[rtype("()")] +pub struct SyncRequestReceived { + pub request_id: InboundRequestId, + pub value: SyncRequestValue, + pub channel: OnceTake>, +} + +#[derive(Message, Clone, Debug)] +#[rtype("()")] +pub struct OutgoingSyncRequestSucceeded { + pub value: SyncResponseValue, +} + +#[derive(Debug, Clone)] +pub struct OutgoingSyncRequestFailed; + /// NetInterface Commands are sent to the network peer over a mspc channel #[derive(Debug)] pub enum NetCommand { @@ -86,6 +115,14 @@ pub enum NetCommand { }, /// Shutdown signal Shutdown, + /// Called from the syning node to request libp2p events from a random peer node starting + /// from the given timestamp. + OutgoingSyncRequest { value: SyncRequestValue }, + /// Send libp2p events back to a peer that requested a sync. + SyncResponse { + value: SyncResponseValue, + channel: OnceTake>, + }, } impl NetCommand { @@ -117,9 +154,13 @@ pub enum NetEvent { message_id: MessageId, }, /// There was an error Dialing a peer - DialError { error: Arc }, + DialError { + error: Arc, + }, /// A connection was established to a peer - ConnectionEstablished { connection_id: ConnectionId }, + ConnectionEstablished { + connection_id: ConnectionId, + }, /// There was an error creating a connection OutgoingConnectionError { connection_id: ConnectionId, @@ -147,7 +188,16 @@ pub enum NetEvent { error: PutOrStoreError, }, /// GossipSubscribed - GossipSubscribed { count: usize, topic: TopicHash }, + GossipSubscribed { + count: usize, + topic: TopicHash, + }, + /// A peer node is requesting gossipsub events since the given timestamp. + /// Use the provided channel to send a `SyncResponse + SyncRequestReceived(SyncRequestReceived), + /// Received gossipsub events from a peer in response to a `SyncRequest`. + OutgoingSyncRequestSucceeded(OutgoingSyncRequestSucceeded), + OutgoingSyncRequestFailed(OutgoingSyncRequestFailed), } #[derive(Clone, Debug)] @@ -255,7 +305,7 @@ mod tests { fn test_enclave_event_gossip_lifecycle() -> anyhow::Result<()> { // event is created locally let event: EnclaveEvent = - EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), 31415); + EnclaveEvent::new_with_timestamp(TestEvent::new("fish", 42).into(), None, 31415); // event is sequenced after bus.publish() adds a sequence number let event: EnclaveEvent = event.into_sequenced(90210); diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index 0b9868cd08..eb7c49065a 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -10,6 +10,7 @@ mod document_publisher; pub mod events; mod net_event_translator; mod net_interface; +mod net_sync_manager; mod repo; pub use cid::Cid; diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index 85ca6c41d7..962827d5bf 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -21,6 +21,7 @@ use e3_events::BusHandle; use e3_events::EType; use e3_events::EnclaveEventData; use e3_events::Event; +use e3_events::EventContextAccessors; use e3_events::EventType; use e3_events::Unsequenced; use e3_events::{CorrelationId, EnclaveEvent, EventId}; @@ -160,7 +161,7 @@ impl Handler for NetEventTranslator { fn handle(&mut self, msg: LibP2pEvent, _: &mut Self::Context) -> Self::Result { let LibP2pEvent(data) = msg; let event: EnclaveEvent = data.try_into()?; - self.sent_events.insert(event.get_id()); + self.sent_events.insert(event.id()); let (data, ts) = event.split(); self.bus.publish_from_remote(data, ts)?; Ok(()) diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 0542845827..e32fba0f59 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -4,14 +4,14 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; +use anyhow::{bail, Result}; use e3_events::CorrelationId; -use e3_utils::ArcBytes; +use e3_utils::{ArcBytes, OnceTake}; use libp2p::{ connection_limits::{self, ConnectionLimits}, futures::StreamExt, gossipsub, - identify::{self, Behaviour as IdentifyBehaviour}, + identify::{Behaviour as IdentifyBehaviour, Config as IdentifyConfig}, identity::Keypair, kad::{ self, @@ -19,15 +19,21 @@ use libp2p::{ Behaviour as KademliaBehaviour, Config as KademliaConfig, GetRecordOk, QueryId, QueryResult, Quorum, Record, RecordKey, }, + request_response::{ + self, cbor::Behaviour as CborRequestResponse, Event as RequestResponseEvent, + Message as RequestResponseMessage, ProtocolSupport, ResponseChannel, + }, swarm::{dial_opts::DialOpts, NetworkBehaviour, SwarmEvent}, StreamProtocol, Swarm, }; +use rand::prelude::IteratorRandom; use std::sync::atomic::AtomicBool; use std::{ collections::HashMap, sync::{atomic::Ordering, Arc}, time::Instant, }; + use std::{io::Error, time::Duration}; use tokio::{select, sync::broadcast, sync::mpsc}; use tracing::{debug, error, info, trace, warn}; @@ -36,7 +42,10 @@ const PROTOCOL_NAME: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0"); const MAX_KADEMLIA_PAYLOAD_MB: usize = 10; const MAX_GOSSIP_MSG_SIZE_KB: usize = 700; -use crate::events::{GossipData, NetCommand}; +use crate::events::{ + GossipData, NetCommand, OutgoingSyncRequestSucceeded, SyncRequestReceived, SyncRequestValue, + SyncResponseValue, +}; use crate::events::{NetEvent, PutOrStoreError}; use crate::{dialer::dial_peers, Cid}; @@ -46,6 +55,7 @@ pub struct NodeBehaviour { kademlia: KademliaBehaviour, connection_limits: connection_limits::Behaviour, identify: IdentifyBehaviour, + sync: CborRequestResponse, } /// Manage the peer to peer connection. This struct wraps a libp2p Swarm and enables communication @@ -167,8 +177,8 @@ fn create_behaviour( ) -> std::result::Result> { let peer_id = key.public().to_peer_id(); let connection_limits = connection_limits::Behaviour::new(ConnectionLimits::default()); - let identify_config = IdentifyBehaviour::new( - identify::Config::new("/enclave/0.0.1".into(), key.public()) + let identify = IdentifyBehaviour::new( + IdentifyConfig::new("/enclave/0.0.1".into(), key.public()) .with_interval(Duration::from_secs(60)), ); @@ -183,7 +193,13 @@ fn create_behaviour( gossipsub::MessageAuthenticity::Signed(key.clone()), gossipsub_config, )?; - + let sync = CborRequestResponse::::new( + [( + StreamProtocol::new("/enclave/sync/0.0.1"), + ProtocolSupport::Full, + )], + request_response::Config::default(), + ); let mut config = KademliaConfig::new(PROTOCOL_NAME); config .set_max_packet_size(MAX_KADEMLIA_PAYLOAD_MB * 1024 * 1024) @@ -203,7 +219,8 @@ fn create_behaviour( gossipsub, kademlia, connection_limits, - identify: identify_config, + identify, + sync, }) } @@ -349,9 +366,11 @@ async fn process_swarm_event( let gossip_data = GossipData::from_bytes(&message.data)?; event_tx.send(NetEvent::GossipData(gossip_data))?; } + SwarmEvent::NewListenAddr { address, .. } => { trace!("Local node is listening on {address}"); } + SwarmEvent::Behaviour(NodeBehaviourEvent::Gossipsub(gossipsub::Event::Subscribed { peer_id, topic, @@ -360,6 +379,34 @@ async fn process_swarm_event( let count = swarm.behaviour().gossipsub.mesh_peers(&topic).count(); event_tx.send(NetEvent::GossipSubscribed { count, topic })?; } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { + message: + RequestResponseMessage::Request { + request, + channel, + request_id, + }, + .. + })) => { + // received a request for events + event_tx.send(NetEvent::SyncRequestReceived(SyncRequestReceived { + request_id, + channel: OnceTake::new(channel), + value: request, + }))?; + } + + SwarmEvent::Behaviour(NodeBehaviourEvent::Sync(RequestResponseEvent::Message { + message: RequestResponseMessage::Response { response, .. }, + .. + })) => { + // received a response to a request for events + event_tx.send(NetEvent::OutgoingSyncRequestSucceeded( + OutgoingSyncRequestSucceeded { value: response }, + ))?; + } + unknown => { trace!("Unknown event: {:?}", unknown); } @@ -405,6 +452,8 @@ async fn process_swarm_command( key, } => handle_get_record(swarm, correlator, correlation_id, key), NetCommand::Shutdown => handle_shutdown(swarm, shutdown_flag), + NetCommand::OutgoingSyncRequest { value } => handle_outgoing_sync_request(swarm, value), + NetCommand::SyncResponse { value, channel } => handle_sync_response(swarm, channel, value), } } @@ -530,6 +579,47 @@ fn handle_shutdown( Ok(()) } +fn handle_outgoing_sync_request( + swarm: &mut Swarm, + value: SyncRequestValue, +) -> Result<()> { + // TODO: + // This is a first pass. + // Lots of stuff to work through here: + // How can I know events are correct? + // How can I trust this peer? + // Can I validate events with another peer? + // Should I use an OrderedSet with a hash and request the hash from a second peer? + + // Pick a random peer + let Some(peer) = swarm + .connected_peers() + .choose(&mut rand::thread_rng()) + .copied() + else { + bail!("No peer found on swarm!") + }; + + // Request events + swarm.behaviour_mut().sync.send_request(&peer, value); + Ok(()) +} + +fn handle_sync_response( + swarm: &mut Swarm, + channel: OnceTake>, + value: SyncResponseValue, +) -> Result<()> { + let channel = channel.try_take()?; + match swarm.behaviour_mut().sync.send_response(channel, value) { + Ok(_) => (), + Err(_res) => { + // TODO: report failure + } + } + Ok(()) +} + /// This correlates query_id and correlation_id. #[derive(Clone)] struct Correlator { diff --git a/crates/net/src/net_sync_manager.rs b/crates/net/src/net_sync_manager.rs new file mode 100644 index 0000000000..7d6c9a5a21 --- /dev/null +++ b/crates/net/src/net_sync_manager.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; +use anyhow::{anyhow, bail, Result}; +use e3_events::{ + prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EnclaveEvent, + EnclaveEventData, Event, GetAggregateEventsAfter, NetSyncEventsReceived, OutgoingSyncRequested, + ReceiveEvents, Unsequenced, +}; +use e3_utils::{retry_with_backoff, to_retry, OnceTake}; +use futures::TryFutureExt; +use libp2p::request_response::ResponseChannel; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tokio::sync::{broadcast, mpsc}; +use tracing::debug; + +use crate::events::{ + call_and_await_response, NetCommand, NetEvent, OutgoingSyncRequestSucceeded, + SyncRequestReceived, SyncRequestValue, SyncResponseValue, +}; + +pub struct NetSyncManager { + /// Enclave EventBus + bus: BusHandle, + /// NetCommand sender to forward commands to the NetInterface + tx: mpsc::Sender, + /// NetEvent receiver to resubscribe for events from the NetInterface. This is in an Arc so + /// that we do not do excessive resubscribes without actually listening for events. + rx: Arc>, + eventstore: Recipient, + requests: HashMap>>, +} + +impl NetSyncManager { + pub fn new( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + eventstore: Recipient, + ) -> Self { + Self { + bus: bus.clone(), + tx: tx.clone(), + rx: rx.clone(), + + eventstore, + requests: HashMap::new(), + } + } + + pub fn setup( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + eventstore: Recipient, + ) -> Addr { + let mut events = rx.resubscribe(); + let addr = Self::new(bus, tx, rx, eventstore).start(); + + // Forward from NetEvent + tokio::spawn({ + debug!("Spawning event receive loop!"); + let addr = addr.clone(); + async move { + while let Ok(event) = events.recv().await { + debug!("Received event {:?}", event); + match event { + NetEvent::OutgoingSyncRequestSucceeded(value) => addr.do_send(value), + NetEvent::SyncRequestReceived(value) => addr.do_send(value), + _ => (), + } + } + } + }); + + addr + } +} + +impl Actor for NetSyncManager { + type Context = actix::Context; +} + +/// Event broadcast from event bus +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + match msg.into_data() { + EnclaveEventData::OutgoingSyncRequested(data) => ctx.notify(data), + _ => (), + } + } +} + +/// SyncRequest is called on start up to fetch remote events +impl Handler for NetSyncManager { + type Result = ResponseFuture<()>; + fn handle(&mut self, msg: OutgoingSyncRequested, ctx: &mut Self::Context) -> Self::Result { + trap_fut( + EType::Net, + &self.bus.clone(), + handle_sync_request_event(self.tx.clone(), self.rx.clone(), msg, ctx.address()), + ) + } +} + +/// We have received the sync response from the remote peer +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: OutgoingSyncRequestSucceeded, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.clone(), || { + self.bus.publish(NetSyncEventsReceived { + events: msg + .value + .events + .iter() + .cloned() + .map(|data| data.try_into()) + .collect::>>>()?, + })?; + + Ok(()) + }); + } +} + +/// We have received a sync request from a remote peer +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: SyncRequestReceived, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus, || { + let id = CorrelationId::new(); + self.requests.insert(id, msg.channel); + self.eventstore.try_send(GetAggregateEventsAfter::new( + id, + msg.value.since, + ctx.address().recipient(), + ))?; + Ok(()) + }); + } +} + +/// Receive Events from EventStore +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.clone(), || { + let Some(channel) = self.requests.get(&msg.id()) else { + bail!("request not found with {}", msg.id()); + }; + + self.tx.try_send(NetCommand::SyncResponse { + value: SyncResponseValue { + events: msg + .events() + .into_iter() + .cloned() + .map(|ev| ev.try_into()) + .collect::>()?, + }, + channel: channel.to_owned(), + })?; + + Ok(()) + }) + } +} + +const SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +async fn sync_request( + net_cmds: mpsc::Sender, + net_events: Arc>, + since: HashMap, +) -> Result { + call_and_await_response( + net_cmds, + net_events, + NetCommand::OutgoingSyncRequest { + value: SyncRequestValue { since }, + }, + |e| match e.clone() { + NetEvent::OutgoingSyncRequestSucceeded(value) => Some(Ok(value)), + NetEvent::OutgoingSyncRequestFailed(error) => { + Some(Err(anyhow!("Outgoing sync request failed: {:?}", error))) + } + _ => None, + }, + SYNC_REQUEST_TIMEOUT, + ) + .await +} + +async fn handle_sync_request_event( + net_cmds: mpsc::Sender, + net_events: Arc>, + event: OutgoingSyncRequested, + address: impl Into>, +) -> Result<()> { + let value = retry_with_backoff( + || { + sync_request( + net_cmds.clone(), + net_events.clone(), + event.since.clone().into_iter().collect(), + ) + .map_err(to_retry) + }, + 4, + 1000, + ) + .await?; + + address.into().try_send(value)?; + Ok(()) +} diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 22fef4d79b..7245c10b69 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::sortition::{GetNodeIndex, Sortition}; +use crate::WithSortitionPartyTicket; use actix::prelude::*; use anyhow::bail; use anyhow::Result; @@ -15,6 +15,7 @@ use e3_events::{ EnclaveEvent, EnclaveEventData, EventType, Shutdown, TicketGenerated, TicketId, }; use e3_request::E3Meta; +use e3_utils::NotifySync; use std::collections::HashMap; use tracing::info; @@ -22,7 +23,6 @@ use tracing::info; /// emits a TicketGenerated event (score sortition) to the event bus pub struct CiphernodeSelector { bus: BusHandle, - sortition: Addr, address: String, e3_cache: Persistable>, } @@ -34,13 +34,11 @@ impl Actor for CiphernodeSelector { impl CiphernodeSelector { pub fn new( bus: &BusHandle, - sortition: &Addr, e3_cache: Persistable>, address: &str, ) -> Self { Self { bus: bus.clone(), - sortition: sortition.clone(), e3_cache, address: address.to_owned(), } @@ -48,14 +46,13 @@ impl CiphernodeSelector { pub async fn attach( bus: &BusHandle, - sortition: &Addr, selector_store: Repository>, address: &str, ) -> Result> { let e3_cache = selector_store.load_or_default(HashMap::new()).await?; - let addr = CiphernodeSelector::new(bus, sortition, e3_cache, address).start(); + let addr = CiphernodeSelector::new(bus, e3_cache, address).start(); - bus.subscribe(EventType::E3Requested, addr.clone().recipient()); + bus.subscribe(EventType::E3RequestComplete, addr.clone().recipient()); bus.subscribe(EventType::CommitteeFinalized, addr.clone().recipient()); bus.subscribe(EventType::Shutdown, addr.clone().recipient()); @@ -68,23 +65,23 @@ impl Handler for CiphernodeSelector { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::E3Requested(data) => ctx.notify(data), - EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), - EnclaveEventData::CommitteeFinalized(data) => ctx.notify(data), - EnclaveEventData::Shutdown(data) => ctx.notify(data), + EnclaveEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data), + EnclaveEventData::Shutdown(data) => self.notify_sync(ctx, data), _ => (), } } } -impl Handler for CiphernodeSelector { - type Result = ResponseFuture<()>; +impl Handler> for CiphernodeSelector { + type Result = (); - fn handle(&mut self, data: E3Requested, _ctx: &mut Self::Context) -> Self::Result { - let address = self.address.clone(); - let sortition = self.sortition.clone(); + fn handle( + &mut self, + data: WithSortitionPartyTicket, + _ctx: &mut Self::Context, + ) -> Self::Result { let bus = self.bus.clone(); - let chain_id = data.e3_id.chain_id(); trap(EType::Sortition, &bus.clone(), || { self.e3_cache.try_mutate(|mut cache| { @@ -98,58 +95,33 @@ impl Handler for CiphernodeSelector { seed: data.seed, threshold_n: data.threshold_n, threshold_m: data.threshold_m, - params: data.params, + params: data.params.clone(), esi_per_ct: data.esi_per_ct, - error_size: data.error_size, + error_size: data.error_size.clone(), }, ); Ok(cache) - }) - }); + })?; - Box::pin(async move { - let seed = data.seed; - let size = data.threshold_n; - info!( - "Calling GetNodeIndex address={} seed={} size={}", - address.clone(), - seed, - size - ); - // TODO: instead of this it would be better to pass the event theough sortition and - // then decorate it with this information WithIndex - if let Ok(found_result) = sortition - .send(GetNodeIndex { - chain_id, - seed, - address: address.clone(), - size, - }) - .await - { - let Some((_party_id, ticket_id)) = found_result else { - info!(node = address, "Ciphernode was not selected"); - return; - }; - - if let Some(tid) = ticket_id { - info!( - node = address, - ticket_id = tid, - "Ticket generated for score sortition" - ); - trap(EType::Sortition, &bus.clone(), || { - bus.publish(TicketGenerated { - e3_id: data.e3_id.clone(), - ticket_id: TicketId::Score(tid), - node: address.clone(), - })?; - Ok(()) - }) - } - } else { - info!("This node is not selected"); + if !data.is_selected() { + info!(node = &data.address(), "Ciphernode was not selected"); + return Ok(()); } + + if let Some(tid) = data.ticket_id() { + info!( + node = &data.address(), + ticket_id = tid, + "Ticket generated for score sortition" + ); + bus.publish(TicketGenerated { + e3_id: data.e3_id.clone(), + ticket_id: TicketId::Score(tid), + node: data.address().to_owned(), + })?; + } + + Ok(()) }) } } diff --git a/crates/sortition/src/sortition.rs b/crates/sortition/src/sortition.rs index 3c63a9d3fb..2720f5c5ef 100644 --- a/crates/sortition/src/sortition.rs +++ b/crates/sortition/src/sortition.rs @@ -5,18 +5,21 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::backends::{SortitionBackend, SortitionList}; +use crate::CiphernodeSelector; use actix::prelude::*; use alloy::primitives::U256; use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, CiphernodeAdded, CiphernodeRemoved, CommitteeFinalized, CommitteePublished, - ConfigurationUpdated, EType, EnclaveEvent, EventType, OperatorActivationChanged, + ConfigurationUpdated, E3Requested, EType, EnclaveEvent, EventType, OperatorActivationChanged, PlaintextOutputPublished, Seed, TicketBalanceUpdated, }; use e3_events::{BusHandle, EnclaveEventData}; +use e3_utils::NotifySync; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::ops::Deref; use tracing::info; use tracing::instrument; use tracing::warn; @@ -94,21 +97,6 @@ impl NodeStateStore { } } -/// Message: ask the `Sortition` whether `address` would be in the -/// committee of size `size` for randomness `seed` on `chain_id`. -#[derive(Message, Clone, Debug, PartialEq, Eq)] -#[rtype(result = "Option<(u64, Option)>")] -pub struct GetNodeIndex { - /// Round seed / randomness used by the sortition algorithm. - pub seed: Seed, - /// Hex-encoded node address (e.g., `"0x..."`). - pub address: String, - /// Committee size (top-N). - pub size: usize, - /// Target chain. - pub chain_id: u64, -} - /// Message: request the current set of registered node addresses for `chain_id`. #[derive(Message, Clone, Debug)] #[rtype(result = "Vec")] @@ -117,6 +105,47 @@ pub struct GetNodes { pub chain_id: u64, } +#[derive(Message, Clone, Debug, PartialEq, Eq)] +#[rtype(result = "()")] +pub struct WithSortitionPartyTicket { + inner: T, + party_ticket_id: Option<(u64, Option)>, + address: String, +} + +impl WithSortitionPartyTicket { + pub fn new(inner: T, party_ticket_id: Option<(u64, Option)>, address: &str) -> Self { + Self { + inner, + party_ticket_id, + address: address.to_owned(), + } + } + + pub fn is_selected(&self) -> bool { + self.party_ticket_id.is_some() + } + + pub fn address(&self) -> &str { + self.address.as_ref() + } + + pub fn ticket_id(&self) -> Option { + self.party_ticket_id.and_then(|(_, ticket_id)| ticket_id) + } + + pub fn party_id(&self) -> Option { + self.party_ticket_id.map(|(party_id, _)| party_id) + } +} + +impl Deref for WithSortitionPartyTicket { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + /// Message to get the finalized committee nodes for a specific E3. #[derive(Message, Clone, Debug)] #[rtype(result = "Vec")] @@ -142,6 +171,10 @@ pub struct Sortition { bus: BusHandle, /// Persistent map of finalized committees per E3 finalized_committees: Persistable>>, + /// Address for the CiphernodeSelector + ciphernode_selector: Addr, + /// Address for the current node + address: String, } /// Parameters for constructing a `Sortition` actor. @@ -155,6 +188,10 @@ pub struct SortitionParams { pub node_state: Persistable>, /// Persistent map of finalized committees per E3 pub finalized_committees: Persistable>>, + /// Address for the CiphernodeSelector + pub ciphernode_selector: Addr, + /// Address for the current node + pub address: String, } impl Sortition { @@ -164,6 +201,8 @@ impl Sortition { node_state: params.node_state, bus: params.bus, finalized_committees: params.finalized_committees, + ciphernode_selector: params.ciphernode_selector, + address: params.address, } } @@ -174,6 +213,8 @@ impl Sortition { node_state_store: Repository>, committees_store: Repository>>, default_backend: SortitionBackend, + ciphernode_selector: Addr, + address: &str, ) -> Result> { let mut backends = backends_store.load_or_default(HashMap::new()).await?; let node_state = node_state_store.load_or_default(HashMap::new()).await?; @@ -189,12 +230,15 @@ impl Sortition { backends, node_state, finalized_committees, + ciphernode_selector, + address: address.to_owned(), }) .start(); // Subscribe to all relevant events bus.subscribe_all( &[ + EventType::E3Requested, EventType::CiphernodeAdded, EventType::CiphernodeRemoved, EventType::TicketBalanceUpdated, @@ -221,6 +265,26 @@ impl Sortition { .ok_or_else(|| anyhow::anyhow!("No backend for chain_id {}", chain_id))?; Ok(backend.nodes()) } + + pub fn get_node_index( + &self, + seed: Seed, + size: usize, + chain_id: u64, + ) -> Option<(u64, Option)> { + let bus = self.bus.clone(); + let map = self.backends.get()?; + let state_map = self.node_state.get()?; + let backend = map.get(&chain_id)?; + let state = state_map.get(&chain_id)?; + + backend + .get_index(seed, size, self.address.clone(), chain_id, state) + .unwrap_or_else(|err| { + bus.err(EType::Sortition, err); + None + }) + } } impl Actor for Sortition { @@ -232,19 +296,37 @@ impl Handler for Sortition { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { - EnclaveEventData::CiphernodeAdded(data) => ctx.notify(data.clone()), - EnclaveEventData::CiphernodeRemoved(data) => ctx.notify(data.clone()), - EnclaveEventData::TicketBalanceUpdated(data) => ctx.notify(data.clone()), - EnclaveEventData::OperatorActivationChanged(data) => ctx.notify(data.clone()), - EnclaveEventData::ConfigurationUpdated(data) => ctx.notify(data.clone()), - EnclaveEventData::CommitteePublished(data) => ctx.notify(data.clone()), - EnclaveEventData::PlaintextOutputPublished(data) => ctx.notify(data.clone()), - EnclaveEventData::CommitteeFinalized(data) => ctx.notify(data.clone()), + EnclaveEventData::E3Requested(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CiphernodeAdded(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CiphernodeRemoved(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::TicketBalanceUpdated(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::OperatorActivationChanged(data) => { + self.notify_sync(ctx, data.clone()) + } + EnclaveEventData::ConfigurationUpdated(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CommitteePublished(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::PlaintextOutputPublished(data) => self.notify_sync(ctx, data.clone()), + EnclaveEventData::CommitteeFinalized(data) => self.notify_sync(ctx, data.clone()), _ => (), } } } +impl Handler for Sortition { + type Result = (); + fn handle(&mut self, msg: E3Requested, ctx: &mut Self::Context) -> Self::Result { + let chain_id = msg.e3_id.chain_id(); + let seed = msg.seed; + let threshold_n = msg.threshold_n; + self.ciphernode_selector + .do_send(WithSortitionPartyTicket::new( + msg, + self.get_node_index(seed, threshold_n, chain_id), + &self.address, + )) + } +} + impl Handler for Sortition { type Result = (); @@ -498,35 +580,6 @@ impl Handler for Sortition { } } -impl Handler for Sortition { - type Result = ResponseFuture)>>; - - fn handle(&mut self, msg: GetNodeIndex, _ctx: &mut Self::Context) -> Self::Result { - let backends_snapshot = self.backends.get(); - let node_state_snapshot = self.node_state.get(); - let bus = self.bus.clone(); - - Box::pin(async move { - if let (Some(map), Some(state_map)) = (backends_snapshot, node_state_snapshot) { - if let (Some(backend), Some(state)) = - (map.get(&msg.chain_id), state_map.get(&msg.chain_id)) - { - backend - .get_index(msg.seed, msg.size, msg.address.clone(), msg.chain_id, state) - .unwrap_or_else(|err| { - bus.err(EType::Sortition, err); - None - }) - } else { - None - } - } else { - None - } - }) - } -} - impl Handler for Sortition { type Result = Vec; diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml new file mode 100644 index 0000000000..97ba4b7fea --- /dev/null +++ b/crates/sync/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "e3-sync" +version.workspace = true +edition.workspace = true +license.workspace = true +description.workspace = true +repository = "https://github.com/gnosisguild/enclave/crates/sync" + +[dependencies] +actix.workspace = true +anyhow.workspace = true +e3-events.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +e3-ciphernode-builder.workspace = true diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs new file mode 100644 index 0000000000..07b21af2e0 --- /dev/null +++ b/crates/sync/src/lib.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +mod sync; + +pub use sync::*; diff --git a/crates/sync/src/sync.rs b/crates/sync/src/sync.rs new file mode 100644 index 0000000000..27fc8efff0 --- /dev/null +++ b/crates/sync/src/sync.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::collections::HashSet; + +use actix::{Actor, Addr, AsyncContext, Handler, Message}; +use anyhow::{Context, Result}; +use e3_events::{ + trap, BusHandle, EType, EventPublisher, EvmEvent, EvmEventConfig, SyncEnd, SyncEvmEvent, + SyncStart, +}; +use tracing::info; + +// NOTE: This is a WIP. We need to synchronize events from EVM as well as libp2p +type ChainId = u64; + +/// Manage the synchronization of events across. +pub struct Synchronizer { + bus: BusHandle, + evm_config: Option, + evm_buffer: Vec, + evm_to_sync: HashSet, + // net_config: NetEventConfig, +} + +impl Synchronizer { + pub fn new(bus: &BusHandle, evm_config: EvmEventConfig) -> Self { + let evm_to_sync = evm_config.chains(); + Self { + evm_config: Some(evm_config), + bus: bus.clone(), + evm_buffer: Vec::new(), + evm_to_sync, + } + } + + pub fn setup(bus: &BusHandle, evm_config: EvmEventConfig) -> Addr { + Self::new(bus, evm_config).start() + } + + fn buffer_evm_event(&mut self, event: EvmEvent) { + info!("buffer evm event({})", event.get_id()); + self.evm_buffer.push(event); + } + + fn handle_sync_complete(&mut self, chain_id: u64) -> Result<()> { + info!("handle sync complete for chain({})", chain_id); + self.evm_to_sync.remove(&chain_id); + info!("{} chains left to sync...", self.evm_to_sync.len()); + if self.evm_to_sync.is_empty() { + self.handle_sync_end()?; + } + Ok(()) + } + + fn handle_sync_end(&mut self) -> Result<()> { + info!("all chains synced draining to bus and running sync end"); + // Order all events (theoretically) + self.evm_buffer.sort_by_key(|i| i.ts()); + + // publish them in order + for evt in self.evm_buffer.drain(..) { + let (data, _, _) = evt.split(); + self.bus.publish(data)?; // Use publish here as historical events will be correctly + // ordered as part of the preparatory process + } + self.bus.publish(SyncEnd::new())?; + Ok(()) + } +} + +impl Actor for Synchronizer { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.notify(Bootstrap); + } +} + +impl Handler for Synchronizer { + type Result = (); + fn handle(&mut self, msg: SyncEvmEvent, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Sync, &self.bus.clone(), || { + match msg { + // Buffer events as the sync actor receives them + SyncEvmEvent::Event(event) => self.buffer_evm_event(event), + // When we hear that sync is complete send all events on chain then publish SyncEnd + SyncEvmEvent::HistoricalSyncComplete(chain_id) => { + self.handle_sync_complete(chain_id)? + } + }; + Ok(()) + }) + } +} + +impl Handler for Synchronizer { + type Result = (); + fn handle(&mut self, _: Bootstrap, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Sync, &self.bus.clone(), || { + let evm_config = self.evm_config.take().context( + "EvmEventConfig was not set likely Bootstrap was called more than once.", + )?; + + // TODO: Get information about what has and has not been synced then fire SyncStart + self.bus.publish(SyncStart::new(ctx.address(), evm_config)) + }) + } +} + +#[derive(Message)] +#[rtype("()")] +pub struct Bootstrap; + +#[cfg(test)] +mod tests { + use super::*; + use e3_ciphernode_builder::EventSystem; + use e3_events::{ + CorrelationId, EnclaveEventData, Event, EvmEventConfig, EvmEventConfigChain, GetEvents, + TestEvent, + }; + use e3_events::{EnclaveEvent, EventContextAccessors}; + use std::time::Duration; + use tokio::time::sleep; + + fn hlc_faucet(bus: &BusHandle, num: usize) -> Result> { + let mut queue = Vec::new(); + for _ in 0..num { + queue.push(bus.ts()?) + } + + Ok(queue.into_iter()) + } + + async fn settle() { + sleep(Duration::from_millis(100)).await; + } + + #[actix::test] + async fn test_synchronizer_full_flow() -> Result<()> { + // Setup event system and synchronizer + let system = EventSystem::new("test").with_fresh_bus(); + let bus = system.handle()?; + let history_collector = bus.history(); + + // Configure test chains + let mut evm_config = EvmEventConfig::new(); + evm_config.insert(1, EvmEventConfigChain::new(0)); + evm_config.insert(2, EvmEventConfigChain::new(0)); + + // Start synchronizer + let sync_addr = Synchronizer::setup(&bus, evm_config); + settle().await; + + // Verify SyncStart was published + let history = history_collector + .send(GetEvents::::new()) + .await?; + let sync_start_count = history + .into_iter() + .filter(|e| matches!(e.get_data(), EnclaveEventData::SyncStart(_))) + .count(); + assert!(sync_start_count > 0, "SyncStart should be dispatched"); + + // Create test events with timestamps + let mut timelord = hlc_faucet(&bus, 100)?; + let (chain_1, chain_2) = (1, 2); + let (block_1, block_2) = (1, 2); + + // Test events - timestamps generated in order + let h_2_1 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("2-first", 1)), + block_1, + timelord.next().unwrap(), + chain_2, + )); + + let h_1_1 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("1-first", 1)), + block_1, + timelord.next().unwrap(), + chain_1, + )); + + let h_1_2 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("1-second", 1)), + block_2, + timelord.next().unwrap(), + chain_1, + )); + + let h_2_2 = SyncEvmEvent::Event(EvmEvent::new( + CorrelationId::new(), + EnclaveEventData::TestEvent(TestEvent::new("2-second", 2)), + block_2, + timelord.next().unwrap(), + chain_2, + )); + + // Chain completion signals + let hc_1 = SyncEvmEvent::HistoricalSyncComplete(chain_1); + let hc_2 = SyncEvmEvent::HistoricalSyncComplete(chain_2); + + // Send events in mixed order to test sorting + sync_addr.send(h_2_2).await?; + sync_addr.send(h_2_1).await?; + sync_addr.send(hc_2).await?; + sync_addr.send(h_1_1).await?; + sync_addr.send(h_1_2).await?; + sync_addr.send(hc_1).await?; + + settle().await; + + // Get final event history and verify ordering + let history = history_collector + .send(GetEvents::::new()) + .await?; + + let events: Vec = history + .into_iter() + .filter(|e| matches!(e.get_data(), EnclaveEventData::TestEvent(_))) + .collect(); + + let event_strings: Vec = events + .into_iter() + .filter_map(|e| { + if let EnclaveEventData::TestEvent(data) = e.into_data() { + Some(data.msg) + } else { + None + } + }) + .collect(); + + // Events should be published in timestamp order + assert_eq!( + event_strings, + vec!["2-first", "1-first", "1-second", "2-second"] + ); + + Ok(()) + } +} diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index 61ef319129..8d577670d1 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -16,7 +16,7 @@ use tokio::time::timeout; type SetupFn<'a> = Box Pin> + 'a>> + 'a>; type ThenFn<'a> = - Box Pin> + 'a>> + 'a>; + Box Pin> + 'a>> + 'a>; /// This builds a ciphernode system using the actor model only. This helps us simulate the network /// in tests that we can run in the /crates/tests crate @@ -85,8 +85,8 @@ impl<'a> CiphernodeSystemBuilder<'a> { } for then_fn in self.thens { - for node in nodes.clone() { - then_fn(node).await?; + for node in nodes.iter() { + then_fn(&node).await?; } } @@ -140,7 +140,7 @@ impl CiphernodeSystem { Ok(CiphernodeHistory(history)) } pub async fn flush_all_history(&self, millis: u64) -> Result<()> { - let nodes = self.0.clone(); + let nodes = &self.0; for node in nodes.iter() { let Some(history) = node.history() else { break; @@ -199,6 +199,7 @@ mod tests { use e3_ciphernode_builder::EventSystem; use e3_data::InMemStore; use e3_events::{EventBus, EventBusConfig}; + use tokio::task::JoinHandle; async fn mock_setup_node(address: String) -> Result { // Create mock actors for the test @@ -207,6 +208,7 @@ mod tests { let history = EventBus::::history(&bus); let errors = EventBus::::error(&bus); let bus = EventSystem::new("test").with_event_bus(bus).handle()?; + let handle: JoinHandle> = tokio::spawn(async { Ok(()) }); Ok(CiphernodeHandle { address, @@ -214,6 +216,8 @@ mod tests { bus, history: Some(history), errors: Some(errors), + join_handle: handle, + peer_id: "-unknown peer id-".to_string(), }) } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 4e3e11f883..90f268185e 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -12,11 +12,13 @@ use e3_ciphernode_builder::{CiphernodeBuilder, EventSystem}; use e3_crypto::Cipher; use e3_events::{ prelude::*, BusHandle, CiphertextOutputPublished, CommitteeFinalized, ConfigurationUpdated, - E3Requested, E3id, EnclaveEventData, OperatorActivationChanged, PlaintextAggregated, - TicketBalanceUpdated, + E3Requested, E3id, EnclaveEvent, EnclaveEventData, OperatorActivationChanged, + PlaintextAggregated, Seed, TakeEvents, TicketBalanceUpdated, }; use e3_fhe_params::{encode_bfv_params, BfvParamSet, BfvPreset}; use e3_multithread::{Multithread, MultithreadReport, ToReport}; +use e3_net::events::{GossipData, NetEvent}; +use e3_net::NetEventTranslator; use e3_test_helpers::ciphernode_system::CiphernodeSystemBuilder; use e3_test_helpers::{create_seed_from_u64, create_shared_rng_from_u64, AddToCommittee}; use e3_trbfv::helpers::calculate_error_size; @@ -25,8 +27,11 @@ use e3_utils::utility_types::ArcBytes; use fhe::bfv::PublicKey; use fhe_traits::{DeserializeParametrized, Serialize}; use num_bigint::BigUint; +use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; use std::time::{Duration, Instant}; use std::{fs, sync::Arc}; +use tokio::sync::{broadcast, mpsc}; pub fn save_snapshot(file_name: &str, bytes: &[u8]) { println!("### WRITING SNAPSHOT TO `{file_name}` ###"); @@ -173,7 +178,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_pubkey_aggregation() .with_sortition_score() .with_threshold_plaintext_aggregation() - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() .await @@ -188,7 +193,7 @@ async fn test_trbfv_actor() -> Result<()> { .with_shared_multithread_report(&multithread_report) .with_trbfv() .with_sortition_score() - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .with_logging() .build() .await @@ -255,6 +260,11 @@ async fn test_trbfv_actor() -> Result<()> { println!("Emitting CommitteeFinalized with {} nodes", committee.len()); + let expected = vec!["E3Requested"]; + let _ = nodes + .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) + .await?; + bus.publish(CommitteeFinalized { e3_id: e3_id.clone(), committee, @@ -263,7 +273,7 @@ async fn test_trbfv_actor() -> Result<()> { let committee_finalized_timer = Instant::now(); - let expected = vec!["E3Requested", "CommitteeFinalized"]; + let expected = vec!["CommitteeFinalized"]; let _ = nodes .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; @@ -540,16 +550,7 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { #[actix::test] async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { - use e3_events::{EnclaveEvent, TakeEvents}; - use e3_net::events::GossipData; - use e3_net::{events::NetEvent, NetEventTranslator}; - use rand::SeedableRng; - use rand_chacha::ChaCha20Rng; - use std::sync::Arc; - use tokio::sync::broadcast; - use tokio::sync::mpsc; - - let seed = e3_events::Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); + let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); // Setup elements in test let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network @@ -563,8 +564,8 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // Capture messages from output on msgs vec let event = E3Requested { e3_id: E3id::new("1235", 1), - threshold_m: 2, - threshold_n: 5, + threshold_m: 3, + threshold_n: 3, seed: seed.clone(), params: ArcBytes::from_bytes(&[1, 2, 3, 4]), ..E3Requested::default() @@ -572,7 +573,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // lets send an event from the network let _ = event_tx.send(NetEvent::GossipData(GossipData::GossipBytes( - bus.event_from(event.clone())?.to_bytes()?, + bus.event_from(event.clone(), None)?.to_bytes()?, ))); // check the history of the event bus @@ -625,7 +626,7 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { let mut builder = CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_trbfv() .with_address(addr) - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() @@ -832,7 +833,7 @@ async fn test_duplicate_e3_id_with_different_chain_id() -> Result<()> { let mut builder = CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_trbfv() .with_address(addr) - .testmode_with_forked_bus(bus.consumer()) + .testmode_with_forked_bus(bus.event_bus()) .testmode_with_history() .testmode_with_errors() .with_pubkey_aggregation() diff --git a/crates/utils/src/actix.rs b/crates/utils/src/actix.rs index f4a18173b3..63cd1dbc02 100644 --- a/crates/utils/src/actix.rs +++ b/crates/utils/src/actix.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, ResponseActFuture, WrapFuture}; +use actix::{Actor, Handler, Message, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; @@ -17,3 +17,24 @@ pub fn bail_result(a: &T, msg: impl Into) -> ResponseActFuture let m: String = msg.into(); Box::pin(async { Err(anyhow!(m)) }.into_actor(a)) } + +/// Extension trait for synchronous message handling +pub trait NotifySync +where + M: Message, + Self: Actor + Handler, +{ + /// Handles a message immediately without queuing. + /// Drop-in replacement for `ctx.notify(msg)` without interleaving other events. + fn notify_sync(&mut self, ctx: &mut Self::Context, msg: M) -> >::Result { + >::handle(self, msg, ctx) + } +} + +// Blanket implementation for all actors that handle the message +impl NotifySync for A +where + A: Actor + Handler, + M: Message, +{ +} diff --git a/crates/utils/src/helpers.rs b/crates/utils/src/helpers.rs index 77d09936ab..9380d01abf 100644 --- a/crates/utils/src/helpers.rs +++ b/crates/utils/src/helpers.rs @@ -3,7 +3,10 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; pub fn to_ordered_vec(source: HashMap) -> Vec where @@ -18,3 +21,44 @@ where // Extract to Vec of ThresholdShares in order pairs.into_iter().map(|(_, value)| value).collect() } + +/// A cloneable wrapper that allows a non-cloneable value to be shared and taken exactly once. +/// +/// Useful for passing oneshot channels or other single-use items through cloneable contexts. +/// +/// # Example +/// ``` +/// use e3_utils::OnceTake; +/// +/// let (tx, rx) = tokio::sync::oneshot::channel::(); +/// let once = OnceTake::new(tx); +/// let cloned = once.clone(); +/// cloned.take().unwrap().send(42).unwrap(); +/// assert!(once.take().is_none()); // already taken +/// ``` +#[derive(Debug)] +pub struct OnceTake(Arc>>); + +impl OnceTake { + /// Wraps an item so it can be cloned and later taken once. + pub fn new(item: T) -> Self { + Self(Arc::new(Mutex::new(Some(item)))) + } + + /// Takes the item, returning `None` if already taken. + pub fn take(&self) -> Option { + self.0.lock().unwrap().take() + } + + /// Takes the item, returning an error if already taken. + pub fn try_take(&self) -> anyhow::Result { + self.take() + .ok_or_else(|| anyhow::anyhow!("Item already taken.")) + } +} + +impl Clone for OnceTake { + fn clone(&self) -> Self { + OnceTake(Arc::clone(&self.0)) + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 761c59c4f4..ec17ce7a9c 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -8,11 +8,13 @@ pub mod actix; pub mod alloy; pub mod formatters; pub mod helpers; +pub mod path; pub mod retry; pub mod utility_types; pub use actix::*; pub use alloy::*; pub use formatters::*; pub use helpers::*; +pub use path::*; pub use retry::*; pub use utility_types::*; diff --git a/crates/utils/src/path.rs b/crates/utils/src/path.rs new file mode 100644 index 0000000000..163f32790f --- /dev/null +++ b/crates/utils/src/path.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::path::PathBuf; + +/// Enumerates a PathBuf by inserting an index before the file extension +/// or at the end if there is no extension +/// +/// Examples: +/// - "/foo/bar/thing.pdf" -> "/foo/bar/thing.0.pdf" +/// - "/foo/bar/thing" -> "/foo/bar/thing.0" +pub fn enumerate_path(path: &PathBuf, index: usize) -> PathBuf { + if let Some(parent) = path.parent() { + if let Some(file_name) = path.file_name() { + if let Some(file_name_str) = file_name.to_str() { + if let Some(dot_pos) = file_name_str.rfind('.') { + // Has extension + let (stem, extension) = file_name_str.split_at(dot_pos); + let new_name = format!("{}.{}{}", stem, index, extension); + parent.join(new_name) + } else { + // No extension + let new_name = format!("{}.{}", file_name_str, index); + parent.join(new_name) + } + } else { + // Invalid UTF-8 in filename, append index directly + let new_name = format!("{}.{}", file_name.to_string_lossy(), index); + parent.join(new_name) + } + } else { + // Path ends with '/', just append index + path.join(format!("{}", index)) + } + } else { + // No parent, just modify the filename directly + if let Some(file_name) = path.file_name() { + if let Some(file_name_str) = file_name.to_str() { + if let Some(dot_pos) = file_name_str.rfind('.') { + // Has extension + let (stem, extension) = file_name_str.split_at(dot_pos); + let new_name = format!("{}.{}{}", stem, index, extension); + PathBuf::from(new_name) + } else { + // No extension + let new_name = format!("{}.{}", file_name_str, index); + PathBuf::from(new_name) + } + } else { + // Invalid UTF-8 in filename, append index directly + let new_name = format!("{}.{}", file_name.to_string_lossy(), index); + PathBuf::from(new_name) + } + } else { + // Empty path, just return the index as a path + PathBuf::from(format!("{}", index)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_enumerate_path_with_extension() { + let path = PathBuf::from("/foo/bar/thing.pdf"); + let result = enumerate_path(&path, 0); + assert_eq!(result, PathBuf::from("/foo/bar/thing.0.pdf")); + } + + #[test] + fn test_enumerate_path_without_extension() { + let path = PathBuf::from("/foo/bar/thing"); + let result = enumerate_path(&path, 5); + assert_eq!(result, PathBuf::from("/foo/bar/thing.5")); + } + + #[test] + fn test_enumerate_path_no_parent() { + let path = PathBuf::from("thing.txt"); + let result = enumerate_path(&path, 1); + assert_eq!(result, PathBuf::from("thing.1.txt")); + } + + #[test] + fn test_enumerate_path_empty() { + let path = PathBuf::from(""); + let result = enumerate_path(&path, 2); + assert_eq!(result, PathBuf::from("2")); + } +} diff --git a/examples/CRISP/server/Dockerfile b/examples/CRISP/server/Dockerfile index 056691e0a0..c2de767e99 100644 --- a/examples/CRISP/server/Dockerfile +++ b/examples/CRISP/server/Dockerfile @@ -85,6 +85,7 @@ COPY crates/request/Cargo.toml crates/request/Cargo.toml COPY crates/sdk/Cargo.toml crates/sdk/Cargo.toml COPY crates/sortition/Cargo.toml crates/sortition/Cargo.toml COPY crates/support-scripts/Cargo.toml crates/support-scripts/Cargo.toml +COPY crates/sync/Cargo.toml crates/sync/Cargo.toml COPY crates/test-helpers/Cargo.toml crates/test-helpers/Cargo.toml COPY crates/tests/Cargo.toml crates/tests/Cargo.toml COPY crates/trbfv/Cargo.toml crates/trbfv/Cargo.toml diff --git a/scripts/run-crisp-test.sh b/scripts/run-crisp-test.sh index 402e542c85..5f94042b91 100755 --- a/scripts/run-crisp-test.sh +++ b/scripts/run-crisp-test.sh @@ -6,4 +6,12 @@ echo "Press any key to continue or Ctrl+C to cancel..." read -rm -rf * && git reset --hard HEAD && git submodule update --init --recursive && pnpm install && pnpm build && cd examples/CRISP && pnpm test:e2e "$@" +rm -rf * && \ + git reset --hard HEAD && \ + git submodule update --init --recursive && \ + pnpm install && \ + cargo build && \ + pnpm build && \ + cd examples/CRISP && \ + pnpm dev:setup && \ + pnpm test:e2e "$@"