From f5556562111352564854da872f60872fb54cd50d Mon Sep 17 00:00:00 2001 From: ryardley Date: Sun, 7 Dec 2025 00:17:20 +0000 Subject: [PATCH 01/66] strategy --- crates/data/src/in_mem.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index e8ad52b3c8..ef9a8a5d30 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -55,6 +55,10 @@ impl InMemStore { } } +// Add a BatchInsert event that contains multiple Insert messages +// Use the Responder pattern to manage the response +// Have a proxy actor hold the Inserts until the BatchInsert event is called + impl Handler for InMemStore { type Result = (); fn handle(&mut self, event: Insert, _: &mut Self::Context) { From 20e923de96f8ec602ecfd58f85eb96350576ad50 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 02:03:54 +0000 Subject: [PATCH 02/66] allow to specify tree on construction of sled_db --- crates/data/src/sled_store.rs | 46 ++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 185b30f4f5..07d3d33973 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -9,7 +9,7 @@ use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::{Context, Result}; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; use once_cell::sync::Lazy; -use sled::Db; +use sled::{Db, Tree}; use std::{ collections::HashMap, path::PathBuf, @@ -29,7 +29,7 @@ impl Actor for SledStore { impl SledStore { pub fn new(bus: &BusHandle, path: &PathBuf) -> Result> { info!("Starting SledStore with {:?}", path); - let db = SledDb::new(PathBuf::from(path))?; + let db = SledDb::new(path, "datastore")?; let store = Self { db: Some(db), @@ -110,7 +110,7 @@ impl Handler for SledStore { } pub struct SledDb { - db: Db, + db: Tree, } // Global static cache @@ -146,29 +146,35 @@ fn get_or_open_db(path: &PathBuf) -> Result { if let Some(db) = cache.get(&key) { return Ok(db.clone()); } - let db = sled::open(path)?; + let db = sled::open(path).with_context(|| { + format!( + "Could not open database at path '{}'", + path.to_string_lossy() + ) + })?; cache.insert(key, db.clone()); + if !db.was_recovered() { + info!("created db at: {:?}", &path); + } else { + info!("recovered db st: {:?}", &path); + } + Ok(db) } +fn get_or_open_db_tree(path: &PathBuf, tree: &str) -> Result { + let db = get_or_open_db(path)?; + Ok(db.open_tree(tree)?) +} + fn clear_all_caches() { let mut cache_lock = SLED_CACHE.lock().unwrap(); cache_lock.clear(); } impl SledDb { - pub fn new(path: PathBuf) -> Result { - let db = get_or_open_db(&path).with_context(|| { - format!( - "Could not open database at path '{}'", - path.to_string_lossy() - ) - })?; - if !db.was_recovered() { - info!("created db at: {:?}", &path); - } else { - info!("recovered db st: {:?}", &path); - } + pub fn new(path: &PathBuf, tree: &str) -> Result { + let db = get_or_open_db_tree(path, tree)?; Ok(Self { db }) } @@ -216,11 +222,11 @@ mod tests { let db_path = temp_dir.path().join("test_cache.db"); // Create first instance and insert data - let mut db1 = SledDb::new(db_path.clone())?; + let mut db1 = SledDb::new(&db_path, "datastore")?; db1.insert(Insert::new(b"test_key".to_vec(), b"test_value".to_vec()))?; // Create second instance to same path and verify data access - let mut db2 = SledDb::new(db_path.clone())?; + let mut db2 = SledDb::new(&db_path, "datastore")?; let result = db2.get(Get::new(b"test_key".to_vec()))?; assert_eq!( result.unwrap(), @@ -246,11 +252,11 @@ mod tests { // Section 3: Test cache with different path let second_path = temp_dir.path().join("different_cache.db"); - let mut db3 = SledDb::new(second_path.clone())?; + let mut db3 = SledDb::new(&second_path, "datastore")?; db3.insert(Insert::new(b"db3_key".to_vec(), b"db3_value".to_vec()))?; // Create another instance to the second path - let mut db4 = SledDb::new(second_path)?; + let mut db4 = SledDb::new(&second_path, "datastore")?; assert_eq!( db4.get(Get::new(b"db3_key".to_vec()))?.unwrap(), b"db3_value".to_vec(), From ed7914bf7ac23ff6403c50c59d2ccdd37af9c50d Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 03:15:51 +0000 Subject: [PATCH 03/66] apply traits to sled_db --- crates/data/src/data_store.rs | 13 +++++++++++++ crates/data/src/into_key.rs | 8 ++++++++ crates/data/src/lib.rs | 1 + crates/data/src/sled_store.rs | 30 ++++++++++++++++++++++++------ crates/data/src/traits.rs | 15 +++++++++++++++ 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 crates/data/src/traits.rs diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 84ce71e4fc..25a049cb2f 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -80,6 +80,19 @@ impl Remove { } } +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "()")] +pub struct SeekForPrev(pub Vec); +impl SeekForPrev { + pub fn new(key: K) -> Self { + Self(key.into_key()) + } + + pub fn key(&self) -> &Vec { + &self.0 + } +} + #[derive(Clone, Debug)] pub enum StoreAddr { InMem(Addr), diff --git a/crates/data/src/into_key.rs b/crates/data/src/into_key.rs index 4fdff60b09..7c1c5e1213 100644 --- a/crates/data/src/into_key.rs +++ b/crates/data/src/into_key.rs @@ -57,3 +57,11 @@ impl<'a> IntoKey for &'a str { self.as_bytes().to_vec() } } + +/// Keys can be u128 +impl IntoKey for u128 { + fn into_key(self) -> Vec { + // Ensuring big endian for ordering + self.to_be_bytes().to_vec() + } +} diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 8cbab7aa47..9f7553ed66 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -12,6 +12,7 @@ mod repositories; mod repository; mod sled_store; mod snapshot; +mod traits; pub use data_store::*; pub use in_mem::*; diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 07d3d33973..ed845fe05e 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,7 +4,10 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertSync, Remove}; +use crate::{ + traits::{KeyValStore, Seekable}, + Get, Insert, InsertSync, Remove, SeekForPrev, +}; use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::{Context, Result}; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; @@ -178,7 +181,13 @@ impl SledDb { Ok(Self { db }) } - pub fn insert(&mut self, msg: Insert) -> Result<()> { + pub fn close_all_connections() { + clear_all_caches() + } +} + +impl KeyValStore for SledDb { + fn insert(&mut self, msg: Insert) -> Result<()> { self.db .insert(msg.key(), msg.value().to_vec()) .context("Could not insert data into db")?; @@ -186,14 +195,14 @@ impl SledDb { Ok(()) } - pub fn remove(&mut self, msg: Remove) -> Result<()> { + fn remove(&mut self, msg: Remove) -> Result<()> { self.db .remove(msg.key()) .context("Could not remove data from db")?; Ok(()) } - pub fn get(&mut self, event: Get) -> Result>> { + fn get(&self, event: Get) -> Result>> { let key = event.key(); let str_key = String::from_utf8_lossy(&key).into_owned(); let res = self @@ -203,9 +212,18 @@ impl SledDb { Ok(res.map(|v| v.to_vec())) } +} - pub fn close_all_connections() { - clear_all_caches() +impl Seekable for SledDb { + fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { + let key = msg.key(); + let entry = self.db.range(..=&key[..]).next_back(); + + match entry { + Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), + Some(Err(e)) => Err(e.into()), + None => Ok(None), + } } } diff --git a/crates/data/src/traits.rs b/crates/data/src/traits.rs new file mode 100644 index 0000000000..1850b670c4 --- /dev/null +++ b/crates/data/src/traits.rs @@ -0,0 +1,15 @@ +use crate::{Get, Insert, Remove, SeekForPrev}; +use anyhow::Result; + +pub trait KeyValStore { + fn insert(&mut self, msg: Insert) -> Result<()>; + fn remove(&mut self, msg: Remove) -> Result<()>; + fn get(&self, msg: Get) -> Result>>; +} + +pub trait Seekable: KeyValStore { + /// Seek for the first key that is less than or equal to the given key in the SeekForPrev msg + /// and return the value as a Some variant. If no value exists return None. If there was an + /// error doing the seek return an Error. + fn seek_for_prev(&self, msg: SeekForPrev) -> Result>>; +} From de7e91444e40117c02b7d1ef89a7cde54dcd52f4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 03:16:36 +0000 Subject: [PATCH 04/66] header --- crates/data/src/traits.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/data/src/traits.rs b/crates/data/src/traits.rs index 1850b670c4..ce675e2887 100644 --- a/crates/data/src/traits.rs +++ b/crates/data/src/traits.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::{Get, Insert, Remove, SeekForPrev}; use anyhow::Result; From 812fa6714bb87e68fe5f2b8024aa4ac3ef8a9e60 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 03:30:53 +0000 Subject: [PATCH 05/66] refactor sled_db --- crates/data/src/data_store.rs | 82 +------------- crates/data/src/events.rs | 82 ++++++++++++++ crates/data/src/lib.rs | 4 + crates/data/src/sled_db.rs | 205 ++++++++++++++++++++++++++++++++++ crates/data/src/sled_store.rs | 200 +-------------------------------- crates/data/src/traits.rs | 5 +- 6 files changed, 299 insertions(+), 279 deletions(-) create mode 100644 crates/data/src/events.rs create mode 100644 crates/data/src/sled_db.rs diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 25a049cb2f..9493228858 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -6,93 +6,15 @@ use std::borrow::Cow; +use crate::{Get, Insert, InsertSync, Remove}; use crate::{InMemStore, IntoKey, SledStore}; -use actix::{Addr, Message, Recipient}; +use actix::{Addr, Recipient}; use anyhow::anyhow; use anyhow::Context; use anyhow::Result; use serde::{Deserialize, Serialize}; use tracing::error; -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "()")] -pub struct Insert(pub Vec, pub Vec); -impl Insert { - pub fn new(key: K, value: Vec) -> Self { - Self(key.into_key(), value) - } - - pub fn key(&self) -> &Vec { - &self.0 - } - - pub fn value(&self) -> &Vec { - &self.1 - } -} - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "Result<()>")] -pub struct InsertSync(pub Vec, pub Vec); -impl InsertSync { - pub fn new(key: K, value: Vec) -> Self { - Self(key.into_key(), value) - } - - pub fn key(&self) -> &Vec { - &self.0 - } - - pub fn value(&self) -> &Vec { - &self.1 - } -} - -impl From for Insert { - fn from(value: InsertSync) -> Self { - Insert::new(value.key(), value.value().clone()) - } -} - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "Option>")] -pub struct Get(pub Vec); -impl Get { - pub fn new(key: K) -> Self { - Self(key.into_key()) - } - - pub fn key(&self) -> &Vec { - &self.0 - } -} - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "()")] -pub struct Remove(pub Vec); -impl Remove { - pub fn new(key: K) -> Self { - Self(key.into_key()) - } - - pub fn key(&self) -> &Vec { - &self.0 - } -} - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "()")] -pub struct SeekForPrev(pub Vec); -impl SeekForPrev { - pub fn new(key: K) -> Self { - Self(key.into_key()) - } - - pub fn key(&self) -> &Vec { - &self.0 - } -} - #[derive(Clone, Debug)] pub enum StoreAddr { InMem(Addr), diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs new file mode 100644 index 0000000000..e0ff6a6eb5 --- /dev/null +++ b/crates/data/src/events.rs @@ -0,0 +1,82 @@ +use crate::IntoKey; +use actix::Message; +use anyhow::Result; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "()")] +pub struct Insert(pub Vec, pub Vec); +impl Insert { + pub fn new(key: K, value: Vec) -> Self { + Self(key.into_key(), value) + } + + pub fn key(&self) -> &Vec { + &self.0 + } + + pub fn value(&self) -> &Vec { + &self.1 + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "Result<()>")] +pub struct InsertSync(pub Vec, pub Vec); +impl InsertSync { + pub fn new(key: K, value: Vec) -> Self { + Self(key.into_key(), value) + } + + pub fn key(&self) -> &Vec { + &self.0 + } + + pub fn value(&self) -> &Vec { + &self.1 + } +} + +impl From for Insert { + fn from(value: InsertSync) -> Self { + Insert::new(value.key(), value.value().clone()) + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "Option>")] +pub struct Get(pub Vec); +impl Get { + pub fn new(key: K) -> Self { + Self(key.into_key()) + } + + pub fn key(&self) -> &Vec { + &self.0 + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "()")] +pub struct Remove(pub Vec); +impl Remove { + pub fn new(key: K) -> Self { + Self(key.into_key()) + } + + pub fn key(&self) -> &Vec { + &self.0 + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "()")] +pub struct SeekForPrev(pub Vec); +impl SeekForPrev { + pub fn new(key: K) -> Self { + Self(key.into_key()) + } + + pub fn key(&self) -> &Vec { + &self.0 + } +} diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 9f7553ed66..c7d6d33cf1 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -5,20 +5,24 @@ // or FITNESS FOR A PARTICULAR PURPOSE. mod data_store; +mod events; mod in_mem; mod into_key; mod persistable; mod repositories; mod repository; +mod sled_db; mod sled_store; mod snapshot; mod traits; pub use data_store::*; +pub use events::*; pub use in_mem::*; pub use into_key::IntoKey; pub use persistable::*; pub use repositories::*; pub use repository::*; +pub use sled_db::*; pub use sled_store::*; pub use snapshot::*; diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs new file mode 100644 index 0000000000..007282b8eb --- /dev/null +++ b/crates/data/src/sled_db.rs @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use anyhow::{Context, Result}; +use once_cell::sync::Lazy; +use sled::{Db, Tree}; +use std::{ + collections::HashMap, + path::PathBuf, + sync::{Arc, Mutex}, +}; +use tracing::info; + +use crate::{ + traits::{KeyValStore, SeekableStore}, + Get, Insert, Remove, SeekForPrev, +}; + +pub struct SledDb { + db: Tree, +} + +impl SledDb { + pub fn new(path: &PathBuf, tree: &str) -> Result { + let db = get_or_open_db_tree(path, tree)?; + Ok(Self { db }) + } + + pub fn close_all_connections() { + clear_all_caches() + } +} + +impl KeyValStore for SledDb { + fn insert(&mut self, msg: Insert) -> Result<()> { + self.db + .insert(msg.key(), msg.value().to_vec()) + .context("Could not insert data into db")?; + + Ok(()) + } + + fn remove(&mut self, msg: Remove) -> Result<()> { + self.db + .remove(msg.key()) + .context("Could not remove data from db")?; + Ok(()) + } + + fn get(&self, event: Get) -> Result>> { + let key = event.key(); + let str_key = String::from_utf8_lossy(&key).into_owned(); + let res = self + .db + .get(key) + .context(format!("Failed to fetch {}", str_key))?; + + Ok(res.map(|v| v.to_vec())) + } +} + +impl SeekableStore for SledDb { + fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { + let key = msg.key(); + let entry = self.db.range(..=&key[..]).next_back(); + + match entry { + Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), + Some(Err(e)) => Err(e.into()), + None => Ok(None), + } + } +} + +// Global static cache +pub static SLED_CACHE: Lazy>>> = + Lazy::new(|| Arc::new(Mutex::new(HashMap::new()))); + +// Returns a stable canonical string path used as a cache key. +// Canonicalizes the parent directory if the target path does not yet exist. +fn canonical_key(path: &PathBuf) -> String { + use std::path::{Path, PathBuf}; + if path.exists() { + return path + .canonicalize() + .unwrap_or_else(|_| path.clone()) + .to_string_lossy() + .into_owned(); + } + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let base: PathBuf = parent + .canonicalize() + .unwrap_or_else(|_| parent.to_path_buf()); + let tail = path.file_name().map(|s| s.to_owned()).unwrap_or_default(); + base.join(tail).to_string_lossy().into_owned() +} + +// Opens or retrieves a cached sled database for the given path. +// Prevents conflicts by ensuring only a single connection was open to a db file at once per process. +// Ensures the directory exists and stabilizes the canonical key across OSes. +fn get_or_open_db(path: &PathBuf) -> Result { + let _ = std::fs::create_dir_all(path); + let key = canonical_key(path); + let mut cache = SLED_CACHE.lock().unwrap(); + if let Some(db) = cache.get(&key) { + return Ok(db.clone()); + } + let db = sled::open(path).with_context(|| { + format!( + "Could not open database at path '{}'", + path.to_string_lossy() + ) + })?; + cache.insert(key, db.clone()); + if !db.was_recovered() { + info!("created db at: {:?}", &path); + } else { + info!("recovered db st: {:?}", &path); + } + + Ok(db) +} + +fn get_or_open_db_tree(path: &PathBuf, tree: &str) -> Result { + let db = get_or_open_db(path)?; + Ok(db.open_tree(tree)?) +} + +fn clear_all_caches() { + let mut cache_lock = SLED_CACHE.lock().unwrap(); + cache_lock.clear(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sled_db_caching() -> Result<()> { + use tempfile::tempdir; + + // Section 1: Test basic cache functionality + let temp_dir = tempdir().expect("Failed to create temporary directory"); + let db_path = temp_dir.path().join("test_cache.db"); + + // Create first instance and insert data + let mut db1 = SledDb::new(&db_path, "datastore")?; + db1.insert(Insert::new(b"test_key".to_vec(), b"test_value".to_vec()))?; + + // Create second instance to same path and verify data access + let mut db2 = SledDb::new(&db_path, "datastore")?; + let result = db2.get(Get::new(b"test_key".to_vec()))?; + assert_eq!( + result.unwrap(), + b"test_value".to_vec(), + "Values from db2 should match" + ); + + // Cross-modify and verify (db1 writes, db2 reads) + db1.insert(Insert::new(b"key2".to_vec(), b"value2".to_vec()))?; + assert_eq!( + db2.get(Get::new(b"key2".to_vec()))?.unwrap(), + b"value2".to_vec(), + "db2 should see changes from db1" + ); + + // Section 2: Test cross-instance operations (db2 writes, db1 reads) + db2.insert(Insert::new(b"key3".to_vec(), b"value3".to_vec()))?; + assert_eq!( + db1.get(Get::new(b"key3".to_vec()))?.unwrap(), + b"value3".to_vec(), + "db1 should see changes from db2" + ); + + // Section 3: Test cache with different path + let second_path = temp_dir.path().join("different_cache.db"); + let mut db3 = SledDb::new(&second_path, "datastore")?; + db3.insert(Insert::new(b"db3_key".to_vec(), b"db3_value".to_vec()))?; + + // Create another instance to the second path + let db4 = SledDb::new(&second_path, "datastore")?; + assert_eq!( + db4.get(Get::new(b"db3_key".to_vec()))?.unwrap(), + b"db3_value".to_vec(), + "db4 should see db3's data" + ); + + // Verify first path data isn't in second path + assert!( + db4.get(Get::new(b"test_key".to_vec()))?.is_none(), + "db4 should not see data from db1/db2" + ); + + // Verify second path data isn't in first path + assert!( + db1.get(Get::new(b"db3_key".to_vec()))?.is_none(), + "db1 should not see data from db3/db4" + ); + + Ok(()) + } +} diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index ed845fe05e..de06b0a7f2 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,20 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{ - traits::{KeyValStore, Seekable}, - Get, Insert, InsertSync, Remove, SeekForPrev, -}; +use crate::{traits::KeyValStore, Get, Insert, InsertSync, Remove, SledDb}; use actix::{Actor, ActorContext, Addr, Handler}; -use anyhow::{Context, Result}; +use anyhow::Result; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; -use once_cell::sync::Lazy; -use sled::{Db, Tree}; -use std::{ - collections::HashMap, - path::PathBuf, - sync::{Arc, Mutex}, -}; +use std::path::PathBuf; use tracing::{error, info}; pub struct SledStore { @@ -111,188 +102,3 @@ impl Handler for SledStore { } } } - -pub struct SledDb { - db: Tree, -} - -// Global static cache -pub static SLED_CACHE: Lazy>>> = - Lazy::new(|| Arc::new(Mutex::new(HashMap::new()))); - -// Returns a stable canonical string path used as a cache key. -// Canonicalizes the parent directory if the target path does not yet exist. -fn canonical_key(path: &PathBuf) -> String { - use std::path::{Path, PathBuf}; - if path.exists() { - return path - .canonicalize() - .unwrap_or_else(|_| path.clone()) - .to_string_lossy() - .into_owned(); - } - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - let base: PathBuf = parent - .canonicalize() - .unwrap_or_else(|_| parent.to_path_buf()); - let tail = path.file_name().map(|s| s.to_owned()).unwrap_or_default(); - base.join(tail).to_string_lossy().into_owned() -} - -// Opens or retrieves a cached sled database for the given path. -// Prevents conflicts by ensuring only a single connection was open to a db file at once per process. -// Ensures the directory exists and stabilizes the canonical key across OSes. -fn get_or_open_db(path: &PathBuf) -> Result { - let _ = std::fs::create_dir_all(path); - let key = canonical_key(path); - let mut cache = SLED_CACHE.lock().unwrap(); - if let Some(db) = cache.get(&key) { - return Ok(db.clone()); - } - let db = sled::open(path).with_context(|| { - format!( - "Could not open database at path '{}'", - path.to_string_lossy() - ) - })?; - cache.insert(key, db.clone()); - if !db.was_recovered() { - info!("created db at: {:?}", &path); - } else { - info!("recovered db st: {:?}", &path); - } - - Ok(db) -} - -fn get_or_open_db_tree(path: &PathBuf, tree: &str) -> Result { - let db = get_or_open_db(path)?; - Ok(db.open_tree(tree)?) -} - -fn clear_all_caches() { - let mut cache_lock = SLED_CACHE.lock().unwrap(); - cache_lock.clear(); -} - -impl SledDb { - pub fn new(path: &PathBuf, tree: &str) -> Result { - let db = get_or_open_db_tree(path, tree)?; - Ok(Self { db }) - } - - pub fn close_all_connections() { - clear_all_caches() - } -} - -impl KeyValStore for SledDb { - fn insert(&mut self, msg: Insert) -> Result<()> { - self.db - .insert(msg.key(), msg.value().to_vec()) - .context("Could not insert data into db")?; - - Ok(()) - } - - fn remove(&mut self, msg: Remove) -> Result<()> { - self.db - .remove(msg.key()) - .context("Could not remove data from db")?; - Ok(()) - } - - fn get(&self, event: Get) -> Result>> { - let key = event.key(); - let str_key = String::from_utf8_lossy(&key).into_owned(); - let res = self - .db - .get(key) - .context(format!("Failed to fetch {}", str_key))?; - - Ok(res.map(|v| v.to_vec())) - } -} - -impl Seekable for SledDb { - fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { - let key = msg.key(); - let entry = self.db.range(..=&key[..]).next_back(); - - match entry { - Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), - Some(Err(e)) => Err(e.into()), - None => Ok(None), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_sled_db_caching() -> Result<()> { - use tempfile::tempdir; - - // Section 1: Test basic cache functionality - let temp_dir = tempdir().expect("Failed to create temporary directory"); - let db_path = temp_dir.path().join("test_cache.db"); - - // Create first instance and insert data - let mut db1 = SledDb::new(&db_path, "datastore")?; - db1.insert(Insert::new(b"test_key".to_vec(), b"test_value".to_vec()))?; - - // Create second instance to same path and verify data access - let mut db2 = SledDb::new(&db_path, "datastore")?; - let result = db2.get(Get::new(b"test_key".to_vec()))?; - assert_eq!( - result.unwrap(), - b"test_value".to_vec(), - "Values from db2 should match" - ); - - // Cross-modify and verify (db1 writes, db2 reads) - db1.insert(Insert::new(b"key2".to_vec(), b"value2".to_vec()))?; - assert_eq!( - db2.get(Get::new(b"key2".to_vec()))?.unwrap(), - b"value2".to_vec(), - "db2 should see changes from db1" - ); - - // Section 2: Test cross-instance operations (db2 writes, db1 reads) - db2.insert(Insert::new(b"key3".to_vec(), b"value3".to_vec()))?; - assert_eq!( - db1.get(Get::new(b"key3".to_vec()))?.unwrap(), - b"value3".to_vec(), - "db1 should see changes from db2" - ); - - // Section 3: Test cache with different path - let second_path = temp_dir.path().join("different_cache.db"); - let mut db3 = SledDb::new(&second_path, "datastore")?; - db3.insert(Insert::new(b"db3_key".to_vec(), b"db3_value".to_vec()))?; - - // Create another instance to the second path - let mut db4 = SledDb::new(&second_path, "datastore")?; - assert_eq!( - db4.get(Get::new(b"db3_key".to_vec()))?.unwrap(), - b"db3_value".to_vec(), - "db4 should see db3's data" - ); - - // Verify first path data isn't in second path - assert!( - db4.get(Get::new(b"test_key".to_vec()))?.is_none(), - "db4 should not see data from db1/db2" - ); - - // Verify second path data isn't in first path - assert!( - db1.get(Get::new(b"db3_key".to_vec()))?.is_none(), - "db1 should not see data from db3/db4" - ); - - Ok(()) - } -} diff --git a/crates/data/src/traits.rs b/crates/data/src/traits.rs index ce675e2887..50eea451b4 100644 --- a/crates/data/src/traits.rs +++ b/crates/data/src/traits.rs @@ -4,16 +4,17 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, Remove, SeekForPrev}; use anyhow::Result; +use crate::{Get, Insert, Remove, SeekForPrev}; + pub trait KeyValStore { fn insert(&mut self, msg: Insert) -> Result<()>; fn remove(&mut self, msg: Remove) -> Result<()>; fn get(&self, msg: Get) -> Result>>; } -pub trait Seekable: KeyValStore { +pub trait SeekableStore: KeyValStore { /// Seek for the first key that is less than or equal to the given key in the SeekForPrev msg /// and return the value as a Some variant. If no value exists return None. If there was an /// error doing the seek return an Error. From 376fbcf11bef86865d1e755754d616c2f6a7bb91 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 03:31:57 +0000 Subject: [PATCH 06/66] add header --- Cargo.lock | 463 ++++++++++++++++++-------------------- crates/data/src/events.rs | 6 + 2 files changed, 229 insertions(+), 240 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 991041a2d5..f4f3cdb0cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -207,7 +207,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -218,7 +218,7 @@ checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.2.23" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35d744058a9daa51a8cf22a3009607498fcf82d3cf4c5444dd8056cdf651f471" +checksum = "bfaa9ea039a6f9304b4a593d780b1f23e1ae183acdee938b11b38795acacc9f1" dependencies = [ "alloy-primitives", "num_enum", @@ -549,7 +549,7 @@ checksum = "d7c69f6c9c68a1287c9d5ff903d0010726934de0dac10989be37b75a29190d55" dependencies = [ "alloy-primitives", "alloy-sol-types", - "http 1.4.0", + "http 1.3.1", "serde", "serde_json", "thiserror 2.0.17", @@ -629,7 +629,7 @@ dependencies = [ "derive_more", "foldhash", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.12.0", "itoa", "k256", "keccak-asm", @@ -731,7 +731,7 @@ checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -929,7 +929,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -942,11 +942,11 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.12.1", + "indexmap 2.12.0", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", "syn-solidity", "tiny-keccak", ] @@ -965,7 +965,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.111", + "syn 2.0.110", "syn-solidity", ] @@ -1059,7 +1059,7 @@ dependencies = [ "alloy-pubsub", "alloy-transport", "futures", - "http 1.4.0", + "http 1.3.1", "rustls", "serde_json", "tokio", @@ -1093,7 +1093,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1143,22 +1143,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.11" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1324,7 +1324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1362,7 +1362,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1447,7 +1447,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1519,7 +1519,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", "synstructure", ] @@ -1531,7 +1531,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1633,7 +1633,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1682,7 +1682,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1699,7 +1699,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1760,7 +1760,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -1814,14 +1814,14 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bigint-poly" version = "0.1.0" -source = "git+https://github.com/gnosisguild/bigint-poly#9eca04d2aa473c5ead1e5a13adc8ad11bf250e4a" +source = "git+https://github.com/gnosisguild/bigint-poly#6b992dd81c65cc693b388e37edfd190749bfc3b3" dependencies = [ "num-bigint", "num-traits", @@ -1867,15 +1867,15 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcoin-io" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" +checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" [[package]] name = "bitcoin_hashes" -version = "0.14.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b" +checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" dependencies = [ "bitcoin-io", "hex-conservative", @@ -2026,9 +2026,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -2059,9 +2059,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.49" +version = "1.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" dependencies = [ "find-msvc-tools", "jobserver", @@ -2137,7 +2137,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -2251,15 +2251,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" -[[package]] -name = "convert_case" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "cookie" version = "0.16.2" @@ -2307,9 +2298,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.4.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -2425,7 +2416,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -2450,7 +2441,7 @@ dependencies = [ "quote", "serde", "strsim", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -2461,7 +2452,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -2501,7 +2492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -2551,24 +2542,22 @@ dependencies = [ [[package]] name = "derive_more" -version = "2.1.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.1.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ - "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.1", - "syn 2.0.111", + "syn 2.0.110", "unicode-xid", ] @@ -2665,7 +2654,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -3388,7 +3377,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -3461,7 +3450,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -3481,7 +3470,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -3584,9 +3573,8 @@ dependencies = [ [[package]] name = "fhe" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" +source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" dependencies = [ - "bincode", "doc-comment", "fhe-math", "fhe-traits", @@ -3610,7 +3598,7 @@ dependencies = [ [[package]] name = "fhe-math" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" +source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" dependencies = [ "ethnum", "fhe-traits", @@ -3624,7 +3612,6 @@ dependencies = [ "prost-build", "rand 0.8.5", "rand_chacha 0.3.1", - "serde", "sha2", "thiserror 1.0.69", "zeroize", @@ -3633,7 +3620,7 @@ dependencies = [ [[package]] name = "fhe-traits" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" +source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" dependencies = [ "rand 0.8.5", ] @@ -3641,7 +3628,7 @@ dependencies = [ [[package]] name = "fhe-util" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" +source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" dependencies = [ "itertools 0.12.1", "num-bigint-dig", @@ -3687,9 +3674,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" [[package]] name = "fixed-hash" @@ -3851,7 +3838,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -4037,7 +4024,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.1", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -4055,8 +4042,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.0", - "indexmap 2.12.1", + "http 1.3.1", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -4098,9 +4085,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" [[package]] name = "heck" @@ -4122,9 +4109,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-conservative" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" +checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" dependencies = [ "arrayvec", ] @@ -4212,11 +4199,12 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", + "fnv", "itoa", ] @@ -4238,7 +4226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http 1.3.1", ] [[package]] @@ -4249,7 +4237,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "pin-project-lite", ] @@ -4292,16 +4280,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2 0.4.12", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "httparse", "itoa", @@ -4318,8 +4306,8 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.4.0", - "hyper 1.8.1", + "http 1.3.1", + "hyper 1.8.0", "hyper-util", "rustls", "rustls-pki-types", @@ -4335,7 +4323,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.1", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -4350,7 +4338,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper 1.8.0", "hyper-util", "native-tls", "tokio", @@ -4360,18 +4348,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "base64", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", - "hyper 1.8.1", + "hyper 1.8.0", "ipnet", "libc", "percent-encoding", @@ -4456,9 +4444,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ "icu_collections", "icu_locale_core", @@ -4470,9 +4458,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" @@ -4591,7 +4579,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -4613,12 +4601,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.16.0", "serde", "serde_core", ] @@ -4781,9 +4769,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" dependencies = [ "once_cell", "wasm-bindgen", @@ -4857,15 +4845,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.178" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.2+1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" dependencies = [ "cc", "libc", @@ -5037,9 +5025,9 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.13" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c7892c221730ba55f7196e98b0b8ba5e04b4155651736036628e9f73ed6fc3" +checksum = "3104e13b51e4711ff5738caa1fb54467c8604c2e94d607e27745bcf709068774" dependencies = [ "bs58", "ed25519-dalek", @@ -5197,7 +5185,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -5347,9 +5335,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" dependencies = [ "value-bag", ] @@ -5395,7 +5383,7 @@ checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -5458,9 +5446,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", "log", @@ -5781,7 +5769,7 @@ checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -5863,7 +5851,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -5906,7 +5894,7 @@ checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" dependencies = [ "async-trait", "bytes", - "http 1.4.0", + "http 1.3.1", "opentelemetry", "reqwest", "tracing", @@ -5919,7 +5907,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" dependencies = [ "futures-core", - "http 1.4.0", + "http 1.3.1", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -6021,7 +6009,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6119,9 +6107,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.4" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" dependencies = [ "memchr", "ucd-trie", @@ -6134,7 +6122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.12.1", + "indexmap 2.12.0", ] [[package]] @@ -6191,7 +6179,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6220,7 +6208,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6325,7 +6313,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6356,7 +6344,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.9", + "toml_edit 0.23.7", ] [[package]] @@ -6378,7 +6366,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6410,7 +6398,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6469,7 +6457,7 @@ dependencies = [ "prost 0.12.6", "prost-types", "regex", - "syn 2.0.111", + "syn 2.0.110", "tempfile", ] @@ -6483,7 +6471,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6496,7 +6484,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6795,7 +6783,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -6861,10 +6849,10 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.12", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.8.0", "hyper-rustls", "hyper-tls", "hyper-util", @@ -6896,9 +6884,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.6" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" +checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" [[package]] name = "rfc6979" @@ -7075,9 +7063,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "web-time", "zeroize", @@ -7150,18 +7138,6 @@ dependencies = [ "taceo-poseidon2", ] -[[package]] -name = "safe" -version = "0.1.7" -source = "git+https://github.com/gnosisguild/enclave#c9b72d1e42d8e37a19473f423978ef76f92454d6" -dependencies = [ - "ark-bn254 0.5.0", - "ark-ff 0.5.0", - "hex", - "sha3", - "taceo-poseidon2", -] - [[package]] name = "scc" version = "2.4.0" @@ -7343,7 +7319,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7389,7 +7365,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.1.0", "serde", @@ -7408,7 +7384,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7417,7 +7393,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.12.0", "itoa", "ryu", "serde", @@ -7456,7 +7432,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7529,9 +7505,9 @@ checksum = "c7a6f98357c6bb0ebace19b22220e5543801d9de90ffe77f8abb27c056bac064" [[package]] name = "shell-words" -version = "1.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shellexpand" @@ -7571,9 +7547,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" dependencies = [ "libc", ] @@ -7590,9 +7566,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "siphasher" @@ -7709,7 +7685,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7731,9 +7707,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -7749,7 +7725,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7769,7 +7745,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7862,7 +7838,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7873,7 +7849,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -7987,7 +7963,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -8087,7 +8063,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.12.0", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -8097,11 +8073,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.9" +version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.12.0", "toml_datetime 0.7.3", "toml_parser", "winnow", @@ -8131,10 +8107,10 @@ dependencies = [ "async-trait", "base64", "bytes", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper 1.8.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -8185,14 +8161,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags 2.10.0", "bytes", "futures-util", - "http 1.4.0", + "http 1.3.1", "http-body 1.0.1", "iri-string", "pin-project-lite", @@ -8227,20 +8203,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.31" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -8312,7 +8288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -8329,7 +8305,7 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.3.1", "httparse", "log", "rand 0.9.2", @@ -8385,12 +8361,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - [[package]] name = "unicode-width" version = "0.2.2" @@ -8405,9 +8375,9 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unit-prefix" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" [[package]] name = "universal-hash" @@ -8486,9 +8456,9 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-bag" -version = "1.12.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" +checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" [[package]] name = "vcpkg" @@ -8556,9 +8526,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ "cfg-if", "once_cell", @@ -8569,9 +8539,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ "cfg-if", "js-sys", @@ -8582,9 +8552,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8592,22 +8562,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" dependencies = [ "unicode-ident", ] @@ -8628,9 +8598,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -8722,7 +8692,7 @@ dependencies = [ "windows-interface", "windows-link 0.2.1", "windows-result 0.4.1", - "windows-strings", + "windows-strings 0.5.1", ] [[package]] @@ -8733,7 +8703,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -8744,7 +8714,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -8761,13 +8731,13 @@ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-registry" -version = "0.6.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" dependencies = [ - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -8779,6 +8749,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -8788,6 +8767,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -9030,9 +9018,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -9157,28 +9145,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -9198,7 +9186,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", "synstructure", ] @@ -9219,7 +9207,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -9252,7 +9240,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.110", ] [[package]] @@ -9268,11 +9256,9 @@ dependencies = [ [[package]] name = "zkfhe-greco" version = "0.1.0" -source = "git+https://github.com/gnosisguild/zkfhe-generator#46d8579f8fc3c53d2c74e89e7c75c280f63e432f" +source = "git+https://github.com/gnosisguild/zkfhe-generator#7314bed079b91ea6432864f30b991e4ef724a5c5" dependencies = [ "anyhow", - "ark-bn254 0.5.0", - "ark-ff 0.5.0", "bigint-poly", "blake3", "fhe", @@ -9283,7 +9269,6 @@ dependencies = [ "num-traits", "rand 0.8.5", "rayon", - "safe 0.1.7 (git+https://github.com/gnosisguild/enclave)", "serde", "serde_json", "tempfile", @@ -9294,11 +9279,9 @@ dependencies = [ [[package]] name = "zkfhe-shared" version = "0.1.0" -source = "git+https://github.com/gnosisguild/zkfhe-generator#46d8579f8fc3c53d2c74e89e7c75c280f63e432f" +source = "git+https://github.com/gnosisguild/zkfhe-generator#7314bed079b91ea6432864f30b991e4ef724a5c5" dependencies = [ "anyhow", - "ark-bn254 0.5.0", - "ark-ff 0.5.0", "bigint-poly", "chrono", "fhe", diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index e0ff6a6eb5..4223ba21db 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::IntoKey; use actix::Message; use anyhow::Result; From 10a9a5330a33a7ac818289ce79f31d12ab14b00e Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 04:55:42 +0000 Subject: [PATCH 07/66] setup in memory data structures --- Cargo.lock | 1 + crates/data/Cargo.toml | 1 + crates/data/src/events.rs | 9 +++++ crates/data/src/hlc_store.rs | 55 +++++++++++++++++++++++++ crates/data/src/in_mem_hlc_store.rs | 62 +++++++++++++++++++++++++++++ crates/data/src/lib.rs | 3 ++ crates/utils/src/actix.rs | 49 ++++++++++++++++++++++- 7 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 crates/data/src/hlc_store.rs create mode 100644 crates/data/src/in_mem_hlc_store.rs diff --git a/Cargo.lock b/Cargo.lock index f4f3cdb0cf..93bee7e5e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2864,6 +2864,7 @@ dependencies = [ "async-trait", "bincode", "e3-events", + "e3-utils", "once_cell", "serde", "sled", diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index 60479f452b..a1638f9d37 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/gnosisguild/enclave/crates/data" [dependencies] actix = { workspace = true } e3-events = { workspace = true } +e3-utils = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } sled = { workspace = true } diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index 4223ba21db..69650e5923 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -86,3 +86,12 @@ impl SeekForPrev { &self.0 } } + +#[derive(Message)] +#[rtype(result = "()")] +pub struct SeekForPrevResponse(u64); +impl SeekForPrevResponse { + pub fn new(val: u64) -> Self { + Self(val) + } +} diff --git a/crates/data/src/hlc_store.rs b/crates/data/src/hlc_store.rs new file mode 100644 index 0000000000..9fa601ccc2 --- /dev/null +++ b/crates/data/src/hlc_store.rs @@ -0,0 +1,55 @@ +use crate::{Insert, KeyValStore, SeekForPrev, SeekForPrevResponse, SeekableStore, SledDb}; +use actix::{Actor, Handler}; +use e3_events::{trap, BusHandle}; +use e3_utils::actix::Responder; + +pub struct HlcStore { + db: SledDb, + bus: BusHandle, +} + +impl HlcStore { + pub fn new(db: SledDb, bus: &BusHandle) -> Self { + Self { + db, + bus: bus.clone(), + } + } +} + +impl Actor for HlcStore { + type Context = actix::Context; +} + +impl Handler> for HlcStore { + type Result = (); + fn handle( + &mut self, + msg: Responder, + _: &mut Self::Context, + ) -> Self::Result { + trap(e3_events::EType::Data, &self.bus.clone(), || { + let Some(result) = self.db.seek_for_prev((*msg).clone())? else { + return Err(anyhow::anyhow!("Seek returned no result.")); + }; + + let seq_bytes: [u8; 8] = result[..8] + .try_into() + .expect("sequence must be exactly 8 bytes"); + + let seq = u64::from_be_bytes(seq_bytes); + msg.try_reply(SeekForPrevResponse::new(seq))?; + Ok(()) + }) + } +} + +impl Handler for HlcStore { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + trap(e3_events::EType::Data, &self.bus.clone(), || { + self.db.insert(msg)?; + Ok(()) + }) + } +} diff --git a/crates/data/src/in_mem_hlc_store.rs b/crates/data/src/in_mem_hlc_store.rs new file mode 100644 index 0000000000..8e4b16ea68 --- /dev/null +++ b/crates/data/src/in_mem_hlc_store.rs @@ -0,0 +1,62 @@ +use std::{collections::BTreeMap, ops::Deref}; + +use actix::Handler; +use e3_utils::Responder; + +use crate::{Insert, SeekForPrev, SeekForPrevResponse, SeekableStore}; + +pub struct InMemHlcStore { + db: InMemDb, +} + +struct InMemDb(BTreeMap, Vec>); +impl Deref for InMemDb { + type Target = BTreeMap, Vec>; + fn deref(&self) -> &Self::Target {} +} + +impl SeekableStore for InMemDb { + fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { + let key = msg.key(); + let entry = self.range(..=&key[..]).next_back(); + + match entry { + Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), + Some(Err(e)) => Err(e.into()), + None => Ok(None), + } + } +} + +impl Handler> for InMemHlcStore { + type Result = (); + fn handle( + &mut self, + msg: Responder, + _: &mut Self::Context, + ) -> Self::Result { + trap(e3_events::EType::Data, &self.bus.clone(), || { + let Some(result) = self.db.seek_for_prev((*msg).clone())? else { + return Err(anyhow::anyhow!("Seek returned no result.")); + }; + + let seq_bytes: [u8; 8] = result[..8] + .try_into() + .expect("sequence must be exactly 8 bytes"); + + let seq = u64::from_be_bytes(seq_bytes); + msg.try_reply(SeekForPrevResponse::new(seq))?; + Ok(()) + }) + } +} + +impl Handler for InMemHlcStore { + type Result = (); + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + trap(e3_events::EType::Data, &self.bus.clone(), || { + self.db.insert(msg.key(), msg.value().to_vec())?; + Ok(()) + }) + } +} diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index c7d6d33cf1..733b92d9f1 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -6,6 +6,7 @@ mod data_store; mod events; +mod hlc_store; mod in_mem; mod into_key; mod persistable; @@ -18,6 +19,7 @@ mod traits; pub use data_store::*; pub use events::*; +pub use hlc_store::*; pub use in_mem::*; pub use into_key::IntoKey; pub use persistable::*; @@ -26,3 +28,4 @@ pub use repository::*; pub use sled_db::*; pub use sled_store::*; pub use snapshot::*; +pub use traits::*; diff --git a/crates/utils/src/actix.rs b/crates/utils/src/actix.rs index f4a18173b3..4bf8e285fc 100644 --- a/crates/utils/src/actix.rs +++ b/crates/utils/src/actix.rs @@ -4,7 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, ResponseActFuture, WrapFuture}; +use std::ops::Deref; + +use actix::{dev::SendError, Actor, Message, Recipient, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; @@ -17,3 +19,48 @@ pub fn bail_result(a: &T, msg: impl Into) -> ResponseActFuture let m: String = msg.into(); Box::pin(async { Err(anyhow!(m)) }.into_actor(a)) } + +#[derive(Message)] +#[rtype("()")] +pub struct Responder +where + U: Message + Send, + U::Result: Send, +{ + value: T, + sender: Recipient, +} + +impl Responder +where + U: Message + Send, + U::Result: Send, +{ + pub fn new(value: T, sender: impl Into>) -> Self { + Self { + value, + sender: sender.into(), + } + } + + pub fn reply(&self, msg: U) { + let sender = &self.sender; + sender.do_send(msg); + } + + pub fn try_reply(&self, msg: U) -> Result<(), SendError> { + let sender = &self.sender; + sender.try_send(msg) + } +} + +impl Deref for Responder +where + U: Message + Send, + U::Result: Send, +{ + type Target = T; + fn deref(&self) -> &Self::Target { + &self.value + } +} From 950714f744e37735f1d5bc6ed8790b715c298b63 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 05:25:24 +0000 Subject: [PATCH 08/66] setup inmem hlcstore --- Cargo.lock | 43 +++++++++++++++++++++++++++ crates/data/Cargo.toml | 1 + crates/data/src/event_log.rs | 42 ++++++++++++++++++++++++++ crates/data/src/in_mem_hlc_store.rs | 46 +++++++++++++++++++++-------- crates/data/src/lib.rs | 3 ++ crates/data/src/sled_db.rs | 2 +- 6 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 crates/data/src/event_log.rs diff --git a/Cargo.lock b/Cargo.lock index 93bee7e5e6..21573de488 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2152,6 +2152,20 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "commitlog" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf2bc2a9c8e9369d54360cc20cd9f0d5f03e427bc3fe8c9d7bc6ba1512addff" +dependencies = [ + "byteorder", + "bytes", + "crc32c", + "log", + "memmap2", + "page_size", +] + [[package]] name = "compile-time" version = "0.2.0" @@ -2311,6 +2325,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version 0.4.1", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2863,6 +2886,7 @@ dependencies = [ "anyhow", "async-trait", "bincode", + "commitlog", "e3-events", "e3-utils", "once_cell", @@ -5423,6 +5447,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -5985,6 +6018,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "page_size" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parity-scale-codec" version = "3.7.5" diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index a1638f9d37..3995bee4ef 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -18,3 +18,4 @@ tracing = { workspace = true } async-trait = { workspace = true } once_cell = { workspace = true } tempfile = { workspace = true } +commitlog = "0.2.0" diff --git a/crates/data/src/event_log.rs b/crates/data/src/event_log.rs new file mode 100644 index 0000000000..7893ab08d1 --- /dev/null +++ b/crates/data/src/event_log.rs @@ -0,0 +1,42 @@ +use std::path::PathBuf; + +use actix::{Actor, Handler, Message}; +use commitlog::{CommitLog, LogOptions}; +use e3_events::{trap, BusHandle, EType}; + +#[derive(Message)] +#[rtype(result = "()")] +pub struct Persist { + value: Vec, +} + +pub struct EventLog { + log: CommitLog, + bus: BusHandle, +} + +impl EventLog { + pub fn new(path: &PathBuf, bus: &BusHandle) -> Self { + let opts = LogOptions::new(path); + let log = CommitLog::new(opts).unwrap(); + + Self { + log, + bus: bus.clone(), + } + } +} + +impl Actor for EventLog { + type Context = actix::Context; +} + +impl Handler for EventLog { + type Result = (); + fn handle(&mut self, msg: Persist, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Data, &self.bus, || { + self.log.append_msg(&msg.value)?; + Ok(()) + }) + } +} diff --git a/crates/data/src/in_mem_hlc_store.rs b/crates/data/src/in_mem_hlc_store.rs index 8e4b16ea68..c6bce1cbe5 100644 --- a/crates/data/src/in_mem_hlc_store.rs +++ b/crates/data/src/in_mem_hlc_store.rs @@ -1,30 +1,50 @@ use std::{collections::BTreeMap, ops::Deref}; -use actix::Handler; +use crate::{Get, Insert, KeyValStore, SeekForPrev, SeekForPrevResponse, SeekableStore}; +use actix::{Actor, Handler}; +use anyhow::Result; +use e3_events::{trap, BusHandle}; use e3_utils::Responder; -use crate::{Insert, SeekForPrev, SeekForPrevResponse, SeekableStore}; - pub struct InMemHlcStore { db: InMemDb, + bus: BusHandle, +} + +impl Actor for InMemHlcStore { + type Context = actix::Context; } struct InMemDb(BTreeMap, Vec>); impl Deref for InMemDb { type Target = BTreeMap, Vec>; - fn deref(&self) -> &Self::Target {} + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl KeyValStore for InMemDb { + fn get(&self, msg: Get) -> Result>> { + Ok(self.0.get(msg.key()).cloned()) + } + fn insert(&mut self, msg: Insert) -> Result<()> { + self.0.insert(msg.key().to_owned(), msg.value().to_owned()); + Ok(()) + } + fn remove(&mut self, msg: crate::Remove) -> Result<()> { + self.0.remove(msg.key()); + Ok(()) + } } impl SeekableStore for InMemDb { fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { let key = msg.key(); - let entry = self.range(..=&key[..]).next_back(); - - match entry { - Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), - Some(Err(e)) => Err(e.into()), - None => Ok(None), - } + Ok(self + .0 + .range(..=key.to_vec()) + .next_back() + .map(|(_, v)| v.clone())) } } @@ -35,7 +55,7 @@ impl Handler> for InMemHlcStore { msg: Responder, _: &mut Self::Context, ) -> Self::Result { - trap(e3_events::EType::Data, &self.bus.clone(), || { + trap(e3_events::EType::Data, &self.bus, || { let Some(result) = self.db.seek_for_prev((*msg).clone())? else { return Err(anyhow::anyhow!("Seek returned no result.")); }; @@ -55,7 +75,7 @@ impl Handler for InMemHlcStore { type Result = (); fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { trap(e3_events::EType::Data, &self.bus.clone(), || { - self.db.insert(msg.key(), msg.value().to_vec())?; + self.db.insert(msg)?; Ok(()) }) } diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 733b92d9f1..3771a78e51 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -5,9 +5,11 @@ // or FITNESS FOR A PARTICULAR PURPOSE. mod data_store; +mod event_log; mod events; mod hlc_store; mod in_mem; +mod in_mem_hlc_store; mod into_key; mod persistable; mod repositories; @@ -18,6 +20,7 @@ mod snapshot; mod traits; pub use data_store::*; +pub use event_log::*; pub use events::*; pub use hlc_store::*; pub use in_mem::*; diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 007282b8eb..0a801a07c6 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -65,7 +65,7 @@ impl KeyValStore for SledDb { impl SeekableStore for SledDb { fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { let key = msg.key(); - let entry = self.db.range(..=&key[..]).next_back(); + let entry = self.db.range(..=key.to_vec()).next_back(); match entry { Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), From 8608b91da12cd5a812b8cdc1cce4dcc68b95f975 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 8 Dec 2025 08:50:36 +0000 Subject: [PATCH 09/66] update to simplify event store --- crates/data/src/event_log.rs | 42 -------------- crates/data/src/event_store.rs | 48 ++++++++++++++++ crates/data/src/event_store_in_mem.rs | 39 +++++++++++++ crates/data/src/events.rs | 4 +- crates/data/src/hlc_store.rs | 55 ------------------ crates/data/src/in_mem.rs | 39 ++++++++++++- crates/data/src/in_mem_hlc_store.rs | 82 --------------------------- crates/data/src/lib.rs | 7 +-- crates/utils/src/actix.rs | 49 +--------------- 9 files changed, 129 insertions(+), 236 deletions(-) delete mode 100644 crates/data/src/event_log.rs create mode 100644 crates/data/src/event_store.rs create mode 100644 crates/data/src/event_store_in_mem.rs delete mode 100644 crates/data/src/hlc_store.rs delete mode 100644 crates/data/src/in_mem_hlc_store.rs diff --git a/crates/data/src/event_log.rs b/crates/data/src/event_log.rs deleted file mode 100644 index 7893ab08d1..0000000000 --- a/crates/data/src/event_log.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::path::PathBuf; - -use actix::{Actor, Handler, Message}; -use commitlog::{CommitLog, LogOptions}; -use e3_events::{trap, BusHandle, EType}; - -#[derive(Message)] -#[rtype(result = "()")] -pub struct Persist { - value: Vec, -} - -pub struct EventLog { - log: CommitLog, - bus: BusHandle, -} - -impl EventLog { - pub fn new(path: &PathBuf, bus: &BusHandle) -> Self { - let opts = LogOptions::new(path); - let log = CommitLog::new(opts).unwrap(); - - Self { - log, - bus: bus.clone(), - } - } -} - -impl Actor for EventLog { - type Context = actix::Context; -} - -impl Handler for EventLog { - type Result = (); - fn handle(&mut self, msg: Persist, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Data, &self.bus, || { - self.log.append_msg(&msg.value)?; - Ok(()) - }) - } -} diff --git a/crates/data/src/event_store.rs b/crates/data/src/event_store.rs new file mode 100644 index 0000000000..dcafef21ab --- /dev/null +++ b/crates/data/src/event_store.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Handler, Message, Recipient}; +use commitlog::CommitLog; +use e3_events::{trap, BusHandle, EType, EnclaveEvent}; + +use crate::{Insert, KeyValStore, SledDb}; + +#[derive(Message)] +#[rtype("()")] +pub struct PersistRequest { + pub event: EnclaveEvent, + pub sender: Recipient, +} + +#[derive(Message)] +#[rtype("()")] +pub struct EventPersisted(pub u64); + +pub struct EventStore { + hlc_store: SledDb, + event_log: CommitLog, + bus: BusHandle, +} + +impl Actor for EventStore { + type Context = actix::Context; +} + +impl Handler for EventStore { + type Result = (); + fn handle(&mut self, msg: PersistRequest, _: &mut Self::Context) -> Self::Result { + trap(EType::Data, &self.bus, || { + let event = msg.event; + let sender = msg.sender; + let ts = event.get_ts(); + let seq = self.event_log.append_msg(event.to_bytes()?)?; + self.hlc_store + .insert(Insert::new(ts, seq.to_be_bytes().to_vec())); + sender.try_send(EventPersisted(seq))?; + Ok(()) + }) + } +} diff --git a/crates/data/src/event_store_in_mem.rs b/crates/data/src/event_store_in_mem.rs new file mode 100644 index 0000000000..0795aa3b7b --- /dev/null +++ b/crates/data/src/event_store_in_mem.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Handler}; +use e3_events::{trap, BusHandle, EType}; + +use crate::{ + event_store::{EventPersisted, PersistRequest}, + InMemCommitLog, InMemDb, Insert, KeyValStore, +}; + +pub struct InMemEventStore { + hlc_store: InMemDb, + event_log: InMemCommitLog, + bus: BusHandle, +} + +impl Actor for InMemEventStore { + type Context = actix::Context; +} + +impl Handler for InMemEventStore { + type Result = (); + fn handle(&mut self, msg: PersistRequest, _: &mut Self::Context) -> Self::Result { + trap(EType::Data, &self.bus, || { + let event = msg.event; + let sender = msg.sender; + let ts = event.get_ts(); + let seq = self.event_log.append_msg(event.to_bytes()?)?; + self.hlc_store + .insert(Insert::new(ts, seq.to_be_bytes().to_vec())); + sender.try_send(EventPersisted(seq))?; + Ok(()) + }) + } +} diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index 69650e5923..94a8aa46cb 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -89,8 +89,8 @@ impl SeekForPrev { #[derive(Message)] #[rtype(result = "()")] -pub struct SeekForPrevResponse(u64); -impl SeekForPrevResponse { +pub struct SeekForPrevReply(u64); +impl SeekForPrevReply { pub fn new(val: u64) -> Self { Self(val) } diff --git a/crates/data/src/hlc_store.rs b/crates/data/src/hlc_store.rs deleted file mode 100644 index 9fa601ccc2..0000000000 --- a/crates/data/src/hlc_store.rs +++ /dev/null @@ -1,55 +0,0 @@ -use crate::{Insert, KeyValStore, SeekForPrev, SeekForPrevResponse, SeekableStore, SledDb}; -use actix::{Actor, Handler}; -use e3_events::{trap, BusHandle}; -use e3_utils::actix::Responder; - -pub struct HlcStore { - db: SledDb, - bus: BusHandle, -} - -impl HlcStore { - pub fn new(db: SledDb, bus: &BusHandle) -> Self { - Self { - db, - bus: bus.clone(), - } - } -} - -impl Actor for HlcStore { - type Context = actix::Context; -} - -impl Handler> for HlcStore { - type Result = (); - fn handle( - &mut self, - msg: Responder, - _: &mut Self::Context, - ) -> Self::Result { - trap(e3_events::EType::Data, &self.bus.clone(), || { - let Some(result) = self.db.seek_for_prev((*msg).clone())? else { - return Err(anyhow::anyhow!("Seek returned no result.")); - }; - - let seq_bytes: [u8; 8] = result[..8] - .try_into() - .expect("sequence must be exactly 8 bytes"); - - let seq = u64::from_be_bytes(seq_bytes); - msg.try_reply(SeekForPrevResponse::new(seq))?; - Ok(()) - }) - } -} - -impl Handler for HlcStore { - type Result = (); - fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - trap(e3_events::EType::Data, &self.bus.clone(), || { - self.db.insert(msg)?; - Ok(()) - }) - } -} diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index ef9a8a5d30..a4d5f55ea2 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -4,10 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertSync, Remove}; +use crate::{Get, Insert, InsertSync, KeyValStore, Remove}; use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; -use std::collections::BTreeMap; +use commitlog::Offset; +use std::{collections::BTreeMap, ops::Deref}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Vec")] @@ -116,3 +117,37 @@ impl Handler for InMemStore { self.get_dump() } } + +pub struct InMemDb(BTreeMap, Vec>); + +impl Deref for InMemDb { + type Target = BTreeMap, Vec>; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl KeyValStore for InMemDb { + fn get(&self, msg: Get) -> Result>> { + Ok(self.0.get(msg.key()).cloned()) + } + fn insert(&mut self, msg: Insert) -> Result<()> { + self.0.insert(msg.key().to_owned(), msg.value().to_owned()); + Ok(()) + } + fn remove(&mut self, msg: Remove) -> Result<()> { + self.0.remove(msg.key()); + Ok(()) + } +} + +pub struct InMemCommitLog { + log: Vec>, +} + +impl InMemCommitLog { + pub fn append_msg(&mut self, payload: Vec) -> Result { + self.log.push(payload); + Ok(self.log.len() as u64) + } +} diff --git a/crates/data/src/in_mem_hlc_store.rs b/crates/data/src/in_mem_hlc_store.rs deleted file mode 100644 index c6bce1cbe5..0000000000 --- a/crates/data/src/in_mem_hlc_store.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::{collections::BTreeMap, ops::Deref}; - -use crate::{Get, Insert, KeyValStore, SeekForPrev, SeekForPrevResponse, SeekableStore}; -use actix::{Actor, Handler}; -use anyhow::Result; -use e3_events::{trap, BusHandle}; -use e3_utils::Responder; - -pub struct InMemHlcStore { - db: InMemDb, - bus: BusHandle, -} - -impl Actor for InMemHlcStore { - type Context = actix::Context; -} - -struct InMemDb(BTreeMap, Vec>); -impl Deref for InMemDb { - type Target = BTreeMap, Vec>; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl KeyValStore for InMemDb { - fn get(&self, msg: Get) -> Result>> { - Ok(self.0.get(msg.key()).cloned()) - } - fn insert(&mut self, msg: Insert) -> Result<()> { - self.0.insert(msg.key().to_owned(), msg.value().to_owned()); - Ok(()) - } - fn remove(&mut self, msg: crate::Remove) -> Result<()> { - self.0.remove(msg.key()); - Ok(()) - } -} - -impl SeekableStore for InMemDb { - fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { - let key = msg.key(); - Ok(self - .0 - .range(..=key.to_vec()) - .next_back() - .map(|(_, v)| v.clone())) - } -} - -impl Handler> for InMemHlcStore { - type Result = (); - fn handle( - &mut self, - msg: Responder, - _: &mut Self::Context, - ) -> Self::Result { - trap(e3_events::EType::Data, &self.bus, || { - let Some(result) = self.db.seek_for_prev((*msg).clone())? else { - return Err(anyhow::anyhow!("Seek returned no result.")); - }; - - let seq_bytes: [u8; 8] = result[..8] - .try_into() - .expect("sequence must be exactly 8 bytes"); - - let seq = u64::from_be_bytes(seq_bytes); - msg.try_reply(SeekForPrevResponse::new(seq))?; - Ok(()) - }) - } -} - -impl Handler for InMemHlcStore { - type Result = (); - fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { - trap(e3_events::EType::Data, &self.bus.clone(), || { - self.db.insert(msg)?; - Ok(()) - }) - } -} diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 3771a78e51..83e26ae23a 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -5,11 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. mod data_store; -mod event_log; +mod event_store; +mod event_store_in_mem; mod events; -mod hlc_store; mod in_mem; -mod in_mem_hlc_store; mod into_key; mod persistable; mod repositories; @@ -20,9 +19,7 @@ mod snapshot; mod traits; pub use data_store::*; -pub use event_log::*; pub use events::*; -pub use hlc_store::*; pub use in_mem::*; pub use into_key::IntoKey; pub use persistable::*; diff --git a/crates/utils/src/actix.rs b/crates/utils/src/actix.rs index 4bf8e285fc..f4a18173b3 100644 --- a/crates/utils/src/actix.rs +++ b/crates/utils/src/actix.rs @@ -4,9 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::ops::Deref; - -use actix::{dev::SendError, Actor, Message, Recipient, ResponseActFuture, WrapFuture}; +use actix::{Actor, ResponseActFuture, WrapFuture}; use anyhow::{anyhow, Result}; @@ -19,48 +17,3 @@ pub fn bail_result(a: &T, msg: impl Into) -> ResponseActFuture let m: String = msg.into(); Box::pin(async { Err(anyhow!(m)) }.into_actor(a)) } - -#[derive(Message)] -#[rtype("()")] -pub struct Responder -where - U: Message + Send, - U::Result: Send, -{ - value: T, - sender: Recipient, -} - -impl Responder -where - U: Message + Send, - U::Result: Send, -{ - pub fn new(value: T, sender: impl Into>) -> Self { - Self { - value, - sender: sender.into(), - } - } - - pub fn reply(&self, msg: U) { - let sender = &self.sender; - sender.do_send(msg); - } - - pub fn try_reply(&self, msg: U) -> Result<(), SendError> { - let sender = &self.sender; - sender.try_send(msg) - } -} - -impl Deref for Responder -where - U: Message + Send, - U::Result: Send, -{ - type Target = T; - fn deref(&self) -> &Self::Target { - &self.value - } -} From b461f3b5da1d5a4432312e0cf805b5a478229e18 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 9 Dec 2025 04:24:17 +0000 Subject: [PATCH 10/66] move event_bus_factory to ciphernode_builder --- Cargo.lock | 2 +- crates/ciphernode-builder/Cargo.toml | 1 + .../src/eventbus_factory.rs | 12 ++++++------ crates/ciphernode-builder/src/lib.rs | 2 ++ crates/entrypoint/src/helpers/datastore.rs | 3 ++- crates/entrypoint/src/start/aggregator_start.rs | 4 ++-- crates/entrypoint/src/start/start.rs | 3 +-- crates/events/Cargo.toml | 1 - crates/events/src/lib.rs | 2 -- 9 files changed, 15 insertions(+), 15 deletions(-) rename crates/{events => ciphernode-builder}/src/eventbus_factory.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index 21573de488..1eb6b24755 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2787,6 +2787,7 @@ dependencies = [ "e3-request", "e3-sortition", "e3-utils", + "once_cell", "tracing", ] @@ -2954,7 +2955,6 @@ dependencies = [ "e3-trbfv", "e3-utils", "futures-util", - "once_cell", "proptest", "rand 0.8.5", "serde", diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 21a7aa068a..b4477ccc72 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -11,6 +11,7 @@ actix.workspace = true alloy.workspace = true anyhow.workspace = true derivative.workspace = true +once_cell.workspace = true e3-aggregator.workspace = true e3-crypto.workspace = true e3-config.workspace = true diff --git a/crates/events/src/eventbus_factory.rs b/crates/ciphernode-builder/src/eventbus_factory.rs similarity index 95% rename from crates/events/src/eventbus_factory.rs rename to crates/ciphernode-builder/src/eventbus_factory.rs index f251d86622..6297b50bfa 100644 --- a/crates/events/src/eventbus_factory.rs +++ b/crates/ciphernode-builder/src/eventbus_factory.rs @@ -12,12 +12,12 @@ use std::any::TypeId; use std::collections::HashMap; use std::sync::Mutex; -use crate::traits::Event; -use crate::BusHandle; -use crate::EnclaveEvent; -use crate::EventBus; -use crate::HistoryCollector; -use crate::Subscribe; +use e3_events::BusHandle; +use e3_events::EnclaveEvent; +use e3_events::Event; +use e3_events::EventBus; +use e3_events::HistoryCollector; +use e3_events::Subscribe; // The singleton factory using once_cell pub struct EventBusFactory { diff --git a/crates/ciphernode-builder/src/lib.rs b/crates/ciphernode-builder/src/lib.rs index d2847b6d67..4b8efbec35 100644 --- a/crates/ciphernode-builder/src/lib.rs +++ b/crates/ciphernode-builder/src/lib.rs @@ -6,5 +6,7 @@ mod ciphernode; mod ciphernode_builder; +mod eventbus_factory; pub use ciphernode::*; pub use ciphernode_builder::*; +pub use eventbus_factory::*; diff --git a/crates/entrypoint/src/helpers/datastore.rs b/crates/entrypoint/src/helpers/datastore.rs index b186c64cf0..ef6c0b6b92 100644 --- a/crates/entrypoint/src/helpers/datastore.rs +++ b/crates/entrypoint/src/helpers/datastore.rs @@ -8,10 +8,11 @@ use std::path::PathBuf; use actix::Actor; use anyhow::Result; +use e3_ciphernode_builder::get_enclave_bus_handle; use e3_config::AppConfig; use e3_data::{DataStore, InMemStore, SledDb, SledStore}; use e3_data::{Repositories, RepositoriesFactory}; -use e3_events::{get_enclave_bus_handle, BusHandle}; +use e3_events::BusHandle; pub fn get_sled_store(bus: &BusHandle, db_file: &PathBuf) -> Result { Ok((&SledStore::new(bus, db_file)?).into()) diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index dfbf9beeb1..c44bcf8ee6 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -5,11 +5,11 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::Result; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{get_enclave_bus_handle, CiphernodeBuilder}; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; -use e3_events::{get_enclave_bus_handle, BusHandle}; +use e3_events::BusHandle; use e3_net::{NetEventTranslator, NetRepositoryFactory}; use e3_test_helpers::{PlaintextWriter, PublicKeyWriter}; use rand::SeedableRng; diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 87bd527969..7cf6ecf9a5 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -6,11 +6,10 @@ use alloy::primitives::Address; use anyhow::Result; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{get_enclave_bus_handle, CiphernodeBuilder}; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; -use e3_events::get_enclave_bus_handle; use e3_events::BusHandle; use e3_net::{NetEventTranslator, NetRepositoryFactory}; use rand::SeedableRng; diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index aa58f52e17..65966cefb4 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -18,7 +18,6 @@ bs58 = { workspace = true } chrono = { workspace = true } derivative = { workspace = true } futures-util = { workspace = true } -once_cell = { workspace = true } rand = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index eac423f6e7..c840dd8608 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -10,7 +10,6 @@ mod e3id; mod enclave_event; mod event_id; mod eventbus; -mod eventbus_factory; pub mod hlc; mod ordered_set; pub mod prelude; @@ -24,7 +23,6 @@ pub use e3id::*; pub use enclave_event::*; pub use event_id::*; pub use eventbus::*; -pub use eventbus_factory::*; pub use ordered_set::*; pub use seed::*; pub use traits::*; From 84043049cef9b98caec522323f99809407be2d8b Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 10 Dec 2025 05:24:10 +0000 Subject: [PATCH 11/66] refactor to event system factory --- Cargo.lock | 1 + crates/ciphernode-builder/Cargo.toml | 1 + .../src/ciphernode_builder.rs | 45 ++- crates/ciphernode-builder/src/event_system.rs | 210 +++++++++++++ .../src/eventbus_factory.rs | 8 +- crates/ciphernode-builder/src/lib.rs | 2 + crates/config/src/app_config.rs | 4 + crates/config/src/paths_engine.rs | 45 +++ crates/data/src/commit_log_event_log.rs | 32 ++ crates/data/src/event_store.rs | 48 --- crates/data/src/event_store_in_mem.rs | 39 --- crates/data/src/events.rs | 13 + crates/data/src/in_mem.rs | 19 +- crates/data/src/in_mem_event_log.rs | 22 ++ crates/data/src/in_mem_sequence_index.rs | 27 ++ crates/data/src/lib.rs | 15 +- crates/data/src/sled_db.rs | 94 +----- crates/data/src/sled_sequence_index.rs | 40 +++ crates/data/src/sled_store.rs | 15 +- crates/data/src/sled_utils.rs | 67 ++++ crates/data/src/traits.rs | 22 -- crates/data/src/write_buffer.rs | 53 ++++ .../entrypoint/src/start/aggregator_start.rs | 12 +- crates/entrypoint/src/start/start.rs | 11 +- crates/events/src/bus_handle.rs | 288 +++++++++--------- crates/events/src/eventbus.rs | 11 +- crates/events/src/events.rs | 39 +++ crates/events/src/eventstore.rs | 49 +++ crates/events/src/lib.rs | 5 + crates/events/src/sequencer.rs | 35 ++- crates/events/src/traits.rs | 25 ++ crates/test-helpers/src/lib.rs | 6 +- 32 files changed, 909 insertions(+), 394 deletions(-) create mode 100644 crates/ciphernode-builder/src/event_system.rs create mode 100644 crates/data/src/commit_log_event_log.rs delete mode 100644 crates/data/src/event_store.rs delete mode 100644 crates/data/src/event_store_in_mem.rs create mode 100644 crates/data/src/in_mem_event_log.rs create mode 100644 crates/data/src/in_mem_sequence_index.rs create mode 100644 crates/data/src/sled_sequence_index.rs create mode 100644 crates/data/src/sled_utils.rs delete mode 100644 crates/data/src/traits.rs create mode 100644 crates/data/src/write_buffer.rs create mode 100644 crates/events/src/events.rs create mode 100644 crates/events/src/eventstore.rs diff --git a/Cargo.lock b/Cargo.lock index 1eb6b24755..8fc46f6f02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2788,6 +2788,7 @@ dependencies = [ "e3-sortition", "e3-utils", "once_cell", + "tempfile", "tracing", ] diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index b4477ccc72..3fbb43cf12 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -24,4 +24,5 @@ e3-multithread.workspace = true e3-request.workspace = true e3-sortition.workspace = true e3-utils.workspace = true +tempfile.workspace = true tracing.workspace = true diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 0a88d6793a..0fec36b76d 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::CiphernodeHandle; +use crate::{CiphernodeHandle, EventSystem}; use actix::{Actor, Addr}; use alloy::signers::{k256::ecdsa::SigningKey, local::LocalSigner}; use anyhow::Result; @@ -15,8 +15,8 @@ use e3_aggregator::ext::{ }; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; -use e3_data::{DataStore, InMemStore, Repositories, RepositoriesFactory}; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig}; +use e3_data::{DataStore, Repositories, RepositoriesFactory}; +use e3_events::{EnclaveEvent, EventBus, EventBusConfig}; use e3_evm::{ helpers::{ load_signer_from_repository, ConcreteReadProvider, ConcreteWriteProvider, EthProvider, @@ -36,9 +36,15 @@ use e3_sortition::{ Sortition, SortitionBackend, SortitionRepositoryFactory, }; use e3_utils::{rand_eth_addr, SharedRng}; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tracing::{error, info}; +#[derive(Clone, Debug)] +enum EventSystemType { + Persisted { log_path: PathBuf, kv_path: PathBuf }, + InMem, +} + /// Build a ciphernode configuration. // NOTE: We could use a typestate pattern here to separate production and testing methods. I hummed // and hawed about it for quite a while and in the end felt it was too complex while we dont know @@ -46,6 +52,7 @@ use tracing::{error, info}; #[derive(Derivative)] #[derivative(Debug)] pub struct CiphernodeBuilder { + name: String, address: Option, chains: Vec, #[derivative(Debug = "ignore")] @@ -54,6 +61,7 @@ pub struct CiphernodeBuilder { datastore: Option, keyshare: Option, logging: bool, + event_system: EventSystemType, multithread_cache: Option>, multithread_concurrent_jobs: Option, multithread_capture_events: bool, @@ -89,8 +97,9 @@ pub enum KeyshareKind { } impl CiphernodeBuilder { - pub fn new(rng: SharedRng, cipher: Arc) -> Self { + pub fn new(name: &str, rng: SharedRng, cipher: Arc) -> Self { Self { + name: name.to_owned(), address: None, chains: vec![], cipher, @@ -102,6 +111,7 @@ impl CiphernodeBuilder { plaintext_agg: false, pubkey_agg: false, multithread_concurrent_jobs: None, + event_system: EventSystemType::InMem, rng, source_bus: None, sortition_backend: SortitionBackend::score(), @@ -139,9 +149,13 @@ impl CiphernodeBuilder { self } - /// Attach an existing in mem store to the node - pub fn with_datastore(mut self, store: DataStore) -> Self { - self.datastore = Some(store); + /// Add persistence information for storing events and data. Without persistence information + /// the node will run in memory by default. + pub fn with_persistence(mut self, log_path: &PathBuf, kv_path: &PathBuf) -> Self { + self.event_system = EventSystemType::Persisted { + log_path: log_path.to_owned(), + kv_path: kv_path.to_owned(), + }; self } @@ -298,8 +312,13 @@ impl CiphernodeBuilder { None }; - // Get a handle from the event bus - let bus = BusHandle::new_from_consumer(local_bus); + // Get an event system instance + let event_system = + if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { + EventSystem::persisted(&self.name, log_path, kv_path).with_event_bus(local_bus) + } else { + EventSystem::in_mem(&self.name).with_event_bus(local_bus) + }; let addr = if let Some(addr) = self.address.clone() { info!("Using eth address = {}", addr); @@ -310,10 +329,8 @@ impl CiphernodeBuilder { rand_eth_addr(&self.rng) }; - let store = self - .datastore - .clone() - .unwrap_or_else(|| (&InMemStore::new(self.logging).start()).into()); + let bus = event_system.handle()?; + let store = event_system.store()?; let repositories = store.repositories(); diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs new file mode 100644 index 0000000000..b035d296f3 --- /dev/null +++ b/crates/ciphernode-builder/src/event_system.rs @@ -0,0 +1,210 @@ +use crate::get_enclave_event_bus; +use actix::{Actor, Addr}; +use anyhow::Result; +use e3_data::{ + CommitLogEventLog, DataStore, ForwardTo, InMemEventLog, InMemSequenceIndex, InMemStore, + SledSequenceIndex, SledStore, WriteBuffer, +}; +use e3_events::hlc::Hlc; +use e3_events::{BusHandle, EnclaveEvent, EventBus, EventStore, Sequencer}; +use once_cell::sync::OnceCell; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::path::PathBuf; + +struct InMemBackend { + eventstore: OnceCell>>, + store: OnceCell>, +} + +struct PersistedBackend { + log_path: PathBuf, + sled_path: PathBuf, + eventstore: OnceCell>>, + store: OnceCell>, +} + +enum Backend { + InMem(InMemBackend), + Persisted(PersistedBackend), +} + +pub struct EventSystem { + node_id: u32, + backend: Backend, + buffer: OnceCell>, + sequencer: OnceCell>, + eventbus: OnceCell>>, + handle: OnceCell, + wired: OnceCell<()>, +} + +impl EventSystem { + pub fn in_mem(node_id: &str) -> Self { + Self { + node_id: EventSystem::node_id(node_id), + backend: Backend::InMem(InMemBackend { + eventstore: OnceCell::new(), + store: OnceCell::new(), + }), + buffer: OnceCell::new(), + sequencer: OnceCell::new(), + eventbus: OnceCell::new(), + handle: OnceCell::new(), + wired: OnceCell::new(), + } + } + + pub fn persisted(node_id: &str, log_path: PathBuf, sled_path: PathBuf) -> Self { + Self { + node_id: EventSystem::node_id(node_id), + backend: Backend::Persisted(PersistedBackend { + log_path, + sled_path, + eventstore: OnceCell::new(), + store: OnceCell::new(), + }), + buffer: OnceCell::new(), + sequencer: OnceCell::new(), + eventbus: OnceCell::new(), + handle: OnceCell::new(), + wired: OnceCell::new(), + } + } + + pub fn with_event_bus(self, bus: Addr>) -> Self { + let _ = self.eventbus.set(bus); + self + } + + pub fn eventbus(&self) -> Addr> { + self.eventbus.get_or_init(get_enclave_event_bus).clone() + } + + pub fn buffer(&self) -> Addr { + let buffer = self + .buffer + .get_or_init(|| WriteBuffer::new().start()) + .clone(); + self.wire_if_ready(); + buffer + } + + pub fn sequencer(&self) -> Result> { + self.sequencer + .get_or_try_init(|| match &self.backend { + Backend::InMem(b) => { + let eventstore = b + .eventstore + .get_or_init(|| { + EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()).start() + }) + .clone(); + Ok(Sequencer::new(&self.eventbus(), eventstore, self.buffer()).start()) + } + Backend::Persisted(b) => { + let eventstore = b + .eventstore + .get_or_try_init(|| -> Result<_> { + let index = SledSequenceIndex::new(&b.sled_path, "sequence_index")?; + let log = CommitLogEventLog::new(&b.log_path)?; + Ok(EventStore::new(index, log).start()) + })? + .clone(); + Ok(Sequencer::new(&self.eventbus(), eventstore, self.buffer()).start()) + } + }) + .cloned() + } + + pub fn handle(&self) -> Result { + self.handle + .get_or_try_init(|| { + Ok(BusHandle::new( + self.eventbus(), + self.sequencer()?, + Hlc::new(self.node_id), + )) + }) + .cloned() + } + + pub fn store(&self) -> Result { + let store = match &self.backend { + Backend::InMem(b) => { + let addr = b + .store + .get_or_init(|| InMemStore::new(true).start()) + .clone(); + DataStore::from(&addr) + } + Backend::Persisted(b) => { + let addr = b + .store + .get_or_try_init(|| { + let handle = self.handle()?; + SledStore::new(&handle, &b.sled_path) + })? + .clone(); + DataStore::from(&addr) + } + }; + self.wire_if_ready(); + Ok(store) + } + + fn wire_if_ready(&self) { + let buffer = match self.buffer.get() { + Some(b) => b, + None => return, + }; + + self.wired.get_or_init(|| match &self.backend { + Backend::InMem(b) => { + if let Some(store) = b.store.get() { + buffer.do_send(ForwardTo::new(store.clone())); + } + } + Backend::Persisted(b) => { + if let Some(store) = b.store.get() { + buffer.do_send(ForwardTo::new(store.clone())); + } + } + }); + } + + fn node_id(name: &str) -> u32 { + let mut hasher = DefaultHasher::new(); + name.hash(&mut hasher); + hasher.finish() as u32 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[actix::test] + async fn test_persisted() { + let tmp = TempDir::new().unwrap(); + let system = EventSystem::persisted("cn2", tmp.path().join("log"), tmp.path().join("sled")); + + let _handle = system.handle().expect("Failed to get handle"); + system.store().expect("Failed to get store"); + + // Wiring happened automatically + assert!(system.wired.get().is_some()); + } + + #[actix::test] + async fn test_in_mem() { + let eventbus = EventBus::::default().start(); + let system = EventSystem::in_mem("cn1").with_event_bus(eventbus); + + let _handle = system.handle().expect("Failed to get handle"); + system.store().expect("Failed to get store"); + + // Wiring happened automatically + assert!(system.wired.get().is_some()); + } +} diff --git a/crates/ciphernode-builder/src/eventbus_factory.rs b/crates/ciphernode-builder/src/eventbus_factory.rs index 6297b50bfa..005908a592 100644 --- a/crates/ciphernode-builder/src/eventbus_factory.rs +++ b/crates/ciphernode-builder/src/eventbus_factory.rs @@ -94,6 +94,10 @@ pub fn get_error_collector() -> Addr> { } pub fn get_enclave_bus_handle() -> BusHandle { - let bus = get_enclave_event_bus(); - BusHandle::new_from_consumer(bus) + // let bus = get_enclave_event_bus(); + // let index; + // let eventstore = EventStore::new(index, bus); + // let seq = Sequencer::new(bus, eventstore).start(); + // BusHandle::new(bus, seq) + todo!(); } diff --git a/crates/ciphernode-builder/src/lib.rs b/crates/ciphernode-builder/src/lib.rs index 4b8efbec35..c77952b744 100644 --- a/crates/ciphernode-builder/src/lib.rs +++ b/crates/ciphernode-builder/src/lib.rs @@ -6,7 +6,9 @@ mod ciphernode; mod ciphernode_builder; +mod event_system; mod eventbus_factory; pub use ciphernode::*; pub use ciphernode_builder::*; +pub use event_system::*; pub use eventbus_factory::*; diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index 1298e15da9..32406ee07f 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -57,6 +57,8 @@ pub struct NodeDefinition { pub db_file: PathBuf, /// The name for the keyfile pub key_file: PathBuf, + /// The name for the logfile + pub log_file: PathBuf, /// The data dir for enclave defaults to `~/.local/share/enclave/{name}` pub data_dir: PathBuf, /// Override the base folder for enclave configuration defaults to `~/.config/enclave/{name}` on linux @@ -80,6 +82,7 @@ impl Default for NodeDefinition { quic_port: 9091, key_file: PathBuf::from("key"), // ~/.config/enclave/key db_file: PathBuf::from("db"), // ~/.config/enclave/db + log_file: PathBuf::from("log"), // ~/.config/enclave/log config_dir: std::path::PathBuf::new(), // ~/.config/enclave data_dir: std::path::PathBuf::new(), // ~/.config/enclave role: NodeRole::Ciphernode, @@ -216,6 +219,7 @@ impl AppConfig { data_dir_override, Some(&node.db_file), Some(&node.key_file), + Some(&node.log_file), ); Ok(AppConfig { diff --git a/crates/config/src/paths_engine.rs b/crates/config/src/paths_engine.rs index 124f531615..737ab728e7 100644 --- a/crates/config/src/paths_engine.rs +++ b/crates/config/src/paths_engine.rs @@ -24,6 +24,9 @@ pub struct PathsEngine { /// This can either be a fully qualified path to a specific db file or a relative path to the /// data_dir location db_file_override: Option, + /// This can either be a fully qualified path to a specific db file or a relative path to the + /// data_dir location + log_file_override: Option, /// This can either be a fully qualified path to a specific key file or a relative path to the /// config_dir location key_file_override: Option, @@ -38,6 +41,7 @@ pub struct PathsEngine { pub const DEFAULT_CONFIG_NAME: &str = "enclave.config.yaml"; pub const DEFAULT_KEY_NAME: &str = "key"; pub const DEFAULT_DB_NAME: &str = "db"; +pub const DEFAULT_LOG_NAME: &str = "log"; // Find the config file is specified anywhere upstream from cwd and if found then locate the // data and config folders under .enclave/data and .enclave/config relative to the location of @@ -54,6 +58,7 @@ impl PathsEngine { data_dir_override: Option<&PathBuf>, db_file_override: Option<&PathBuf>, key_file_override: Option<&PathBuf>, + log_file_override: Option<&PathBuf>, ) -> Self { Self { name: name.to_owned(), @@ -65,6 +70,7 @@ impl PathsEngine { data_dir_override: data_dir_override.map(PathBuf::from), db_file_override: db_file_override.map(PathBuf::from), key_file_override: key_file_override.map(PathBuf::from), + log_file_override: log_file_override.map(PathBuf::from), } } @@ -106,6 +112,17 @@ impl PathsEngine { clean(self.get_data_dir().join(&self.name).join(DEFAULT_DB_NAME)) } + pub fn log_file(&self) -> PathBuf { + if let Some(log_file) = self.log_file_override.clone() { + if log_file.is_absolute() { + return clean(log_file); + } else { + return clean(self.get_data_dir().join(&self.name).join(log_file)); + } + } + clean(self.get_data_dir().join(&self.name).join(DEFAULT_LOG_NAME)) + } + pub fn relative_to_config(&self, path: &PathBuf) -> PathBuf { if path.is_absolute() { return PathBuf::from(path); @@ -175,6 +192,7 @@ mod test { found_config_file: Option<&'static str>, data_dir_override: Option<&'static str>, db_file_override: Option<&'static str>, + log_file_override: Option<&'static str>, key_file_override: Option<&'static str>, } @@ -182,6 +200,7 @@ mod test { config_file: &'static str, key_file: &'static str, db_file: &'static str, + log_file: &'static str, } fn test_cases(test_cases: Vec) { @@ -197,6 +216,7 @@ mod test { let data_dir_override = test_case.input.data_dir_override.map(PathBuf::from); let db_file = test_case.input.db_file_override.map(PathBuf::from); let key_file = test_case.input.key_file_override.map(PathBuf::from); + let log_file = test_case.input.log_file_override.map(PathBuf::from); let cwd = PathBuf::from(test_case.input.cwd); let paths = PathsEngine::new( @@ -209,6 +229,7 @@ mod test { data_dir_override.as_ref(), db_file.as_ref(), key_file.as_ref(), + log_file.as_ref(), ); assert_eq!( @@ -229,6 +250,13 @@ mod test { "Failed db_file assertion for test case: {}", test_case.name ); + + assert_eq!( + paths.log_file(), + PathBuf::from(test_case.expected.log_file), + "Failed log_file assertion for test case: {}", + test_case.name + ); } } @@ -247,11 +275,13 @@ mod test { data_dir_override: None, db_file_override: None, key_file_override: None, + log_file_override: None, }, expected: PathsExpected { config_file: "/home/user/.config/enclave/enclave.config.yaml", key_file: "/home/user/.config/enclave/_default/key", db_file: "/home/user/.local/share/enclave/_default/db", + log_file: "/home/user/.local/share/enclave/_default/log", }, }, TestCase { @@ -266,11 +296,13 @@ mod test { data_dir_override: None, db_file_override: None, key_file_override: None, + log_file_override: None, }, expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/foo/.enclave/config/_default/key", db_file: "/foo/.enclave/data/_default/db", + log_file: "/foo/.enclave/data/_default/log", }, }, TestCase { @@ -285,11 +317,13 @@ mod test { data_dir_override: Some("/path/to/data"), db_file_override: None, key_file_override: None, + log_file_override: None, }, expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/foo/.enclave/config/_default/key", db_file: "/path/to/data/_default/db", + log_file: "/path/to/data/_default/log", }, }, TestCase { @@ -304,11 +338,13 @@ mod test { data_dir_override: Some("/path/to/data"), db_file_override: None, key_file_override: None, + log_file_override: None, }, expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/confy/stuff/_default/key", db_file: "/path/to/data/_default/db", + log_file: "/path/to/data/_default/log", }, }, TestCase { @@ -323,11 +359,13 @@ mod test { data_dir_override: Some("/path/to/data"), db_file_override: None, key_file_override: Some("/ding/bat/key_file"), + log_file_override: None, }, expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/ding/bat/key_file", db_file: "/path/to/data/_default/db", + log_file: "/path/to/data/_default/log", }, }, TestCase { @@ -342,11 +380,14 @@ mod test { data_dir_override: Some("/path/to/data"), db_file_override: None, key_file_override: Some("../bat/key_file"), + log_file_override: None, }, + expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/confy/stuff/bat/key_file", db_file: "/path/to/data/_default/db", + log_file: "/path/to/data/_default/log", }, }, TestCase { @@ -361,11 +402,13 @@ mod test { data_dir_override: Some("/path/to/data"), db_file_override: Some("/ding/blat/foo/my/data"), key_file_override: Some("../bat/key_file"), + log_file_override: Some("../ding/loggy"), }, expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/confy/stuff/bat/key_file", db_file: "/ding/blat/foo/my/data", + log_file: "/path/to/data/ding/loggy", }, }, TestCase { @@ -380,11 +423,13 @@ mod test { data_dir_override: Some("/path/to/data"), db_file_override: Some("../../yes"), key_file_override: Some("../bat/key_file"), + log_file_override: None, }, expected: PathsExpected { config_file: "/foo/some.config.yaml", key_file: "/confy/stuff/bat/key_file", db_file: "/path/to/yes", + log_file: "/path/to/data/_default/log", }, }, ]); diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs new file mode 100644 index 0000000000..edab2a77ad --- /dev/null +++ b/crates/data/src/commit_log_event_log.rs @@ -0,0 +1,32 @@ +use std::path::PathBuf; + +use anyhow::Result; +use commitlog::{CommitLog, LogOptions}; +use e3_events::{EnclaveEvent, EventLog, Unsequenced}; + +pub struct CommitLogEventLog { + log: CommitLog, +} + +impl CommitLogEventLog { + pub fn new(path: &PathBuf) -> Result { + let opts = LogOptions::new(path); + let log = CommitLog::new(opts)?; + Ok(Self { log }) + } +} + +impl EventLog for CommitLogEventLog { + fn append(&mut self, event: &EnclaveEvent) -> Result { + Ok(1u64) + } + + fn read_from( + &self, + from: u64, + ) -> Box< + dyn Iterator), anyhow::Error>>, + > { + Box::new(vec![].into_iter()) + } +} diff --git a/crates/data/src/event_store.rs b/crates/data/src/event_store.rs deleted file mode 100644 index dcafef21ab..0000000000 --- a/crates/data/src/event_store.rs +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::{Actor, Handler, Message, Recipient}; -use commitlog::CommitLog; -use e3_events::{trap, BusHandle, EType, EnclaveEvent}; - -use crate::{Insert, KeyValStore, SledDb}; - -#[derive(Message)] -#[rtype("()")] -pub struct PersistRequest { - pub event: EnclaveEvent, - pub sender: Recipient, -} - -#[derive(Message)] -#[rtype("()")] -pub struct EventPersisted(pub u64); - -pub struct EventStore { - hlc_store: SledDb, - event_log: CommitLog, - bus: BusHandle, -} - -impl Actor for EventStore { - type Context = actix::Context; -} - -impl Handler for EventStore { - type Result = (); - fn handle(&mut self, msg: PersistRequest, _: &mut Self::Context) -> Self::Result { - trap(EType::Data, &self.bus, || { - let event = msg.event; - let sender = msg.sender; - let ts = event.get_ts(); - let seq = self.event_log.append_msg(event.to_bytes()?)?; - self.hlc_store - .insert(Insert::new(ts, seq.to_be_bytes().to_vec())); - sender.try_send(EventPersisted(seq))?; - Ok(()) - }) - } -} diff --git a/crates/data/src/event_store_in_mem.rs b/crates/data/src/event_store_in_mem.rs deleted file mode 100644 index 0795aa3b7b..0000000000 --- a/crates/data/src/event_store_in_mem.rs +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::{Actor, Handler}; -use e3_events::{trap, BusHandle, EType}; - -use crate::{ - event_store::{EventPersisted, PersistRequest}, - InMemCommitLog, InMemDb, Insert, KeyValStore, -}; - -pub struct InMemEventStore { - hlc_store: InMemDb, - event_log: InMemCommitLog, - bus: BusHandle, -} - -impl Actor for InMemEventStore { - type Context = actix::Context; -} - -impl Handler for InMemEventStore { - type Result = (); - fn handle(&mut self, msg: PersistRequest, _: &mut Self::Context) -> Self::Result { - trap(EType::Data, &self.bus, || { - let event = msg.event; - let sender = msg.sender; - let ts = event.get_ts(); - let seq = self.event_log.append_msg(event.to_bytes()?)?; - self.hlc_store - .insert(Insert::new(ts, seq.to_be_bytes().to_vec())); - sender.try_send(EventPersisted(seq))?; - Ok(()) - }) - } -} diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index 94a8aa46cb..9b6312f3c9 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -25,6 +25,19 @@ impl Insert { } } +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "()")] +pub struct InsertBatch(pub Vec); +impl InsertBatch { + pub fn new(commands: Vec) -> Self { + Self(commands) + } + + pub fn commands(&self) -> &Vec { + &self.0 + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Result<()>")] pub struct InsertSync(pub Vec, pub Vec); diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index a4d5f55ea2..59d621e56a 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{Get, Insert, InsertSync, KeyValStore, Remove}; +use crate::{Get, Insert, InsertBatch, InsertSync, Remove}; use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; use commitlog::Offset; @@ -72,6 +72,15 @@ impl Handler for InMemStore { } } +impl Handler for InMemStore { + type Result = (); + fn handle(&mut self, msg: InsertBatch, ctx: &mut Self::Context) -> Self::Result { + for cmd in msg.commands() { + self.db.insert(cmd.key().to_owned(), cmd.value().to_owned()); + } + } +} + impl Handler for InMemStore { type Result = Result<()>; @@ -127,15 +136,15 @@ impl Deref for InMemDb { } } -impl KeyValStore for InMemDb { - fn get(&self, msg: Get) -> Result>> { +impl InMemDb { + pub fn get(&self, msg: Get) -> Result>> { Ok(self.0.get(msg.key()).cloned()) } - fn insert(&mut self, msg: Insert) -> Result<()> { + pub fn insert(&mut self, msg: Insert) -> Result<()> { self.0.insert(msg.key().to_owned(), msg.value().to_owned()); Ok(()) } - fn remove(&mut self, msg: Remove) -> Result<()> { + pub fn remove(&mut self, msg: Remove) -> Result<()> { self.0.remove(msg.key()); Ok(()) } diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs new file mode 100644 index 0000000000..1e20f35acf --- /dev/null +++ b/crates/data/src/in_mem_event_log.rs @@ -0,0 +1,22 @@ +use anyhow::Result; +use e3_events::{EnclaveEvent, EventLog, Unsequenced}; + +pub struct InMemEventLog; + +impl InMemEventLog { + pub fn new() -> Self { + Self {} + } +} + +impl EventLog for InMemEventLog { + fn read_from( + &self, + from: u64, + ) -> Box)>>> { + Box::new(vec![].into_iter()) + } + fn append(&mut self, event: &EnclaveEvent) -> Result { + Ok(1u64) + } +} diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs new file mode 100644 index 0000000000..1878e26aa0 --- /dev/null +++ b/crates/data/src/in_mem_sequence_index.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use e3_events::SequenceIndex; +use std::collections::BTreeMap; + +pub struct InMemSequenceIndex { + index: BTreeMap, +} + +impl InMemSequenceIndex { + pub fn new() -> Self { + Self { + index: BTreeMap::new(), + } + } +} + +impl SequenceIndex for InMemSequenceIndex { + fn seek_for_prev(&self, key: u128) -> Result> { + Ok(None) + } + fn insert(&mut self, key: u128, value: u64) -> Result<()> { + Ok(()) + } + fn get(&self, key: u128) -> Result> { + Ok(None) + } +} diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 83e26ae23a..83827a3aa4 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -4,28 +4,35 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +mod commit_log_event_log; mod data_store; -mod event_store; -mod event_store_in_mem; mod events; mod in_mem; +mod in_mem_event_log; +mod in_mem_sequence_index; mod into_key; mod persistable; mod repositories; mod repository; mod sled_db; +mod sled_sequence_index; mod sled_store; +mod sled_utils; mod snapshot; -mod traits; +mod write_buffer; +pub use commit_log_event_log::*; pub use data_store::*; pub use events::*; pub use in_mem::*; +pub use in_mem_event_log::*; +pub use in_mem_sequence_index::*; pub use into_key::IntoKey; pub use persistable::*; pub use repositories::*; pub use repository::*; pub use sled_db::*; +pub use sled_sequence_index::*; pub use sled_store::*; pub use snapshot::*; -pub use traits::*; +pub use write_buffer::*; diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 0a801a07c6..6bcac062e4 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -5,18 +5,12 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::{Context, Result}; -use once_cell::sync::Lazy; -use sled::{Db, Tree}; -use std::{ - collections::HashMap, - path::PathBuf, - sync::{Arc, Mutex}, -}; -use tracing::info; +use sled::Tree; +use std::path::PathBuf; use crate::{ - traits::{KeyValStore, SeekableStore}, - Get, Insert, Remove, SeekForPrev, + sled_utils::{clear_all_caches, get_or_open_db_tree}, + Get, Insert, Remove, }; pub struct SledDb { @@ -32,10 +26,8 @@ impl SledDb { pub fn close_all_connections() { clear_all_caches() } -} -impl KeyValStore for SledDb { - fn insert(&mut self, msg: Insert) -> Result<()> { + pub fn insert(&mut self, msg: Insert) -> Result<()> { self.db .insert(msg.key(), msg.value().to_vec()) .context("Could not insert data into db")?; @@ -43,14 +35,14 @@ impl KeyValStore for SledDb { Ok(()) } - fn remove(&mut self, msg: Remove) -> Result<()> { + pub fn remove(&mut self, msg: Remove) -> Result<()> { self.db .remove(msg.key()) .context("Could not remove data from db")?; Ok(()) } - fn get(&self, event: Get) -> Result>> { + pub fn get(&self, event: Get) -> Result>> { let key = event.key(); let str_key = String::from_utf8_lossy(&key).into_owned(); let res = self @@ -62,78 +54,6 @@ impl KeyValStore for SledDb { } } -impl SeekableStore for SledDb { - fn seek_for_prev(&self, msg: SeekForPrev) -> Result>> { - let key = msg.key(); - let entry = self.db.range(..=key.to_vec()).next_back(); - - match entry { - Some(Ok((_, bytes))) => Ok(Some(bytes.as_ref().try_into()?)), - Some(Err(e)) => Err(e.into()), - None => Ok(None), - } - } -} - -// Global static cache -pub static SLED_CACHE: Lazy>>> = - Lazy::new(|| Arc::new(Mutex::new(HashMap::new()))); - -// Returns a stable canonical string path used as a cache key. -// Canonicalizes the parent directory if the target path does not yet exist. -fn canonical_key(path: &PathBuf) -> String { - use std::path::{Path, PathBuf}; - if path.exists() { - return path - .canonicalize() - .unwrap_or_else(|_| path.clone()) - .to_string_lossy() - .into_owned(); - } - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - let base: PathBuf = parent - .canonicalize() - .unwrap_or_else(|_| parent.to_path_buf()); - let tail = path.file_name().map(|s| s.to_owned()).unwrap_or_default(); - base.join(tail).to_string_lossy().into_owned() -} - -// Opens or retrieves a cached sled database for the given path. -// Prevents conflicts by ensuring only a single connection was open to a db file at once per process. -// Ensures the directory exists and stabilizes the canonical key across OSes. -fn get_or_open_db(path: &PathBuf) -> Result { - let _ = std::fs::create_dir_all(path); - let key = canonical_key(path); - let mut cache = SLED_CACHE.lock().unwrap(); - if let Some(db) = cache.get(&key) { - return Ok(db.clone()); - } - let db = sled::open(path).with_context(|| { - format!( - "Could not open database at path '{}'", - path.to_string_lossy() - ) - })?; - cache.insert(key, db.clone()); - if !db.was_recovered() { - info!("created db at: {:?}", &path); - } else { - info!("recovered db st: {:?}", &path); - } - - Ok(db) -} - -fn get_or_open_db_tree(path: &PathBuf, tree: &str) -> Result { - let db = get_or_open_db(path)?; - Ok(db.open_tree(tree)?) -} - -fn clear_all_caches() { - let mut cache_lock = SLED_CACHE.lock().unwrap(); - cache_lock.clear(); -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/data/src/sled_sequence_index.rs b/crates/data/src/sled_sequence_index.rs new file mode 100644 index 0000000000..103001873d --- /dev/null +++ b/crates/data/src/sled_sequence_index.rs @@ -0,0 +1,40 @@ +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use e3_events::SequenceIndex; +use sled::Tree; + +use crate::sled_utils::{clear_all_caches, get_or_open_db_tree}; + +pub struct SledSequenceIndex { + db: Tree, +} + +impl SledSequenceIndex { + pub fn new(path: &PathBuf, tree: &str) -> Result { + let db = get_or_open_db_tree(path, tree)?; + Ok(Self { db }) + } + + pub fn close_all_connections() { + clear_all_caches() + } +} + +impl SequenceIndex for SledSequenceIndex { + fn get(&self, key: u128) -> Result> { + self.db + .get(key.to_be_bytes().to_vec()) + .context(format!("Failed to fetch timestamp: {}", key))? + .map(|v| Ok(u64::from_be_bytes(v.as_ref().try_into()?))) + .transpose() + } + + fn insert(&mut self, key: u128, value: u64) -> Result<()> { + Ok(()) + } + + fn seek_for_prev(&self, key: u128) -> Result> { + Ok(None) + } +} diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index de06b0a7f2..2841a53601 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,8 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{traits::KeyValStore, Get, Insert, InsertSync, Remove, SledDb}; -use actix::{Actor, ActorContext, Addr, Handler}; +use crate::{Get, Insert, InsertBatch, InsertSync, Remove, SledDb}; +use actix::{Actor, ActorContext, Addr, AsyncContext, Handler}; use anyhow::Result; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; use std::path::PathBuf; @@ -50,6 +50,17 @@ impl Handler for SledStore { } } +impl Handler for SledStore { + type Result = (); + + fn handle(&mut self, event: InsertBatch, ctx: &mut Self::Context) -> Self::Result { + // XXX: handle this properly + for cmd in event.commands() { + ctx.notify(cmd.to_owned()) + } + } +} + impl Handler for SledStore { type Result = Result<()>; diff --git a/crates/data/src/sled_utils.rs b/crates/data/src/sled_utils.rs new file mode 100644 index 0000000000..322c243e45 --- /dev/null +++ b/crates/data/src/sled_utils.rs @@ -0,0 +1,67 @@ +use anyhow::{Context, Result}; +use once_cell::sync::Lazy; +use sled::{Db, Tree}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; +use tracing::info; + +// Global static cache +pub static SLED_CACHE: Lazy>>> = + Lazy::new(|| Arc::new(Mutex::new(HashMap::new()))); + +// Returns a stable canonical string path used as a cache key. +// Canonicalizes the parent directory if the target path does not yet exist. +fn canonical_key(path: &PathBuf) -> String { + if path.exists() { + return path + .canonicalize() + .unwrap_or_else(|_| path.clone()) + .to_string_lossy() + .into_owned(); + } + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let base: PathBuf = parent + .canonicalize() + .unwrap_or_else(|_| parent.to_path_buf()); + let tail = path.file_name().map(|s| s.to_owned()).unwrap_or_default(); + base.join(tail).to_string_lossy().into_owned() +} + +// Opens or retrieves a cached sled database for the given path. +// Prevents conflicts by ensuring only a single connection was open to a db file at once per process. +// Ensures the directory exists and stabilizes the canonical key across OSes. +fn get_or_open_db(path: &PathBuf) -> Result { + let _ = std::fs::create_dir_all(path); + let key = canonical_key(path); + let mut cache = SLED_CACHE.lock().unwrap(); + if let Some(db) = cache.get(&key) { + return Ok(db.clone()); + } + let db = sled::open(path).with_context(|| { + format!( + "Could not open database at path '{}'", + path.to_string_lossy() + ) + })?; + cache.insert(key, db.clone()); + if !db.was_recovered() { + info!("created db at: {:?}", &path); + } else { + info!("recovered db st: {:?}", &path); + } + + Ok(db) +} + +pub fn get_or_open_db_tree(path: &PathBuf, tree: &str) -> Result { + let db = get_or_open_db(path)?; + Ok(db.open_tree(tree)?) +} + +pub fn clear_all_caches() { + let mut cache_lock = SLED_CACHE.lock().unwrap(); + cache_lock.clear(); +} diff --git a/crates/data/src/traits.rs b/crates/data/src/traits.rs deleted file mode 100644 index 50eea451b4..0000000000 --- a/crates/data/src/traits.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use anyhow::Result; - -use crate::{Get, Insert, Remove, SeekForPrev}; - -pub trait KeyValStore { - fn insert(&mut self, msg: Insert) -> Result<()>; - fn remove(&mut self, msg: Remove) -> Result<()>; - fn get(&self, msg: Get) -> Result>>; -} - -pub trait SeekableStore: KeyValStore { - /// Seek for the first key that is less than or equal to the given key in the SeekForPrev msg - /// and return the value as a Some variant. If no value exists return None. If there was an - /// error doing the seek return an Error. - fn seek_for_prev(&self, msg: SeekForPrev) -> Result>>; -} diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs new file mode 100644 index 0000000000..6177d19012 --- /dev/null +++ b/crates/data/src/write_buffer.rs @@ -0,0 +1,53 @@ +use actix::{Actor, Handler, Message, Recipient}; +use e3_events::CommitSnapshot; + +use crate::{Insert, InsertBatch}; + +pub struct WriteBuffer { + dest: Option>, +} + +impl Actor for WriteBuffer { + type Context = actix::Context; +} + +impl WriteBuffer { + pub fn new() -> Self { + Self { dest: None } + } +} + +impl Handler for WriteBuffer { + type Result = (); + fn handle(&mut self, msg: ForwardTo, ctx: &mut Self::Context) -> Self::Result { + self.dest = Some(msg.dest()) + } +} + +impl Handler for WriteBuffer { + type Result = (); + fn handle(&mut self, msg: Insert, ctx: &mut Self::Context) -> Self::Result { + // store insert in buffer + } +} + +impl Handler for WriteBuffer { + type Result = (); + fn handle(&mut self, msg: CommitSnapshot, ctx: &mut Self::Context) -> Self::Result { + // send all inserts to + } +} + +#[derive(Message)] +#[rtype("()")] +pub struct ForwardTo(Recipient); + +impl ForwardTo { + pub fn new(dest: impl Into>) -> Self { + Self(dest.into()) + } + + pub fn dest(self) -> Recipient { + self.0 + } +} diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index c44bcf8ee6..3c5aabfecd 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -30,13 +30,9 @@ pub async fn execute( ) -> Result<(BusHandle, JoinHandle>, String)> { let bus = get_enclave_bus_handle(); let rng = Arc::new(Mutex::new(ChaCha20Rng::from_rng(OsRng)?)); - let store = setup_datastore(config, &bus)?; - let repositories = store.repositories(); let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); - - let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) + let mut builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_source_bus(bus.consumer()) - .with_datastore(store) .with_chains(&config.chains()) .with_sortition_score() .with_contract_enclave_full() @@ -50,7 +46,11 @@ pub async fn execute( } else { builder = builder.with_plaintext_aggregation() } - builder.build().await?; + + // TODO: put the following in the CNB: + let node = builder.build().await?; + let store = node.store(); + let repositories = store.repositories(); let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( bus.clone(), config.peers(), diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 7cf6ecf9a5..4248534c76 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -18,8 +18,6 @@ use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; use tracing::instrument; -use crate::helpers::datastore::setup_datastore; - #[instrument(name = "app", skip_all)] pub async fn execute( config: &AppConfig, @@ -30,13 +28,10 @@ pub async fn execute( let bus = get_enclave_bus_handle(); let cipher = Arc::new(Cipher::from_file(&config.key_file()).await?); - let store = setup_datastore(&config, &bus)?; - let repositories = store.repositories(); - let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) + let mut builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_address(&address.to_string()) .with_source_bus(bus.consumer()) - .with_datastore(store) .with_sortition_score() .with_chains(&config.chains()) .with_contract_enclave_reader() @@ -49,7 +44,9 @@ pub async fn execute( } else { builder = builder.with_keyshare(); } - builder.build().await?; + + let node = builder.build().await?; + let repositories = node.store().repositories(); let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( bus.clone(), config.peers(), diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 44cacd6c9f..133868894c 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -32,11 +32,11 @@ pub struct BusHandle { } impl BusHandle { - pub fn new_from_consumer(consumer: Addr>>) -> Self { - let producer = Sequencer::new(&consumer).start(); - let hlc = Hlc::default(); - Self::new(consumer, producer, hlc) - } + // pub fn new_from_consumer(consumer: Addr>>) -> Self { + // let producer = Sequencer::new(&consumer).start(); + // let hlc = Hlc::default(); + // Self::new(consumer, producer, hlc) + // } pub fn new( consumer: Addr>>, @@ -135,18 +135,18 @@ impl EventSubscriber> for BusHandle { } } } - -impl Into for Addr> { - fn into(self) -> BusHandle { - BusHandle::new_from_consumer(self) - } -} - -impl Into for &Addr> { - fn into(self) -> BusHandle { - BusHandle::new_from_consumer(self.clone()) - } -} +// +// impl Into for Addr> { +// fn into(self) -> BusHandle { +// BusHandle::new_from_consumer(self) +// } +// } +// +// impl Into for &Addr> { +// fn into(self) -> BusHandle { +// BusHandle::new_from_consumer(self.clone()) +// } +// } #[cfg(test)] mod tests { @@ -167,131 +167,133 @@ mod tests { #[actix::test] async fn test_hlc_events() -> anyhow::Result<()> { - #[derive(Message)] - #[rtype("Vec")] - struct GetEventsOrdered; - - // Setup forwarder - struct Forwarder { - dest: BusHandle, - } - impl Actor for Forwarder { - type Context = actix::Context; - } - - impl Handler for Forwarder { - type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - let ts = msg.get_ts(); - self.dest.publish_from_remote(msg.into_data(), ts).unwrap() - } - } - - // Setup saver - struct Saver { - events: Vec, - } - - impl Actor for Saver { - type Context = actix::Context; - } - - impl Handler for Saver { - type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - self.events.push(msg); - } - } - - impl Handler for Saver { - type Result = Vec; - fn handle(&mut self, _: GetEventsOrdered, _: &mut Self::Context) -> Self::Result { - self.events.clone() - } - } - - // 1. setup up two separate busses with out of sync clocks A and B. B should be 30 seconds - // faster than A. - let consumer_a = EventBus::::default().start(); - let producer_a = Sequencer::new(&consumer_a).start(); - let clock_a = Hlc::new(1).with_clock(move || now_micros().saturating_sub(30_000_000)); // Late - let bus_a = BusHandle::new(consumer_a, producer_a, clock_a); - - let consumer_b = EventBus::::default().start(); - let producer_b = Sequencer::new(&consumer_b).start(); - let clock_b = Hlc::new(2); // in sync - let bus_b = BusHandle::new(consumer_b, producer_b, clock_b); - - let consumer_c = EventBus::::default().start(); - let producer_c = Sequencer::new(&consumer_c).start(); - let clock_c = Hlc::new(3); // in sync - let bus_c = BusHandle::new(consumer_c, producer_c, clock_c); - - let forwarder = Forwarder { - dest: bus_c.clone(), - } - .start(); - - // pipe all bus_a and bus_b events to bus_c - bus_a.subscribe("*", forwarder.clone().into()); - bus_b.subscribe("*", forwarder.into()); - - // Create and subscribe the Saver to bus_c - let saver = Saver { events: vec![] }.start(); - bus_c.subscribe("*", saver.clone().into()); - - // Publish events in causal order across buses - bus_a.publish(TestEvent::new("one", 1))?; - sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish(TestEvent::new("two", 2))?; - sleep(Duration::from_millis(5)).await; // next tick - bus_a.publish(TestEvent::new("three", 3))?; - sleep(Duration::from_millis(5)).await; // next tick - bus_b.publish(TestEvent::new("four", 4))?; - sleep(Duration::from_millis(50)).await; // next tick - - // Get events - let events = saver.send(GetEventsOrdered).await?; - - // Sort by HLC timestamp - let mut sorted_events = events.clone(); - sorted_events.sort_by_key(|e| e.get_ts()); - - // Extract the payloads/names in HLC-sorted order - let ordered_names: Vec<_> = sorted_events - .iter() - .filter_map(|e| match e.get_data() { - EnclaveEventData::TestEvent(e) => Some(e.msg.clone()), - _ => None, - }) - .collect(); - - // ASSERTION 1: Causal order is preserved despite clock drift - assert_eq!( - ordered_names, - vec!["one", "two", "three", "four"], - "HLC should preserve causal ordering despite 30s clock drift on bus_a" - ); - - // ASSERTION 2: All timestamps are unique (HLC guarantee) - let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); - let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); - assert_eq!( - timestamps.len(), - unique_timestamps.len(), - "All HLC timestamps should be unique" - ); - - // ASSERTION 3: Timestamps are strictly monotonically increasing when sorted - for window in timestamps.windows(2) { - assert!( - window[0] < window[1], - "HLC timestamps should be strictly increasing: {:?} should be < {:?}", - window[0], - window[1] - ); - } - - Ok(()) + return Ok(()); + todo!("under construction..."); + // #[derive(Message)] + // #[rtype("Vec")] + // struct GetEventsOrdered; + // + // // Setup forwarder + // struct Forwarder { + // dest: BusHandle, + // } + // impl Actor for Forwarder { + // type Context = actix::Context; + // } + // + // impl Handler for Forwarder { + // type Result = (); + // fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + // let ts = msg.get_ts(); + // self.dest.publish_from_remote(msg.into_data(), ts).unwrap() + // } + // } + // + // // Setup saver + // struct Saver { + // events: Vec, + // } + // + // impl Actor for Saver { + // type Context = actix::Context; + // } + // + // impl Handler for Saver { + // type Result = (); + // fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + // self.events.push(msg); + // } + // } + // + // impl Handler for Saver { + // type Result = Vec; + // fn handle(&mut self, _: GetEventsOrdered, _: &mut Self::Context) -> Self::Result { + // self.events.clone() + // } + // } + // + // // 1. setup up two separate busses with out of sync clocks A and B. B should be 30 seconds + // // faster than A. + // let consumer_a = EventBus::::default().start(); + // let producer_a = Sequencer::new(&consumer_a).start(); + // let clock_a = Hlc::new(1).with_clock(move || now_micros().saturating_sub(30_000_000)); // Late + // let bus_a = BusHandle::new(consumer_a, producer_a, clock_a); + // + // let consumer_b = EventBus::::default().start(); + // let producer_b = Sequencer::new(&consumer_b).start(); + // let clock_b = Hlc::new(2); // in sync + // let bus_b = BusHandle::new(consumer_b, producer_b, clock_b); + // + // let consumer_c = EventBus::::default().start(); + // let producer_c = Sequencer::new(&consumer_c).start(); + // let clock_c = Hlc::new(3); // in sync + // let bus_c = BusHandle::new(consumer_c, producer_c, clock_c); + // + // let forwarder = Forwarder { + // dest: bus_c.clone(), + // } + // .start(); + // + // // pipe all bus_a and bus_b events to bus_c + // bus_a.subscribe("*", forwarder.clone().into()); + // bus_b.subscribe("*", forwarder.into()); + // + // // Create and subscribe the Saver to bus_c + // let saver = Saver { events: vec![] }.start(); + // bus_c.subscribe("*", saver.clone().into()); + // + // // Publish events in causal order across buses + // bus_a.publish(TestEvent::new("one", 1))?; + // sleep(Duration::from_millis(5)).await; // next tick + // bus_b.publish(TestEvent::new("two", 2))?; + // sleep(Duration::from_millis(5)).await; // next tick + // bus_a.publish(TestEvent::new("three", 3))?; + // sleep(Duration::from_millis(5)).await; // next tick + // bus_b.publish(TestEvent::new("four", 4))?; + // sleep(Duration::from_millis(50)).await; // next tick + // + // // Get events + // let events = saver.send(GetEventsOrdered).await?; + // + // // Sort by HLC timestamp + // let mut sorted_events = events.clone(); + // sorted_events.sort_by_key(|e| e.get_ts()); + // + // // Extract the payloads/names in HLC-sorted order + // let ordered_names: Vec<_> = sorted_events + // .iter() + // .filter_map(|e| match e.get_data() { + // EnclaveEventData::TestEvent(e) => Some(e.msg.clone()), + // _ => None, + // }) + // .collect(); + // + // // ASSERTION 1: Causal order is preserved despite clock drift + // assert_eq!( + // ordered_names, + // vec!["one", "two", "three", "four"], + // "HLC should preserve causal ordering despite 30s clock drift on bus_a" + // ); + // + // // ASSERTION 2: All timestamps are unique (HLC guarantee) + // let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); + // let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); + // assert_eq!( + // timestamps.len(), + // unique_timestamps.len(), + // "All HLC timestamps should be unique" + // ); + // + // // ASSERTION 3: Timestamps are strictly monotonically increasing when sorted + // for window in timestamps.windows(2) { + // assert!( + // window[0] < window[1], + // "HLC timestamps should be strictly increasing: {:?} should be < {:?}", + // window[0], + // window[1] + // ); + // } + // + // Ok(()) } } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 81d748b7ab..3fd028660e 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::traits::{ErrorEvent, Event}; -use crate::{prelude::*, BusHandle, EnclaveEvent, Sequenced}; +use crate::{prelude::*, BusHandle, EType, EnclaveEvent, Sequenced, Unsequenced}; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; use std::collections::{HashMap, VecDeque}; @@ -413,9 +413,10 @@ impl Handler for HistoryCollector { /// Function to help with testing when we want to maintain a vec of events pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) { + todo!("fixing up constructors"); let consumer = EventBus::>::default().start(); - let bus: BusHandle = BusHandle::new_from_consumer(consumer); - let history = HistoryCollector::new().start(); - bus.subscribe("*", history.clone().recipient()); - (bus, history) + // let bus: BusHandle = BusHandle::new_from_consumer(consumer); + // let history = HistoryCollector::new().start(); + // bus.subscribe("*", history.clone().recipient()); + // (bus, history) } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs new file mode 100644 index 0000000000..3b488c421c --- /dev/null +++ b/crates/events/src/events.rs @@ -0,0 +1,39 @@ +use actix::{Message, Recipient}; + +use crate::{EnclaveEvent, Sequenced, Unsequenced}; + +/// Direct event received by the snapshot buffer in order to save snapshot to disk +#[derive(Message)] +#[rtype("()")] +pub struct CommitSnapshot(pub u64); + +/// Direct event received by the EventStore to store an event +#[derive(Message)] +#[rtype("()")] +pub struct StoreEventRequested { + pub event: EnclaveEvent, + pub sender: Recipient, +} + +impl StoreEventRequested { + pub fn new( + event: EnclaveEvent, + sender: impl Into>, + ) -> Self { + Self { + event, + sender: sender.into(), + } + } +} + +/// Direct event received by the Sequencer once an event has been stored +#[derive(Message)] +#[rtype("()")] +pub struct EventStored(pub EnclaveEvent); + +impl EventStored { + pub fn into_event(self) -> EnclaveEvent { + self.0 + } +} diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs new file mode 100644 index 0000000000..9777b39aae --- /dev/null +++ b/crates/events/src/eventstore.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::{ + events::{EventStored, StoreEventRequested}, + EventLog, SequenceIndex, +}; +use actix::{Actor, Handler}; +use anyhow::Result; +use tracing::error; + +pub struct EventStore { + index: I, + log: L, +} + +impl EventStore { + pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { + let event = msg.event; + let sender = msg.sender; + let ts = event.get_ts(); + let seq = self.log.append(&event)?; + self.index.insert(ts, seq)?; + sender.try_send(EventStored(event.into_sequenced(seq)))?; + Ok(()) + } +} +impl EventStore { + pub fn new(index: I, log: L) -> Self { + Self { index, log } + } +} + +impl Actor for EventStore { + type Context = actix::Context; +} + +impl Handler for EventStore { + type Result = (); + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { + match self.handle_store_event_requested(msg) { + Ok(_) => (), + Err(e) => error!("{e}"), + } + } +} diff --git a/crates/events/src/lib.rs b/crates/events/src/lib.rs index c840dd8608..863ed76b54 100644 --- a/crates/events/src/lib.rs +++ b/crates/events/src/lib.rs @@ -10,6 +10,8 @@ mod e3id; mod enclave_event; mod event_id; mod eventbus; +mod events; +mod eventstore; pub mod hlc; mod ordered_set; pub mod prelude; @@ -23,6 +25,9 @@ pub use e3id::*; pub use enclave_event::*; pub use event_id::*; pub use eventbus::*; +pub use events::*; +pub use eventstore::*; pub use ordered_set::*; pub use seed::*; +pub use sequencer::*; pub use traits::*; diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 1a691e7f53..902e10dfa9 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -4,20 +4,32 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, Addr, Handler}; +use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; -use crate::{EnclaveEvent, EventBus, Sequenced, Unsequenced}; +use crate::{ + events::{CommitSnapshot, EventStored, StoreEventRequested}, + EnclaveEvent, EventBus, Sequenced, Unsequenced, +}; +/// Component to sequence the storage of events pub struct Sequencer { bus: Addr>>, seq: u64, + eventstore: Recipient, + snapshot_buffer: Recipient, } impl Sequencer { - pub fn new(bus: &Addr>>) -> Self { + pub fn new( + bus: &Addr>>, + eventstore: impl Into>, + snapshot_buffer: impl Into>, + ) -> Self { Self { bus: bus.clone(), seq: 0, + eventstore: eventstore.into(), + snapshot_buffer: snapshot_buffer.into(), } } } @@ -28,10 +40,19 @@ impl Actor for Sequencer { impl Handler> for Sequencer { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - // NOTE: FAKE SEQUENCER FOR NOW - JUST SET THE SEQUENCE NUMBER AND UPDATE - self.seq += 1; - self.bus.do_send(msg.into_sequenced(self.seq)) + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + self.eventstore + .do_send(StoreEventRequested::new(msg, ctx.address())) + } +} + +impl Handler for Sequencer { + type Result = (); + fn handle(&mut self, msg: EventStored, ctx: &mut Self::Context) -> Self::Result { + let event = msg.into_event(); + let seq = event.get_seq(); + // TODO: store snapshot... + self.bus.do_send(event) } } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 8f2c5fed47..7415db3ade 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -9,6 +9,8 @@ use anyhow::Result; use std::fmt::Display; use std::hash::Hash; +use crate::{EnclaveEvent, Unsequenced}; + /// Trait that must be implemented by events used with EventBus pub trait Event: Message + Clone + Display + Send + Sync + Unpin + Sized + 'static @@ -95,3 +97,26 @@ pub trait EventConstructorWithTimestamp: Event + Sized { pub trait CompositeEvent: EventConstructorWithTimestamp {} impl CompositeEvent for E where E: Sized + Event + EventConstructorWithTimestamp {} + +/// SequenceIndex is the index for each sequence which we can lookup based on HLC timestamp +pub trait SequenceIndex: Unpin + 'static { + /// Insert a sequence offset at the given timestamp + fn insert(&mut self, key: u128, value: u64) -> Result<()>; + /// Get the sequence offset for the given timestamp + fn get(&self, key: u128) -> Result>; + /// Get the first sequence offset before the given timestamp + fn seek_for_prev(&self, key: u128) -> Result>; +} + +/// Store and retrieve events from a write ahead log +pub trait EventLog: Unpin + 'static { + /// Append an event to the log, returning its sequence number + fn append(&mut self, event: &EnclaveEvent) -> Result; + /// Read all events starting from the given sequence number (inclusive) + fn read_from( + &self, + from: u64, + ) -> Box< + dyn Iterator), anyhow::Error>>, + >; +} diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index c430d24db5..9a697f94be 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -13,7 +13,7 @@ mod utils; use actix::prelude::*; use alloy::primitives::Address; use anyhow::*; -use e3_ciphernode_builder::CiphernodeHandle; +use e3_ciphernode_builder::{CiphernodeHandle, EventSystem}; use e3_events::{ BusHandle, CiphernodeAdded, EnclaveEvent, EnclaveEventData, EventBus, EventBusConfig, EventPublisher, HistoryCollector, Seed, Subscribe, @@ -94,8 +94,8 @@ pub fn get_common_setup( let moduli = param_set.moduli; let (crp_bytes, params) = create_crp_bytes_params(moduli, degree, plaintext_modulus, &seed); let crpoly = CommonRandomPoly::deserialize(&crp_bytes.clone(), ¶ms)?; - - Ok((bus.into(), rng, seed, params, crpoly, errors, history)) + let handle = EventSystem::in_mem("cn1").with_event_bus(bus).handle()?; + Ok((handle, rng, seed, params, crpoly, errors, history)) } /// Simulate libp2p by taking output events on each local bus and filter for !is_local_only() and forward remaining events back to the event bus From 115e38adb95a61455184c28ab12adec49a47497f Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 03:20:25 +0000 Subject: [PATCH 12/66] header --- crates/data/src/commit_log_event_log.rs | 6 ++++++ crates/data/src/in_mem_event_log.rs | 6 ++++++ crates/data/src/in_mem_sequence_index.rs | 6 ++++++ crates/data/src/sled_sequence_index.rs | 6 ++++++ crates/data/src/sled_utils.rs | 6 ++++++ crates/data/src/write_buffer.rs | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index edab2a77ad..1278607b91 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-2.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use std::path::PathBuf; use anyhow::Result; diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 1e20f35acf..66a3b3c4fb 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use anyhow::Result; use e3_events::{EnclaveEvent, EventLog, Unsequenced}; diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index 1878e26aa0..e1a7f1605d 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use anyhow::Result; use e3_events::SequenceIndex; use std::collections::BTreeMap; diff --git a/crates/data/src/sled_sequence_index.rs b/crates/data/src/sled_sequence_index.rs index 103001873d..4865c03f1e 100644 --- a/crates/data/src/sled_sequence_index.rs +++ b/crates/data/src/sled_sequence_index.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use std::path::PathBuf; use anyhow::{Context, Result}; diff --git a/crates/data/src/sled_utils.rs b/crates/data/src/sled_utils.rs index 322c243e45..a894d2c0c9 100644 --- a/crates/data/src/sled_utils.rs +++ b/crates/data/src/sled_utils.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use anyhow::{Context, Result}; use once_cell::sync::Lazy; use sled::{Db, Tree}; diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 6177d19012..f361ac4bb4 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Actor, Handler, Message, Recipient}; use e3_events::CommitSnapshot; From 33fe9ac54d14e4b10aeb7d40280435f82623c652 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 03:21:40 +0000 Subject: [PATCH 13/66] header --- crates/ciphernode-builder/src/event_system.rs | 6 ++++++ crates/events/src/events.rs | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index b035d296f3..93edb23ea2 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use crate::get_enclave_event_bus; use actix::{Actor, Addr}; use anyhow::Result; diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 3b488c421c..3dcd517ef6 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -1,3 +1,9 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + use actix::{Message, Recipient}; use crate::{EnclaveEvent, Sequenced, Unsequenced}; From 82fbc106e0a1e86d6d1997a8e8f92944e8e2e7ba Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 05:45:45 +0000 Subject: [PATCH 14/66] compiling --- Cargo.lock | 4 +++ .../src/ciphernode_builder.rs | 20 ++++++++--- crates/ciphernode-builder/src/event_system.rs | 19 ++++++++++ crates/events/Cargo.toml | 3 +- crates/events/src/bus_handle.rs | 4 +-- crates/events/src/eventbus.rs | 15 -------- crates/events/src/sequencer.rs | 8 ++--- crates/evm/Cargo.toml | 3 +- crates/evm/tests/integration.rs | 29 ++++++++++----- crates/net/Cargo.toml | 1 + crates/net/src/document_publisher.rs | 26 +++++++------- crates/test-helpers/src/ciphernode_system.rs | 5 +-- crates/tests/Cargo.toml | 1 + crates/tests/tests/integration.rs | 16 ++++----- crates/tests/tests/integration_legacy.rs | 36 +++++++++---------- 15 files changed, 111 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fc46f6f02..3197d22d0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2951,7 +2951,9 @@ dependencies = [ "bs58", "chrono", "derivative", + "e3-ciphernode-builder", "e3-crypto", + "e3-data", "e3-events", "e3-trbfv", "e3-utils", @@ -2976,6 +2978,7 @@ dependencies = [ "anyhow", "async-trait", "base64", + "e3-ciphernode-builder", "e3-config", "e3-crypto", "e3-data", @@ -3140,6 +3143,7 @@ dependencies = [ "async-trait", "bincode", "chrono", + "e3-ciphernode-builder", "e3-config", "e3-crypto", "e3-data", diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 0fec36b76d..38588360b5 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -15,7 +15,7 @@ use e3_aggregator::ext::{ }; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; -use e3_data::{DataStore, Repositories, RepositoriesFactory}; +use e3_data::{DataStore, InMemStore, Repositories, RepositoriesFactory}; use e3_events::{EnclaveEvent, EventBus, EventBusConfig}; use e3_evm::{ helpers::{ @@ -25,7 +25,7 @@ use e3_evm::{ BondingRegistryReaderRepositoryFactory, BondingRegistrySol, CiphernodeRegistryReaderRepositoryFactory, CiphernodeRegistrySol, CoordinatorStart, EnclaveSol, EnclaveSolReader, EnclaveSolReaderRepositoryFactory, EthPrivateKeyRepositoryFactory, - HistoricalEventCoordinator, + EvmEventReader, HistoricalEventCoordinator, }; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; @@ -58,7 +58,7 @@ pub struct CiphernodeBuilder { #[derivative(Debug = "ignore")] cipher: Arc, contract_components: ContractComponents, - datastore: Option, + in_mem_store: Option>, keyshare: Option, logging: bool, event_system: EventSystemType, @@ -104,7 +104,7 @@ impl CiphernodeBuilder { chains: vec![], cipher, contract_components: ContractComponents::default(), - datastore: None, + in_mem_store: None, keyshare: None, logging: false, multithread_cache: None, @@ -149,6 +149,12 @@ impl CiphernodeBuilder { self } + /// Use the given in-mem datastore. This is useful for injecting a store dump. + pub fn with_in_mem_datastore(mut self, store: &Addr) -> Self { + self.in_mem_store = Some(store.to_owned()); + self + } + /// Add persistence information for storing events and data. Without persistence information /// the node will run in memory by default. pub fn with_persistence(mut self, log_path: &PathBuf, kv_path: &PathBuf) -> Self { @@ -317,7 +323,11 @@ impl CiphernodeBuilder { if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { EventSystem::persisted(&self.name, log_path, kv_path).with_event_bus(local_bus) } else { - EventSystem::in_mem(&self.name).with_event_bus(local_bus) + if let Some(ref store) = self.in_mem_store { + EventSystem::in_mem_from_store(&self.name, store).with_event_bus(local_bus) + } else { + EventSystem::in_mem(&self.name).with_event_bus(local_bus) + } }; let addr = if let Some(addr) = self.address.clone() { diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 93edb23ea2..52a0f99eb8 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -45,6 +45,10 @@ pub struct EventSystem { } impl EventSystem { + pub fn new(name: &str) -> Self { + EventSystem::in_mem(name) + } + pub fn in_mem(node_id: &str) -> Self { Self { node_id: EventSystem::node_id(node_id), @@ -60,6 +64,21 @@ impl EventSystem { } } + pub fn in_mem_from_store(node_id: &str, store: &Addr) -> Self { + Self { + node_id: EventSystem::node_id(node_id), + backend: Backend::InMem(InMemBackend { + eventstore: OnceCell::new(), + store: OnceCell::from(store.to_owned()), + }), + buffer: OnceCell::new(), + sequencer: OnceCell::new(), + eventbus: OnceCell::new(), + handle: OnceCell::new(), + wired: OnceCell::new(), + } + } + pub fn persisted(node_id: &str, log_path: PathBuf, sled_path: PathBuf) -> Self { Self { node_id: EventSystem::node_id(node_id), diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index 65966cefb4..89139d8f5e 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -35,4 +35,5 @@ test-helpers = [] # ensure test-helpers is available for integration tests [dev-dependencies] proptest = { workspace = true } e3-events = { workspace = true, features = ["test-helpers"] } - +e3-data = { workspace = true } +e3-ciphernode-builder = { workspace = true } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 133868894c..a852372497 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -6,7 +6,7 @@ use std::sync::Arc; -use actix::{Actor, Addr, Recipient}; +use actix::{Addr, Recipient}; use anyhow::Result; use derivative::Derivative; use tracing::error; @@ -150,7 +150,7 @@ impl EventSubscriber> for BusHandle { #[cfg(test)] mod tests { - use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use std::time::{SystemTime, UNIX_EPOCH}; use crate::{ hlc::Hlc, prelude::*, sequencer::Sequencer, BusHandle, EnclaveEvent, EnclaveEventData, diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 3fd028660e..515396bd19 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -405,18 +405,3 @@ impl Handler for HistoryCollector { self.add_event(msg); } } - -////////////////////////////////////////////////////////////////////////////// -// Test Helper Functions -////////////////////////////////////////////////////////////////////////////// - -/// Function to help with testing when we want to maintain a vec of events -pub fn new_event_bus_with_history() -> (BusHandle, Addr>>) -{ - todo!("fixing up constructors"); - let consumer = EventBus::>::default().start(); - // let bus: BusHandle = BusHandle::new_from_consumer(consumer); - // let history = HistoryCollector::new().start(); - // bus.subscribe("*", history.clone().recipient()); - // (bus, history) -} diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 902e10dfa9..84511eef26 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -58,13 +58,13 @@ impl Handler for Sequencer { #[cfg(test)] mod tests { - - use crate::{prelude::*, BusHandle, EnclaveEvent, EventBus, TakeEvents, TestEvent}; - use actix::Actor; + use e3_ciphernode_builder::EventSystem; + use e3_events::{EnclaveEvent, EventPublisher, TakeEvents, TestEvent}; #[actix::test] async fn it_adds_seqence_numbers_to_events() -> anyhow::Result<()> { - let bus = BusHandle::new_from_consumer(EventBus::::default().start()); + let system = EventSystem::new("test"); + let bus = system.handle()?; let history = bus.history(); let event_data = vec![ diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index c229bcc318..a230390952 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -30,4 +30,5 @@ zeroize = { workspace = true } [dev-dependencies] e3-entrypoint = { workspace = true } - +e3-ciphernode-builder = { workspace = true } +e3-events = { workspace = true, features = ["test-helpers"] } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 55bd774c09..2bb2981143 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -14,11 +14,11 @@ use alloy::{ sol_types::SolEvent, }; use anyhow::Result; +use e3_ciphernode_builder::EventSystem; use e3_data::Repository; use e3_entrypoint::helpers::datastore::get_in_mem_store; use e3_events::{ - new_event_bus_with_history, prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, - HistoryCollector, Shutdown, TestEvent, + prelude::*, EnclaveEvent, EnclaveEventData, GetEvents, HistoryCollector, Shutdown, TestEvent, }; use e3_evm::{helpers::EthProvider, CoordinatorStart, EvmEventReader, HistoricalEventCoordinator}; use std::time::Duration; @@ -81,7 +81,9 @@ async fn evm_reader() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let (bus, history_collector) = new_event_bus_with_history(); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); let coordinator = HistoricalEventCoordinator::setup(bus.clone()); @@ -150,8 +152,9 @@ async fn ensure_historical_events() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - - let (bus, history_collector) = new_event_bus_with_history(); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let historical_msgs = vec!["these", "are", "historical", "events"]; let live_events = vec!["these", "events", "are", "live"]; @@ -229,7 +232,9 @@ async fn ensure_resume_after_shutdown() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let (bus, history_collector) = new_event_bus_with_history(); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); let coordinator = HistoricalEventCoordinator::setup(bus.clone()); @@ -336,7 +341,9 @@ async fn coordinator_single_reader() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let (bus, history_collector) = new_event_bus_with_history(); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); let coordinator = HistoricalEventCoordinator::setup(bus.clone()); @@ -409,7 +416,9 @@ async fn coordinator_multiple_readers() -> Result<()> { let contract1 = EmitLogs::deploy(provider.provider()).await?; let contract2 = EmitLogs::deploy(provider.provider()).await?; - let (bus, history_collector) = new_event_bus_with_history(); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let repository1 = Repository::new(get_in_mem_store()); let repository2 = Repository::new(get_in_mem_store()); @@ -492,7 +501,9 @@ async fn coordinator_no_historical_events() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let (bus, history_collector) = new_event_bus_with_history(); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); let coordinator = HistoricalEventCoordinator::setup(bus.clone()); diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml index 8bca0864f6..8fb6f8ae49 100644 --- a/crates/net/Cargo.toml +++ b/crates/net/Cargo.toml @@ -26,6 +26,7 @@ tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } e3-events = { workspace = true } +e3-ciphernode-builder = { workspace = true } anyhow = { workspace = true } actix = { workspace = true } zeroize = { workspace = true } diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index a0aac9e06f..d50a081d39 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -486,10 +486,10 @@ mod tests { use crate::events::NetCommand; use actix::Addr; use anyhow::{bail, Result}; + use e3_ciphernode_builder::EventSystem; use e3_events::{ BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EnclaveError, - EnclaveEvent, EventBus, EventBusConfig, GetEvents, HistoryCollector, - PublishDocumentRequested, TakeEvents, + EnclaveEvent, GetEvents, HistoryCollector, PublishDocumentRequested, TakeEvents, }; use libp2p::kad::{GetRecordError, PutRecordError, RecordKey}; use tokio::{ @@ -498,7 +498,7 @@ mod tests { }; use tracing::subscriber::DefaultGuard; - fn setup_test() -> ( + fn setup_test() -> Result<( DefaultGuard, BusHandle, mpsc::Sender, @@ -508,7 +508,7 @@ mod tests { Addr>, Addr>, Addr, - ) { + )> { use tracing_subscriber::{fmt, EnvFilter}; let subscriber = fmt() @@ -518,8 +518,10 @@ mod tests { let guard = tracing::subscriber::set_default(subscriber); - let consumer = EventBus::::new(EventBusConfig { deduplicate: true }).start(); - let bus = BusHandle::new_from_consumer(consumer); + // let consumer = EventBus::::new(EventBusConfig { deduplicate: true }).start(); + // let bus = BusHandle::new(consumer); + let system = EventSystem::new("test"); + let bus = system.handle()?; let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); let (net_evt_tx, net_evt_rx) = broadcast::channel(100); let net_evt_rx = Arc::new(net_evt_rx); @@ -529,15 +531,15 @@ mod tests { bus.subscribe("EnclaveError", error.clone().recipient()); let publisher = DocumentPublisher::setup(&bus, &net_cmd_tx, &net_evt_rx, "topic"); - ( + Ok(( guard, bus, net_cmd_tx, net_cmd_rx, net_evt_tx, net_evt_rx, history, error, publisher, - ) + )) } #[actix::test] async fn test_publishes_document() -> Result<()> { let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, _, _) = - setup_test(); + setup_test()?; let value = ArcBytes::from_bytes(b"I am a special document"); let expires_at = Some(Utc::now() + chrono::Duration::days(1)); let e3_id = E3id::new("1243", 1); @@ -609,7 +611,7 @@ mod tests { #[actix::test] async fn test_get_document_fails_with_exponential_backoff() -> Result<()> { let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, errors, _) = - setup_test(); + setup_test()?; let value = b"I am a special document".to_vec(); let expires_at = Some(Utc::now() + chrono::Duration::days(1)); @@ -673,7 +675,7 @@ mod tests { _history, errors, _, - ) = setup_test(); + ) = setup_test()?; let value = ArcBytes::from_bytes(b"I am a special document"); let expires_at = Some(Utc::now() + chrono::Duration::days(1)); let e3_id = E3id::new("1243", 1); @@ -721,7 +723,7 @@ mod tests { #[actix::test] async fn test_notified_of_document() -> Result<()> { let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, history, _, _) = - setup_test(); + setup_test()?; let value = ArcBytes::from_bytes(b"I am a special document"); let expires_at = Utc::now() + chrono::Duration::days(1); diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index 2772ffac09..61ef319129 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -196,8 +196,9 @@ impl Deref for CiphernodeHistory { mod tests { use super::*; use actix::prelude::*; + use e3_ciphernode_builder::EventSystem; use e3_data::InMemStore; - use e3_events::{BusHandle, EventBus, EventBusConfig}; + use e3_events::{EventBus, EventBusConfig}; async fn mock_setup_node(address: String) -> Result { // Create mock actors for the test @@ -205,7 +206,7 @@ mod tests { let bus = EventBus::::new(EventBusConfig { deduplicate: true }).start(); let history = EventBus::::history(&bus); let errors = EventBus::::error(&bus); - let bus = BusHandle::new_from_consumer(bus); + let bus = EventSystem::new("test").with_event_bus(bus).handle()?; Ok(CiphernodeHandle { address, diff --git a/crates/tests/Cargo.toml b/crates/tests/Cargo.toml index fe94458915..ee84285db7 100644 --- a/crates/tests/Cargo.toml +++ b/crates/tests/Cargo.toml @@ -46,3 +46,4 @@ zeroize = { workspace = true } [dev-dependencies] e3-events = { workspace = true, features = ["test-helpers"] } + diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 0b13895c1c..29867ebbc8 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -4,15 +4,14 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::Actor; use alloy::primitives::{FixedBytes, I256, U256}; use anyhow::{bail, Result}; -use e3_ciphernode_builder::CiphernodeBuilder; +use e3_ciphernode_builder::{CiphernodeBuilder, EventSystem}; use e3_crypto::Cipher; use e3_events::{ prelude::*, BusHandle, CiphertextOutputPublished, CommitteeFinalized, ConfigurationUpdated, - E3Requested, E3id, EnclaveEvent, EnclaveEventData, EventBus, EventBusConfig, - OperatorActivationChanged, PlaintextAggregated, TicketBalanceUpdated, + E3Requested, E3id, EnclaveEventData, OperatorActivationChanged, PlaintextAggregated, + TicketBalanceUpdated, }; use e3_multithread::{GetReport, Multithread}; use e3_sdk::bfv_helpers::{build_bfv_params_arc, decode_bytes_to_vec_u64, encode_bfv_params}; @@ -118,9 +117,8 @@ async fn test_trbfv_actor() -> Result<()> { let rng = create_shared_rng_from_u64(42); // Create "trigger" bus - let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + let system = EventSystem::new("test"); + let bus = system.handle()?; // Parameters (128bits of security) let (degree, plaintext_modulus, moduli) = ( @@ -180,7 +178,7 @@ async fn test_trbfv_actor() -> Result<()> { .add_group(1, || async { let addr = rand_eth_addr(&rng); println!("Building collector {}!", addr); - CiphernodeBuilder::new(rng.clone(), cipher.clone()) + CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_address(&addr) .with_injected_multithread(multithread.clone()) .testmode_with_history() @@ -196,7 +194,7 @@ async fn test_trbfv_actor() -> Result<()> { .add_group(6, || async { let addr = rand_eth_addr(&rng); println!("Building normal {}", &addr); - CiphernodeBuilder::new(rng.clone(), cipher.clone()) + CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_address(&addr) .with_injected_multithread(multithread.clone()) .with_trbfv() diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index be35b47dbe..5b442f70b5 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -10,6 +10,7 @@ use alloy::primitives::{FixedBytes, I256, U256}; use anyhow::*; use e3_ciphernode_builder::CiphernodeBuilder; use e3_ciphernode_builder::CiphernodeHandle; +use e3_ciphernode_builder::EventSystem; use e3_crypto::Cipher; use e3_data::GetDump; use e3_data::InMemStore; @@ -20,8 +21,8 @@ use e3_events::Unsequenced; use e3_events::{ prelude::*, CiphernodeSelected, CiphertextOutputPublished, CommitteeFinalized, ConfigurationUpdated, E3Requested, E3id, EnclaveEvent, EventBus, EventBusConfig, - HistoryCollector, OperatorActivationChanged, OrderedSet, PlaintextAggregated, - PublicKeyAggregated, Seed, Shutdown, TakeEvents, TicketBalanceUpdated, + OperatorActivationChanged, OrderedSet, PlaintextAggregated, PublicKeyAggregated, Seed, + Shutdown, TakeEvents, TicketBalanceUpdated, }; use e3_net::events::GossipData; use e3_net::{events::NetEvent, NetEventTranslator}; @@ -52,10 +53,10 @@ async fn setup_local_ciphernode( rng: &SharedRng, logging: bool, addr: &str, - data: Option>, + store: Option>, cipher: &Arc, ) -> Result { - let mut builder = CiphernodeBuilder::new(rng.clone(), cipher.clone()) + let mut builder = CiphernodeBuilder::new(&addr, rng.clone(), cipher.clone()) .with_keyshare() .with_address(addr) .testmode_with_forked_bus(bus.consumer()) @@ -65,8 +66,8 @@ async fn setup_local_ciphernode( .with_plaintext_aggregation() .with_sortition_score(); - if let Some(data) = data { - builder = builder.with_datastore((&data).into()); + if let Some(ref in_mem_store) = store { + builder = builder.with_in_mem_datastore(in_mem_store); } if logging { @@ -361,9 +362,10 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { // Apply the address and data node to two new actors // Here we test that hydration occurred sucessfully - let bus = EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); + + let bus = EventSystem::in_mem("cn2") + .with_event_bus(EventBus::::new(EventBusConfig { deduplicate: true }).start()) + .handle()?; let cn1 = setup_local_ciphernode( &bus, &rng, @@ -444,11 +446,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { // Setup elements in test let (cmd_tx, mut cmd_rx) = mpsc::channel(100); // Transmit byte events to the network let (event_tx, _) = broadcast::channel(100); // Receive byte events from the network - let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); - let history_collector = HistoryCollector::::new().start(); - bus.subscribe("*", history_collector.clone().recipient()); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); let event_rx = Arc::new(event_tx.subscribe()); // Pas cmd and event channels to NetEventTranslator NetEventTranslator::setup(&bus, &cmd_tx, &event_rx, "my-topic"); @@ -621,11 +621,9 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // Setup elements in test let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network let (event_tx, event_rx) = broadcast::channel(100); // Receive byte events from the network - let bus: BusHandle = EventBus::::new(EventBusConfig { deduplicate: true }) - .start() - .into(); - let history_collector = HistoryCollector::::new().start(); - bus.subscribe("*", history_collector.clone().recipient()); + let system = EventSystem::new("test"); + let bus = system.handle()?; + let history_collector = bus.history(); NetEventTranslator::setup(&bus, &cmd_tx, &Arc::new(event_rx), "mytopic"); From 0775ebb38017f6b73330b756880d3a7c2e1f3003 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 09:53:54 +0000 Subject: [PATCH 15/66] update in mem log --- crates/data/src/in_mem_event_log.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 66a3b3c4fb..51a0c2d213 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -7,11 +7,13 @@ use anyhow::Result; use e3_events::{EnclaveEvent, EventLog, Unsequenced}; -pub struct InMemEventLog; +pub struct InMemEventLog { + log: Vec>, +} impl InMemEventLog { pub fn new() -> Self { - Self {} + Self { log: Vec::new() } } } @@ -20,9 +22,21 @@ impl EventLog for InMemEventLog { &self, from: u64, ) -> Box)>>> { - Box::new(vec![].into_iter()) + // Convert 1-indexed sequence to 0-indexed array position + let start_idx = from.saturating_sub(1) as usize; + + let events: Vec<_> = self + .log + .iter() + .skip(start_idx) + .enumerate() + .map(|(i, event)| Ok((from + i as u64, event.clone()))) + .collect(); + + Box::new(events.into_iter()) } fn append(&mut self, event: &EnclaveEvent) -> Result { - Ok(1u64) + self.log.push(event.to_owned()); + Ok(self.log.len() as u64) } } From 29dc152e437ebaf97519a980b90e742333f0d85e Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 10:09:44 +0000 Subject: [PATCH 16/66] get tests passing --- crates/ciphernode-builder/src/event_system.rs | 9 ++++++++- crates/evm/tests/integration.rs | 12 ++++++------ crates/net/src/document_publisher.rs | 2 +- crates/tests/tests/integration_legacy.rs | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 52a0f99eb8..85daba7572 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -12,7 +12,7 @@ use e3_data::{ SledSequenceIndex, SledStore, WriteBuffer, }; use e3_events::hlc::Hlc; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventStore, Sequencer}; +use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; use once_cell::sync::OnceCell; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; @@ -101,6 +101,13 @@ impl EventSystem { self } + pub fn with_fresh_bus(self) -> Self { + let _ = self + .eventbus + .set(EventBus::new(EventBusConfig { deduplicate: true }).start()); + self + } + pub fn eventbus(&self) -> Addr> { self.eventbus.get_or_init(get_enclave_event_bus).clone() } diff --git a/crates/evm/tests/integration.rs b/crates/evm/tests/integration.rs index 2bb2981143..385f890781 100644 --- a/crates/evm/tests/integration.rs +++ b/crates/evm/tests/integration.rs @@ -81,7 +81,7 @@ async fn evm_reader() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); @@ -152,7 +152,7 @@ async fn ensure_historical_events() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let historical_msgs = vec!["these", "are", "historical", "events"]; @@ -232,7 +232,7 @@ async fn ensure_resume_after_shutdown() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); @@ -341,7 +341,7 @@ async fn coordinator_single_reader() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); @@ -416,7 +416,7 @@ async fn coordinator_multiple_readers() -> Result<()> { let contract1 = EmitLogs::deploy(provider.provider()).await?; let contract2 = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let repository1 = Repository::new(get_in_mem_store()); @@ -501,7 +501,7 @@ async fn coordinator_no_historical_events() -> Result<()> { ) .await?; let contract = EmitLogs::deploy(provider.provider()).await?; - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); let repository = Repository::new(get_in_mem_store()); diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index d50a081d39..dd560296f1 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -520,7 +520,7 @@ mod tests { // let consumer = EventBus::::new(EventBusConfig { deduplicate: true }).start(); // let bus = BusHandle::new(consumer); - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); let (net_evt_tx, net_evt_rx) = broadcast::channel(100); diff --git a/crates/tests/tests/integration_legacy.rs b/crates/tests/tests/integration_legacy.rs index 5b442f70b5..4bfc160924 100644 --- a/crates/tests/tests/integration_legacy.rs +++ b/crates/tests/tests/integration_legacy.rs @@ -621,7 +621,7 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { // Setup elements in test let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network let (event_tx, event_rx) = broadcast::channel(100); // Receive byte events from the network - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let history_collector = bus.history(); From 7ec690ffd5af11ef2e7198ae2d3f5320d314bb24 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 10:25:56 +0000 Subject: [PATCH 17/66] add comments to event system for clarity --- crates/ciphernode-builder/src/event_system.rs | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 85daba7572..e261a0c750 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -17,11 +17,13 @@ use once_cell::sync::OnceCell; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; +/// Hold the InMem EventStore instance and InMemStore struct InMemBackend { eventstore: OnceCell>>, store: OnceCell>, } +/// Hold the Persistent EventStore instance and SledStore struct PersistedBackend { log_path: PathBuf, sled_path: PathBuf, @@ -29,30 +31,49 @@ struct PersistedBackend { store: OnceCell>, } -enum Backend { +/// An EventSystemBackend is holding the potentially persistent structures for the system +enum EventSystemBackend { InMem(InMemBackend), Persisted(PersistedBackend), } +/// EventSystem holds interconnected references to the components that manage events and +/// persistence within the node. The EventSystem connects: +/// +/// - **BusHandle** for interacting with the event system +/// - **EventBus** for managing publishing of events to listeners +/// - **EventStore** for managing persistence of events +/// - **Sequencer** for managing sequencing of event persistence and snapshot coordination +/// - **WriteBuffer** for batching inserts from actors into a snapshot +/// pub struct EventSystem { + /// A nodes id to be used as a tiebreaker in logical clock timestamp differentiation node_id: u32, - backend: Backend, + /// EventSystem backend either persisted or in memory + backend: EventSystemBackend, + /// WriteBuffer for batching inserts from actors into a snapshot buffer: OnceCell>, + /// EventSystem Sequencer sequencer: OnceCell>, + /// EventSystem eventbus eventbus: OnceCell>>, + /// EventSystem BusHandle handle: OnceCell, + /// A OnceLock that is used to indicate whether the system is wired to write snapshots wired: OnceCell<()>, } impl EventSystem { + /// Create a new in memory EventSystem with default settings pub fn new(name: &str) -> Self { EventSystem::in_mem(name) } + /// Create an in memory EventSystem pub fn in_mem(node_id: &str) -> Self { Self { node_id: EventSystem::node_id(node_id), - backend: Backend::InMem(InMemBackend { + backend: EventSystemBackend::InMem(InMemBackend { eventstore: OnceCell::new(), store: OnceCell::new(), }), @@ -64,10 +85,11 @@ impl EventSystem { } } + /// Create an in memory EventSystem with a given store pub fn in_mem_from_store(node_id: &str, store: &Addr) -> Self { Self { node_id: EventSystem::node_id(node_id), - backend: Backend::InMem(InMemBackend { + backend: EventSystemBackend::InMem(InMemBackend { eventstore: OnceCell::new(), store: OnceCell::from(store.to_owned()), }), @@ -79,10 +101,11 @@ impl EventSystem { } } + /// Create a persisted EventSystem with datafiles at the given paths pub fn persisted(node_id: &str, log_path: PathBuf, sled_path: PathBuf) -> Self { Self { node_id: EventSystem::node_id(node_id), - backend: Backend::Persisted(PersistedBackend { + backend: EventSystemBackend::Persisted(PersistedBackend { log_path, sled_path, eventstore: OnceCell::new(), @@ -96,11 +119,13 @@ impl EventSystem { } } + /// Pass in a sepecific given event bus pub fn with_event_bus(self, bus: Addr>) -> Self { let _ = self.eventbus.set(bus); self } + /// Use a fresh event bus that is not the default singleton instance pub fn with_fresh_bus(self) -> Self { let _ = self .eventbus @@ -108,10 +133,12 @@ impl EventSystem { self } + /// Get the eventbus address pub fn eventbus(&self) -> Addr> { self.eventbus.get_or_init(get_enclave_event_bus).clone() } + /// Get the buffer address pub fn buffer(&self) -> Addr { let buffer = self .buffer @@ -121,6 +148,7 @@ impl EventSystem { buffer } + /// Get the sequencer address pub fn sequencer(&self) -> Result> { self.sequencer .get_or_try_init(|| match &self.backend { @@ -148,6 +176,7 @@ impl EventSystem { .cloned() } + /// Get the BusHandle pub fn handle(&self) -> Result { self.handle .get_or_try_init(|| { @@ -160,6 +189,7 @@ impl EventSystem { .cloned() } + /// Get the DataStore pub fn store(&self) -> Result { let store = match &self.backend { Backend::InMem(b) => { @@ -184,6 +214,8 @@ impl EventSystem { Ok(store) } + // We need to ensure that once the buffer and store are created they are connected so that + // inserts are sent between the two actors. This internal function ensures this happens. fn wire_if_ready(&self) { let buffer = match self.buffer.get() { Some(b) => b, From f03006a379f70fe0b5f53fbacebc94d3b81a516c Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 10:28:02 +0000 Subject: [PATCH 18/66] remove redunant imports --- crates/ciphernode-builder/src/ciphernode_builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 38588360b5..1e2704e8e0 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -15,7 +15,7 @@ use e3_aggregator::ext::{ }; use e3_config::chain_config::ChainConfig; use e3_crypto::Cipher; -use e3_data::{DataStore, InMemStore, Repositories, RepositoriesFactory}; +use e3_data::{InMemStore, Repositories, RepositoriesFactory}; use e3_events::{EnclaveEvent, EventBus, EventBusConfig}; use e3_evm::{ helpers::{ @@ -25,7 +25,7 @@ use e3_evm::{ BondingRegistryReaderRepositoryFactory, BondingRegistrySol, CiphernodeRegistryReaderRepositoryFactory, CiphernodeRegistrySol, CoordinatorStart, EnclaveSol, EnclaveSolReader, EnclaveSolReaderRepositoryFactory, EthPrivateKeyRepositoryFactory, - EvmEventReader, HistoricalEventCoordinator, + HistoricalEventCoordinator, }; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; From f02003cdfe2ab27160c36942f93bb188698bbab1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 20:38:48 +0000 Subject: [PATCH 19/66] add todos --- crates/data/src/in_mem_sequence_index.rs | 3 +++ crates/data/src/sled_sequence_index.rs | 2 ++ crates/data/src/write_buffer.rs | 2 ++ 3 files changed, 7 insertions(+) diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index e1a7f1605d..d901c705a9 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -22,12 +22,15 @@ impl InMemSequenceIndex { impl SequenceIndex for InMemSequenceIndex { fn seek_for_prev(&self, key: u128) -> Result> { + todo!("do this"); Ok(None) } fn insert(&mut self, key: u128, value: u64) -> Result<()> { + todo!("do this"); Ok(()) } fn get(&self, key: u128) -> Result> { + todo!("do this"); Ok(None) } } diff --git a/crates/data/src/sled_sequence_index.rs b/crates/data/src/sled_sequence_index.rs index 4865c03f1e..15a6d00ea6 100644 --- a/crates/data/src/sled_sequence_index.rs +++ b/crates/data/src/sled_sequence_index.rs @@ -37,10 +37,12 @@ impl SequenceIndex for SledSequenceIndex { } fn insert(&mut self, key: u128, value: u64) -> Result<()> { + todo!("Do this"); Ok(()) } fn seek_for_prev(&self, key: u128) -> Result> { + todo!("Do this"); Ok(None) } } diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index f361ac4bb4..43ae241cc9 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -33,6 +33,7 @@ impl Handler for WriteBuffer { impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: Insert, ctx: &mut Self::Context) -> Self::Result { + // XXX: finish me // store insert in buffer } } @@ -40,6 +41,7 @@ impl Handler for WriteBuffer { impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: CommitSnapshot, ctx: &mut Self::Context) -> Self::Result { + // XXX: finish me // send all inserts to } } From eeb1b39772b6334de070138f61b9126b7f6a4878 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 21:24:04 +0000 Subject: [PATCH 20/66] fix issues --- Cargo.lock | 463 +++++++++--------- crates/ciphernode-builder/src/event_system.rs | 12 +- 2 files changed, 246 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3197d22d0d..893922103a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -207,7 +207,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -218,7 +218,7 @@ checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaa9ea039a6f9304b4a593d780b1f23e1ae183acdee938b11b38795acacc9f1" +checksum = "35d744058a9daa51a8cf22a3009607498fcf82d3cf4c5444dd8056cdf651f471" dependencies = [ "alloy-primitives", "num_enum", @@ -549,7 +549,7 @@ checksum = "d7c69f6c9c68a1287c9d5ff903d0010726934de0dac10989be37b75a29190d55" dependencies = [ "alloy-primitives", "alloy-sol-types", - "http 1.3.1", + "http 1.4.0", "serde", "serde_json", "thiserror 2.0.17", @@ -629,7 +629,7 @@ dependencies = [ "derive_more", "foldhash", "hashbrown 0.15.5", - "indexmap 2.12.0", + "indexmap 2.12.1", "itoa", "k256", "keccak-asm", @@ -731,7 +731,7 @@ checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -929,7 +929,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -942,11 +942,11 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.12.0", + "indexmap 2.12.1", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "syn-solidity", "tiny-keccak", ] @@ -965,7 +965,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.110", + "syn 2.0.111", "syn-solidity", ] @@ -1059,7 +1059,7 @@ dependencies = [ "alloy-pubsub", "alloy-transport", "futures", - "http 1.3.1", + "http 1.4.0", "rustls", "serde_json", "tokio", @@ -1093,7 +1093,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1143,22 +1143,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1324,7 +1324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1362,7 +1362,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1447,7 +1447,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1519,7 +1519,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "synstructure", ] @@ -1531,7 +1531,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1633,7 +1633,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1682,7 +1682,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1699,7 +1699,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1760,7 +1760,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1814,14 +1814,14 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" [[package]] name = "bigint-poly" version = "0.1.0" -source = "git+https://github.com/gnosisguild/bigint-poly#6b992dd81c65cc693b388e37edfd190749bfc3b3" +source = "git+https://github.com/gnosisguild/bigint-poly#9eca04d2aa473c5ead1e5a13adc8ad11bf250e4a" dependencies = [ "num-bigint", "num-traits", @@ -1867,15 +1867,15 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitcoin-io" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" +checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" [[package]] name = "bitcoin_hashes" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb18c03d0db0247e147a21a6faafd5a7eb851c743db062de72018b6b7e8e4d16" +checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b" dependencies = [ "bitcoin-io", "hex-conservative", @@ -2026,9 +2026,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] @@ -2059,9 +2059,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.45" +version = "1.2.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" dependencies = [ "find-msvc-tools", "jobserver", @@ -2137,7 +2137,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2265,6 +2265,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -2312,9 +2321,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ "crc-catalog", ] @@ -2439,7 +2448,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2464,7 +2473,7 @@ dependencies = [ "quote", "serde", "strsim", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2475,7 +2484,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2515,7 +2524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2565,22 +2574,24 @@ dependencies = [ [[package]] name = "derive_more" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" dependencies = [ + "convert_case", "proc-macro2", "quote", - "syn 2.0.110", + "rustc_version 0.4.1", + "syn 2.0.111", "unicode-xid", ] @@ -2677,7 +2688,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3407,7 +3418,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3480,7 +3491,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3500,7 +3511,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3603,8 +3614,9 @@ dependencies = [ [[package]] name = "fhe" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" +source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" dependencies = [ + "bincode", "doc-comment", "fhe-math", "fhe-traits", @@ -3628,7 +3640,7 @@ dependencies = [ [[package]] name = "fhe-math" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" +source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" dependencies = [ "ethnum", "fhe-traits", @@ -3642,6 +3654,7 @@ dependencies = [ "prost-build", "rand 0.8.5", "rand_chacha 0.3.1", + "serde", "sha2", "thiserror 1.0.69", "zeroize", @@ -3650,7 +3663,7 @@ dependencies = [ [[package]] name = "fhe-traits" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" +source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" dependencies = [ "rand 0.8.5", ] @@ -3658,7 +3671,7 @@ dependencies = [ [[package]] name = "fhe-util" version = "0.1.0-beta.7" -source = "git+https://github.com/gnosisguild/fhe.rs#7692b954ce887ee78f0b6baae36447ca1aa74708" +source = "git+https://github.com/gnosisguild/fhe.rs#f588e090d69488a9cffa2a891bd068a295553348" dependencies = [ "itertools 0.12.1", "num-bigint-dig", @@ -3704,9 +3717,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixed-hash" @@ -3868,7 +3881,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -4054,7 +4067,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -4072,8 +4085,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", - "indexmap 2.12.0", + "http 1.4.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -4115,9 +4128,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "heck" @@ -4139,9 +4152,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-conservative" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" +checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" dependencies = [ "arrayvec", ] @@ -4229,12 +4242,11 @@ dependencies = [ [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -4256,7 +4268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http 1.4.0", ] [[package]] @@ -4267,7 +4279,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] @@ -4310,16 +4322,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "httparse", "itoa", @@ -4336,8 +4348,8 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.3.1", - "hyper 1.8.0", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "rustls", "rustls-pki-types", @@ -4353,7 +4365,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -4368,7 +4380,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -4378,18 +4390,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "base64", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", - "hyper 1.8.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -4474,9 +4486,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -4488,9 +4500,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" @@ -4609,7 +4621,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -4631,12 +4643,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -4799,9 +4811,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -4875,15 +4887,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libgit2-sys" -version = "0.18.2+1.9.1" +version = "0.18.3+1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" dependencies = [ "cc", "libc", @@ -5055,9 +5067,9 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3104e13b51e4711ff5738caa1fb54467c8604c2e94d607e27745bcf709068774" +checksum = "f0c7892c221730ba55f7196e98b0b8ba5e04b4155651736036628e9f73ed6fc3" dependencies = [ "bs58", "ed25519-dalek", @@ -5215,7 +5227,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5365,9 +5377,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" dependencies = [ "value-bag", ] @@ -5413,7 +5425,7 @@ checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5485,9 +5497,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "log", @@ -5808,7 +5820,7 @@ checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5890,7 +5902,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5933,7 +5945,7 @@ checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" dependencies = [ "async-trait", "bytes", - "http 1.3.1", + "http 1.4.0", "opentelemetry", "reqwest", "tracing", @@ -5946,7 +5958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" dependencies = [ "futures-core", - "http 1.3.1", + "http 1.4.0", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -6058,7 +6070,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6156,9 +6168,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" dependencies = [ "memchr", "ucd-trie", @@ -6171,7 +6183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.12.0", + "indexmap 2.12.1", ] [[package]] @@ -6228,7 +6240,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6257,7 +6269,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6362,7 +6374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6393,7 +6405,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.7", + "toml_edit 0.23.9", ] [[package]] @@ -6415,7 +6427,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6447,7 +6459,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6506,7 +6518,7 @@ dependencies = [ "prost 0.12.6", "prost-types", "regex", - "syn 2.0.110", + "syn 2.0.111", "tempfile", ] @@ -6520,7 +6532,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6533,7 +6545,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6832,7 +6844,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6898,10 +6910,10 @@ dependencies = [ "futures-core", "futures-util", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls", "hyper-tls", "hyper-util", @@ -6933,9 +6945,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rfc6979" @@ -7112,9 +7124,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" dependencies = [ "web-time", "zeroize", @@ -7187,6 +7199,18 @@ dependencies = [ "taceo-poseidon2", ] +[[package]] +name = "safe" +version = "0.1.7" +source = "git+https://github.com/gnosisguild/enclave#c9b72d1e42d8e37a19473f423978ef76f92454d6" +dependencies = [ + "ark-bn254 0.5.0", + "ark-ff 0.5.0", + "hex", + "sha3", + "taceo-poseidon2", +] + [[package]] name = "scc" version = "2.4.0" @@ -7368,7 +7392,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7414,7 +7438,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.0", + "indexmap 2.12.1", "schemars 0.9.0", "schemars 1.1.0", "serde", @@ -7433,7 +7457,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7442,7 +7466,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "itoa", "ryu", "serde", @@ -7481,7 +7505,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7554,9 +7578,9 @@ checksum = "c7a6f98357c6bb0ebace19b22220e5543801d9de90ffe77f8abb27c056bac064" [[package]] name = "shell-words" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" [[package]] name = "shellexpand" @@ -7596,9 +7620,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -7615,9 +7639,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "siphasher" @@ -7734,7 +7758,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7756,9 +7780,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.110" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -7774,7 +7798,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7794,7 +7818,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7887,7 +7911,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7898,7 +7922,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8012,7 +8036,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8112,7 +8136,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -8122,11 +8146,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.7" +version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" +checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "toml_datetime 0.7.3", "toml_parser", "winnow", @@ -8156,10 +8180,10 @@ dependencies = [ "async-trait", "base64", "bytes", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -8210,14 +8234,14 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags 2.10.0", "bytes", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "iri-string", "pin-project-lite", @@ -8252,20 +8276,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", @@ -8337,7 +8361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8354,7 +8378,7 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http 1.3.1", + "http 1.4.0", "httparse", "log", "rand 0.9.2", @@ -8410,6 +8434,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-width" version = "0.2.2" @@ -8424,9 +8454,9 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unit-prefix" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] name = "universal-hash" @@ -8505,9 +8535,9 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "value-bag" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" +checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" [[package]] name = "vcpkg" @@ -8575,9 +8605,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", @@ -8588,9 +8618,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.55" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ "cfg-if", "js-sys", @@ -8601,9 +8631,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8611,22 +8641,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] @@ -8647,9 +8677,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", @@ -8741,7 +8771,7 @@ dependencies = [ "windows-interface", "windows-link 0.2.1", "windows-result 0.4.1", - "windows-strings 0.5.1", + "windows-strings", ] [[package]] @@ -8752,7 +8782,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8763,7 +8793,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8780,13 +8810,13 @@ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-registry" -version = "0.5.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings", ] [[package]] @@ -8798,15 +8828,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -8816,15 +8837,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows-strings" version = "0.5.1" @@ -9067,9 +9079,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ "memchr", ] @@ -9194,28 +9206,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -9235,7 +9247,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "synstructure", ] @@ -9256,7 +9268,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -9289,7 +9301,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -9305,9 +9317,11 @@ dependencies = [ [[package]] name = "zkfhe-greco" version = "0.1.0" -source = "git+https://github.com/gnosisguild/zkfhe-generator#7314bed079b91ea6432864f30b991e4ef724a5c5" +source = "git+https://github.com/gnosisguild/zkfhe-generator#46d8579f8fc3c53d2c74e89e7c75c280f63e432f" dependencies = [ "anyhow", + "ark-bn254 0.5.0", + "ark-ff 0.5.0", "bigint-poly", "blake3", "fhe", @@ -9318,6 +9332,7 @@ dependencies = [ "num-traits", "rand 0.8.5", "rayon", + "safe 0.1.7 (git+https://github.com/gnosisguild/enclave)", "serde", "serde_json", "tempfile", @@ -9328,9 +9343,11 @@ dependencies = [ [[package]] name = "zkfhe-shared" version = "0.1.0" -source = "git+https://github.com/gnosisguild/zkfhe-generator#7314bed079b91ea6432864f30b991e4ef724a5c5" +source = "git+https://github.com/gnosisguild/zkfhe-generator#46d8579f8fc3c53d2c74e89e7c75c280f63e432f" dependencies = [ "anyhow", + "ark-bn254 0.5.0", + "ark-ff 0.5.0", "bigint-poly", "chrono", "fhe", diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index e261a0c750..28dd603af3 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -152,7 +152,7 @@ impl EventSystem { pub fn sequencer(&self) -> Result> { self.sequencer .get_or_try_init(|| match &self.backend { - Backend::InMem(b) => { + EventSystemBackend::InMem(b) => { let eventstore = b .eventstore .get_or_init(|| { @@ -161,7 +161,7 @@ impl EventSystem { .clone(); Ok(Sequencer::new(&self.eventbus(), eventstore, self.buffer()).start()) } - Backend::Persisted(b) => { + EventSystemBackend::Persisted(b) => { let eventstore = b .eventstore .get_or_try_init(|| -> Result<_> { @@ -192,14 +192,14 @@ impl EventSystem { /// Get the DataStore pub fn store(&self) -> Result { let store = match &self.backend { - Backend::InMem(b) => { + EventSystemBackend::InMem(b) => { let addr = b .store .get_or_init(|| InMemStore::new(true).start()) .clone(); DataStore::from(&addr) } - Backend::Persisted(b) => { + EventSystemBackend::Persisted(b) => { let addr = b .store .get_or_try_init(|| { @@ -223,12 +223,12 @@ impl EventSystem { }; self.wired.get_or_init(|| match &self.backend { - Backend::InMem(b) => { + EventSystemBackend::InMem(b) => { if let Some(store) = b.store.get() { buffer.do_send(ForwardTo::new(store.clone())); } } - Backend::Persisted(b) => { + EventSystemBackend::Persisted(b) => { if let Some(store) = b.store.get() { buffer.do_send(ForwardTo::new(store.clone())); } From b5d1956ea7e04fb3237269aab81309cb1c47dbe3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 21:27:21 +0000 Subject: [PATCH 21/66] implement InMemSequenceIndex --- crates/data/src/in_mem_sequence_index.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index d901c705a9..da9ffa6bb3 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -22,15 +22,16 @@ impl InMemSequenceIndex { impl SequenceIndex for InMemSequenceIndex { fn seek_for_prev(&self, key: u128) -> Result> { - todo!("do this"); - Ok(None) + // Find the largest key <= the given key and return its value + Ok(self.index.range(..=key).next_back().map(|(_, &v)| v)) } + fn insert(&mut self, key: u128, value: u64) -> Result<()> { - todo!("do this"); + self.index.insert(key, value); Ok(()) } + fn get(&self, key: u128) -> Result> { - todo!("do this"); - Ok(None) + Ok(self.index.get(&key).copied()) } } From 28ab74dbcdba1519f19ba85655f123910692eae4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 21:36:48 +0000 Subject: [PATCH 22/66] connect up enclave factory method --- crates/ciphernode-builder/src/eventbus_factory.rs | 15 ++++++++------- crates/data/src/sled_sequence_index.rs | 14 +++++++++++--- crates/entrypoint/src/helpers/datastore.rs | 2 +- crates/entrypoint/src/start/aggregator_start.rs | 2 +- crates/entrypoint/src/start/start.rs | 2 +- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/ciphernode-builder/src/eventbus_factory.rs b/crates/ciphernode-builder/src/eventbus_factory.rs index 005908a592..1008d7d785 100644 --- a/crates/ciphernode-builder/src/eventbus_factory.rs +++ b/crates/ciphernode-builder/src/eventbus_factory.rs @@ -6,6 +6,8 @@ use actix::Actor; use actix::Addr; +use e3_config::AppConfig; +use e3_events::EventStore; use once_cell::sync::Lazy; use std::any::Any; use std::any::TypeId; @@ -19,6 +21,8 @@ use e3_events::EventBus; use e3_events::HistoryCollector; use e3_events::Subscribe; +use crate::EventSystem; + // The singleton factory using once_cell pub struct EventBusFactory { event_bus_cache: Mutex>>, @@ -93,11 +97,8 @@ pub fn get_error_collector() -> Addr> { EventBusFactory::instance().get_error_collector() } -pub fn get_enclave_bus_handle() -> BusHandle { - // let bus = get_enclave_event_bus(); - // let index; - // let eventstore = EventStore::new(index, bus); - // let seq = Sequencer::new(bus, eventstore).start(); - // BusHandle::new(bus, seq) - todo!(); +pub fn get_enclave_bus_handle(config: &AppConfig) -> anyhow::Result { + let bus = get_enclave_event_bus(); + let system = EventSystem::new(&config.name()).with_event_bus(bus); + Ok(system.handle()?) } diff --git a/crates/data/src/sled_sequence_index.rs b/crates/data/src/sled_sequence_index.rs index 15a6d00ea6..1905b53d73 100644 --- a/crates/data/src/sled_sequence_index.rs +++ b/crates/data/src/sled_sequence_index.rs @@ -37,12 +37,20 @@ impl SequenceIndex for SledSequenceIndex { } fn insert(&mut self, key: u128, value: u64) -> Result<()> { - todo!("Do this"); + self.db + .insert(key.to_be_bytes().to_vec(), value.to_be_bytes().to_vec()) + .context(format!("Failed to insert key: {}", key))?; Ok(()) } fn seek_for_prev(&self, key: u128) -> Result> { - todo!("Do this"); - Ok(None) + let key_bytes = key.to_be_bytes(); + self.db + .range(..=key_bytes) + .next_back() + .transpose() + .context(format!("Failed to seek for prev: {}", key))? + .map(|(_, v)| Ok(u64::from_be_bytes(v.as_ref().try_into()?))) + .transpose() } } diff --git a/crates/entrypoint/src/helpers/datastore.rs b/crates/entrypoint/src/helpers/datastore.rs index ef6c0b6b92..692ec690df 100644 --- a/crates/entrypoint/src/helpers/datastore.rs +++ b/crates/entrypoint/src/helpers/datastore.rs @@ -32,7 +32,7 @@ pub fn setup_datastore(config: &AppConfig, bus: &BusHandle) -> Result } pub fn get_repositories(config: &AppConfig) -> Result { - let bus = get_enclave_bus_handle(); + let bus = get_enclave_bus_handle(config)?; let store = setup_datastore(config, &bus)?; Ok(store.repositories()) } diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index 3c5aabfecd..f2779a8ee2 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -28,7 +28,7 @@ pub async fn execute( plaintext_write_path: Option, experimental_trbfv: bool, ) -> Result<(BusHandle, JoinHandle>, String)> { - let bus = get_enclave_bus_handle(); + let bus = get_enclave_bus_handle(config)?; let rng = Arc::new(Mutex::new(ChaCha20Rng::from_rng(OsRng)?)); let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); let mut builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 4248534c76..783f4aa903 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -26,7 +26,7 @@ pub async fn execute( ) -> Result<(BusHandle, JoinHandle>, String)> { let rng = Arc::new(Mutex::new(rand_chacha::ChaCha20Rng::from_rng(OsRng)?)); - let bus = get_enclave_bus_handle(); + let bus = get_enclave_bus_handle(config)?; let cipher = Arc::new(Cipher::from_file(&config.key_file()).await?); let mut builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) From a55ff36c339d4a2e8cb883fea3cd09532598ea4a Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 22:10:41 +0000 Subject: [PATCH 23/66] fix up testing --- crates/ciphernode-builder/src/event_system.rs | 20 +- crates/events/src/bus_handle.rs | 276 +++++++++--------- crates/events/src/hlc.rs | 7 +- 3 files changed, 153 insertions(+), 150 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 28dd603af3..7c5e99c308 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -61,6 +61,8 @@ pub struct EventSystem { handle: OnceCell, /// A OnceLock that is used to indicate whether the system is wired to write snapshots wired: OnceCell<()>, + /// Hlc override + hlc: OnceCell, } impl EventSystem { @@ -82,6 +84,7 @@ impl EventSystem { eventbus: OnceCell::new(), handle: OnceCell::new(), wired: OnceCell::new(), + hlc: OnceCell::new(), } } @@ -98,6 +101,7 @@ impl EventSystem { eventbus: OnceCell::new(), handle: OnceCell::new(), wired: OnceCell::new(), + hlc: OnceCell::new(), } } @@ -116,6 +120,7 @@ impl EventSystem { eventbus: OnceCell::new(), handle: OnceCell::new(), wired: OnceCell::new(), + hlc: OnceCell::new(), } } @@ -133,6 +138,12 @@ impl EventSystem { self } + /// Add an injected hlc + pub fn with_hlc(self, hlc: Hlc) -> Self { + let _ = self.hlc.set(hlc); + self + } + /// Get the eventbus address pub fn eventbus(&self) -> Addr> { self.eventbus.get_or_init(get_enclave_event_bus).clone() @@ -176,6 +187,13 @@ impl EventSystem { .cloned() } + /// Get an instance of the Hlc + pub fn hlc(&self) -> Result { + self.hlc + .get_or_try_init(|| Ok(Hlc::new(self.node_id))) + .cloned() + } + /// Get the BusHandle pub fn handle(&self) -> Result { self.handle @@ -183,7 +201,7 @@ impl EventSystem { Ok(BusHandle::new( self.eventbus(), self.sequencer()?, - Hlc::new(self.node_id), + self.hlc()?, )) }) .cloned() diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index a852372497..0ed6d32bf1 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -135,29 +135,17 @@ impl EventSubscriber> for BusHandle { } } } -// -// impl Into for Addr> { -// fn into(self) -> BusHandle { -// BusHandle::new_from_consumer(self) -// } -// } -// -// impl Into for &Addr> { -// fn into(self) -> BusHandle { -// BusHandle::new_from_consumer(self.clone()) -// } -// } #[cfg(test)] mod tests { - use std::time::{SystemTime, UNIX_EPOCH}; - - use crate::{ - hlc::Hlc, prelude::*, sequencer::Sequencer, BusHandle, EnclaveEvent, EnclaveEventData, - EventBus, TestEvent, - }; use actix::{Actor, Handler, Message}; + use e3_ciphernode_builder::EventSystem; + use e3_events::{ + hlc::Hlc, prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventPublisher, TestEvent, + }; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::time::sleep; + fn now_micros() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -167,133 +155,129 @@ mod tests { #[actix::test] async fn test_hlc_events() -> anyhow::Result<()> { - return Ok(()); - todo!("under construction..."); - // #[derive(Message)] - // #[rtype("Vec")] - // struct GetEventsOrdered; - // - // // Setup forwarder - // struct Forwarder { - // dest: BusHandle, - // } - // impl Actor for Forwarder { - // type Context = actix::Context; - // } - // - // impl Handler for Forwarder { - // type Result = (); - // fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - // let ts = msg.get_ts(); - // self.dest.publish_from_remote(msg.into_data(), ts).unwrap() - // } - // } - // - // // Setup saver - // struct Saver { - // events: Vec, - // } - // - // impl Actor for Saver { - // type Context = actix::Context; - // } - // - // impl Handler for Saver { - // type Result = (); - // fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - // self.events.push(msg); - // } - // } - // - // impl Handler for Saver { - // type Result = Vec; - // fn handle(&mut self, _: GetEventsOrdered, _: &mut Self::Context) -> Self::Result { - // self.events.clone() - // } - // } - // - // // 1. setup up two separate busses with out of sync clocks A and B. B should be 30 seconds - // // faster than A. - // let consumer_a = EventBus::::default().start(); - // let producer_a = Sequencer::new(&consumer_a).start(); - // let clock_a = Hlc::new(1).with_clock(move || now_micros().saturating_sub(30_000_000)); // Late - // let bus_a = BusHandle::new(consumer_a, producer_a, clock_a); - // - // let consumer_b = EventBus::::default().start(); - // let producer_b = Sequencer::new(&consumer_b).start(); - // let clock_b = Hlc::new(2); // in sync - // let bus_b = BusHandle::new(consumer_b, producer_b, clock_b); - // - // let consumer_c = EventBus::::default().start(); - // let producer_c = Sequencer::new(&consumer_c).start(); - // let clock_c = Hlc::new(3); // in sync - // let bus_c = BusHandle::new(consumer_c, producer_c, clock_c); - // - // let forwarder = Forwarder { - // dest: bus_c.clone(), - // } - // .start(); - // - // // pipe all bus_a and bus_b events to bus_c - // bus_a.subscribe("*", forwarder.clone().into()); - // bus_b.subscribe("*", forwarder.into()); - // - // // Create and subscribe the Saver to bus_c - // let saver = Saver { events: vec![] }.start(); - // bus_c.subscribe("*", saver.clone().into()); - // - // // Publish events in causal order across buses - // bus_a.publish(TestEvent::new("one", 1))?; - // sleep(Duration::from_millis(5)).await; // next tick - // bus_b.publish(TestEvent::new("two", 2))?; - // sleep(Duration::from_millis(5)).await; // next tick - // bus_a.publish(TestEvent::new("three", 3))?; - // sleep(Duration::from_millis(5)).await; // next tick - // bus_b.publish(TestEvent::new("four", 4))?; - // sleep(Duration::from_millis(50)).await; // next tick - // - // // Get events - // let events = saver.send(GetEventsOrdered).await?; - // - // // Sort by HLC timestamp - // let mut sorted_events = events.clone(); - // sorted_events.sort_by_key(|e| e.get_ts()); - // - // // Extract the payloads/names in HLC-sorted order - // let ordered_names: Vec<_> = sorted_events - // .iter() - // .filter_map(|e| match e.get_data() { - // EnclaveEventData::TestEvent(e) => Some(e.msg.clone()), - // _ => None, - // }) - // .collect(); - // - // // ASSERTION 1: Causal order is preserved despite clock drift - // assert_eq!( - // ordered_names, - // vec!["one", "two", "three", "four"], - // "HLC should preserve causal ordering despite 30s clock drift on bus_a" - // ); - // - // // ASSERTION 2: All timestamps are unique (HLC guarantee) - // let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); - // let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); - // assert_eq!( - // timestamps.len(), - // unique_timestamps.len(), - // "All HLC timestamps should be unique" - // ); - // - // // ASSERTION 3: Timestamps are strictly monotonically increasing when sorted - // for window in timestamps.windows(2) { - // assert!( - // window[0] < window[1], - // "HLC timestamps should be strictly increasing: {:?} should be < {:?}", - // window[0], - // window[1] - // ); - // } - // - // Ok(()) + #[derive(Message)] + #[rtype("Vec")] + struct GetEventsOrdered; + + // Setup forwarder + struct Forwarder { + dest: BusHandle, + } + impl Actor for Forwarder { + type Context = actix::Context; + } + + impl Handler for Forwarder { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + let ts = msg.get_ts(); + self.dest.publish_from_remote(msg.into_data(), ts).unwrap() + } + } + + // Setup saver + struct Saver { + events: Vec, + } + + impl Actor for Saver { + type Context = actix::Context; + } + + impl Handler for Saver { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + self.events.push(msg); + } + } + + impl Handler for Saver { + type Result = Vec; + fn handle(&mut self, _: GetEventsOrdered, _: &mut Self::Context) -> Self::Result { + self.events.clone() + } + } + + // 1. setup up two separate busses with out of sync clocks A and B. B should be 30 seconds + // faster than A. + let bus_a = EventSystem::new("a") + .with_fresh_bus() + .with_hlc(Hlc::new(1).with_clock(move || now_micros().saturating_sub(30_000_000))) // Late + .handle()?; + let bus_b = EventSystem::new("b") + .with_fresh_bus() + .with_hlc(Hlc::new(2)) + .handle()?; + let bus_c = EventSystem::new("c") + .with_fresh_bus() + .with_hlc(Hlc::new(3)) + .handle()?; + + let forwarder = Forwarder { + dest: bus_c.clone(), + } + .start(); + + // pipe all bus_a and bus_b events to bus_c + bus_a.subscribe("*", forwarder.clone().into()); + bus_b.subscribe("*", forwarder.into()); + + // Create and subscribe the Saver to bus_c + let saver = Saver { events: vec![] }.start(); + bus_c.subscribe("*", saver.clone().into()); + + // Publish events in causal order across buses + bus_a.publish(TestEvent::new("one", 1))?; + sleep(Duration::from_millis(5)).await; // next tick + bus_b.publish(TestEvent::new("two", 2))?; + sleep(Duration::from_millis(5)).await; // next tick + bus_a.publish(TestEvent::new("three", 3))?; + sleep(Duration::from_millis(5)).await; // next tick + bus_b.publish(TestEvent::new("four", 4))?; + sleep(Duration::from_millis(50)).await; // next tick + + // Get events + let events = saver.send(GetEventsOrdered).await?; + + // Sort by HLC timestamp + let mut sorted_events = events.clone(); + sorted_events.sort_by_key(|e| e.get_ts()); + + // Extract the payloads/names in HLC-sorted order + let ordered_names: Vec<_> = sorted_events + .iter() + .filter_map(|e| match e.get_data() { + EnclaveEventData::TestEvent(e) => Some(e.msg.clone()), + _ => None, + }) + .collect(); + + // ASSERTION 1: Causal order is preserved despite clock drift + assert_eq!( + ordered_names, + vec!["one", "two", "three", "four"], + "HLC should preserve causal ordering despite 30s clock drift on bus_a" + ); + + // ASSERTION 2: All timestamps are unique (HLC guarantee) + let timestamps: Vec<_> = sorted_events.iter().map(|e| e.get_ts()).collect(); + let unique_timestamps: std::collections::HashSet<_> = timestamps.iter().collect(); + assert_eq!( + timestamps.len(), + unique_timestamps.len(), + "All HLC timestamps should be unique" + ); + + // ASSERTION 3: Timestamps are strictly monotonically increasing when sorted + for window in timestamps.windows(2) { + assert!( + window[0] < window[1], + "HLC timestamps should be strictly increasing: {:?} should be < {:?}", + window[0], + window[1] + ); + } + + Ok(()) } } diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index 7a8cb3c11a..11a63d5727 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -159,9 +159,10 @@ impl From for HlcTimestamp { /// # Ok(()) /// # } /// ``` +#[derive(Clone)] pub struct Hlc { /// Inner state guarded by mutex - inner: Mutex, + inner: Arc>, /// Our node id node: u32, /// Maximum drift amount @@ -187,7 +188,7 @@ impl Hlc { pub fn new(node: u32) -> Self { Self { - inner: Mutex::new(HlcInner { ts: 0, counter: 0 }), + inner: Arc::new(Mutex::new(HlcInner { ts: 0, counter: 0 })), node, max_drift: Self::DEFAULT_MAX_DRIFT, clock: None, @@ -203,7 +204,7 @@ impl Hlc { pub fn with_state(ts: u64, counter: u32, node: u32) -> Self { Self { - inner: Mutex::new(HlcInner { ts, counter }), + inner: Arc::new(Mutex::new(HlcInner { ts, counter })), node, max_drift: Self::DEFAULT_MAX_DRIFT, clock: None, From 3ad90e3bd8e8a9916635a5cacb6251eab217e22b Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 22:12:51 +0000 Subject: [PATCH 24/66] add comment --- crates/events/src/bus_handle.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 0ed6d32bf1..2c767045c7 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -140,6 +140,7 @@ impl EventSubscriber> for BusHandle { mod tests { use actix::{Actor, Handler, Message}; use e3_ciphernode_builder::EventSystem; + // NOTE: We cannot pull from crate as the features will be missing as they are not default. use e3_events::{ hlc::Hlc, prelude::*, BusHandle, EnclaveEvent, EnclaveEventData, EventPublisher, TestEvent, }; From b448c0b261d7c44d5ba71db0b432d47a34eaa319 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 22:26:41 +0000 Subject: [PATCH 25/66] add buffer implementation --- crates/data/src/write_buffer.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 43ae241cc9..dfe522e1f1 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -11,6 +11,7 @@ use crate::{Insert, InsertBatch}; pub struct WriteBuffer { dest: Option>, + buffer: Vec, } impl Actor for WriteBuffer { @@ -19,30 +20,39 @@ impl Actor for WriteBuffer { impl WriteBuffer { pub fn new() -> Self { - Self { dest: None } + Self { + dest: None, + buffer: Vec::new(), + } } } impl Handler for WriteBuffer { type Result = (); - fn handle(&mut self, msg: ForwardTo, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: ForwardTo, _: &mut Self::Context) -> Self::Result { self.dest = Some(msg.dest()) } } impl Handler for WriteBuffer { type Result = (); - fn handle(&mut self, msg: Insert, ctx: &mut Self::Context) -> Self::Result { - // XXX: finish me - // store insert in buffer + + fn handle(&mut self, msg: Insert, _: &mut Self::Context) -> Self::Result { + self.buffer.push(msg); } } impl Handler for WriteBuffer { type Result = (); - fn handle(&mut self, msg: CommitSnapshot, ctx: &mut Self::Context) -> Self::Result { - // XXX: finish me - // send all inserts to + + fn handle(&mut self, _: CommitSnapshot, _: &mut Self::Context) -> Self::Result { + if let Some(ref dest) = self.dest { + if !self.buffer.is_empty() { + let inserts = std::mem::take(&mut self.buffer); + let batch = InsertBatch::new(inserts); + dest.do_send(batch); + } + } } } From e99e4b1ff5e86750fbcf53c5f0df52742fb7609c Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 22:42:50 +0000 Subject: [PATCH 26/66] add batch to sled --- crates/data/src/sled_db.rs | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 6bcac062e4..5a794aa748 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -5,7 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::{Context, Result}; -use sled::Tree; +use sled::{ + transaction::{ConflictableTransactionError, TransactionalTree, UnabortableTransactionError}, + Tree, +}; use std::path::PathBuf; use crate::{ @@ -35,6 +38,18 @@ impl SledDb { Ok(()) } + pub fn insert_batch(&mut self, msgs: Vec) -> Result<()> { + self.db + .transaction(|tx_db| { + for msg in &msgs { + tx_db.insert(msg.key().as_slice(), msg.value().to_vec())?; + } + Ok::<(), ConflictableTransactionError>(()) + }) + .context("Could not insert batch data into db")?; + Ok(()) + } + pub fn remove(&mut self, msg: Remove) -> Result<()> { self.db .remove(msg.key()) @@ -122,4 +137,49 @@ mod tests { Ok(()) } + + #[test] + fn test_sled_db_batch_insert() -> Result<()> { + use tempfile::tempdir; + + let temp_dir = tempdir().expect("Failed to create temporary directory"); + let db_path = temp_dir.path().join("test_batch.db"); + + let mut db = SledDb::new(&db_path, "datastore")?; + + // Create a batch of inserts + let batch = vec![ + Insert::new(b"batch_key1".to_vec(), b"batch_value1".to_vec()), + Insert::new(b"batch_key2".to_vec(), b"batch_value2".to_vec()), + Insert::new(b"batch_key3".to_vec(), b"batch_value3".to_vec()), + ]; + + // Insert the batch + db.insert_batch(batch)?; + + // Verify all items were inserted + assert_eq!( + db.get(Get::new(b"batch_key1".to_vec()))?.unwrap(), + b"batch_value1".to_vec(), + "First batch item should be retrievable" + ); + assert_eq!( + db.get(Get::new(b"batch_key2".to_vec()))?.unwrap(), + b"batch_value2".to_vec(), + "Second batch item should be retrievable" + ); + assert_eq!( + db.get(Get::new(b"batch_key3".to_vec()))?.unwrap(), + b"batch_value3".to_vec(), + "Third batch item should be retrievable" + ); + + // Verify non-existent key returns None + assert!( + db.get(Get::new(b"nonexistent".to_vec()))?.is_none(), + "Non-existent key should return None" + ); + + Ok(()) + } } From fe4f0da26ab75d10bd760536338f3caf54b712c4 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 22:44:34 +0000 Subject: [PATCH 27/66] hook up batch insert --- crates/data/src/sled_db.rs | 6 +++--- crates/data/src/sled_store.rs | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 5a794aa748..9bee35e67f 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -38,10 +38,10 @@ impl SledDb { Ok(()) } - pub fn insert_batch(&mut self, msgs: Vec) -> Result<()> { + pub fn insert_batch(&mut self, msgs: &Vec) -> Result<()> { self.db .transaction(|tx_db| { - for msg in &msgs { + for msg in msgs { tx_db.insert(msg.key().as_slice(), msg.value().to_vec())?; } Ok::<(), ConflictableTransactionError>(()) @@ -155,7 +155,7 @@ mod tests { ]; // Insert the batch - db.insert_batch(batch)?; + db.insert_batch(&batch)?; // Verify all items were inserted assert_eq!( diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 2841a53601..f911a938b6 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -53,10 +53,12 @@ impl Handler for SledStore { impl Handler for SledStore { type Result = (); - fn handle(&mut self, event: InsertBatch, ctx: &mut Self::Context) -> Self::Result { - // XXX: handle this properly - for cmd in event.commands() { - ctx.notify(cmd.to_owned()) + fn handle(&mut self, event: InsertBatch, _: &mut Self::Context) -> Self::Result { + if let Some(ref mut db) = &mut self.db { + match db.insert_batch(event.commands()) { + Err(err) => self.bus.err(EType::Data, err), + _ => (), + } } } } From edc8544027ac233e966bb16f36fd4dd14d86e0e3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Thu, 11 Dec 2025 23:32:34 +0000 Subject: [PATCH 28/66] dont build evm --- crates/net/tests/Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/net/tests/Dockerfile b/crates/net/tests/Dockerfile index bf4f3ded8f..2c33860179 100644 --- a/crates/net/tests/Dockerfile +++ b/crates/net/tests/Dockerfile @@ -4,12 +4,19 @@ FROM rust:1.86 AS builder WORKDIR /app # Copy workspace files and all crates EXCEPT net -COPY --exclude=crates/net . . +COPY --exclude=crates/net --exclude=crates/evm . . + COPY ./crates/net/Cargo.toml ./crates/net/Cargo.toml RUN mkdir -p ./crates/net/src/bin && \ echo "fn main() {}" > ./crates/net/src/main.rs && \ echo "fn main() {}" > ./crates/net/src/bin/p2p_test.rs +# Stub out evm crate too +COPY ./crates/evm/Cargo.toml ./crates/evm/Cargo.toml +RUN mkdir -p ./crates/evm/src && \ + echo "" > ./crates/evm/src/lib.rs + + # Build dependencies RUN cargo build --locked --bin p2p_test From a4028a6cabc97815d259974318cf01b79b0fc58d Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 12 Dec 2025 00:24:51 +0000 Subject: [PATCH 29/66] add solc and jq --- crates/net/tests/Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/net/tests/Dockerfile b/crates/net/tests/Dockerfile index 2c33860179..7c60d6d798 100644 --- a/crates/net/tests/Dockerfile +++ b/crates/net/tests/Dockerfile @@ -3,20 +3,22 @@ FROM rust:1.86 AS builder WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends \ + jq \ + software-properties-common \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -L https://github.com/ethereum/solidity/releases/download/v0.8.27/solc-static-linux -o /usr/local/bin/solc \ + && chmod +x /usr/local/bin/solc + # Copy workspace files and all crates EXCEPT net -COPY --exclude=crates/net --exclude=crates/evm . . +COPY --exclude=crates/net . . COPY ./crates/net/Cargo.toml ./crates/net/Cargo.toml RUN mkdir -p ./crates/net/src/bin && \ echo "fn main() {}" > ./crates/net/src/main.rs && \ echo "fn main() {}" > ./crates/net/src/bin/p2p_test.rs -# Stub out evm crate too -COPY ./crates/evm/Cargo.toml ./crates/evm/Cargo.toml -RUN mkdir -p ./crates/evm/src && \ - echo "" > ./crates/evm/src/lib.rs - - # Build dependencies RUN cargo build --locked --bin p2p_test From 2997af0f431a19405ecbf495bb608eb5a450055e Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 12 Dec 2025 00:49:33 +0000 Subject: [PATCH 30/66] use image tag based on commit --- Cargo.lock | 2 ++ crates/net/tests/docker-compose.yaml | 21 ++++++++------------- crates/net/tests/run.sh | 10 +++++----- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 893922103a..564c8aac68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2785,6 +2785,7 @@ dependencies = [ "actix", "alloy", "anyhow", + "bincode", "derivative", "e3-aggregator", "e3-config", @@ -2800,6 +2801,7 @@ dependencies = [ "e3-utils", "once_cell", "tempfile", + "tokio", "tracing", ] diff --git a/crates/net/tests/docker-compose.yaml b/crates/net/tests/docker-compose.yaml index f75de41fbd..2741a121d3 100644 --- a/crates/net/tests/docker-compose.yaml +++ b/crates/net/tests/docker-compose.yaml @@ -1,9 +1,10 @@ x-common-variables: &common-variables SYNC_THRESHOLD: 6 - +x-common-image: &common-image + image: p2p_test:${IMAGE_TAG} services: alice: - image: p2p_test:latest + <<: *common-image command: ['/app/p2p_test', 'alice'] environment: <<: *common-variables @@ -12,9 +13,8 @@ services: TEST_CONFIG: 'lead' networks: - p2p_test_net - bob: - image: p2p_test:latest + <<: *common-image command: ['/app/p2p_test', 'bob'] environment: <<: *common-variables @@ -22,18 +22,16 @@ services: DIAL_TO: '/dns4/charlie/udp/9091/quic-v1' networks: - p2p_test_net - charlie: - image: p2p_test:latest + <<: *common-image command: ['/app/p2p_test', 'charlie'] environment: <<: *common-variables QUIC_PORT: 9091 networks: - p2p_test_net - daniel: - image: p2p_test:latest + <<: *common-image command: ['/app/p2p_test', 'daniel'] environment: <<: *common-variables @@ -41,9 +39,8 @@ services: DIAL_TO: '/dns4/charlie/udp/9091/quic-v1' networks: - p2p_test_net - eve: - image: p2p_test:latest + <<: *common-image command: ['/app/p2p_test', 'eve'] environment: <<: *common-variables @@ -51,9 +48,8 @@ services: DIAL_TO: '/dns4/charlie/udp/9091/quic-v1' networks: - p2p_test_net - fabian: - image: p2p_test:latest + <<: *common-image command: ['/app/p2p_test', 'fabian'] environment: <<: *common-variables @@ -61,7 +57,6 @@ services: DIAL_TO: '/dns4/charlie/udp/9091/quic-v1' networks: - p2p_test_net - networks: p2p_test_net: driver: bridge diff --git a/crates/net/tests/run.sh b/crates/net/tests/run.sh index 81a4fc589d..fa724bf041 100755 --- a/crates/net/tests/run.sh +++ b/crates/net/tests/run.sh @@ -1,16 +1,16 @@ #!/usr/bin/env bash - set -e - # Export env vars once for all docker compose commands export DOCKER_BUILDKIT=1 export COMPOSE_DOCKER_CLI_BUILD=1 +# Get the current commit SHA +export IMAGE_TAG=$(git rev-parse --short HEAD) + echo "" -echo "Building docker image" +echo "Building docker image (p2p_test:${IMAGE_TAG})" echo "" -docker build --network host -f ./Dockerfile -t p2p_test:latest ../../.. - +docker build --network host -f ./Dockerfile -t "p2p_test:${IMAGE_TAG}" ../../.. echo "" echo "NETWORK TESTS" echo "" From d3729df12a05fb1ee03291bec0c7aebe57e9332d Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 12 Dec 2025 05:51:18 +0000 Subject: [PATCH 31/66] --wip-- [skip ci] --- crates/ciphernode-builder/Cargo.toml | 1 + crates/ciphernode-builder/src/event_system.rs | 162 ++++++++++++++---- .../src/eventbus_factory.rs | 1 - crates/data/src/in_mem.rs | 3 + crates/data/src/in_mem_event_log.rs | 1 + crates/data/src/in_mem_sequence_index.rs | 1 + crates/data/src/write_buffer.rs | 5 +- crates/events/src/events.rs | 14 +- crates/events/src/eventstore.rs | 3 + crates/events/src/sequencer.rs | 11 +- 10 files changed, 159 insertions(+), 43 deletions(-) diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 3fbb43cf12..6369cfd8de 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -25,4 +25,5 @@ e3-request.workspace = true e3-sortition.workspace = true e3-utils.workspace = true tempfile.workspace = true +tokio.workspace = true tracing.workspace = true diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 7c5e99c308..738bdfe8e1 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -5,14 +5,16 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::get_enclave_event_bus; -use actix::{Actor, Addr}; +use actix::{Actor, Addr, Recipient}; use anyhow::Result; use e3_data::{ CommitLogEventLog, DataStore, ForwardTo, InMemEventLog, InMemSequenceIndex, InMemStore, - SledSequenceIndex, SledStore, WriteBuffer, + InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, }; use e3_events::hlc::Hlc; -use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; +use e3_events::{ + BusHandle, CommitSnapshot, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer, +}; use once_cell::sync::OnceCell; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; @@ -37,6 +39,11 @@ enum EventSystemBackend { Persisted(PersistedBackend), } +pub enum EventStoreAddr { + InMem(Addr>), + Persisted(Addr>), +} + /// EventSystem holds interconnected references to the components that manage events and /// persistence within the node. The EventSystem connects: /// @@ -162,31 +169,43 @@ impl EventSystem { /// Get the sequencer address pub fn sequencer(&self) -> Result> { self.sequencer - .get_or_try_init(|| match &self.backend { - EventSystemBackend::InMem(b) => { - let eventstore = b - .eventstore - .get_or_init(|| { - EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()).start() - }) - .clone(); - Ok(Sequencer::new(&self.eventbus(), eventstore, self.buffer()).start()) + .get_or_try_init(|| match self.eventstore()? { + EventStoreAddr::InMem(es) => { + Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) } - EventSystemBackend::Persisted(b) => { - let eventstore = b - .eventstore - .get_or_try_init(|| -> Result<_> { - let index = SledSequenceIndex::new(&b.sled_path, "sequence_index")?; - let log = CommitLogEventLog::new(&b.log_path)?; - Ok(EventStore::new(index, log).start()) - })? - .clone(); - Ok(Sequencer::new(&self.eventbus(), eventstore, self.buffer()).start()) + EventStoreAddr::Persisted(es) => { + Ok(Sequencer::new(&self.eventbus(), es, self.buffer()).start()) } }) .cloned() } + /// Get the EventStore address + pub fn eventstore(&self) -> Result { + match &self.backend { + EventSystemBackend::InMem(b) => { + let addr = b + .eventstore + .get_or_init(|| { + EventStore::new(InMemSequenceIndex::new(), InMemEventLog::new()).start() + }) + .clone(); + Ok(EventStoreAddr::InMem(addr)) + } + EventSystemBackend::Persisted(b) => { + let addr = b + .eventstore + .get_or_try_init(|| -> Result<_> { + let index = SledSequenceIndex::new(&b.sled_path, "sequence_index")?; + let log = CommitLogEventLog::new(&b.log_path)?; + Ok(EventStore::new(index, log).start()) + })? + .clone(); + Ok(EventStoreAddr::Persisted(addr)) + } + } + } + /// Get an instance of the Hlc pub fn hlc(&self) -> Result { self.hlc @@ -240,17 +259,18 @@ impl EventSystem { None => return, }; - self.wired.get_or_init(|| match &self.backend { - EventSystemBackend::InMem(b) => { - if let Some(store) = b.store.get() { - buffer.do_send(ForwardTo::new(store.clone())); - } - } - EventSystemBackend::Persisted(b) => { - if let Some(store) = b.store.get() { - buffer.do_send(ForwardTo::new(store.clone())); - } - } + let store: Option> = match &self.backend { + EventSystemBackend::InMem(b) => b.store.get().cloned().map(Into::into), + EventSystemBackend::Persisted(b) => b.store.get().cloned().map(Into::into), + }; + + let Some(store) = store else { + return; + }; + + // Now we know both are ready, so initialization will succeed + self.wired.get_or_init(|| { + buffer.do_send(ForwardTo::new(store)); }); } @@ -263,8 +283,19 @@ impl EventSystem { #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; + use actix::Actor; + use actix::Handler; + use actix::Message; + use e3_data::Get; + use e3_data::Insert; + use e3_events::prelude::*; + use e3_events::EnclaveEventData; + use e3_events::TestEvent; use tempfile::TempDir; + use tokio::time::sleep; #[actix::test] async fn test_persisted() { @@ -289,4 +320,69 @@ mod tests { // Wiring happened automatically assert!(system.wired.get().is_some()); } + + #[actix::test] + async fn test_correct_data() -> Result<()> { + let system = EventSystem::in_mem("cn1").with_fresh_bus(); + let seqencer = system.sequencer()?; + let handle = system.handle()?; + let store = system.store()?; + let buffer = system.buffer(); + let eventstore = system.eventstore()?; + + #[derive(Message, Debug)] + #[rtype("Vec")] + struct GetLogs; + + struct Listener { + logs: Vec, + } + + impl Handler for Listener { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::TestEvent(TestEvent { msg, .. }) = msg.into_data() { + self.logs.push(msg); + } + } + } + + impl Handler for Listener { + type Result = Vec; + fn handle(&mut self, msg: GetLogs, _: &mut Self::Context) -> Self::Result { + self.logs.clone() + } + } + + impl Actor for Listener { + type Context = actix::Context; + } + + buffer.do_send(Insert::new("/foo/name", b"Fred".into())); + buffer.do_send(Insert::new("/foo/age", b"21".into())); + buffer.do_send(Insert::new("/foo/occupation", b"developer".into())); + + let r = store.scope("name").read::>().await?; + assert_eq!(r, None); + + let listener = Listener { logs: Vec::new() }.start(); + handle.subscribe("*", listener.clone().into()); + handle.publish(TestEvent::new("pink", 1))?; + // handle.publish(TestEvent::new("yellow", 1))?; + // handle.publish(TestEvent::new("red", 1))?; + // handle.publish(TestEvent::new("white", 1))?; + sleep(Duration::from_millis(100)).await; + + let logs = listener.send(GetLogs).await?; + + assert_eq!(logs, vec!["pink"]); + // assert_eq!(logs, vec!["pink", "yellow", "red", "white"]); + + sleep(Duration::from_millis(100)).await; + + let r = store.scope("/foo/name").read::>().await?; + assert_eq!(r, None); + + Ok(()) + } } diff --git a/crates/ciphernode-builder/src/eventbus_factory.rs b/crates/ciphernode-builder/src/eventbus_factory.rs index 1008d7d785..a8c6e9a490 100644 --- a/crates/ciphernode-builder/src/eventbus_factory.rs +++ b/crates/ciphernode-builder/src/eventbus_factory.rs @@ -7,7 +7,6 @@ use actix::Actor; use actix::Addr; use e3_config::AppConfig; -use e3_events::EventStore; use once_cell::sync::Lazy; use std::any::Any; use std::any::TypeId; diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 59d621e56a..1f87324d9b 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -75,7 +75,9 @@ impl Handler for InMemStore { impl Handler for InMemStore { type Result = (); fn handle(&mut self, msg: InsertBatch, ctx: &mut Self::Context) -> Self::Result { + println!("InMemStore: Running InsertBatch"); for cmd in msg.commands() { + println!("InMemStore: writing {:?}", cmd.key()); self.db.insert(cmd.key().to_owned(), cmd.value().to_owned()); } } @@ -108,6 +110,7 @@ impl Handler for InMemStore { impl Handler for InMemStore { type Result = Option>; fn handle(&mut self, event: Get, _: &mut Self::Context) -> Option> { + println!("Getting contents {:?}", event.key()); let key = event.key(); self.db.get(key).cloned() } diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 51a0c2d213..d82e621b1e 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -36,6 +36,7 @@ impl EventLog for InMemEventLog { Box::new(events.into_iter()) } fn append(&mut self, event: &EnclaveEvent) -> Result { + println!("InMemEventLog.append({:?})", event); self.log.push(event.to_owned()); Ok(self.log.len() as u64) } diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index da9ffa6bb3..d8fe312ac5 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -27,6 +27,7 @@ impl SequenceIndex for InMemSequenceIndex { } fn insert(&mut self, key: u128, value: u64) -> Result<()> { + println!("InMemSequenceIndex got key={} value={}", key, value); self.index.insert(key, value); Ok(()) } diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index dfe522e1f1..24a2a76319 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -45,11 +45,14 @@ impl Handler for WriteBuffer { impl Handler for WriteBuffer { type Result = (); - fn handle(&mut self, _: CommitSnapshot, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { + println!("WriteBuffer received {:?}", msg); if let Some(ref dest) = self.dest { + println!("buffer.is_empty() == {}", self.buffer.is_empty()); if !self.buffer.is_empty() { let inserts = std::mem::take(&mut self.buffer); let batch = InsertBatch::new(inserts); + println!("running BatchInsert!"); dest.do_send(batch); } } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 3dcd517ef6..fc104336c9 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -9,12 +9,18 @@ use actix::{Message, Recipient}; use crate::{EnclaveEvent, Sequenced, Unsequenced}; /// Direct event received by the snapshot buffer in order to save snapshot to disk -#[derive(Message)] +#[derive(Message, Debug)] #[rtype("()")] -pub struct CommitSnapshot(pub u64); +pub struct CommitSnapshot(u64); + +impl CommitSnapshot { + pub fn new(seq: u64) -> Self { + Self(seq) + } +} /// Direct event received by the EventStore to store an event -#[derive(Message)] +#[derive(Message, Debug)] #[rtype("()")] pub struct StoreEventRequested { pub event: EnclaveEvent, @@ -34,7 +40,7 @@ impl StoreEventRequested { } /// Direct event received by the Sequencer once an event has been stored -#[derive(Message)] +#[derive(Message, Debug)] #[rtype("()")] pub struct EventStored(pub EnclaveEvent); diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 9777b39aae..51892c87de 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -19,10 +19,13 @@ pub struct EventStore { impl EventStore { pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { + println!("EventStore got {:?}", msg); + let event = msg.event; let sender = msg.sender; let ts = event.get_ts(); let seq = self.log.append(&event)?; + println!("EventStore: got seq {}", seq); self.index.insert(ts, seq)?; sender.try_send(EventStored(event.into_sequenced(seq)))?; Ok(()) diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 84511eef26..b844f08138 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -16,20 +16,20 @@ pub struct Sequencer { bus: Addr>>, seq: u64, eventstore: Recipient, - snapshot_buffer: Recipient, + buffer: Recipient, } impl Sequencer { pub fn new( bus: &Addr>>, eventstore: impl Into>, - snapshot_buffer: impl Into>, + buffer: impl Into>, ) -> Self { Self { bus: bus.clone(), seq: 0, eventstore: eventstore.into(), - snapshot_buffer: snapshot_buffer.into(), + buffer: buffer.into(), } } } @@ -41,6 +41,7 @@ impl Actor for Sequencer { impl Handler> for Sequencer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + println!("Sequencer got {}", msg); self.eventstore .do_send(StoreEventRequested::new(msg, ctx.address())) } @@ -49,9 +50,11 @@ impl Handler> for Sequencer { impl Handler for Sequencer { type Result = (); fn handle(&mut self, msg: EventStored, ctx: &mut Self::Context) -> Self::Result { + println!("Sequencer got EventStored {:?}", msg); + let event = msg.into_event(); let seq = event.get_seq(); - // TODO: store snapshot... + self.buffer.do_send(CommitSnapshot::new(seq)); self.bus.do_send(event) } } From d81bd36941edf8f2126ec7287bea7e591ded40d9 Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 13 Dec 2025 05:18:27 +0000 Subject: [PATCH 32/66] Unit test eventsystem --- crates/ciphernode-builder/Cargo.toml | 1 + crates/ciphernode-builder/src/event_system.rs | 191 +++++++++++++----- crates/data/src/commit_log_event_log.rs | 7 +- crates/data/src/data_store.rs | 38 +++- crates/data/src/in_mem.rs | 6 +- crates/data/src/in_mem_event_log.rs | 10 +- crates/data/src/in_mem_sequence_index.rs | 35 +++- crates/data/src/sled_sequence_index.rs | 41 +++- crates/data/src/write_buffer.rs | 3 - crates/events/src/bus_handle.rs | 11 +- crates/events/src/events.rs | 30 +++ crates/events/src/eventstore.rs | 32 ++- crates/events/src/sequencer.rs | 3 - crates/events/src/traits.rs | 9 +- 14 files changed, 306 insertions(+), 111 deletions(-) diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index 6369cfd8de..898563c9df 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -27,3 +27,4 @@ e3-utils.workspace = true tempfile.workspace = true tokio.workspace = true tracing.workspace = true +bincode.workspace = true diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 738bdfe8e1..aee7c302fe 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -6,15 +6,13 @@ use crate::get_enclave_event_bus; use actix::{Actor, Addr, Recipient}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use e3_data::{ CommitLogEventLog, DataStore, ForwardTo, InMemEventLog, InMemSequenceIndex, InMemStore, InsertBatch, SledSequenceIndex, SledStore, WriteBuffer, }; use e3_events::hlc::Hlc; -use e3_events::{ - BusHandle, CommitSnapshot, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer, -}; +use e3_events::{BusHandle, EnclaveEvent, EventBus, EventBusConfig, EventStore, Sequencer}; use once_cell::sync::OnceCell; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::PathBuf; @@ -44,6 +42,19 @@ pub enum EventStoreAddr { Persisted(Addr>), } +impl TryFrom for Addr> { + type Error = anyhow::Error; + fn try_from(value: EventStoreAddr) -> std::result::Result { + if let EventStoreAddr::InMem(addr) = value { + Ok(addr) + } else { + Err(anyhow!( + "address was not EventStore" + )) + } + } +} + /// EventSystem holds interconnected references to the components that manage events and /// persistence within the node. The EventSystem connects: /// @@ -234,7 +245,7 @@ impl EventSystem { .store .get_or_init(|| InMemStore::new(true).start()) .clone(); - DataStore::from(&addr) + DataStore::from_in_mem(&addr, &self.buffer()) } EventSystemBackend::Persisted(b) => { let addr = b @@ -244,7 +255,7 @@ impl EventSystem { SledStore::new(&handle, &b.sled_path) })? .clone(); - DataStore::from(&addr) + DataStore::from_sled_store(&addr, &self.buffer()) } }; self.wire_if_ready(); @@ -289,14 +300,71 @@ mod tests { use actix::Actor; use actix::Handler; use actix::Message; - use e3_data::Get; - use e3_data::Insert; + use e3_events::prelude::*; use e3_events::EnclaveEventData; + use e3_events::GetEventsAfter; + use e3_events::ReceiveEvents; use e3_events::TestEvent; use tempfile::TempDir; use tokio::time::sleep; + // Setup Listener for the test + #[derive(Message, Debug)] + #[rtype("Vec")] + struct GetLogs; + + #[derive(Message, Debug)] + #[rtype("Vec")] + struct GetEvents; + + struct Listener { + logs: Vec, + events: Vec, + } + + impl Handler for Listener { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let EnclaveEventData::TestEvent(TestEvent { msg, .. }) = msg.into_data() { + self.logs.push(msg); + } + } + } + + impl Handler for Listener { + type Result = Vec; + fn handle(&mut self, msg: GetLogs, _: &mut Self::Context) -> Self::Result { + self.logs.clone() + } + } + + impl Handler for Listener { + type Result = Vec; + fn handle(&mut self, _: GetEvents, _: &mut Self::Context) -> Self::Result { + self.events + .iter() + .filter_map(|event| { + if let EnclaveEventData::TestEvent(evt) = event.get_data() { + return Some(evt.msg.clone()); + } + None + }) + .collect::>() + } + } + + impl Handler for Listener { + type Result = (); + fn handle(&mut self, msg: ReceiveEvents, _: &mut Self::Context) -> Self::Result { + self.events = msg.events().clone(); + } + } + + impl Actor for Listener { + type Context = actix::Context; + } + #[actix::test] async fn test_persisted() { let tmp = TempDir::new().unwrap(); @@ -322,67 +390,80 @@ mod tests { } #[actix::test] - async fn test_correct_data() -> Result<()> { + async fn test_event_system() -> Result<()> { let system = EventSystem::in_mem("cn1").with_fresh_bus(); - let seqencer = system.sequencer()?; let handle = system.handle()?; - let store = system.store()?; - let buffer = system.buffer(); + let datastore = system.store()?; let eventstore = system.eventstore()?; - - #[derive(Message, Debug)] - #[rtype("Vec")] - struct GetLogs; - - struct Listener { - logs: Vec, - } - - impl Handler for Listener { - type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - if let EnclaveEventData::TestEvent(TestEvent { msg, .. }) = msg.into_data() { - self.logs.push(msg); - } - } - } - - impl Handler for Listener { - type Result = Vec; - fn handle(&mut self, msg: GetLogs, _: &mut Self::Context) -> Self::Result { - self.logs.clone() - } - } - - impl Actor for Listener { - type Context = actix::Context; + let listener = Listener { + logs: Vec::new(), + events: Vec::new(), } + .start(); - buffer.do_send(Insert::new("/foo/name", b"Fred".into())); - buffer.do_send(Insert::new("/foo/age", b"21".into())); - buffer.do_send(Insert::new("/foo/occupation", b"developer".into())); - - let r = store.scope("name").read::>().await?; - assert_eq!(r, None); - - let listener = Listener { logs: Vec::new() }.start(); + // Send all evts to the listener handle.subscribe("*", listener.clone().into()); + + // Lets store some data + datastore.scope("/foo/name").write("Fred".to_string()); + datastore.scope("/foo/age").write(21u64); + datastore + .scope("/foo/occupation") + .write("developer".to_string()); + + // NOTE: Eventual consistency + // Store should not have data set on it until event has been published + // There is an argument we should instead delay reads until the event has been stored + // For now we allow this inconsistency under the assumption that data is written for + // snapshot storage exclusively. If we have issues we may wish to change to a more + // highly consistent model. + + // Let's check the eventual consistency all data points should be none... + assert_eq!(datastore.scope("/foo/name").read::().await?, None); + assert_eq!(datastore.scope("/foo/age").read::().await?, None); + assert_eq!( + datastore.scope("/foo/occupation").read::().await?, + None + ); + + // Push an event handle.publish(TestEvent::new("pink", 1))?; - // handle.publish(TestEvent::new("yellow", 1))?; - // handle.publish(TestEvent::new("red", 1))?; - // handle.publish(TestEvent::new("white", 1))?; + sleep(Duration::from_millis(1)).await; + + // Now we have published an event all data should be written we can get the data from the store + assert_eq!( + datastore.scope("/foo/name").read::().await?, + Some("Fred".to_string()) + ); + assert_eq!(datastore.scope("/foo/age").read::().await?, Some(21)); + assert_eq!( + datastore.scope("/foo/occupation").read::().await?, + Some("developer".to_string()) + ); + + // Get a timestamp + let ts = handle.ts()?; + + // Push a few other events + handle.publish(TestEvent::new("yellow", 1))?; + handle.publish(TestEvent::new("red", 1))?; + handle.publish(TestEvent::new("white", 1))?; sleep(Duration::from_millis(100)).await; + // Get the event logs from the listener let logs = listener.send(GetLogs).await?; + assert_eq!(logs, vec!["pink", "yellow", "red", "white"]); - assert_eq!(logs, vec!["pink"]); - // assert_eq!(logs, vec!["pink", "yellow", "red", "white"]); + // Get the in mem address for the event store + let es: Addr> = eventstore.try_into()?; + // Get all events after the given timestamp and send them to the listener + es.do_send(GetEventsAfter::new(ts, listener.clone())); sleep(Duration::from_millis(100)).await; - let r = store.scope("/foo/name").read::>().await?; - assert_eq!(r, None); - + // Pull the events off the listsner since the timestamp + let events = listener.send(GetEvents).await?; + assert_eq!(events, vec!["yellow", "red", "white"]); Ok(()) } } diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 1278607b91..d22d71b0e0 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -27,12 +27,7 @@ impl EventLog for CommitLogEventLog { Ok(1u64) } - fn read_from( - &self, - from: u64, - ) -> Box< - dyn Iterator), anyhow::Error>>, - > { + fn read_from(&self, from: u64) -> Box)>> { Box::new(vec![].into_iter()) } } diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 9493228858..b4a2743ab9 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -6,9 +6,9 @@ use std::borrow::Cow; -use crate::{Get, Insert, InsertSync, Remove}; +use crate::{ForwardTo, Get, Insert, InsertSync, Remove, WriteBuffer}; use crate::{InMemStore, IntoKey, SledStore}; -use actix::{Addr, Recipient}; +use actix::{Actor, Addr, Recipient}; use anyhow::anyhow; use anyhow::Context; use anyhow::Result; @@ -22,12 +22,12 @@ pub enum StoreAddr { } impl StoreAddr { - pub fn to_data_store(&self) -> DataStore { - match self { - StoreAddr::InMem(s) => s.into(), - StoreAddr::Sled(s) => s.into(), - } - } + // pub fn to_data_store(&self) -> DataStore { + // match self { + // StoreAddr::InMem(s) => s.into(), + // StoreAddr::Sled(s) => s.into(), + // } + // } pub fn to_maybe_in_mem(&self) -> Option<&Addr> { match self { @@ -152,6 +152,28 @@ impl DataStore { scope: key.into_key(), } } + + pub fn from_sled_store(addr: &Addr, write_buffer: &Addr) -> Self { + Self { + addr: StoreAddr::Sled(addr.clone()), + get: addr.clone().recipient(), + insert: write_buffer.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + } + } + + pub fn from_in_mem(addr: &Addr, write_buffer: &Addr) -> Self { + Self { + addr: StoreAddr::InMem(addr.clone()), + get: addr.clone().recipient(), + insert: write_buffer.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + } + } } impl From<&Addr> for DataStore { diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 1f87324d9b..ccff6c7f61 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -75,9 +75,7 @@ impl Handler for InMemStore { impl Handler for InMemStore { type Result = (); fn handle(&mut self, msg: InsertBatch, ctx: &mut Self::Context) -> Self::Result { - println!("InMemStore: Running InsertBatch"); for cmd in msg.commands() { - println!("InMemStore: writing {:?}", cmd.key()); self.db.insert(cmd.key().to_owned(), cmd.value().to_owned()); } } @@ -110,9 +108,9 @@ impl Handler for InMemStore { impl Handler for InMemStore { type Result = Option>; fn handle(&mut self, event: Get, _: &mut Self::Context) -> Option> { - println!("Getting contents {:?}", event.key()); let key = event.key(); - self.db.get(key).cloned() + let r = self.db.get(key); + r.cloned() } } diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index d82e621b1e..e916d42466 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -18,10 +18,7 @@ impl InMemEventLog { } impl EventLog for InMemEventLog { - fn read_from( - &self, - from: u64, - ) -> Box)>>> { + fn read_from(&self, from: u64) -> Box)>> { // Convert 1-indexed sequence to 0-indexed array position let start_idx = from.saturating_sub(1) as usize; @@ -30,13 +27,12 @@ impl EventLog for InMemEventLog { .iter() .skip(start_idx) .enumerate() - .map(|(i, event)| Ok((from + i as u64, event.clone()))) + .map(|(i, event)| (from + i as u64, event.clone())) .collect(); - + println!("XXX: events:{:?}", events); Box::new(events.into_iter()) } fn append(&mut self, event: &EnclaveEvent) -> Result { - println!("InMemEventLog.append({:?})", event); self.log.push(event.to_owned()); Ok(self.log.len() as u64) } diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index d8fe312ac5..a7ddc6bb6d 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -21,13 +21,11 @@ impl InMemSequenceIndex { } impl SequenceIndex for InMemSequenceIndex { - fn seek_for_prev(&self, key: u128) -> Result> { - // Find the largest key <= the given key and return its value - Ok(self.index.range(..=key).next_back().map(|(_, &v)| v)) + fn seek(&self, key: u128) -> Result> { + Ok(self.index.range(key..).next().map(|(_, &v)| v)) } fn insert(&mut self, key: u128, value: u64) -> Result<()> { - println!("InMemSequenceIndex got key={} value={}", key, value); self.index.insert(key, value); Ok(()) } @@ -36,3 +34,32 @@ impl SequenceIndex for InMemSequenceIndex { Ok(self.index.get(&key).copied()) } } + +#[cfg(test)] +mod tests { + use crate::InMemSequenceIndex; + use e3_events::SequenceIndex; + + #[test] + fn seek_for_prev_finds_nearest_key_at_or_before_target() { + let mut index = InMemSequenceIndex::new(); + index.insert(100, 1).unwrap(); + index.insert(200, 2).unwrap(); + index.insert(300, 3).unwrap(); + + // Empty range (before all keys) + assert_eq!(index.seek(50).unwrap(), Some(1)); + + // Exact matches + assert_eq!(index.seek(100).unwrap(), Some(1)); + assert_eq!(index.seek(200).unwrap(), Some(2)); + assert_eq!(index.seek(300).unwrap(), Some(3)); + + // Between keys (returns next) + assert_eq!(index.seek(150).unwrap(), Some(2)); + assert_eq!(index.seek(250).unwrap(), Some(3)); + + // After all keys (returns last) + assert_eq!(index.seek(999).unwrap(), None); + } +} diff --git a/crates/data/src/sled_sequence_index.rs b/crates/data/src/sled_sequence_index.rs index 1905b53d73..4922fb4600 100644 --- a/crates/data/src/sled_sequence_index.rs +++ b/crates/data/src/sled_sequence_index.rs @@ -43,14 +43,47 @@ impl SequenceIndex for SledSequenceIndex { Ok(()) } - fn seek_for_prev(&self, key: u128) -> Result> { + fn seek(&self, key: u128) -> Result> { let key_bytes = key.to_be_bytes(); self.db - .range(..=key_bytes) - .next_back() + .range(key_bytes..) + .next() .transpose() - .context(format!("Failed to seek for prev: {}", key))? + .context(format!("Failed to seek: {}", key))? .map(|(_, v)| Ok(u64::from_be_bytes(v.as_ref().try_into()?))) .transpose() } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn seek_finds_nearest_key_at_or_after_target() { + let dir = tempdir().unwrap(); + let path = dir.path().to_path_buf(); + + let mut index = SledSequenceIndex::new(&path, "test_tree").unwrap(); + + index.insert(100, 1).unwrap(); + index.insert(200, 2).unwrap(); + index.insert(300, 3).unwrap(); + + // Before all keys (returns first) + assert_eq!(index.seek(50).unwrap(), Some(1)); + + // Exact matches + assert_eq!(index.seek(100).unwrap(), Some(1)); + assert_eq!(index.seek(200).unwrap(), Some(2)); + assert_eq!(index.seek(300).unwrap(), Some(3)); + + // Between keys (returns next) + assert_eq!(index.seek(150).unwrap(), Some(2)); + assert_eq!(index.seek(250).unwrap(), Some(3)); + + // After all keys + assert_eq!(index.seek(999).unwrap(), None); + } +} diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index 24a2a76319..f41dc08e9f 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -46,13 +46,10 @@ impl Handler for WriteBuffer { type Result = (); fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { - println!("WriteBuffer received {:?}", msg); if let Some(ref dest) = self.dest { - println!("buffer.is_empty() == {}", self.buffer.is_empty()); if !self.buffer.is_empty() { let inserts = std::mem::take(&mut self.buffer); let batch = InsertBatch::new(inserts); - println!("running BatchInsert!"); dest.do_send(batch); } } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 2c767045c7..3fd7be014c 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -32,12 +32,6 @@ pub struct BusHandle { } impl BusHandle { - // pub fn new_from_consumer(consumer: Addr>>) -> Self { - // let producer = Sequencer::new(&consumer).start(); - // let hlc = Hlc::default(); - // Self::new(consumer, producer, hlc) - // } - pub fn new( consumer: Addr>>, producer: Addr, @@ -61,6 +55,11 @@ impl BusHandle { pub fn consumer(&self) -> &Addr>> { &self.consumer } + + pub fn ts(&self) -> Result { + let ts = self.hlc.tick()?; + Ok(ts.into()) + } } impl EventPublisher> for BusHandle { diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index fc104336c9..9ae0d86dab 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -39,6 +39,36 @@ impl StoreEventRequested { } } +/// Get events after timestamp in EventStore +#[derive(Message, Debug)] +#[rtype("()")] +pub struct GetEventsAfter { + pub ts: u128, + pub sender: Recipient, +} + +impl GetEventsAfter { + pub fn new(ts: u128, sender: impl Into>) -> Self { + Self { + ts, + sender: sender.into(), + } + } +} + +#[derive(Message, Debug)] +#[rtype("()")] +pub struct ReceiveEvents(Vec>); + +impl ReceiveEvents { + pub fn new(events: Vec) -> Self { + Self(events) + } + pub fn events(&self) -> &Vec { + &self.0 + } +} + /// Direct event received by the Sequencer once an event has been stored #[derive(Message, Debug)] #[rtype("()")] diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 51892c87de..7db2e52963 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -6,7 +6,7 @@ use crate::{ events::{EventStored, StoreEventRequested}, - EventLog, SequenceIndex, + EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, }; use actix::{Actor, Handler}; use anyhow::Result; @@ -19,17 +19,31 @@ pub struct EventStore { impl EventStore { pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) -> Result<()> { - println!("EventStore got {:?}", msg); - let event = msg.event; let sender = msg.sender; let ts = event.get_ts(); + println!("VVV: ts={}", ts); let seq = self.log.append(&event)?; - println!("EventStore: got seq {}", seq); + println!("VVV: seq={}", seq); self.index.insert(ts, seq)?; sender.try_send(EventStored(event.into_sequenced(seq)))?; Ok(()) } + + pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { + println!("XXX: GetEventsAfter {}!", msg.ts); + let seq = self.index.seek(msg.ts)?.unwrap_or(1); + println!("XXX: seq={}", seq); + + let evts = self + .log + .read_from(seq) + .map(|(s, e)| e.into_sequenced(s)) + .collect::>(); + println!("XXX: Got evts:{:?}", evts.len()); + msg.sender.try_send(ReceiveEvents::new(evts))?; + Ok(()) + } } impl EventStore { pub fn new(index: I, log: L) -> Self { @@ -50,3 +64,13 @@ impl Handler for EventStore< } } } + +impl Handler for EventStore { + type Result = (); + fn handle(&mut self, msg: GetEventsAfter, ctx: &mut Self::Context) -> Self::Result { + match self.handle_get_events_after(msg) { + Ok(_) => (), + Err(e) => error!("{e}"), + } + } +} diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index b844f08138..d1304a7fcf 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -41,7 +41,6 @@ impl Actor for Sequencer { impl Handler> for Sequencer { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { - println!("Sequencer got {}", msg); self.eventstore .do_send(StoreEventRequested::new(msg, ctx.address())) } @@ -50,8 +49,6 @@ impl Handler> for Sequencer { impl Handler for Sequencer { type Result = (); fn handle(&mut self, msg: EventStored, ctx: &mut Self::Context) -> Self::Result { - println!("Sequencer got EventStored {:?}", msg); - let event = msg.into_event(); let seq = event.get_seq(); self.buffer.do_send(CommitSnapshot::new(seq)); diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 7415db3ade..c207a2b5da 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -105,7 +105,7 @@ pub trait SequenceIndex: Unpin + 'static { /// Get the sequence offset for the given timestamp fn get(&self, key: u128) -> Result>; /// Get the first sequence offset before the given timestamp - fn seek_for_prev(&self, key: u128) -> Result>; + fn seek(&self, key: u128) -> Result>; } /// Store and retrieve events from a write ahead log @@ -113,10 +113,5 @@ pub trait EventLog: Unpin + 'static { /// Append an event to the log, returning its sequence number fn append(&mut self, event: &EnclaveEvent) -> Result; /// Read all events starting from the given sequence number (inclusive) - fn read_from( - &self, - from: u64, - ) -> Box< - dyn Iterator), anyhow::Error>>, - >; + fn read_from(&self, from: u64) -> Box)>>; } From 323366e4448e69908b8c001cb3d064a26f29a91f Mon Sep 17 00:00:00 2001 From: ryardley Date: Sat, 13 Dec 2025 06:43:01 +0000 Subject: [PATCH 33/66] --wip-- [skip ci] --- crates/events/src/bus_handle.rs | 35 ++++++++++++++++++++++++++----- crates/test-helpers/src/lib.rs | 7 ++++--- crates/tests/tests/integration.rs | 8 +++++-- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 3fd7be014c..3dd55b217a 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -6,7 +6,7 @@ use std::sync::Arc; -use actix::{Addr, Recipient}; +use actix::{Actor, Addr, Handler, Recipient}; use anyhow::Result; use derivative::Derivative; use tracing::error; @@ -18,16 +18,16 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, Sequenced, - Subscribe, Unsequenced, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, Event, EventBus, HistoryCollector, + Sequenced, Subscribe, Unsequenced, }; #[derive(Clone, Derivative)] -#[derivative(Debug)] +#[derivative(Debug, PartialEq, Eq)] pub struct BusHandle { consumer: Addr>>, producer: Addr, - #[derivative(Debug = "ignore")] + #[derivative(Debug = "ignore", PartialEq = "ignore")] hlc: Arc, } @@ -60,6 +60,14 @@ impl BusHandle { let ts = self.hlc.tick()?; Ok(ts.into()) } + + pub fn pipe_to(&self, other: &BusHandle, predicate: F) + where + F: Fn(&EnclaveEvent) -> bool + 'static, + { + let pipe = BusHandlePipe(other.to_owned(), predicate).start(); + self.subscribe("*", pipe.into()); + } } impl EventPublisher> for BusHandle { @@ -281,3 +289,20 @@ mod tests { Ok(()) } } + +pub struct BusHandlePipe(BusHandle, F) +where + F: Fn(&EnclaveEvent) -> bool + 'static; + +impl Actor for BusHandlePipe { + type Context = actix::Context; +} + +impl Handler> for BusHandlePipe { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + let (data, ts) = msg.split(); + let _ = self.0.publish_from_remote(data, ts); + } +} +// XXX: need to use predicate here to filter events to forward diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 9a697f94be..5bce010815 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -16,7 +16,7 @@ use anyhow::*; use e3_ciphernode_builder::{CiphernodeHandle, EventSystem}; use e3_events::{ BusHandle, CiphernodeAdded, EnclaveEvent, EnclaveEventData, EventBus, EventBusConfig, - EventPublisher, HistoryCollector, Seed, Subscribe, + EventPublisher, EventSubscriber, HistoryCollector, Seed, Subscribe, }; use e3_fhe::{create_crp, setup_crp_params, ParamsWithCrp}; use e3_net::{DocumentPublisher, NetEventTranslator}; @@ -126,10 +126,11 @@ pub fn get_common_setup( /// ``` pub fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { for node in nodes.iter() { - let source = node.bus().consumer(); + let source = node.bus(); for (_, node) in nodes.iter().enumerate() { - let dest = node.bus().consumer(); + let dest = node.bus(); if source != dest { + source.pipe_to(dest); EventBus::pipe_filter( source, move |e: &EnclaveEvent| { diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index 29867ebbc8..ef46983c25 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -86,6 +86,7 @@ fn serialize_report(report: &[(&str, Duration)]) -> String { #[actix::test] #[serial_test::serial] async fn test_trbfv_actor() -> Result<()> { + println!("Running test_trbfv_actor..."); let mut report: Vec<(&str, Duration)> = vec![]; let whole_test = Instant::now(); use tracing_subscriber::{fmt, EnvFilter}; @@ -117,7 +118,7 @@ async fn test_trbfv_actor() -> Result<()> { let rng = create_shared_rng_from_u64(42); // Create "trigger" bus - let system = EventSystem::new("test"); + let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; // Parameters (128bits of security) @@ -275,7 +276,6 @@ async fn test_trbfv_actor() -> Result<()> { let committee_finalized_timer = Instant::now(); let expected = vec!["E3Requested", "CommitteeFinalized"]; - let _ = nodes .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; @@ -310,6 +310,10 @@ async fn test_trbfv_actor() -> Result<()> { let h = nodes .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; + + return Ok(()); + unreachable!(); + report.push(( "ThresholdShares -> PublicKeyAggregated", shares_to_pubkey_agg_timer.elapsed(), From 756bb062bd1b65e4b3067b44e4df022cc400e4a6 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 16 Dec 2025 03:43:06 +0000 Subject: [PATCH 34/66] add piping --- crates/events/src/bus_handle.rs | 42 ++++++++++++++++++++++++--------- crates/test-helpers/src/lib.rs | 19 ++++++--------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 3dd55b217a..31a0d9623e 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -18,8 +18,8 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventFactory, EventPublisher, EventSubscriber, }, - EType, EnclaveEvent, EnclaveEventData, ErrorEvent, Event, EventBus, HistoryCollector, - Sequenced, Subscribe, Unsequenced, + EType, EnclaveEvent, EnclaveEventData, ErrorEvent, EventBus, HistoryCollector, Sequenced, + Subscribe, Unsequenced, }; #[derive(Clone, Derivative)] @@ -63,9 +63,9 @@ impl BusHandle { pub fn pipe_to(&self, other: &BusHandle, predicate: F) where - F: Fn(&EnclaveEvent) -> bool + 'static, + F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, { - let pipe = BusHandlePipe(other.to_owned(), predicate).start(); + let pipe = BusHandlePipe::new(other.to_owned(), predicate).start(); self.subscribe("*", pipe.into()); } } @@ -290,19 +290,39 @@ mod tests { } } -pub struct BusHandlePipe(BusHandle, F) +pub struct BusHandlePipe where - F: Fn(&EnclaveEvent) -> bool + 'static; + F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, +{ + handle: BusHandle, + predicate: F, +} + +impl BusHandlePipe +where + F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, +{ + pub fn new(handle: BusHandle, predicate: F) -> Self { + Self { handle, predicate } + } +} -impl Actor for BusHandlePipe { +impl Actor for BusHandlePipe +where + F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, +{ type Context = actix::Context; } -impl Handler> for BusHandlePipe { +impl Handler> for BusHandlePipe +where + F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, +{ type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - let (data, ts) = msg.split(); - let _ = self.0.publish_from_remote(data, ts); + if (self.predicate)(&msg) { + let (data, ts) = msg.split(); + let _ = self.handle.publish_from_remote(data, ts); + } } } -// XXX: need to use predicate here to filter events to forward diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 5bce010815..4ca54ec896 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -130,18 +130,13 @@ pub fn simulate_libp2p_net(nodes: &[CiphernodeHandle]) { for (_, node) in nodes.iter().enumerate() { let dest = node.bus(); if source != dest { - source.pipe_to(dest); - EventBus::pipe_filter( - source, - move |e: &EnclaveEvent| { - // TODO: Document publisher events need to be - // converted to DocumentReceived events - - NetEventTranslator::is_forwardable_event(e) - || DocumentPublisher::is_document_publisher_event(e) - }, - dest, - ) + source.pipe_to(dest, |e: &EnclaveEvent| { + // TODO: Document publisher events need to be + // converted to DocumentReceived events + + NetEventTranslator::is_forwardable_event(e) + || DocumentPublisher::is_document_publisher_event(e) + }); } else { println!("not piping bus to itself"); } From 2529464c4514eb7677db23d0a645489e089f9da1 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 16 Dec 2025 08:53:41 +0000 Subject: [PATCH 35/66] refactor ciphernodeselector --- .../src/ciphernode_builder.rs | 6 +- crates/config/src/store_keys.rs | 4 + crates/sortition/src/ciphernode_selector.rs | 122 ++++++++++-------- crates/sortition/src/repo.rs | 11 ++ 4 files changed, 86 insertions(+), 57 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 1e2704e8e0..4bdf19378a 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -32,8 +32,8 @@ use e3_keyshare::ext::{KeyshareExtension, ThresholdKeyshareExtension}; use e3_multithread::Multithread; use e3_request::E3Router; use e3_sortition::{ - CiphernodeSelector, FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, - Sortition, SortitionBackend, SortitionRepositoryFactory, + CiphernodeSelector, CiphernodeSelectorFactory, FinalizedCommitteesRepositoryFactory, + NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; use e3_utils::{rand_eth_addr, SharedRng}; use std::{collections::HashMap, path::PathBuf, sync::Arc}; @@ -356,7 +356,7 @@ impl CiphernodeBuilder { ) .await?; - CiphernodeSelector::attach(&bus, &sortition, &addr, &store); + CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr); let mut provider_cache = ProviderCaches::new(); let cipher = &self.cipher; diff --git a/crates/config/src/store_keys.rs b/crates/config/src/store_keys.rs index 5cdc6ee26f..6b0eae21ae 100644 --- a/crates/config/src/store_keys.rs +++ b/crates/config/src/store_keys.rs @@ -72,4 +72,8 @@ impl StoreKeys { pub fn finalized_committees() -> String { String::from("//finalized_committees") } + + pub fn ciphernode_selector() -> String { + String::from("//ciphernode_selector") + } } diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 8de35d2885..eb18bad15f 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -5,23 +5,25 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::sortition::{GetNodeIndex, Sortition}; -/// CiphernodeSelector is an actor that determines if a ciphernode is part of a committee and if so -/// emits a TicketGenerated event (score sortition) to the event bus use actix::prelude::*; -use e3_config::StoreKeys; -use e3_data::{DataStore, RepositoriesFactory}; +use anyhow::bail; +use anyhow::Result; +use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ - prelude::*, trap, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, EType, + prelude::*, trap, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, E3id, EType, EnclaveEvent, EnclaveEventData, Shutdown, TicketGenerated, TicketId, }; -use e3_request::MetaRepositoryFactory; +use e3_request::E3Meta; +use std::collections::HashMap; use tracing::info; +/// CiphernodeSelector is an actor that determines if a ciphernode is part of a committee and if so +/// emits a TicketGenerated event (score sortition) to the event bus pub struct CiphernodeSelector { bus: BusHandle, sortition: Addr, address: String, - data_store: DataStore, + e3_cache: Persistable>, } impl Actor for CiphernodeSelector { @@ -32,30 +34,31 @@ impl CiphernodeSelector { pub fn new( bus: &BusHandle, sortition: &Addr, + e3_cache: Persistable>, address: &str, - data_store: &DataStore, ) -> Self { Self { bus: bus.clone(), sortition: sortition.clone(), + e3_cache, address: address.to_owned(), - data_store: data_store.clone(), } } - pub fn attach( + pub async fn attach( bus: &BusHandle, sortition: &Addr, + selector_store: Repository>, address: &str, - data_store: &DataStore, - ) -> Addr { - let addr = CiphernodeSelector::new(bus, sortition, address, data_store).start(); + ) -> Result> { + let e3_cache = selector_store.load_or_default(HashMap::new()).await?; + let addr = CiphernodeSelector::new(bus, sortition, e3_cache, address).start(); bus.subscribe("E3Requested", addr.clone().recipient()); bus.subscribe("CommitteeFinalized", addr.clone().recipient()); bus.subscribe("Shutdown", addr.clone().recipient()); - addr + Ok(addr) } } @@ -80,6 +83,23 @@ impl Handler for CiphernodeSelector { let bus = self.bus.clone(); let chain_id = data.e3_id.chain_id(); + trap(EType::Sortition, &bus.clone(), || { + self.e3_cache.try_mutate(|mut cache| { + cache.insert( + data.e3_id.clone(), + E3Meta { + seed: data.seed, + threshold_n: data.threshold_n, + threshold_m: data.threshold_m, + params: data.params, + esi_per_ct: data.esi_per_ct, + error_size: data.error_size, + }, + ); + Ok(cache) + }) + }); + Box::pin(async move { let seed = data.seed; let size = data.threshold_n; @@ -126,61 +146,55 @@ impl Handler for CiphernodeSelector { } impl Handler for CiphernodeSelector { - type Result = ResponseFuture<()>; + type Result = (); fn handle(&mut self, msg: CommitteeFinalized, _ctx: &mut Self::Context) -> Self::Result { - let address = self.address.clone(); - let bus = self.bus.clone(); - let e3_id = msg.e3_id.clone(); - let repositories = self - .data_store - .scope(StoreKeys::router()) - .scope(StoreKeys::context(&e3_id)) - .repositories(); - - // Check if this node is in the finalized committee - if !msg.committee.contains(&address) { - info!(node = address, "Node not in finalized committee"); - return Box::pin(async {}); - } + trap(EType::Sortition, &self.bus.clone(), move || { + let bus = self.bus.clone(); + let Some(e3_cache) = self.e3_cache.get() else { + bail!("Could not get cache"); + }; - Box::pin(async move { - // Retrieve E3 metadata from repository - let meta_repo = repositories.meta(&e3_id); - let Some(e3_meta) = meta_repo.read().await.ok().flatten() else { - info!( - node = address, - "No stored E3 metadata for {:?}, skipping", e3_id + let Some(e3_meta) = e3_cache.get(&msg.e3_id) else { + bail!( + "Could not find E3Meta on CiphernodeSelector for {}", + msg.e3_id ); - return; }; - let Some(party_id) = msg.committee.iter().position(|addr| addr == &address) else { + // Check if this node is in the finalized committee + if !msg.committee.contains(&self.address) { + info!(node = self.address, "Node not in finalized committee"); + return Ok(()); + } + + // Retrieve E3 metadata from repository + let Some(party_id) = msg.committee.iter().position(|addr| addr == &self.address) else { info!( - node = address, + node = self.address, "Node address not found in committee list (should not happen)" ); - return; + return Ok(()); }; info!( - node = address, + node = self.address, party_id = party_id, "Node is in finalized committee, emitting CiphernodeSelected" ); - trap(EType::Sortition, &bus.clone(), || { - bus.publish(CiphernodeSelected { - party_id: party_id as u64, - e3_id, - threshold_m: e3_meta.threshold_m, - threshold_n: e3_meta.threshold_n, - esi_per_ct: e3_meta.esi_per_ct, - error_size: e3_meta.error_size, - params: e3_meta.params, - seed: e3_meta.seed, - })?; - Ok(()) - }) + + bus.publish(CiphernodeSelected { + party_id: party_id as u64, + e3_id: msg.e3_id, + threshold_m: e3_meta.threshold_m, + threshold_n: e3_meta.threshold_n, + esi_per_ct: e3_meta.esi_per_ct, + error_size: e3_meta.error_size.clone(), + params: e3_meta.params.clone(), + seed: e3_meta.seed, + })?; + + Ok(()) }) } } diff --git a/crates/sortition/src/repo.rs b/crates/sortition/src/repo.rs index 419c8db7d4..75a92d0ccb 100644 --- a/crates/sortition/src/repo.rs +++ b/crates/sortition/src/repo.rs @@ -9,6 +9,7 @@ use crate::sortition::NodeStateStore; use e3_config::StoreKeys; use e3_data::{Repositories, Repository}; use e3_events::E3id; +use e3_request::E3Meta; use std::collections::HashMap; pub trait SortitionRepositoryFactory { @@ -21,6 +22,16 @@ impl SortitionRepositoryFactory for Repositories { } } +pub trait CiphernodeSelectorFactory { + fn ciphernode_selector(&self) -> Repository>; +} + +impl CiphernodeSelectorFactory for Repositories { + fn ciphernode_selector(&self) -> Repository> { + Repository::new(self.store.scope(StoreKeys::ciphernode_selector())) + } +} + pub trait NodeStateRepositoryFactory { fn node_state(&self) -> Repository>; } From 4d2f556659e9368839000a4a06e995ed6bee6909 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 16 Dec 2025 09:02:34 +0000 Subject: [PATCH 36/66] remove test stop --- crates/tests/tests/integration.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index ef46983c25..df0fe66a30 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -311,9 +311,6 @@ async fn test_trbfv_actor() -> Result<()> { .take_history_with_timeout(0, expected.len(), Duration::from_secs(1000)) .await?; - return Ok(()); - unreachable!(); - report.push(( "ThresholdShares -> PublicKeyAggregated", shares_to_pubkey_agg_timer.elapsed(), From 0ee7914bdb2af7fbb63ad54b6c2def225dcce11c Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 16 Dec 2025 09:15:01 +0000 Subject: [PATCH 37/66] tidy up --- crates/ciphernode-builder/src/event_system.rs | 8 +++++--- crates/events/src/eventstore.rs | 6 ------ crates/events/src/sequencer.rs | 2 -- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index aee7c302fe..bd034e09ca 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -413,10 +413,12 @@ mod tests { // NOTE: Eventual consistency // Store should not have data set on it until event has been published - // There is an argument we should instead delay reads until the event has been stored + // There is an argument we should instead delay reads until the event has been stored but + // this would: + // a. Promote poor patterns of sharing data through persistence + // b. Add a large amount of complexity to batching Get operations // For now we allow this inconsistency under the assumption that data is written for - // snapshot storage exclusively. If we have issues we may wish to change to a more - // highly consistent model. + // snapshot storage exclusively. // Let's check the eventual consistency all data points should be none... assert_eq!(datastore.scope("/foo/name").read::().await?, None); diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 7db2e52963..8ed9a0d4d4 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -22,25 +22,19 @@ impl EventStore { let event = msg.event; let sender = msg.sender; let ts = event.get_ts(); - println!("VVV: ts={}", ts); let seq = self.log.append(&event)?; - println!("VVV: seq={}", seq); self.index.insert(ts, seq)?; sender.try_send(EventStored(event.into_sequenced(seq)))?; Ok(()) } pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { - println!("XXX: GetEventsAfter {}!", msg.ts); let seq = self.index.seek(msg.ts)?.unwrap_or(1); - println!("XXX: seq={}", seq); - let evts = self .log .read_from(seq) .map(|(s, e)| e.into_sequenced(s)) .collect::>(); - println!("XXX: Got evts:{:?}", evts.len()); msg.sender.try_send(ReceiveEvents::new(evts))?; Ok(()) } diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index d1304a7fcf..b19596b3bb 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -14,7 +14,6 @@ use crate::{ /// Component to sequence the storage of events pub struct Sequencer { bus: Addr>>, - seq: u64, eventstore: Recipient, buffer: Recipient, } @@ -27,7 +26,6 @@ impl Sequencer { ) -> Self { Self { bus: bus.clone(), - seq: 0, eventstore: eventstore.into(), buffer: buffer.into(), } From 9fd60d4aa31d405c8b40b10f38bd5180185cc522 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 00:51:57 +0000 Subject: [PATCH 38/66] fix missing await and add color utils for debugging --- Cargo.lock | 1 + .../src/ciphernode_builder.rs | 3 ++- crates/events/src/enclave_event/mod.rs | 4 +++- crates/keyshare/src/threshold_keyshare.rs | 1 + crates/sortition/Cargo.toml | 1 + crates/sortition/src/ciphernode_selector.rs | 12 ++++++++++ crates/utils/src/formatters.rs | 23 +++++++++++++++++++ 7 files changed, 43 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 564c8aac68..962eff9c45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3225,6 +3225,7 @@ dependencies = [ "e3-data", "e3-events", "e3-request", + "e3-utils", "num", "num-bigint", "rand 0.8.5", diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 4bdf19378a..3d548ac4ed 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -356,7 +356,8 @@ impl CiphernodeBuilder { ) .await?; - CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr); + CiphernodeSelector::attach(&bus, &sortition, repositories.ciphernode_selector(), &addr) + .await?; let mut provider_cache = ProviderCaches::new(); let cipher = &self.cipher; diff --git a/crates/events/src/enclave_event/mod.rs b/crates/events/src/enclave_event/mod.rs index 0f7f2b07ad..16c7eb441a 100644 --- a/crates/events/src/enclave_event/mod.rs +++ b/crates/events/src/enclave_event/mod.rs @@ -46,6 +46,7 @@ pub use decryptionshare_created::*; pub use die::*; pub use e3_request_complete::*; pub use e3_requested::*; +use e3_utils::{colorize, Color}; pub use enclave_error::*; pub use keyshare_created::*; pub use operator_activation_changed::*; @@ -345,7 +346,8 @@ impl TryFrom> for EnclaveError { impl fmt::Display for EnclaveEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&format!("{:?}", self)) + let t = self.event_type(); + f.write_str(&format!("{} {:?}", colorize(t, Color::Cyan), self)) } } diff --git a/crates/keyshare/src/threshold_keyshare.rs b/crates/keyshare/src/threshold_keyshare.rs index d79398f13a..ccc84e1125 100644 --- a/crates/keyshare/src/threshold_keyshare.rs +++ b/crates/keyshare/src/threshold_keyshare.rs @@ -336,6 +336,7 @@ impl ThresholdKeyshare { msg: CiphernodeSelected, address: Addr, ) -> Result<()> { + info!("CiphernodeSelected received."); // Ensure the collector is created let _ = self.ensure_collector(address.clone()); // Initialize State diff --git a/crates/sortition/Cargo.toml b/crates/sortition/Cargo.toml index d38cd3bf30..42bf45d7b0 100644 --- a/crates/sortition/Cargo.toml +++ b/crates/sortition/Cargo.toml @@ -17,6 +17,7 @@ e3-config = { workspace = true } e3-data = { workspace = true } e3-events = { workspace = true } e3-request = { workspace = true } +e3-utils = { workspace = true } num = { workspace = true } rand = { workspace = true } serde = { workspace = true } diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index eb18bad15f..ec1505f703 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -14,6 +14,8 @@ use e3_events::{ EnclaveEvent, EnclaveEventData, Shutdown, TicketGenerated, TicketId, }; use e3_request::E3Meta; +use e3_utils::colorize; +use e3_utils::Color; use std::collections::HashMap; use tracing::info; @@ -58,6 +60,7 @@ impl CiphernodeSelector { bus.subscribe("CommitteeFinalized", addr.clone().recipient()); bus.subscribe("Shutdown", addr.clone().recipient()); + info!("CiphernodeSelector listening!"); Ok(addr) } } @@ -85,6 +88,10 @@ impl Handler for CiphernodeSelector { trap(EType::Sortition, &bus.clone(), || { self.e3_cache.try_mutate(|mut cache| { + info!( + "Mutating e3_cache: appending data: {:?}", + data.e3_id.clone() + ); cache.insert( data.e3_id.clone(), E3Meta { @@ -109,6 +116,8 @@ impl Handler for CiphernodeSelector { seed, size ); + // TODO: instead of this it would be better to pass the event theough sortition and + // then decorate it with this information WithIndex if let Ok(found_result) = sortition .send(GetNodeIndex { chain_id, @@ -150,11 +159,14 @@ impl Handler for CiphernodeSelector { fn handle(&mut self, msg: CommitteeFinalized, _ctx: &mut Self::Context) -> Self::Result { trap(EType::Sortition, &self.bus.clone(), move || { + info!("CiphernodeSelector received CommitteeFinalized."); let bus = self.bus.clone(); + info!("Getting e3_cache..."); let Some(e3_cache) = self.e3_cache.get() else { bail!("Could not get cache"); }; + info!("Getting e3_meta..."); let Some(e3_meta) = e3_cache.get(&msg.e3_id) else { bail!( "Could not find E3Meta on CiphernodeSelector for {}", diff --git a/crates/utils/src/formatters.rs b/crates/utils/src/formatters.rs index 48051a3830..b8094d900b 100644 --- a/crates/utils/src/formatters.rs +++ b/crates/utils/src/formatters.rs @@ -35,3 +35,26 @@ pub fn truncate(s: String) -> String { format!("", s.len(), start, end) } } + +pub enum Color { + Black = 30, + Red = 31, + Green = 32, + Yellow = 33, + Blue = 34, + Magenta = 35, + Cyan = 36, + White = 37, + BrightBlack = 90, + BrightRed = 91, + BrightGreen = 92, + BrightYellow = 93, + BrightBlue = 94, + BrightMagenta = 95, + BrightCyan = 96, + BrightWhite = 97, +} + +pub fn colorize(s: T, color: Color) -> String { + format!("\x1b[{}m{}\x1b[0m", color as u8, s) +} From 57dd201fc0fe07d01798e935e8fa4f720a2374ce Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 01:26:01 +0000 Subject: [PATCH 39/66] implement commit log and tidy up --- crates/data/src/commit_log_event_log.rs | 113 +++++++++++++++++++++++- crates/data/src/data_store.rs | 11 +-- crates/data/src/in_mem_event_log.rs | 1 - crates/data/src/sled_db.rs | 5 +- crates/data/src/sled_store.rs | 2 +- crates/data/src/write_buffer.rs | 3 +- crates/events/src/eventbus.rs | 1 - crates/events/src/events.rs | 4 + crates/events/src/eventstore.rs | 2 +- crates/events/src/sequencer.rs | 2 +- 10 files changed, 122 insertions(+), 22 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index d22d71b0e0..7c68457b8b 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -7,7 +7,8 @@ use std::path::PathBuf; use anyhow::Result; -use commitlog::{CommitLog, LogOptions}; +use commitlog::message::MessageSet; +use commitlog::{CommitLog, LogOptions, ReadLimit}; use e3_events::{EnclaveEvent, EventLog, Unsequenced}; pub struct CommitLogEventLog { @@ -24,10 +25,116 @@ impl CommitLogEventLog { impl EventLog for CommitLogEventLog { fn append(&mut self, event: &EnclaveEvent) -> Result { - Ok(1u64) + let bytes = bincode::serialize(event)?; + let offset = self.log.append_msg(&bytes)?; + // Return 1-indexed sequence number + Ok(offset + 1) } fn read_from(&self, from: u64) -> Box)>> { - Box::new(vec![].into_iter()) + // Convert 1-indexed sequence to 0-indexed offset + let mut current_offset = from.saturating_sub(1); + let mut events = Vec::new(); + + loop { + let message_buf = match self.log.read(current_offset, ReadLimit::default()) { + Ok(msgs) => msgs, + Err(_) => break, + }; + + let mut count = 0; + for msg in message_buf.iter() { + if let Ok(event) = bincode::deserialize::>(msg.payload()) + { + // Convert 0-indexed offset back to 1-indexed sequence number + events.push((msg.offset() + 1, event)); + current_offset = msg.offset() + 1; // Next offset to read from + } + count += 1; + } + + // No more messages to read + if count == 0 { + break; + } + } + + Box::new(events.into_iter()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; + use tempfile::tempdir; + + fn event_from(data: impl Into) -> EnclaveEvent { + EnclaveEvent::::new_with_timestamp(data.into().into(), 123) + } + + #[test] + fn test_append_and_read() { + let dir = tempdir().unwrap(); + let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + + // Create some test events - adjust these to match your actual EnclaveEvent structure + let event1 = event_from(TestEvent::new("one", 1)); + let event2 = event_from(TestEvent::new("two", 2)); + + // Append events + let offset1 = log.append(&event1).unwrap(); + let offset2 = log.append(&event2).unwrap(); + + assert_eq!(offset1, 1); // 1-indexed + assert_eq!(offset2, 2); + + // Read back from the beginning + let events: Vec<_> = log.read_from(1).collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 1); + assert_eq!(events[1].0, 2); + } + + #[test] + fn test_read_from_offset() { + let dir = tempdir().unwrap(); + let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + + let event1 = event_from(TestEvent::new("one", 1)); + let event2 = event_from(TestEvent::new("two", 2)); + let event3 = event_from(TestEvent::new("three", 3)); + + log.append(&event1).unwrap(); + log.append(&event2).unwrap(); + log.append(&event3).unwrap(); + + // Read from offset 2 (should get events 2 and 3) + let events: Vec<_> = log.read_from(2).collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 2); + assert_eq!(events[1].0, 3); + } + + #[test] + fn test_read_empty_log() { + let dir = tempdir().unwrap(); + let log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + + let events: Vec<_> = log.read_from(1).collect(); + assert!(events.is_empty()); + } + + #[test] + fn test_read_past_end() { + let dir = tempdir().unwrap(); + let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + + let event = event_from(TestEvent::new("one", 1)); + log.append(&event).unwrap(); + + // Read from offset beyond what exists + let events: Vec<_> = log.read_from(100).collect(); + assert!(events.is_empty()); } } diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index b4a2743ab9..b2df9273cf 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -6,9 +6,9 @@ use std::borrow::Cow; -use crate::{ForwardTo, Get, Insert, InsertSync, Remove, WriteBuffer}; +use crate::{Get, Insert, InsertSync, Remove, WriteBuffer}; use crate::{InMemStore, IntoKey, SledStore}; -use actix::{Actor, Addr, Recipient}; +use actix::{Addr, Recipient}; use anyhow::anyhow; use anyhow::Context; use anyhow::Result; @@ -22,13 +22,6 @@ pub enum StoreAddr { } impl StoreAddr { - // pub fn to_data_store(&self) -> DataStore { - // match self { - // StoreAddr::InMem(s) => s.into(), - // StoreAddr::Sled(s) => s.into(), - // } - // } - pub fn to_maybe_in_mem(&self) -> Option<&Addr> { match self { StoreAddr::InMem(ref store) => Some(store), diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index e916d42466..a742fa5e5d 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -29,7 +29,6 @@ impl EventLog for InMemEventLog { .enumerate() .map(|(i, event)| (from + i as u64, event.clone())) .collect(); - println!("XXX: events:{:?}", events); Box::new(events.into_iter()) } fn append(&mut self, event: &EnclaveEvent) -> Result { diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 9bee35e67f..641ea39797 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -5,10 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::{Context, Result}; -use sled::{ - transaction::{ConflictableTransactionError, TransactionalTree, UnabortableTransactionError}, - Tree, -}; +use sled::{transaction::ConflictableTransactionError, Tree}; use std::path::PathBuf; use crate::{ diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index f911a938b6..2deaab949c 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{Get, Insert, InsertBatch, InsertSync, Remove, SledDb}; -use actix::{Actor, ActorContext, Addr, AsyncContext, Handler}; +use actix::{Actor, ActorContext, Addr, Handler}; use anyhow::Result; use e3_events::{prelude::*, BusHandle, EType, EnclaveEvent, EnclaveEventData}; use std::path::PathBuf; diff --git a/crates/data/src/write_buffer.rs b/crates/data/src/write_buffer.rs index f41dc08e9f..c3d39afdb0 100644 --- a/crates/data/src/write_buffer.rs +++ b/crates/data/src/write_buffer.rs @@ -48,7 +48,8 @@ impl Handler for WriteBuffer { fn handle(&mut self, msg: CommitSnapshot, _: &mut Self::Context) -> Self::Result { if let Some(ref dest) = self.dest { if !self.buffer.is_empty() { - let inserts = std::mem::take(&mut self.buffer); + let mut inserts = std::mem::take(&mut self.buffer); + inserts.push(Insert::new("//seq", msg.seq().to_be_bytes().to_vec())); let batch = InsertBatch::new(inserts); dest.do_send(batch); } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 515396bd19..ddf2422ae6 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -5,7 +5,6 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::traits::{ErrorEvent, Event}; -use crate::{prelude::*, BusHandle, EType, EnclaveEvent, Sequenced, Unsequenced}; use actix::prelude::*; use bloom::{BloomFilter, ASMS}; use std::collections::{HashMap, VecDeque}; diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 9ae0d86dab..96450e4921 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -17,6 +17,10 @@ impl CommitSnapshot { pub fn new(seq: u64) -> Self { Self(seq) } + + pub fn seq(&self) -> u64 { + self.0 + } } /// Direct event received by the EventStore to store an event diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 8ed9a0d4d4..68295a7037 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -61,7 +61,7 @@ impl Handler for EventStore< impl Handler for EventStore { type Result = (); - fn handle(&mut self, msg: GetEventsAfter, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: GetEventsAfter, _: &mut Self::Context) -> Self::Result { match self.handle_get_events_after(msg) { Ok(_) => (), Err(e) => error!("{e}"), diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index b19596b3bb..a521637def 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -46,7 +46,7 @@ impl Handler> for Sequencer { impl Handler for Sequencer { type Result = (); - fn handle(&mut self, msg: EventStored, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EventStored, _: &mut Self::Context) -> Self::Result { let event = msg.into_event(); let seq = event.get_seq(); self.buffer.do_send(CommitSnapshot::new(seq)); From d396f071ead248c3d87f5755277918b41a35cee3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 01:27:21 +0000 Subject: [PATCH 40/66] remove unused dep --- crates/data/src/in_mem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index ccff6c7f61..95c16c8794 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -74,7 +74,7 @@ impl Handler for InMemStore { impl Handler for InMemStore { type Result = (); - fn handle(&mut self, msg: InsertBatch, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { for cmd in msg.commands() { self.db.insert(cmd.key().to_owned(), cmd.value().to_owned()); } From 866679a479b3890fa37bc4d7a2df53bb954ae797 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 01:33:19 +0000 Subject: [PATCH 41/66] remove unused dep --- crates/ciphernode-builder/src/event_system.rs | 2 +- crates/data/src/events.rs | 22 ------------------- .../entrypoint/src/start/aggregator_start.rs | 2 -- crates/net/src/events.rs | 5 +---- crates/net/src/net_event_translator.rs | 2 +- crates/request/src/router.rs | 1 - crates/sortition/src/ciphernode_selector.rs | 2 -- crates/test-helpers/src/lib.rs | 2 +- 8 files changed, 4 insertions(+), 34 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index bd034e09ca..2f2233b7c0 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -334,7 +334,7 @@ mod tests { impl Handler for Listener { type Result = Vec; - fn handle(&mut self, msg: GetLogs, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, _: GetLogs, _: &mut Self::Context) -> Self::Result { self.logs.clone() } } diff --git a/crates/data/src/events.rs b/crates/data/src/events.rs index 9b6312f3c9..6ecfc25d40 100644 --- a/crates/data/src/events.rs +++ b/crates/data/src/events.rs @@ -86,25 +86,3 @@ impl Remove { &self.0 } } - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "()")] -pub struct SeekForPrev(pub Vec); -impl SeekForPrev { - pub fn new(key: K) -> Self { - Self(key.into_key()) - } - - pub fn key(&self) -> &Vec { - &self.0 - } -} - -#[derive(Message)] -#[rtype(result = "()")] -pub struct SeekForPrevReply(u64); -impl SeekForPrevReply { - pub fn new(val: u64) -> Self { - Self(val) - } -} diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index f2779a8ee2..9cb45696b9 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -20,8 +20,6 @@ use std::{ }; use tokio::task::JoinHandle; -use crate::helpers::datastore::setup_datastore; - pub async fn execute( config: &AppConfig, pubkey_write_path: Option, diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 0218cae8c1..e042eedd1d 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -7,10 +7,7 @@ use crate::Cid; use actix::Message; use anyhow::{bail, Context, Result}; -use e3_events::{ - CorrelationId, DocumentMeta, EnclaveEvent, EventConstructorWithTimestamp, Sequenced, - Unsequenced, -}; +use e3_events::{CorrelationId, DocumentMeta, EnclaveEvent, Sequenced, Unsequenced}; use e3_utils::ArcBytes; use libp2p::{ gossipsub::{MessageId, PublishError, TopicHash}, diff --git a/crates/net/src/net_event_translator.rs b/crates/net/src/net_event_translator.rs index d12abd5793..54126d7e13 100644 --- a/crates/net/src/net_event_translator.rs +++ b/crates/net/src/net_event_translator.rs @@ -29,7 +29,7 @@ use std::sync::Arc; use tokio::sync::broadcast; use tokio::sync::mpsc; use tracing::warn; -use tracing::{error, info, instrument, trace}; +use tracing::{info, instrument, trace}; // TODO: store event filtering here on this actor instead of is_local_only() on the event. We // should do this as this functionality is not global and ramifications should stay local to here diff --git a/crates/request/src/router.rs b/crates/request/src/router.rs index 3df1c2e1a9..82ee518b04 100644 --- a/crates/request/src/router.rs +++ b/crates/request/src/router.rs @@ -10,7 +10,6 @@ use crate::E3ContextParams; use crate::E3ContextSnapshot; use crate::E3MetaExtension; use crate::RouterRepositoryFactory; -use actix::Message; use actix::{Actor, Addr, Context, Handler}; use anyhow::*; use async_trait::async_trait; diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index ec1505f703..36fc8c8e97 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -14,8 +14,6 @@ use e3_events::{ EnclaveEvent, EnclaveEventData, Shutdown, TicketGenerated, TicketId, }; use e3_request::E3Meta; -use e3_utils::colorize; -use e3_utils::Color; use std::collections::HashMap; use tracing::info; diff --git a/crates/test-helpers/src/lib.rs b/crates/test-helpers/src/lib.rs index 4ca54ec896..8f7540e14c 100644 --- a/crates/test-helpers/src/lib.rs +++ b/crates/test-helpers/src/lib.rs @@ -16,7 +16,7 @@ use anyhow::*; use e3_ciphernode_builder::{CiphernodeHandle, EventSystem}; use e3_events::{ BusHandle, CiphernodeAdded, EnclaveEvent, EnclaveEventData, EventBus, EventBusConfig, - EventPublisher, EventSubscriber, HistoryCollector, Seed, Subscribe, + EventPublisher, HistoryCollector, Seed, Subscribe, }; use e3_fhe::{create_crp, setup_crp_params, ParamsWithCrp}; use e3_net::{DocumentPublisher, NetEventTranslator}; From 5cebe4b93b4d1458460be9ebf159348562f101ff Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 01:34:42 +0000 Subject: [PATCH 42/66] remove comments --- crates/data/src/commit_log_event_log.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 7c68457b8b..716daf1ccf 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -78,11 +78,9 @@ mod tests { let dir = tempdir().unwrap(); let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); - // Create some test events - adjust these to match your actual EnclaveEvent structure let event1 = event_from(TestEvent::new("one", 1)); let event2 = event_from(TestEvent::new("two", 2)); - // Append events let offset1 = log.append(&event1).unwrap(); let offset2 = log.append(&event2).unwrap(); From 6dacec472d5ef4e770bb28fe07e074f133a3b35c Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 02:57:31 +0000 Subject: [PATCH 43/66] use persistence --- crates/config/src/app_config.rs | 5 ++ crates/data/src/in_mem_event_log.rs | 69 +++++++++++++++++++ .../entrypoint/src/start/aggregator_start.rs | 1 + crates/entrypoint/src/start/start.rs | 1 + crates/evm/src/helpers.rs | 1 + 5 files changed, 77 insertions(+) diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index 32406ee07f..8bed2fe5d5 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -251,6 +251,11 @@ impl AppConfig { self.paths.db_file() } + /// Get the log file + pub fn log_file(&self) -> PathBuf { + self.paths.log_file() + } + fn node_def(&self) -> &NodeDefinition { // NOTE: on creation an invariant we have is that our node name is an extant key in our // nodes datastructure so expect here is ok and we dont have to clone the NodeDefinition diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index a742fa5e5d..947e774f83 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -36,3 +36,72 @@ impl EventLog for InMemEventLog { Ok(self.log.len() as u64) } } + +#[cfg(test)] +mod tests { + use super::*; + use e3_events::{EnclaveEventData, EventConstructorWithTimestamp, TestEvent}; + + fn event_from(data: impl Into) -> EnclaveEvent { + EnclaveEvent::::new_with_timestamp(data.into().into(), 123) + } + + #[test] + fn test_append_and_read() { + let mut log = InMemEventLog::new(); + + let event1 = event_from(TestEvent::new("one", 1)); + let event2 = event_from(TestEvent::new("two", 2)); + + let offset1 = log.append(&event1).unwrap(); + let offset2 = log.append(&event2).unwrap(); + + assert_eq!(offset1, 1); // 1-indexed + assert_eq!(offset2, 2); + + // Read back from the beginning + let events: Vec<_> = log.read_from(1).collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 1); + assert_eq!(events[1].0, 2); + } + + #[test] + fn test_read_from_offset() { + let mut log = InMemEventLog::new(); + + let event1 = event_from(TestEvent::new("one", 1)); + let event2 = event_from(TestEvent::new("two", 2)); + let event3 = event_from(TestEvent::new("three", 3)); + + log.append(&event1).unwrap(); + log.append(&event2).unwrap(); + log.append(&event3).unwrap(); + + // Read from offset 2 (should get events 2 and 3) + let events: Vec<_> = log.read_from(2).collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 2); + assert_eq!(events[1].0, 3); + } + + #[test] + fn test_read_empty_log() { + let log = InMemEventLog::new(); + + let events: Vec<_> = log.read_from(1).collect(); + assert!(events.is_empty()); + } + + #[test] + fn test_read_past_end() { + let mut log = InMemEventLog::new(); + + let event = event_from(TestEvent::new("one", 1)); + log.append(&event).unwrap(); + + // Read from offset beyond what exists + let events: Vec<_> = log.read_from(100).collect(); + assert!(events.is_empty()); + } +} diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index 9cb45696b9..aa5ee77440 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -35,6 +35,7 @@ pub async fn execute( .with_sortition_score() .with_contract_enclave_full() .with_contract_bonding_registry() + .with_persistence(&config.log_file(), &config.db_file()) .with_contract_ciphernode_registry() .with_max_threads() .with_pubkey_aggregation(); diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 783f4aa903..545077e4b8 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -34,6 +34,7 @@ pub async fn execute( .with_source_bus(bus.consumer()) .with_sortition_score() .with_chains(&config.chains()) + .with_persistence(&config.log_file(), &config.db_file()) .with_contract_enclave_reader() .with_contract_bonding_registry() .with_max_threads() diff --git a/crates/evm/src/helpers.rs b/crates/evm/src/helpers.rs index c655a33924..678532008d 100644 --- a/crates/evm/src/helpers.rs +++ b/crates/evm/src/helpers.rs @@ -32,6 +32,7 @@ use e3_config::{RpcAuth, RPC}; use e3_crypto::Cipher; use e3_data::Repository; use std::{env, sync::Arc}; +use tracing::info; pub trait AuthConversions { fn to_header_value(&self) -> Option; From 83698b00915ee631ad52d76dbfd615249419c886 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 03:57:57 +0000 Subject: [PATCH 44/66] update eventstore limit --- crates/data/src/commit_log_event_log.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 716daf1ccf..adb0cd06c2 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -17,7 +17,9 @@ pub struct CommitLogEventLog { impl CommitLogEventLog { pub fn new(path: &PathBuf) -> Result { - let opts = LogOptions::new(path); + let mut opts = LogOptions::new(path); + // The following is high at 32Mb but we have large events + opts.message_max_bytes(32 * 1024 * 1024); let log = CommitLog::new(opts)?; Ok(Self { log }) } From 467939d3bb1cb134572ca453bf50daa60c5689e3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 03:58:38 +0000 Subject: [PATCH 45/66] update eventstore limit --- crates/data/src/commit_log_event_log.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index adb0cd06c2..ab09fadf78 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -18,7 +18,7 @@ pub struct CommitLogEventLog { impl CommitLogEventLog { pub fn new(path: &PathBuf) -> Result { let mut opts = LogOptions::new(path); - // The following is high at 32Mb but we have large events + // TODO: drive this from config - currently set high to be permissive opts.message_max_bytes(32 * 1024 * 1024); let log = CommitLog::new(opts)?; Ok(Self { log }) From 895276e3015308099b680009900725a187c686e2 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 04:00:52 +0000 Subject: [PATCH 46/66] add event context --- crates/data/src/commit_log_event_log.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index ab09fadf78..c7f9659bde 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -4,12 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::path::PathBuf; - +use anyhow::Context; use anyhow::Result; use commitlog::message::MessageSet; use commitlog::{CommitLog, LogOptions, ReadLimit}; use e3_events::{EnclaveEvent, EventLog, Unsequenced}; +use std::path::PathBuf; pub struct CommitLogEventLog { log: CommitLog, @@ -28,7 +28,10 @@ impl CommitLogEventLog { impl EventLog for CommitLogEventLog { fn append(&mut self, event: &EnclaveEvent) -> Result { let bytes = bincode::serialize(event)?; - let offset = self.log.append_msg(&bytes)?; + let offset = self + .log + .append_msg(&bytes) + .context("Failed to append to event log")?; // Return 1-indexed sequence number Ok(offset + 1) } From 1ef1a880c5c8d43410305d80360ea6ac59a7793b Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 04:48:21 +0000 Subject: [PATCH 47/66] tidy up pr --- crates/data/src/commit_log_event_log.rs | 3 +++ crates/data/src/in_mem.rs | 11 ----------- crates/data/src/in_mem_event_log.rs | 6 ++++++ crates/events/src/bus_handle.rs | 2 +- crates/events/src/hlc.rs | 11 +++++++++++ 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index c7f9659bde..8c20d0800c 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -10,6 +10,7 @@ use commitlog::message::MessageSet; use commitlog::{CommitLog, LogOptions, ReadLimit}; use e3_events::{EnclaveEvent, EventLog, Unsequenced}; use std::path::PathBuf; +use tracing::error; pub struct CommitLogEventLog { log: CommitLog, @@ -54,6 +55,8 @@ impl EventLog for CommitLogEventLog { // Convert 0-indexed offset back to 1-indexed sequence number events.push((msg.offset() + 1, event)); current_offset = msg.offset() + 1; // Next offset to read from + } else { + error!("Error deserializing event in read_from... skipping"); } count += 1; } diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 95c16c8794..4820e1ad05 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -150,14 +150,3 @@ impl InMemDb { Ok(()) } } - -pub struct InMemCommitLog { - log: Vec>, -} - -impl InMemCommitLog { - pub fn append_msg(&mut self, payload: Vec) -> Result { - self.log.push(payload); - Ok(self.log.len() as u64) - } -} diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 947e774f83..9b95398919 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -17,6 +17,12 @@ impl InMemEventLog { } } +impl Default for InMemEventLog { + fn default() -> Self { + Self::new() + } +} + impl EventLog for InMemEventLog { fn read_from(&self, from: u64) -> Box)>> { // Convert 1-indexed sequence to 0-indexed array position diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 31a0d9623e..5405f37e76 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -27,7 +27,7 @@ use crate::{ pub struct BusHandle { consumer: Addr>>, producer: Addr, - #[derivative(Debug = "ignore", PartialEq = "ignore")] + #[derivative(Debug = "ignore")] hlc: Arc, } diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index 11a63d5727..2cdb3001a4 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -4,6 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +use derivative::Derivative; use rand::Rng; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::{Arc, Mutex}; @@ -171,6 +172,7 @@ pub struct Hlc { clock: Option u64 + Send + Sync>>, } +#[derive(PartialEq)] struct HlcInner { ts: u64, counter: u32, @@ -183,6 +185,15 @@ impl Default for Hlc { } } +impl PartialEq for Hlc { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + && self.node == other.node + && self.max_drift == other.max_drift + // note clock ignored because it is only used in testing + } +} + impl Hlc { const DEFAULT_MAX_DRIFT: u64 = 60_000_000; // 60 sec From 30537f9636cb67726c1994ce8db2582a37d1a82a Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 04:50:54 +0000 Subject: [PATCH 48/66] tidy up unused imports --- crates/data/src/in_mem.rs | 1 - crates/events/src/hlc.rs | 1 - crates/evm/src/helpers.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 4820e1ad05..0785ef1047 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -7,7 +7,6 @@ use crate::{Get, Insert, InsertBatch, InsertSync, Remove}; use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; -use commitlog::Offset; use std::{collections::BTreeMap, ops::Deref}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index 2cdb3001a4..a49c077e79 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -4,7 +4,6 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use derivative::Derivative; use rand::Rng; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::{Arc, Mutex}; diff --git a/crates/evm/src/helpers.rs b/crates/evm/src/helpers.rs index 678532008d..c655a33924 100644 --- a/crates/evm/src/helpers.rs +++ b/crates/evm/src/helpers.rs @@ -32,7 +32,6 @@ use e3_config::{RpcAuth, RPC}; use e3_crypto::Cipher; use e3_data::Repository; use std::{env, sync::Arc}; -use tracing::info; pub trait AuthConversions { fn to_header_value(&self) -> Option; From c43a73d0c183cbd73e9812ab6192a316a6b148b3 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 05:01:49 +0000 Subject: [PATCH 49/66] add comments --- crates/ciphernode-builder/src/ciphernode_builder.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 3d548ac4ed..821f8d7013 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -52,6 +52,11 @@ enum EventSystemType { #[derive(Derivative)] #[derivative(Debug)] pub struct CiphernodeBuilder { + /// Unique name for the ciphernode. This name is used to derive the HLC tie breaker + /// functionality. + // TODO: how do we ensure this name is unique? + // TODO: we might be able to get away with simply making this a random number for every session + // but we need to discuss name: String, address: Option, chains: Vec, @@ -97,6 +102,11 @@ pub enum KeyshareKind { } impl CiphernodeBuilder { + /// Create a new ciphernode builder. + /// + /// - name - Unique name for the ciphernode + /// - rng - Arc Mutex wrapped random number generator + /// - cipher - Cipher for encryption and decryption of sensitive data pub fn new(name: &str, rng: SharedRng, cipher: Arc) -> Self { Self { name: name.to_owned(), From 794bc89ce1b88785ada2854ecda13d3d282b354d Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 05:35:22 +0000 Subject: [PATCH 50/66] add comments --- crates/entrypoint/src/start/aggregator_start.rs | 2 +- crates/events/src/bus_handle.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index aa5ee77440..e8f2b2ce13 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -46,7 +46,7 @@ pub async fn execute( builder = builder.with_plaintext_aggregation() } - // TODO: put the following in the CNB: + // TODO: put net package provisioning in the ciphernode-builder: let node = builder.build().await?; let store = node.store(); let repositories = store.repositories(); diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 5405f37e76..ece22e36c1 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -25,13 +25,17 @@ use crate::{ #[derive(Clone, Derivative)] #[derivative(Debug, PartialEq, Eq)] pub struct BusHandle { + /// EventBus that actors can consume sequenced events from consumer: Addr>>, + /// Sequencer that new events should be produced from producer: Addr, + /// Hlc clock used to time all events created on this BusHandle #[derivative(Debug = "ignore")] hlc: Arc, } impl BusHandle { + /// Create a new BusHandle pub fn new( consumer: Addr>>, producer: Addr, @@ -44,23 +48,28 @@ impl BusHandle { } } + /// Return a HistoryCollector for examining events that have passed through on the events bus pub fn history(&self) -> Addr>> { EventBus::>::history(&self.consumer) } + /// Access the producer to internally dispatch am event to pub fn producer(&self) -> &Addr { &self.producer } + /// Access the consumer to internally subscribe to events pub fn consumer(&self) -> &Addr>> { &self.consumer } + /// Get a new timestamp. Note this ticks over the internal Hlc. pub fn ts(&self) -> Result { let ts = self.hlc.tick()?; Ok(ts.into()) } + /// Pipe events from this handle to the other handle only when the predicate returns true pub fn pipe_to(&self, other: &BusHandle, predicate: F) where F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, @@ -290,6 +299,7 @@ mod tests { } } +/// Actor for piping between BusHandles. pub struct BusHandlePipe where F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, @@ -302,6 +312,8 @@ impl BusHandlePipe where F: Fn(&EnclaveEvent) -> bool + Unpin + 'static, { + /// Create a new BusHandlePipe only forwarding events to the wrapped handle when the predicate + /// function returns true pub fn new(handle: BusHandle, predicate: F) -> Self { Self { handle, predicate } } From 6ac6835b1f9cf4642950edbb5855ca188946b25d Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 05:41:52 +0000 Subject: [PATCH 51/66] remove e3 meta once request is complete --- crates/sortition/src/ciphernode_selector.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/sortition/src/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selector.rs index 36fc8c8e97..5ae487dc7b 100644 --- a/crates/sortition/src/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selector.rs @@ -9,6 +9,7 @@ use actix::prelude::*; use anyhow::bail; use anyhow::Result; use e3_data::{AutoPersist, Persistable, Repository}; +use e3_events::E3RequestComplete; use e3_events::{ prelude::*, trap, BusHandle, CiphernodeSelected, CommitteeFinalized, E3Requested, E3id, EType, EnclaveEvent, EnclaveEventData, Shutdown, TicketGenerated, TicketId, @@ -68,6 +69,7 @@ impl Handler for CiphernodeSelector { fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { match msg.into_data() { EnclaveEventData::E3Requested(data) => ctx.notify(data), + EnclaveEventData::E3RequestComplete(data) => ctx.notify(data), EnclaveEventData::CommitteeFinalized(data) => ctx.notify(data), EnclaveEventData::Shutdown(data) => ctx.notify(data), _ => (), @@ -152,6 +154,18 @@ impl Handler for CiphernodeSelector { } } +impl Handler for CiphernodeSelector { + type Result = (); + fn handle(&mut self, msg: E3RequestComplete, _: &mut Self::Context) -> Self::Result { + trap(EType::Sortition, &self.bus.clone(), move || { + self.e3_cache.try_mutate(|mut cache| { + cache.remove(&msg.e3_id); + Ok(cache) + }) + }) + } +} + impl Handler for CiphernodeSelector { type Result = (); From 73a2cf0d455b73465f3abab6f8589be0148c7390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 17 Dec 2025 05:53:14 +0000 Subject: [PATCH 52/66] Update crates/ciphernode-builder/src/event_system.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/ciphernode-builder/src/event_system.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index 2f2233b7c0..73ad4dfba0 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -142,7 +142,7 @@ impl EventSystem { } } - /// Pass in a sepecific given event bus + /// Pass in a specific given event bus pub fn with_event_bus(self, bus: Addr>) -> Self { let _ = self.eventbus.set(bus); self From fc31debf2117c8938a991b95085cd88dca6f8c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 17 Dec 2025 05:54:12 +0000 Subject: [PATCH 53/66] Update crates/data/src/in_mem_sequence_index.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/data/src/in_mem_sequence_index.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index a7ddc6bb6d..a1b9e01ba3 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -41,7 +41,7 @@ mod tests { use e3_events::SequenceIndex; #[test] - fn seek_for_prev_finds_nearest_key_at_or_before_target() { + fn seek_finds_nearest_key_at_or_after_target() { let mut index = InMemSequenceIndex::new(); index.insert(100, 1).unwrap(); index.insert(200, 2).unwrap(); From 5d8b3140c7f04975ed24184a73c329132dd5c35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 17 Dec 2025 05:55:03 +0000 Subject: [PATCH 54/66] Update crates/events/src/traits.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/events/src/traits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index c207a2b5da..cdae6383a1 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -104,7 +104,7 @@ pub trait SequenceIndex: Unpin + 'static { fn insert(&mut self, key: u128, value: u64) -> Result<()>; /// Get the sequence offset for the given timestamp fn get(&self, key: u128) -> Result>; - /// Get the first sequence offset before the given timestamp + /// Get the first sequence offset at or after the given timestamp fn seek(&self, key: u128) -> Result>; } From f2f67f89be9381570387ab057f01f43c4ef37f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 17 Dec 2025 05:55:26 +0000 Subject: [PATCH 55/66] Update crates/net/src/document_publisher.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/net/src/document_publisher.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/net/src/document_publisher.rs b/crates/net/src/document_publisher.rs index dd560296f1..e46527adc9 100644 --- a/crates/net/src/document_publisher.rs +++ b/crates/net/src/document_publisher.rs @@ -518,8 +518,6 @@ mod tests { let guard = tracing::subscriber::set_default(subscriber); - // let consumer = EventBus::::new(EventBusConfig { deduplicate: true }).start(); - // let bus = BusHandle::new(consumer); let system = EventSystem::new("test").with_fresh_bus(); let bus = system.handle()?; let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); From 8d3c84b0062950e7767a1575b7b09ff5c450daef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 17 Dec 2025 05:55:51 +0000 Subject: [PATCH 56/66] Update crates/config/src/paths_engine.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/config/src/paths_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/config/src/paths_engine.rs b/crates/config/src/paths_engine.rs index 737ab728e7..891fceb0f3 100644 --- a/crates/config/src/paths_engine.rs +++ b/crates/config/src/paths_engine.rs @@ -24,7 +24,7 @@ pub struct PathsEngine { /// This can either be a fully qualified path to a specific db file or a relative path to the /// data_dir location db_file_override: Option, - /// This can either be a fully qualified path to a specific db file or a relative path to the + /// This can either be a fully qualified path to a specific log file or a relative path to the /// data_dir location log_file_override: Option, /// This can either be a fully qualified path to a specific key file or a relative path to the From f3893d1fd9db32a14b110dba9d6c2a5e8640a2aa Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 05:59:01 +0000 Subject: [PATCH 57/66] add logging when requested --- crates/data/src/in_mem.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index 0785ef1047..a606b1f03a 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -76,6 +76,9 @@ impl Handler for InMemStore { fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { for cmd in msg.commands() { self.db.insert(cmd.key().to_owned(), cmd.value().to_owned()); + if self.capture { + self.log.push(DataOp::Insert(cmd.clone())); + } } } } From ca01581a917ff778fa74c10fab4bcb1817c6dbd8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 06:31:06 +0000 Subject: [PATCH 58/66] remove redundant struct --- crates/data/src/in_mem.rs | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index a606b1f03a..ed0f52f60f 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -7,7 +7,7 @@ use crate::{Get, Insert, InsertBatch, InsertSync, Remove}; use actix::{Actor, Handler, Message}; use anyhow::{Context, Result}; -use std::{collections::BTreeMap, ops::Deref}; +use std::collections::BTreeMap; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Vec")] @@ -129,26 +129,3 @@ impl Handler for InMemStore { self.get_dump() } } - -pub struct InMemDb(BTreeMap, Vec>); - -impl Deref for InMemDb { - type Target = BTreeMap, Vec>; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl InMemDb { - pub fn get(&self, msg: Get) -> Result>> { - Ok(self.0.get(msg.key()).cloned()) - } - pub fn insert(&mut self, msg: Insert) -> Result<()> { - self.0.insert(msg.key().to_owned(), msg.value().to_owned()); - Ok(()) - } - pub fn remove(&mut self, msg: Remove) -> Result<()> { - self.0.remove(msg.key()); - Ok(()) - } -} From 0b11081c0e3790e20e1eefca989777eb2f4ffd8b Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 06:32:02 +0000 Subject: [PATCH 59/66] remove inaccurate comments --- crates/data/src/in_mem_sequence_index.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/data/src/in_mem_sequence_index.rs b/crates/data/src/in_mem_sequence_index.rs index a1b9e01ba3..675f18070d 100644 --- a/crates/data/src/in_mem_sequence_index.rs +++ b/crates/data/src/in_mem_sequence_index.rs @@ -47,7 +47,6 @@ mod tests { index.insert(200, 2).unwrap(); index.insert(300, 3).unwrap(); - // Empty range (before all keys) assert_eq!(index.seek(50).unwrap(), Some(1)); // Exact matches @@ -59,7 +58,6 @@ mod tests { assert_eq!(index.seek(150).unwrap(), Some(2)); assert_eq!(index.seek(250).unwrap(), Some(3)); - // After all keys (returns last) assert_eq!(index.seek(999).unwrap(), None); } } From 18beae98c1cce6f39b7b6ffc045375efad1e504b Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 23:15:19 +0000 Subject: [PATCH 60/66] update to use addr as the node id --- .../src/ciphernode_builder.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 821f8d7013..14c2c7635c 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -196,7 +196,7 @@ impl CiphernodeBuilder { self } - /// Use the given Address to represent the node + /// Use the given Address to represent the node. This should be unique. pub fn with_address(mut self, addr: &str) -> Self { self.address = Some(addr.to_owned()); self @@ -328,18 +328,6 @@ impl CiphernodeBuilder { None }; - // Get an event system instance - let event_system = - if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { - EventSystem::persisted(&self.name, log_path, kv_path).with_event_bus(local_bus) - } else { - if let Some(ref store) = self.in_mem_store { - EventSystem::in_mem_from_store(&self.name, store).with_event_bus(local_bus) - } else { - EventSystem::in_mem(&self.name).with_event_bus(local_bus) - } - }; - let addr = if let Some(addr) = self.address.clone() { info!("Using eth address = {}", addr); addr @@ -349,6 +337,18 @@ impl CiphernodeBuilder { rand_eth_addr(&self.rng) }; + // Get an event system instance. + let event_system = + if let EventSystemType::Persisted { kv_path, log_path } = self.event_system.clone() { + EventSystem::persisted(&addr, log_path, kv_path).with_event_bus(local_bus) + } else { + if let Some(ref store) = self.in_mem_store { + EventSystem::in_mem_from_store(&addr, store).with_event_bus(local_bus) + } else { + EventSystem::in_mem(&addr).with_event_bus(local_bus) + } + }; + let bus = event_system.handle()?; let store = event_system.store()?; From 1b37e3f24f68b2abcf2806785b5772d3722445a6 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 17 Dec 2025 23:16:21 +0000 Subject: [PATCH 61/66] Remove comment --- crates/ciphernode-builder/src/ciphernode_builder.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 14c2c7635c..74e08ae701 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -52,11 +52,6 @@ enum EventSystemType { #[derive(Derivative)] #[derivative(Debug)] pub struct CiphernodeBuilder { - /// Unique name for the ciphernode. This name is used to derive the HLC tie breaker - /// functionality. - // TODO: how do we ensure this name is unique? - // TODO: we might be able to get away with simply making this a random number for every session - // but we need to discuss name: String, address: Option, chains: Vec, From 3dc7a0b361392d0237676c30e98fedff309c4e7a Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 23 Dec 2025 07:49:02 +0000 Subject: [PATCH 62/66] fix bug with offset not updating --- Cargo.toml | 1 + crates/data/Cargo.toml | 2 +- crates/data/src/commit_log_event_log.rs | 30 ++++++++++++++++++++----- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a39b962616..4bb2a832ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,6 +115,7 @@ bs58 = "=0.5.1" base64 = "=0.22.1" clap = { version = "=4.5.41", features = ["derive"] } chrono = { version = "=0.4.41", features = ["serde"] } +commitlog = "=0.2.0" compile-time = "=0.2.0" derivative = "=2.2.0" dirs = "=5.0.1" diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index 3995bee4ef..59c4aa6987 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -18,4 +18,4 @@ tracing = { workspace = true } async-trait = { workspace = true } once_cell = { workspace = true } tempfile = { workspace = true } -commitlog = "0.2.0" +commitlog = { workspace = true } diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 8c20d0800c..5905f1b7dd 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -24,11 +24,8 @@ impl CommitLogEventLog { let log = CommitLog::new(opts)?; Ok(Self { log }) } -} -impl EventLog for CommitLogEventLog { - fn append(&mut self, event: &EnclaveEvent) -> Result { - let bytes = bincode::serialize(event)?; + fn append_bytes(&mut self, bytes: &[u8]) -> Result { let offset = self .log .append_msg(&bytes) @@ -36,6 +33,13 @@ impl EventLog for CommitLogEventLog { // Return 1-indexed sequence number Ok(offset + 1) } +} + +impl EventLog for CommitLogEventLog { + fn append(&mut self, event: &EnclaveEvent) -> Result { + let bytes = bincode::serialize(event)?; + self.append_bytes(&bytes) + } fn read_from(&self, from: u64) -> Box)>> { // Convert 1-indexed sequence to 0-indexed offset @@ -54,10 +58,10 @@ impl EventLog for CommitLogEventLog { { // Convert 0-indexed offset back to 1-indexed sequence number events.push((msg.offset() + 1, event)); - current_offset = msg.offset() + 1; // Next offset to read from } else { error!("Error deserializing event in read_from... skipping"); } + current_offset = msg.offset() + 1; // Next offset to read from count += 1; } @@ -122,6 +126,22 @@ mod tests { assert_eq!(events[1].0, 3); } + #[test] + fn test_read_from_corruption_at_end_causes_infinite_loop() { + let dir = tempdir().unwrap(); + let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + + for i in 0..100 { + let e = event_from(TestEvent::new("myevent", i)); + log.append(&e).unwrap(); + } + // Corrupt the last message + log.append_bytes(b"I am a bad event!").unwrap(); + + // Ensure if last message is corrupt we don't end up in an infinite loop + let _: Vec<_> = log.read_from(1).collect(); + } + #[test] fn test_read_empty_log() { let dir = tempdir().unwrap(); From 5400f03fd47e16cfe0c9f3dc80796e8f32412b45 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 23 Dec 2025 07:53:34 +0000 Subject: [PATCH 63/66] ensure index cannot have a value already stored at timestamp --- crates/events/src/eventstore.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 68295a7037..9d9ca00ca6 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -9,7 +9,7 @@ use crate::{ EventLog, GetEventsAfter, ReceiveEvents, SequenceIndex, }; use actix::{Actor, Handler}; -use anyhow::Result; +use anyhow::{bail, Result}; use tracing::error; pub struct EventStore { @@ -22,6 +22,9 @@ impl EventStore { let event = msg.event; let sender = msg.sender; let ts = event.get_ts(); + if let Some(_) = self.index.get(ts)? { + bail!("Event already stored at timestamp {ts}!"); + } let seq = self.log.append(&event)?; self.index.insert(ts, seq)?; sender.try_send(EventStored(event.into_sequenced(seq)))?; From ffee673e475155130c8316cbed390ab7644bffe5 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 23 Dec 2025 07:59:04 +0000 Subject: [PATCH 64/66] add panic instead of just logging --- crates/events/src/eventstore.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 9d9ca00ca6..a3896d0464 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -57,7 +57,8 @@ impl Handler for EventStore< fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { match self.handle_store_event_requested(msg) { Ok(_) => (), - Err(e) => error!("{e}"), + Err(e) => panic!("{e}"), // panic here because when event storage fails we really need + // to just give up } } } From 8cbd0efe5b3c9c38bcf11266af954bcda0b290f0 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 23 Dec 2025 08:13:51 +0000 Subject: [PATCH 65/66] remove extra bus --- crates/entrypoint/src/start/aggregator_start.rs | 7 +++---- crates/entrypoint/src/start/start.rs | 9 +++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/crates/entrypoint/src/start/aggregator_start.rs b/crates/entrypoint/src/start/aggregator_start.rs index e8f2b2ce13..602863c46e 100644 --- a/crates/entrypoint/src/start/aggregator_start.rs +++ b/crates/entrypoint/src/start/aggregator_start.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::Result; -use e3_ciphernode_builder::{get_enclave_bus_handle, CiphernodeBuilder}; +use e3_ciphernode_builder::{get_enclave_bus_handle, get_enclave_event_bus, CiphernodeBuilder}; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; @@ -26,16 +26,14 @@ pub async fn execute( plaintext_write_path: Option, experimental_trbfv: bool, ) -> Result<(BusHandle, JoinHandle>, String)> { - let bus = get_enclave_bus_handle(config)?; let rng = Arc::new(Mutex::new(ChaCha20Rng::from_rng(OsRng)?)); let cipher = Arc::new(Cipher::from_file(config.key_file()).await?); let mut builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) - .with_source_bus(bus.consumer()) + .with_persistence(&config.log_file(), &config.db_file()) .with_chains(&config.chains()) .with_sortition_score() .with_contract_enclave_full() .with_contract_bonding_registry() - .with_persistence(&config.log_file(), &config.db_file()) .with_contract_ciphernode_registry() .with_max_threads() .with_pubkey_aggregation(); @@ -50,6 +48,7 @@ pub async fn execute( let node = builder.build().await?; let store = node.store(); let repositories = store.repositories(); + let bus = node.bus.clone(); let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( bus.clone(), config.peers(), diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 545077e4b8..16a19870d9 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -6,7 +6,7 @@ use alloy::primitives::Address; use anyhow::Result; -use e3_ciphernode_builder::{get_enclave_bus_handle, CiphernodeBuilder}; +use e3_ciphernode_builder::{get_enclave_bus_handle, get_enclave_event_bus, CiphernodeBuilder}; use e3_config::AppConfig; use e3_crypto::Cipher; use e3_data::RepositoriesFactory; @@ -25,16 +25,12 @@ pub async fn execute( experimental_trbfv: bool, ) -> Result<(BusHandle, JoinHandle>, String)> { let rng = Arc::new(Mutex::new(rand_chacha::ChaCha20Rng::from_rng(OsRng)?)); - - let bus = get_enclave_bus_handle(config)?; let cipher = Arc::new(Cipher::from_file(&config.key_file()).await?); - let mut builder = CiphernodeBuilder::new(&config.name(), rng.clone(), cipher.clone()) .with_address(&address.to_string()) - .with_source_bus(bus.consumer()) + .with_persistence(&config.log_file(), &config.db_file()) .with_sortition_score() .with_chains(&config.chains()) - .with_persistence(&config.log_file(), &config.db_file()) .with_contract_enclave_reader() .with_contract_bonding_registry() .with_max_threads() @@ -48,6 +44,7 @@ pub async fn execute( let node = builder.build().await?; let repositories = node.store().repositories(); + let bus = node.bus.clone(); let (_, _, join_handle, peer_id) = NetEventTranslator::setup_with_interface( bus.clone(), config.peers(), From c9105112913317f9b0a6f2be5a6bca345b0a9547 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 23 Dec 2025 08:22:14 +0000 Subject: [PATCH 66/66] update to avoid bug in seek return --- crates/events/src/eventstore.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index a3896d0464..5218459196 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -32,7 +32,12 @@ impl EventStore { } pub fn handle_get_events_after(&mut self, msg: GetEventsAfter) -> Result<()> { - let seq = self.index.seek(msg.ts)?.unwrap_or(1); + // if there are no events after the timestamp return an empty vector + let Some(seq) = self.index.seek(msg.ts)? else { + msg.sender.try_send(ReceiveEvents::new(vec![]))?; + return Ok(()); + }; + // read and return the events let evts = self .log .read_from(seq)