feat: implements keeper crate - state orchestrator

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "accountsdb",
+    "keeper",
     "ledger",
     "nucleus",
     "programs/magic-root-interface",
@@ -25,6 +26,7 @@ version = "0.1.0"
 
 [workspace.dependencies]
 accountsdb = { path = "accountsdb", package = "magicblock-accountsdb" }
+keeper = { path = "keeper", package = "magicblock-keeper" }
 ledger = { path = "ledger", package = "magicblock-ledger" }
 magic-root-interface = { path = "programs/magic-root-interface" }
 magic-root-program = { path = "programs/magic-root-program" }
@@ -140,7 +142,6 @@ solana-transaction-context = { path = "solana/transaction-context" }
 missing_docs = "deny"
 rust_2018_idioms = { level = "warn", priority = -1 }
 unreachable_pub = "warn"
-unsafe_op_in_unsafe_fn = "allow"
 unused_lifetimes = "warn"
 unused_macro_rules = "warn"
 unused_qualifications = "warn"

keeper/Cargo.toml

@@ -0,0 +1,83 @@
+[package]
+name = "magicblock-keeper"
+
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[lib]
+name = "keeper"
+
+[features]
+# Exposes `keeper::testkit`, the shared keeper-level test harness, as normal code
+# so downstream crates can import it via a dev-dependency (no new crate needed).
+testkit = [
+    "dep:solana-instruction",
+    "dep:tempfile",
+    "dep:v42-calculator-interface",
+    "nucleus/testkit"
+]
+
+[dependencies]
+accountsdb = { workspace = true }
+ledger = { workspace = true }
+nucleus = { workspace = true, features = [
+    "ledger-schema",
+    "metrics",
+    "notifier",
+    "runtime",
+    "shutdown"
+] }
+
+
+tempfile = { workspace = true, optional = true }
+v42-calculator-interface = { workspace = true, optional = true }
+
+ahash = { workspace = true }
+arc-swap = { workspace = true }
+derive_more = { workspace = true, features = ["from"] }
+flume = { workspace = true }
+parking_lot = { workspace = true }
+scc = { workspace = true }
+serde = { workspace = true }
+tar = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["rt", "sync"] }
+tracing = { workspace = true }
+zstd = { workspace = true }
+
+agave-feature-set = { workspace = true, features = ["agave-unstable-api"] }
+agave-transaction-view = { workspace = true }
+solana-account = { workspace = true, features = ["bincode"] }
+solana-feature-gate-interface = { workspace = true }
+solana-hash = { workspace = true }
+solana-instruction = { workspace = true, optional = true }
+solana-keypair = { workspace = true }
+solana-message = { workspace = true }
+solana-program-runtime = { workspace = true }
+solana-pubkey = { workspace = true }
+solana-sdk-ids = { workspace = true }
+solana-signature = { workspace = true }
+solana-signer = { workspace = true }
+solana-svm = { workspace = true }
+solana-sysvar = { workspace = true }
+solana-transaction-error = { workspace = true }
+
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+nucleus = { workspace = true, features = ["testkit"] }
+tempfile = { workspace = true }
+v42-calculator-interface = { workspace = true }
+
+solana-instruction = { workspace = true }
+solana-keypair = { workspace = true }
+solana-signer = { workspace = true }
+tokio = { workspace = true, features = ["macros"] }
+
+[lints]
+workspace = true

keeper/README.md

@@ -0,0 +1,49 @@
+# `magicblock-keeper`
+
+`accountsdb` and `ledger` each own one half of the engine's durable state, but
+something has to open them together, keep them consistent, and present them as a
+single thing the rest of the engine can use. That's the keeper. It owns both
+stores plus the read-side caches and subscription fanout, and it deliberately
+holds *no storage policy of its own* — every routing and retention decision stays
+in the crates below it. The keeper is the seam, not a third store.
+
+## Startup and recovery
+
+`KeeperBuilder::build` opens the ledger and accountsdb and then seeds the accounts
+the engine can't run without: active features, native builtins, the configured
+upgradeable programs (with their ELF), caller-provided accounts, and sysvars.
+
+Recovery is the part worth understanding. If accountsdb validation reports
+corruption on open, the keeper doesn't give up — it saves the active (corrupt)
+tree for inspection, restores the latest archived snapshot from the retained
+superblocks, and revalidates. This is why accountsdb snapshots are archived
+*alongside* sealed superblocks: the ledger's retained history is also the supply
+of restore points.
+
+The blockstore parameters carry the expected block time and a non-zero slot
+interval, which is what drives superblock sealing.
+
+## Finalize
+
+`finalize` seals the next superblock with the current accounts checksum and
+transaction count, then writes and archives an accountsdb snapshot under that
+superblock. It requires **exclusive write access** while it runs — the snapshot
+has to capture a single coherent point in time, and a concurrent write would let
+it record a state that never actually existed.
+
+## Caches and subscriptions
+
+Caches are slot-based and lazily evicted on insertion, so stale slots fall out as
+new ones arrive rather than needing a sweeper. Subscriptions (accounts, programs,
+signatures, logs, transactions) are plain broadcast channels and hold no durable
+state — they only fan out what's already happening.
+
+## `testkit` feature
+
+Enables `keeper::testkit`, the shared keeper-level test harness (building a real
+`Keeper` over throwaway directories, plus the loadable v42 calculator program that
+`build.rs` compiles). It also provides shared v42 account and transaction-view
+helpers for keeper-backed integration tests. Downstream test suites enable it as
+a dev-dependency — `keeper = { workspace = true, features = ["testkit"] }` —
+instead of re-deriving the setup. The v42 ELF lives here, so only keeper builds
+it.

keeper/build.rs

@@ -0,0 +1,62 @@
+//! Builds the v42 calculator SBF program for runtime tests.
+
+use std::{env, io, path::PathBuf, process::Command};
+
+const PROGRAM_DIR: &str = "programs/v42-calculator-program";
+const PROGRAM: &str = "programs/v42-calculator-program/Cargo.toml";
+const SO: &str = "v42_calculator_program.so";
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let manifest_dir = PathBuf::from(
+        env::var_os("CARGO_MANIFEST_DIR")
+            .ok_or_else(|| io::Error::other("CARGO_MANIFEST_DIR is not set"))?,
+    );
+    let workspace = manifest_dir.parent().ok_or_else(|| {
+        io::Error::other(format!(
+            "CARGO_MANIFEST_DIR has no parent: {}",
+            manifest_dir.display()
+        ))
+    })?;
+    let manifest = workspace.join(PROGRAM);
+    let artifact = workspace.join("target/deploy").join(SO);
+    println!(
+        "cargo:rerun-if-changed={}",
+        workspace.join(PROGRAM_DIR).display()
+    );
+
+    let output = Command::new("cargo")
+        .arg("build-sbf")
+        .arg("--manifest-path")
+        .arg(&manifest)
+        .arg("--arch")
+        .arg("v3")
+        .current_dir(workspace)
+        // `cargo clippy` exports these wrappers pointing at `clippy-driver`; left in
+        // place they hijack the SBF toolchain's rustc, which can't resolve the
+        // `sbpf*-solana` target. Strip them so `build-sbf` uses its own toolchain.
+        .env_remove("RUSTC_WRAPPER")
+        .env_remove("RUSTC_WORKSPACE_WRAPPER")
+        .output()
+        .map_err(|e| io::Error::other(format!("failed to run `cargo build-sbf`: {e}")))?;
+
+    if !output.status.success() {
+        return Err(io::Error::other(format!(
+            "`cargo build-sbf` failed with {}\nstdout:\n{}\nstderr:\n{}",
+            output.status,
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ))
+        .into());
+    }
+    if !artifact.is_file() {
+        Err(io::Error::other(format!(
+            "missing SBF artifact: {}",
+            artifact.display()
+        )))?;
+    }
+    println!(
+        "cargo:rustc-env=V42_CALCULATOR_PROGRAM_SO={}",
+        artifact.display()
+    );
+    Ok(())
+}