From bbd32ab23113bd1c7c275986a3efef471018a9c6 Mon Sep 17 00:00:00 2001 From: Babur Makhmudov Date: Mon, 6 Jul 2026 18:57:40 +0400 Subject: [PATCH] refactor: remove crates moved to magicblock engine --- Cargo.toml | 77 +- magicblock-account-cloner/Cargo.toml | 31 - magicblock-account-cloner/README.md | 101 -- magicblock-account-cloner/src/lib.rs | 1136 ----------------- magicblock-account-cloner/src/util.rs | 12 - magicblock-accounts-db/Cargo.toml | 41 - magicblock-accounts-db/README.md | 43 - magicblock-accounts-db/src/error.rs | 52 - magicblock-accounts-db/src/index.rs | 519 -------- magicblock-accounts-db/src/index/iterator.rs | 119 -- magicblock-accounts-db/src/index/table.rs | 108 -- magicblock-accounts-db/src/index/tests.rs | 478 ------- magicblock-accounts-db/src/index/utils.rs | 93 -- magicblock-accounts-db/src/lib.rs | 650 ---------- magicblock-accounts-db/src/reset.rs | 73 -- magicblock-accounts-db/src/snapshot.rs | 500 -------- magicblock-accounts-db/src/storage.rs | 642 ---------- magicblock-accounts-db/src/tests.rs | 911 ------------- magicblock-accounts-db/src/traits.rs | 25 - magicblock-accounts/Cargo.toml | 18 - magicblock-accounts/README.md | 41 - magicblock-accounts/src/config.rs | 18 - magicblock-accounts/src/errors.rs | 69 - magicblock-accounts/src/lib.rs | 6 - magicblock-accounts/src/traits.rs | 18 - magicblock-processor/Cargo.toml | 62 - magicblock-processor/README.md | 96 -- magicblock-processor/src/builtins.rs | 78 -- magicblock-processor/src/executor/callback.rs | 44 - magicblock-processor/src/executor/mod.rs | 301 ----- .../src/executor/processing.rs | 511 -------- magicblock-processor/src/lib.rs | 273 ---- magicblock-processor/src/loader.rs | 71 -- .../src/scheduler/coordinator.rs | 388 ------ magicblock-processor/src/scheduler/locks.rs | 84 -- magicblock-processor/src/scheduler/mod.rs | 687 ---------- magicblock-processor/src/scheduler/state.rs | 115 -- magicblock-processor/src/scheduler/tests.rs | 491 ------- .../tests/ephemeral_accounts.rs | 1115 ---------------- magicblock-processor/tests/execution.rs | 122 -- magicblock-processor/tests/fees.rs | 275 ---- .../tests/post_delegation_actions.rs | 301 ----- magicblock-processor/tests/replay.rs | 121 -- .../tests/replica_ordering.rs | 466 ------- magicblock-processor/tests/scheduling.rs | 529 -------- magicblock-processor/tests/security.rs | 87 -- magicblock-processor/tests/simulation.rs | 198 --- test-kit/Cargo.toml | 33 - test-kit/src/lib.rs | 539 -------- test-kit/src/macros.rs | 108 -- tools/genx/Cargo.toml | 18 - tools/genx/README.md | 18 - tools/genx/src/main.rs | 49 - tools/genx/src/test_validator.rs | 158 --- tools/keypair-base58/Cargo.toml | 11 - tools/keypair-base58/README.md | 10 - tools/keypair-base58/src/main.rs | 26 - tools/ledger-stats/Cargo.toml | 25 - tools/ledger-stats/README.md | 191 --- tools/ledger-stats/src/account.rs | 53 - tools/ledger-stats/src/accounts.rs | 213 ---- tools/ledger-stats/src/blockhash.rs | 31 - tools/ledger-stats/src/counts.rs | 67 - tools/ledger-stats/src/lib.rs | 3 - tools/ledger-stats/src/main.rs | 184 --- tools/ledger-stats/src/transaction_details.rs | 217 ---- tools/ledger-stats/src/transaction_logs.rs | 77 -- tools/ledger-stats/src/utils.rs | 72 -- 68 files changed, 18 insertions(+), 14281 deletions(-) delete mode 100644 magicblock-account-cloner/Cargo.toml delete mode 100644 magicblock-account-cloner/README.md delete mode 100644 magicblock-account-cloner/src/lib.rs delete mode 100644 magicblock-account-cloner/src/util.rs delete mode 100644 magicblock-accounts-db/Cargo.toml delete mode 100644 magicblock-accounts-db/README.md delete mode 100644 magicblock-accounts-db/src/error.rs delete mode 100644 magicblock-accounts-db/src/index.rs delete mode 100644 magicblock-accounts-db/src/index/iterator.rs delete mode 100644 magicblock-accounts-db/src/index/table.rs delete mode 100644 magicblock-accounts-db/src/index/tests.rs delete mode 100644 magicblock-accounts-db/src/index/utils.rs delete mode 100644 magicblock-accounts-db/src/lib.rs delete mode 100644 magicblock-accounts-db/src/reset.rs delete mode 100644 magicblock-accounts-db/src/snapshot.rs delete mode 100644 magicblock-accounts-db/src/storage.rs delete mode 100644 magicblock-accounts-db/src/tests.rs delete mode 100644 magicblock-accounts-db/src/traits.rs delete mode 100644 magicblock-accounts/Cargo.toml delete mode 100644 magicblock-accounts/README.md delete mode 100644 magicblock-accounts/src/config.rs delete mode 100644 magicblock-accounts/src/errors.rs delete mode 100644 magicblock-accounts/src/lib.rs delete mode 100644 magicblock-accounts/src/traits.rs delete mode 100644 magicblock-processor/Cargo.toml delete mode 100644 magicblock-processor/README.md delete mode 100644 magicblock-processor/src/builtins.rs delete mode 100644 magicblock-processor/src/executor/callback.rs delete mode 100644 magicblock-processor/src/executor/mod.rs delete mode 100644 magicblock-processor/src/executor/processing.rs delete mode 100644 magicblock-processor/src/lib.rs delete mode 100644 magicblock-processor/src/loader.rs delete mode 100644 magicblock-processor/src/scheduler/coordinator.rs delete mode 100644 magicblock-processor/src/scheduler/locks.rs delete mode 100644 magicblock-processor/src/scheduler/mod.rs delete mode 100644 magicblock-processor/src/scheduler/state.rs delete mode 100644 magicblock-processor/src/scheduler/tests.rs delete mode 100644 magicblock-processor/tests/ephemeral_accounts.rs delete mode 100644 magicblock-processor/tests/execution.rs delete mode 100644 magicblock-processor/tests/fees.rs delete mode 100644 magicblock-processor/tests/post_delegation_actions.rs delete mode 100644 magicblock-processor/tests/replay.rs delete mode 100644 magicblock-processor/tests/replica_ordering.rs delete mode 100644 magicblock-processor/tests/scheduling.rs delete mode 100644 magicblock-processor/tests/security.rs delete mode 100644 magicblock-processor/tests/simulation.rs delete mode 100644 test-kit/Cargo.toml delete mode 100644 test-kit/src/lib.rs delete mode 100644 test-kit/src/macros.rs delete mode 100644 tools/genx/Cargo.toml delete mode 100644 tools/genx/README.md delete mode 100644 tools/genx/src/main.rs delete mode 100644 tools/genx/src/test_validator.rs delete mode 100644 tools/keypair-base58/Cargo.toml delete mode 100644 tools/keypair-base58/README.md delete mode 100644 tools/keypair-base58/src/main.rs delete mode 100644 tools/ledger-stats/Cargo.toml delete mode 100644 tools/ledger-stats/README.md delete mode 100644 tools/ledger-stats/src/account.rs delete mode 100644 tools/ledger-stats/src/accounts.rs delete mode 100644 tools/ledger-stats/src/blockhash.rs delete mode 100644 tools/ledger-stats/src/counts.rs delete mode 100644 tools/ledger-stats/src/lib.rs delete mode 100644 tools/ledger-stats/src/main.rs delete mode 100644 tools/ledger-stats/src/transaction_details.rs delete mode 100644 tools/ledger-stats/src/transaction_logs.rs delete mode 100644 tools/ledger-stats/src/utils.rs diff --git a/Cargo.toml b/Cargo.toml index 451715d7d..039d61602 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,6 @@ split-debuginfo = "packed" [workspace] members = [ - "magicblock-account-cloner", - "magicblock-accounts", - "magicblock-accounts-db", "magicblock-aml", "magicblock-aperture", "magicblock-api", @@ -16,7 +13,6 @@ members = [ "magicblock-committor-service", "magicblock-config", "magicblock-core", - "magicblock-ledger", "magicblock-magic-program-api", "magicblock-metrics", "magicblock-processor", @@ -29,12 +25,7 @@ members = [ "magicblock-validator-admin", "magicblock-version", "programs/guinea", - "programs/magicblock", - "test-kit", - "tools/genx", - "tools/keypair-base58", - "tools/ledger-stats", - "tools/magicblock-tui-client", + "programs/magicblock" ] # This prevents a Travis CI error when building for Windows. @@ -43,7 +34,7 @@ resolver = "2" [workspace.package] # Solana 4.x versions authors = ["MagicBlock Maintainers "] -edition = "2021" +edition = "2024" homepage = "https://www.magicblock.xyz" license = "Business Source License 1.1" repository = "https://github.com/magicblock-labs/ephemeral-validator" @@ -107,11 +98,9 @@ magicblock-accounts-db = { path = "./magicblock-accounts-db" } magicblock-aml = { path = "./magicblock-aml" } magicblock-aperture = { path = "./magicblock-aperture" } magicblock-api = { path = "./magicblock-api" } -magicblock-chainlink = { path = "./magicblock-chainlink", features = [ - "dev-context", -] } +magicblock-chainlink = { path = "./magicblock-chainlink", features = ["dev-context"] } magicblock-committor-program = { path = "./magicblock-committor-program", features = [ - "no-entrypoint", + "no-entrypoint" ] } magicblock-committor-service = { path = "./magicblock-committor-service" } magicblock-config = { path = "./magicblock-config" } @@ -138,18 +127,14 @@ num_cpus = "1.16.0" parking_lot = "0.12" paste = "1.0" prometheus = "0.13.4" -# Keep in sync with `solana-storage-proto` codegen. - solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "190146af2ac0890848b50e8fc9d6c926c8205b5e", features = [ - "dev-context-only-utils", + "dev-context-only-utils" ] } solana-transaction-error = { version = "3.0" } solana-transaction-status = { version = "4.0" } solana-transaction-status-client-types = "4.0" -solana-zk-elgamal-proof-program = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-zk-elgamal-proof-program = { version = "=4.0.0", features = ["agave-unstable-api"] } static_assertions = "1.1.0" tempfile = "3.10.1" test-kit = { path = "./test-kit" } @@ -163,10 +148,7 @@ tonic-prost-build = "0.14" tracing = "0.1" tracing-log = { version = "0.2", features = ["log-tracer"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } -twox-hash = { version = "2.1", default-features = false, features = [ - "alloc", - "xxhash3_64", -] } +twox-hash = { version = "2.1", default-features = false, features = ["alloc", "xxhash3_64"] } url = "2.5.0" # SPL Token crates used across the workspace @@ -179,14 +161,10 @@ prost = "0.14" protobuf-src = "1.1" rand = "0.9" reqwest = "0.12" -rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } rusqlite = { version = "0.37.0", features = ["bundled"] } # bundled sqlite 3.44 rustc-hash = "2.1" rustc_version = "0.4" -rustls = { version = "0.23", default-features = false, features = [ - "aws_lc_rs", - "std", -] } +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std"] } scc = "2.4" semver = "1.0.22" serde = "1.0.217" @@ -199,22 +177,14 @@ solana-account-decoder = { version = "4.0" } solana-account-decoder-client-types = { version = "4.0" } solana-account-info = { version = "3.1" } solana-address-lookup-table-interface = { version = "3.0" } -solana-bpf-loader-program = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-bpf-loader-program = { version = "=4.0.0", features = ["agave-unstable-api"] } solana-clock = { version = "3.0" } solana-cluster-type = { version = "3.1" } solana-commitment-config = { version = "3.1" } -solana-compute-budget = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } -solana-compute-budget-instruction = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-compute-budget = { version = "=4.0.0", features = ["agave-unstable-api"] } +solana-compute-budget-instruction = { version = "=4.0.0", features = ["agave-unstable-api"] } solana-compute-budget-interface = { version = "3.0" } -solana-compute-budget-program = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-compute-budget-program = { version = "=4.0.0", features = ["agave-unstable-api"] } solana-cpi = { version = "3.1" } solana-feature-gate-interface = { version = "3.1" } solana-feature-set = { package = "agave-feature-set", version = "4.0" } @@ -228,14 +198,12 @@ solana-instructions-sysvar = { version = "3.0" } solana-keypair = { version = "=3.1.0" } solana-loader-v3-interface = { version = "6.1.0" } solana-loader-v4-interface = { version = "3.1" } -solana-loader-v4-program = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-loader-v4-program = { version = "=4.0.0", features = ["agave-unstable-api"] } solana-log-collector = { package = "solana-svm-log-collector", version = "=4.0.0", features = [ - "agave-unstable-api", + "agave-unstable-api" ] } solana-measure = { package = "solana-svm-measure", version = "=4.0.0", features = [ - "agave-unstable-api", + "agave-unstable-api" ] } solana-message = { version = "3.0" } solana-metrics = { version = "4.0", features = ["agave-unstable-api"] } @@ -259,19 +227,14 @@ solana-sha256-hasher = { version = "3.1" } solana-signature = { version = "3.1" } solana-signer = { version = "3.0" } solana-slot-hashes = { version = "3.0" } -solana-storage-proto = { path = "storage-proto" } solana-svm-callback = { version = "=4.0.0", features = ["agave-unstable-api"] } -solana-svm-transaction = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-svm-transaction = { version = "=4.0.0", features = ["agave-unstable-api"] } solana-system-interface = { version = "3.1" } -solana-system-program = { version = "=4.0.0", features = [ - "agave-unstable-api", -] } +solana-system-program = { version = "=4.0.0", features = ["agave-unstable-api"] } solana-system-transaction = { version = "3.0" } solana-sysvar = { version = "3.0" } solana-timings = { package = "solana-svm-timings", version = "=4.0.0", features = [ - "agave-unstable-api", + "agave-unstable-api" ] } solana-transaction = { version = "3.0" } @@ -283,11 +246,7 @@ rev = "190146af2ac0890848b50e8fc9d6c926c8205b5e" [patch.crates-io] solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "190146af2ac0890848b50e8fc9d6c926c8205b5e" } solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "190146af2ac0890848b50e8fc9d6c926c8205b5e" } -solana-storage-proto = { path = "storage-proto" } solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "190146af2ac0890848b50e8fc9d6c926c8205b5e" } solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "190146af2ac0890848b50e8fc9d6c926c8205b5e" } # Fork is used to enable `disable_manual_compaction` usage -# Fork is based on commit d4e9e16 of rocksdb (parent commit of 0.23.0 release) -# without patching update isn't possible due to conflict with solana deps libsodium-rs = { git = "https://github.com/jedisct1/libsodium-rs.git", rev = "0397a6c5785233f9f2ac91f3eedc3cceb74e0060" } -rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } diff --git a/magicblock-account-cloner/Cargo.toml b/magicblock-account-cloner/Cargo.toml deleted file mode 100644 index 4ef318fe9..000000000 --- a/magicblock-account-cloner/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "magicblock-account-cloner" -version.workspace = true -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[dependencies] -async-trait = { workspace = true } -bincode = { workspace = true } -tracing = { workspace = true } -magicblock-chainlink = { workspace = true } -magicblock-core = { workspace = true } -magicblock-ledger = { workspace = true } -magicblock-magic-program-api = { workspace = true } -magicblock-program = { workspace = true } -rand = { workspace = true } -solana-account = { workspace = true } -solana-hash = { workspace = true } -solana-instruction = { workspace = true } -solana-loader-v4-interface = { workspace = true } -solana-pubkey = { workspace = true } -solana-sdk-ids = { workspace = true } -solana-signature = { workspace = true } -solana-signer = { workspace = true } -solana-sysvar = { workspace = true } -solana-transaction = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } diff --git a/magicblock-account-cloner/README.md b/magicblock-account-cloner/README.md deleted file mode 100644 index 81449c653..000000000 --- a/magicblock-account-cloner/README.md +++ /dev/null @@ -1,101 +0,0 @@ - -# Summary - -Implements logic for fetching remote accounts and dumping them into the local bank - -Accounts come in 3 different important flavors: -- `FeePayer` accounts, which never contain data, are on-curve and owned by the system program. They can be used as wallet accounts to pay fees. -- `Undelegated` accounts, which do contain data and can never be written to in the ephemeral -- `Delegated` accounts, which have a valid delegation record, therefore can be locally modified - -Here are all possible cases: -- `if !properly_delegated && !has_data && is_on_curve && is_system_program_owned` -> `FeePayer` -- `if !properly_delegated && has_data` -> `Undelegated` -- `if properly_delegated && !has_data` -> `Delegated` -- `if properly_delegated && has_data` -> `Delegated` - -# Logic Overview - -The cloning pipeline is made out of a few components: -- The cloner (highest level) -> crate `magicblock-account-cloner` - - The fetcher (read on-chain latest account state) -> crate `magicblock-account-fetcher` - - The updates (subscribe to on-chain account changes) -> crate `magicblock-account-updates` - - The dumper (apply cloned state to the bank) -> crate `magicblock-account-dumper` - -## Cloning logic - -Different types of event will trigger cloning actions: -- `Transaction event`: A transaction is received in the validator -- `Update event`: An on-chain account has changed - -The important states stored for each account are: -- RemoteAccountUpdatesWorker.`last_known_update_slot` -> a map of which slot was the account was last updated at -- RemoteAccountUpdatesWorker.`first_subscribed_slot` -> a map of which slot was the account first subscribed at -- RemoteAccountClonerWorker.`last_clone_output` -> a cache of the latest clone's result (contains the on-chain slot at which it happened) - -### Transaction event: new transaction received - -When a transaction is received by the validator, each account of the transaction is cloned separately in parrallel. - -Each account's clone request is pushed into a queue and executed on a worker thread dedicated to the cloner. - -We can detect if an account needs to be cloned based on if the `last_known_update_slot` is more recent than the slot from which the last clone's state originated from. - -For each account, the logic goes as follow: - -- A) If the account was never seen before or changes to the account were detected since last clone (checks `last_known_update_slot` and compares it to the `last_clone_output`) - - 0) Validate that we actually want to clone that account (is it blacklisted?) - - 1) Start subscribing to on-chain changes for this account (so we can detect change for future clones) - - This will do nothing if we already subscribed to the account before - - This will set the `first_subscribed_slot` for that account if it's the first time we see it - - 2) Fetch the latest on-chain account state - - This will retry until we fetched the state of a more recent slot than `first_subscribed_slot` - - After 5 failed retry and 200 ms sleep in between each, we fail the clone - - 3) Differentiate based on the account's fetched flavor (we will use the "dumper"): - - A) If Undelegated: Simply dump the latest up-to-date fetched data to the bank (programs also fetched/updated) - - B) If FeePayer: Dump the account as-is, but with special lamport value - - C) If Delegated: If the account's latest delegation_slot is NOT the same as the last clone's delegation_slot, dump the latest state, otherwise ignore the change and use the cache - - 4) Save the result of the clone to the cache - -- B) If the account has already been cloned (and it has not changed on-chain since last clone) - - 0) Do nothing, use the cache of the latest clone's result into `last_clone_output` - -### Update event: On-chain change detected - -When an on-chain account's subscription notices a change: - -- We update the `last_known_update_slot` for that account -- On the next clone for that account, it will force the logic (A) instead of (B) -- This is because the last clone's slot inside the cache will now be too old - -## Update logic - -During the cloner's step `A.1`, an account is added to the set of monitored acounts. -Once an account has been cloned, we keep monitoring for on-chain changes forever. - -Each account's monitoring request is pushed into a queue and executed on a worker thread dedicated to the updates. -The worker maintains a list of "Shard", each shard manages a single RPC websocket connection: -- Shards are constantly created and deleted -- Each shard subscribe to EVERY monitored account at all times - -On startup, we subscribe to the RPC's `Clock` changes, in order to know which slot is the latest confirmed slot for the RPC. - -For each account monitoring request, an "accountSubscribe" websocket subscription is created through an RPC call. -For each account monitoring request, we set the `first_subscribed_slot` to the last `Clock`'s slot at the time of subscription. - -For each update received in the websocket subscription, we save the slot at which the update occured: This is what we call the `last_known_update_slot`. - -Note: multiple RPC connections are maintained at all times, and all subscriptions are refreshed every 5 minutes: -- one RPC websocket connection is destroyed (and all subscriptions dropped) -- one RPC websocket connection is created (and all subscription re-opened) - -## Fetch logic - -During the cloner's step `A.2`, an account fetch request is submitted to the "fetcher". -Each account's fetch request is pushed into a queue and executed on a worker thread dedicated to the fetcher. - -For each fetched account, we simply use the `getMultipleAccount` solana's RPC call on both the account itself and its delegation record. -Note that we use the `minContextSlot` parameter is passed to try to enforce that the state being fetched is more recent than the latest subscription's slot. -The `minContextSlot` passed as parameter is the most recent confirmed slot which was received from the "Update" subscriptions (it's the `first_subscribed_slot`). - -The fetcher structure in the validator's repository is mostly used for queuing and scheduling purposeds, most of the actual RPC request logic and parsing is done in the cunjuntoi repository implementation of the "AccountChainSnapshotProvider" diff --git a/magicblock-account-cloner/src/lib.rs b/magicblock-account-cloner/src/lib.rs deleted file mode 100644 index 7e90c68eb..000000000 --- a/magicblock-account-cloner/src/lib.rs +++ /dev/null @@ -1,1136 +0,0 @@ -//! Chainlink cloner - clones accounts from remote chain to ephemeral validator. -//! -//! # Account Cloning -//! -//! Accounts are cloned via direct encoding in transactions: -//! - Small accounts (<63KB): Single `CloneAccount` instruction -//! - Large accounts (>=63KB): `CloneAccountInit` → `CloneAccountContinue`* sequence -//! -//! # Program Cloning -//! -//! Programs use a buffer-based approach to handle loader-specific logic: -//! -//! ## V1 Programs (bpf_loader) -//! Converted to V3 (upgradeable loader) format: -//! 1. Clone ELF to buffer account -//! 2. `FinalizeV1ProgramFromBuffer` creates program + program_data accounts -//! -//! ## V4 Programs (loader_v4) -//! 1. Clone ELF to buffer account -//! 2. `FinalizeProgramFromBuffer` creates program account with LoaderV4 header -//! 3. `LoaderV4::Deploy` is called -//! 4. `SetProgramAuthority` sets the chain's authority -//! -//! # Buffer Account -//! -//! The buffer is a temporary account that holds the raw ELF data during cloning. -//! It's derived as a PDA: `["buffer", program_id]` owned by validator authority. - -use async_trait::async_trait; -use magicblock_chainlink::{ - cloner::{ - errors::{ClonerError, ClonerResult}, - AccountCloneRequest, Cloner, - }, - remote_account_provider::program_account::{ - LoadedProgram, RemoteProgramLoader, - }, -}; -use magicblock_core::link::transactions::{ - with_encoded, TransactionSchedulerHandle, -}; -use magicblock_ledger::LatestBlock; -use magicblock_magic_program_api::{ - args::ScheduleTaskArgs, - instruction::{AccountCloneFields, MagicBlockInstruction}, - MAGIC_CONTEXT_PUBKEY, -}; -use magicblock_program::{ - instruction_utils::InstructionUtils, - validator::{validator_authority, validator_authority_id}, -}; -use solana_account::ReadableAccount; -use solana_hash::Hash; -use solana_instruction::{AccountMeta, Instruction}; -use solana_loader_v4_interface::{ - instruction::LoaderV4Instruction, state::LoaderV4Status, -}; -use solana_pubkey::Pubkey; -use solana_sdk_ids::{bpf_loader_upgradeable, loader_v4}; -use solana_signature::Signature; -use solana_signer::Signer; -use solana_sysvar::rent::Rent; -use solana_transaction::Transaction; -use tracing::*; - -/// Max data that fits in a single transaction (~63KB) -pub const MAX_INLINE_DATA_SIZE: usize = 63 * 1024; -const MAX_INLINE_TRANSACTION_SIZE: usize = u16::MAX as usize; - -mod util; - -pub use util::derive_buffer_pubkey; - -pub struct ChainlinkCloner { - tx_scheduler: TransactionSchedulerHandle, - block: LatestBlock, -} - -impl ChainlinkCloner { - pub fn new( - tx_scheduler: TransactionSchedulerHandle, - block: LatestBlock, - ) -> Self { - Self { - tx_scheduler, - block, - } - } - - // ----------------- - // Transaction Helpers - // ----------------- - - async fn send_tx(&self, tx: Transaction) -> ClonerResult { - let sig = tx.signatures[0]; - self.tx_scheduler.execute(with_encoded(tx)?).await?; - Ok(sig) - } - - // ----------------- - fn create_signed_tx( - &self, - ixs: &[Instruction], - blockhash: Hash, - ) -> Transaction { - let kp = validator_authority(); - Transaction::new_signed_with_payer( - ixs, - Some(&kp.pubkey()), - &[&kp], - blockhash, - ) - } - - fn transaction_size(tx: &Transaction) -> ClonerResult { - Ok(bincode::serialized_size(tx)?.try_into()?) - } - - fn ensure_transaction_fits( - pubkey: Pubkey, - tx: &Transaction, - ) -> ClonerResult { - let tx_size = Self::transaction_size(tx)?; - if tx_size > MAX_INLINE_TRANSACTION_SIZE { - return Err(ClonerError::CloneTransactionTooLarge { - pubkey, - size: tx_size, - max_size: MAX_INLINE_TRANSACTION_SIZE, - }); - } - Ok(tx_size) - } - - fn ensure_transactions_fit( - pubkey: Pubkey, - txs: &[Transaction], - ) -> ClonerResult<()> { - for tx in txs { - Self::ensure_transaction_fits(pubkey, tx)?; - } - Ok(()) - } - - // ----------------- - // Instruction Builders (delegates to InstructionUtils) - // ----------------- - - fn clone_ix( - pubkey: Pubkey, - data: Vec, - fields: AccountCloneFields, - actions: Vec, - ) -> Instruction { - InstructionUtils::clone_account_instruction( - pubkey, data, fields, actions, - ) - } - - fn clone_init_ix( - pubkey: Pubkey, - total_len: u32, - initial_data: Vec, - fields: AccountCloneFields, - ) -> Instruction { - InstructionUtils::clone_account_init_instruction( - pubkey, - total_len, - initial_data, - fields, - ) - } - - fn clone_continue_ix( - pubkey: Pubkey, - offset: u32, - data: Vec, - is_last: bool, - actions: Vec, - needs_undelegation: bool, - ) -> Instruction { - InstructionUtils::clone_account_continue_instruction( - pubkey, - offset, - data, - is_last, - actions, - needs_undelegation, - ) - } - - fn cleanup_ix(pubkey: Pubkey) -> Instruction { - InstructionUtils::cleanup_partial_clone_instruction(pubkey) - } - - fn post_delegation_action_ix( - delegated_account_pubkey: Pubkey, - actions: Vec, - ) -> Instruction { - InstructionUtils::post_delegation_action_executor_instruction( - delegated_account_pubkey, - actions, - ) - } - - fn finalize_program_ix( - program: Pubkey, - buffer: Pubkey, - remote_slot: u64, - ) -> Instruction { - InstructionUtils::finalize_program_from_buffer_instruction( - program, - buffer, - remote_slot, - ) - } - - fn set_authority_ix(program: Pubkey, authority: Pubkey) -> Instruction { - InstructionUtils::set_program_authority_instruction(program, authority) - } - - fn schedule_undelegation_ix(pubkey: Pubkey) -> Instruction { - InstructionUtils::schedule_cloned_account_undelegation_instruction( - pubkey, - ) - } - - // ----------------- - // Clone Fields Helper - // ----------------- - - fn clone_fields(request: &AccountCloneRequest) -> AccountCloneFields { - AccountCloneFields { - lamports: request.account.lamports(), - owner: *request.account.owner(), - executable: request.account.executable(), - delegated: request.account.delegated(), - confined: request.account.confined(), - remote_slot: request.account.remote_slot(), - } - } - - // Account Cloning - // ----------------- - - fn build_small_account_tx( - &self, - request: &AccountCloneRequest, - blockhash: Hash, - ) -> Transaction { - let fields = Self::clone_fields(request); - let actions: Vec = - request.delegation_actions.clone().into(); - let clone_actions = if request.needs_undelegation { - Vec::new() - } else { - actions.clone() - }; - let clone_ix = Self::clone_ix( - request.pubkey, - request.account.data().to_vec(), - fields, - clone_actions, - ); - - // TODO(#625): Re-enable frequency commits when proper limits are in place: - // 1. Allow configuring a higher minimum frequency - // 2. Stop committing accounts if they have been committed more than X times - // where X corresponds to what we can charge - // - // To re-enable, uncomment the following and use `ixs` instead of `[clone_ix]`: - // let ixs = self.maybe_add_crank_commits_ix(request, clone_ix); - let mut ixs = vec![clone_ix]; - if request.needs_undelegation { - ixs.push(Self::schedule_undelegation_ix(request.pubkey)); - } else if !actions.is_empty() { - ixs.push(Self::post_delegation_action_ix(request.pubkey, actions)); - } - - self.create_signed_tx(&ixs, blockhash) - } - - /// Builds crank commits instruction for periodic account commits. - /// Currently disabled - see https://github.com/magicblock-labs/magicblock-validator/issues/625 - #[allow(dead_code)] - fn build_crank_commits_ix( - pubkey: Pubkey, - commit_frequency_ms: i64, - ) -> Instruction { - let task_id: i64 = rand::random(); - let schedule_commit_ix = Instruction::new_with_bincode( - magicblock_program::ID, - &MagicBlockInstruction::ScheduleCommit, - vec![ - AccountMeta::new(validator_authority_id(), true), - AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), - AccountMeta::new_readonly(pubkey, false), - ], - ); - InstructionUtils::schedule_task_instruction( - &validator_authority_id(), - ScheduleTaskArgs { - task_id, - execution_interval_millis: commit_frequency_ms, - iterations: i64::MAX, - instructions: vec![schedule_commit_ix], - }, - ) - } - - fn build_large_account_txs( - &self, - request: &AccountCloneRequest, - blockhash: Hash, - ) -> Vec { - let data = request.account.data(); - let fields = Self::clone_fields(request); - let mut txs = Vec::new(); - - // Init tx with first chunk - let first_chunk = data[..MAX_INLINE_DATA_SIZE.min(data.len())].to_vec(); - let init_ix = Self::clone_init_ix( - request.pubkey, - // we assume the cloned accounts do not have data field - // not exceeding the max solana limit, which is always true - // since the source of cloning is always base chain - data.len() as u32, - first_chunk, - fields, - ); - txs.push(self.create_signed_tx(&[init_ix], blockhash)); - - let actions: Vec = - request.delegation_actions.clone().into(); - let clone_actions = if request.needs_undelegation { - Vec::new() - } else { - actions.clone() - }; - - // Continue txs for remaining chunks - let has_post_delegation_action = - !clone_actions.is_empty() || request.needs_undelegation; - let mut offset = MAX_INLINE_DATA_SIZE; - while offset < data.len() { - let end = (offset + MAX_INLINE_DATA_SIZE).min(data.len()); - let chunk = data[offset..end].to_vec(); - let is_last = end == data.len(); - let final_without_actions = is_last && !has_post_delegation_action; - - let continue_ix = Self::clone_continue_ix( - request.pubkey, - offset as u32, - chunk, - final_without_actions, - Vec::new(), - false, - ); - txs.push(self.create_signed_tx(&[continue_ix], blockhash)); - offset = end; - } - - if request.needs_undelegation || !actions.is_empty() { - let continue_ix = Self::clone_continue_ix( - request.pubkey, - data.len() as u32, - Vec::new(), - true, - clone_actions, - request.needs_undelegation, - ); - let ix = if request.needs_undelegation { - Self::schedule_undelegation_ix(request.pubkey) - } else { - Self::post_delegation_action_ix(request.pubkey, actions) - }; - txs.push(self.create_signed_tx(&[continue_ix, ix], blockhash)); - } - - txs - } - - async fn send_cleanup(&self, pubkey: Pubkey) { - let blockhash = self.block.load().blockhash; - let tx = self.create_signed_tx(&[Self::cleanup_ix(pubkey)], blockhash); - if let Err(e) = self.send_tx(tx).await { - error!(pubkey = %pubkey, error = ?e, "Failed to cleanup partial clone"); - } - } - - // ----------------- - // Program Cloning - // ----------------- - - fn build_program_txs( - &self, - program: LoadedProgram, - blockhash: Hash, - ) -> ClonerResult>> { - match program.loader { - RemoteProgramLoader::V1 => { - self.build_v1_program_txs(program, blockhash) - } - _ => self.build_v4_program_txs(program, blockhash), - } - } - - /// Helper to build buffer fields for program cloning. - fn buffer_fields(data_len: usize) -> AccountCloneFields { - let lamports = Rent::default().minimum_balance(data_len); - AccountCloneFields { - lamports, - owner: solana_sdk_ids::system_program::id(), - ..Default::default() - } - } - - /// Helper to build program transactions (shared between V1 and V4). - fn build_program_txs_from_finalize( - &self, - buffer_pubkey: Pubkey, - program_data: Vec, - finalize_ixs: Vec, - blockhash: Hash, - ) -> Vec { - let buffer_fields = Self::buffer_fields(program_data.len()); - - if program_data.len() <= MAX_INLINE_DATA_SIZE { - // Small: single transaction with clone + finalize - let mut ixs = vec![Self::clone_ix( - buffer_pubkey, - program_data, - buffer_fields, - Vec::new(), - )]; - ixs.extend(finalize_ixs); - vec![self.create_signed_tx(&ixs, blockhash)] - } else { - // Large: multi-transaction flow - self.build_large_program_txs( - buffer_pubkey, - program_data, - buffer_fields, - finalize_ixs, - blockhash, - ) - } - } - - /// V1 programs are converted to V3 (upgradeable loader) format. - /// Supports programs of any size via multi-transaction cloning. - /// - /// NOTE: we don't support modifying this kind of program once it was - /// deployed into our validator once. - /// By nature of being immutable on chain this should never happen. - /// Thus we avoid having to run the upgrade instruction and get - /// away with just directly modifying the program and program data accounts. - fn build_v1_program_txs( - &self, - program: LoadedProgram, - blockhash: Hash, - ) -> ClonerResult>> { - let program_id = program.program_id; - let chain_authority = program.authority; - let remote_slot = program.remote_slot; - - debug!(program_id = %program_id, "Loading V1 program as V3 format"); - - let elf_data = program.program_data; - let (buffer_pubkey, _) = derive_buffer_pubkey(&program_id); - let (program_data_addr, _) = Pubkey::find_program_address( - &[program_id.as_ref()], - &bpf_loader_upgradeable::id(), - ); - - // Finalization instruction - // Must wrap in disable/enable executable check since finalize sets executable=true - let finalize_ixs = vec![ - InstructionUtils::disable_executable_check_instruction( - &validator_authority_id(), - ), - InstructionUtils::finalize_v1_program_from_buffer_instruction( - program_id, - program_data_addr, - buffer_pubkey, - remote_slot, - chain_authority, - ), - InstructionUtils::enable_executable_check_instruction( - &validator_authority_id(), - ), - ]; - - let txs = self.build_program_txs_from_finalize( - buffer_pubkey, - elf_data, - finalize_ixs, - blockhash, - ); - - Ok(Some(txs)) - } - - /// V2/V3/V4 programs use LoaderV4 with proper deploy flow. - /// Supports programs of any size via multi-transaction cloning. - fn build_v4_program_txs( - &self, - program: LoadedProgram, - blockhash: Hash, - ) -> ClonerResult>> { - let program_id = program.program_id; - let chain_authority = program.authority; - let remote_slot = program.remote_slot; - - // Skip retracted programs - if matches!(program.loader_status, LoaderV4Status::Retracted) { - debug!(program_id = %program_id, "Program is retracted on chain"); - return Ok(None); - } - - debug!(program_id = %program_id, "Deploying program with V4 loader"); - - let program_data = program.program_data; - let (buffer_pubkey, _) = derive_buffer_pubkey(&program_id); - - let deploy_ix = Instruction { - program_id: loader_v4::id(), - accounts: vec![ - AccountMeta::new(program_id, false), - AccountMeta::new_readonly(validator_authority_id(), true), - ], - data: bincode::serialize(&LoaderV4Instruction::Deploy)?, - }; - - // Finalization instructions (always in last tx) - // Must wrap in disable/enable executable check since finalize sets executable=true - let finalize_ixs = vec![ - InstructionUtils::disable_executable_check_instruction( - &validator_authority_id(), - ), - Self::finalize_program_ix(program_id, buffer_pubkey, remote_slot), - deploy_ix, - Self::set_authority_ix(program_id, chain_authority), - InstructionUtils::enable_executable_check_instruction( - &validator_authority_id(), - ), - ]; - - let txs = self.build_program_txs_from_finalize( - buffer_pubkey, - program_data, - finalize_ixs, - blockhash, - ); - - Ok(Some(txs)) - } - - /// Builds multi-transaction flow for large programs (any loader). - fn build_large_program_txs( - &self, - buffer_pubkey: Pubkey, - program_data: Vec, - fields: AccountCloneFields, - finalize_ixs: Vec, - blockhash: Hash, - ) -> Vec { - let total_len = program_data.len() as u32; - let num_chunks = - total_len.div_ceil(MAX_INLINE_DATA_SIZE as u32) as usize; - - // First chunk via Init - let first_chunk = - &program_data[..MAX_INLINE_DATA_SIZE.min(program_data.len())]; - let init_ix = Self::clone_init_ix( - buffer_pubkey, - total_len, - first_chunk.to_vec(), - fields, - ); - let mut txs = vec![self.create_signed_tx(&[init_ix], blockhash)]; - - // Middle chunks (all except last) - let last_offset = (num_chunks - 1) * MAX_INLINE_DATA_SIZE; - for offset in - (MAX_INLINE_DATA_SIZE..last_offset).step_by(MAX_INLINE_DATA_SIZE) - { - let chunk = &program_data[offset..offset + MAX_INLINE_DATA_SIZE]; - let continue_ix = Self::clone_continue_ix( - buffer_pubkey, - offset as u32, - chunk.to_vec(), - false, - Vec::new(), - false, - ); - txs.push(self.create_signed_tx(&[continue_ix], blockhash)); - } - - // Last chunk with finalize instructions - let last_chunk = &program_data[last_offset..]; - let mut ixs = vec![Self::clone_continue_ix( - buffer_pubkey, - last_offset as u32, - last_chunk.to_vec(), - true, - Vec::new(), - false, - )]; - ixs.extend(finalize_ixs); - txs.push(self.create_signed_tx(&ixs, blockhash)); - - txs - } -} - -/// Shared account metas for clone instructions. -#[async_trait] -impl Cloner for ChainlinkCloner { - async fn evict_account(&self, pubkey: Pubkey) -> ClonerResult<()> { - let blockhash = self.block.load().blockhash; - let evict_ix = InstructionUtils::evict_account_instruction(pubkey); - let tx = self.create_signed_tx(&[evict_ix], blockhash); - self.send_tx(tx).await.map_err(|err| { - ClonerError::FailedToEvictAccount(pubkey, Box::new(err)) - })?; - Ok(()) - } - - async fn clone_account( - &self, - request: AccountCloneRequest, - ) -> ClonerResult { - let blockhash = self.block.load().blockhash; - let data_len = request.account.data().len(); - - if data_len <= MAX_INLINE_DATA_SIZE { - let tx = self.build_small_account_tx(&request, blockhash); - let tx_size = Self::transaction_size(&tx)?; - if tx_size > MAX_INLINE_TRANSACTION_SIZE - && !request.delegation_actions.is_empty() - { - debug!( - pubkey = %request.pubkey, - data_len, - tx_size, - "Small account clone transaction is too large, using chunked clone" - ); - } else { - let signature = self.send_tx(tx).await.map_err(|err| { - ClonerError::FailedToCloneRegularAccount( - request.pubkey, - Box::new(err), - ) - })?; - - return Ok(signature); - } - } - - // Large account: multi-tx with cleanup on failure - let txs = self.build_large_account_txs(&request, blockhash); - Self::ensure_transactions_fit(request.pubkey, &txs)?; - - let mut last_sig = None; - for tx in txs { - match self.send_tx(tx).await { - Ok(sig) => { - last_sig.replace(sig); - } - Err(e) => { - self.send_cleanup(request.pubkey).await; - return Err(ClonerError::FailedToCloneRegularAccount( - request.pubkey, - Box::new(e), - )); - } - } - } - - Ok(last_sig.unwrap_or_default()) - } - - async fn clone_program( - &self, - program: LoadedProgram, - ) -> ClonerResult { - let blockhash = self.block.load().blockhash; - let program_id = program.program_id; - - let Some(txs) = - self.build_program_txs(program, blockhash).map_err(|e| { - ClonerError::FailedToCreateCloneProgramTransaction( - program_id, - Box::new(e), - ) - })? - else { - // Program was retracted - return Ok(Signature::default()); - }; - - // Both V1 and V4 use buffer_pubkey for multi-tx cloning - let buffer_pubkey = derive_buffer_pubkey(&program_id).0; - - let mut last_sig = None; - for tx in txs { - match self.send_tx(tx).await { - Ok(sig) => { - last_sig.replace(sig); - } - Err(e) => { - self.send_cleanup(buffer_pubkey).await; - return Err(ClonerError::FailedToCloneProgram( - program_id, - Box::new(e), - )); - } - } - } - - // After cloning a program we need to wait at least one slot for it to become - // usable, so we do that here - let current_slot = self.block.load().slot; - let mut block_updated = self.block.subscribe(); - while self.block.load().slot == current_slot { - let _ = block_updated.recv().await; - } - - Ok(last_sig.unwrap_or_default()) - } -} - -#[cfg(test)] -mod tests { - use magicblock_chainlink::cloner::DelegationActions; - use magicblock_core::link::link; - use magicblock_magic_program_api::{ - instruction::{ - MagicBlockInstruction, PostDelegationActionExecutorInstruction, - }, - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, - }; - use solana_account::AccountSharedData; - use solana_sdk_ids::system_program; - - use super::*; - - fn cloner() -> ChainlinkCloner { - magicblock_program::validator::generate_validator_authority_if_needed(); - let (dispatch, _) = link(); - ChainlinkCloner::new( - dispatch.transaction_scheduler, - LatestBlock::default(), - ) - } - - fn request( - pubkey: Pubkey, - data: Vec, - actions: Vec, - ) -> AccountCloneRequest { - let mut account = - AccountSharedData::new(1_000, data.len(), &system_program::id()); - account.set_data_from_slice(&data); - account.set_delegated(true); - AccountCloneRequest { - pubkey, - account, - commit_frequency_ms: None, - delegation_actions: DelegationActions::from(actions), - delegated_to_other: None, - needs_undelegation: false, - } - } - - fn action() -> Instruction { - Instruction { - program_id: Pubkey::new_unique(), - accounts: vec![AccountMeta::new(Pubkey::new_unique(), true)], - data: vec![1, 2, 3], - } - } - - fn action_with_data_len(data_len: usize) -> Instruction { - Instruction { - program_id: Pubkey::new_unique(), - accounts: vec![AccountMeta::new(Pubkey::new_unique(), true)], - data: vec![1; data_len], - } - } - - fn instruction_program_id(tx: &Transaction, ix_idx: usize) -> Pubkey { - let ix = &tx.message().instructions[ix_idx]; - tx.message().account_keys[ix.program_id_index as usize] - } - - fn instruction_data(tx: &Transaction, ix_idx: usize) -> &[u8] { - &tx.message().instructions[ix_idx].data - } - - #[test] - fn transaction_size_matches_bincode_serialized_len() { - let pubkey = Pubkey::new_unique(); - let tx = cloner().build_small_account_tx( - &request(pubkey, vec![1, 2, 3], vec![action()]), - Hash::default(), - ); - - assert_eq!( - ChainlinkCloner::transaction_size(&tx).unwrap(), - bincode::serialize(&tx).unwrap().len() - ); - } - - #[test] - fn oversized_chunked_action_transaction_fails_preflight() { - let pubkey = Pubkey::new_unique(); - let actions = - vec![action_with_data_len(MAX_INLINE_TRANSACTION_SIZE / 2 + 1024)]; - let txs = cloner().build_large_account_txs( - &request(pubkey, vec![1, 2, 3], actions), - Hash::default(), - ); - let final_tx_size = - ChainlinkCloner::transaction_size(txs.last().unwrap()).unwrap(); - - assert!(final_tx_size > MAX_INLINE_TRANSACTION_SIZE); - - let err = - ChainlinkCloner::ensure_transactions_fit(pubkey, &txs).unwrap_err(); - match err { - ClonerError::CloneTransactionTooLarge { - pubkey: err_pubkey, - size, - max_size, - } => { - assert_eq!(err_pubkey, pubkey); - assert_eq!(size, final_tx_size); - assert_eq!(max_size, MAX_INLINE_TRANSACTION_SIZE); - } - err => panic!("unexpected error: {err:?}"), - } - } - - #[test] - fn undelegation_ix_uses_specific_magic_instruction() { - magicblock_program::validator::generate_validator_authority_if_needed(); - let pubkey = Pubkey::new_unique(); - let ix = ChainlinkCloner::schedule_undelegation_ix(pubkey); - - assert_eq!(ix.program_id, POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID); - assert_eq!(ix.accounts.len(), 4); - assert_eq!( - ix.accounts[0].pubkey, - magicblock_program::validator::validator_authority_id() - ); - assert!(ix.accounts[0].is_signer); - assert!(!ix.accounts[0].is_writable); - assert_eq!(ix.accounts[1].pubkey, pubkey); - assert!(ix.accounts[1].is_writable); - assert_eq!( - ix.accounts[2].pubkey, - solana_sdk_ids::sysvar::instructions::id() - ); - assert!(!ix.accounts[2].is_writable); - assert_eq!(ix.accounts[3].pubkey, MAGIC_CONTEXT_PUBKEY); - assert!(ix.accounts[3].is_writable); - - match bincode::deserialize(&ix.data).unwrap() { - PostDelegationActionExecutorInstruction::ScheduleUndelegation { - .. - } => {} - _ => panic!("expected schedule undelegation instruction"), - } - } - - #[test] - fn small_undelegation_clone_schedules_in_same_tx_without_actions() { - let pubkey = Pubkey::new_unique(); - let mut request = request(pubkey, vec![1, 2, 3], vec![action()]); - request.needs_undelegation = true; - let tx = cloner().build_small_account_tx(&request, Hash::default()); - - assert_eq!(tx.message().instructions.len(), 2); - assert_eq!(instruction_program_id(&tx, 0), magicblock_program::ID); - assert_eq!( - instruction_program_id(&tx, 1), - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID - ); - - match bincode::deserialize(instruction_data(&tx, 0)).unwrap() { - MagicBlockInstruction::CloneAccount { - pubkey: clone_pubkey, - actions, - fields, - .. - } => { - assert_eq!(clone_pubkey, pubkey); - assert!(fields.delegated); - assert!(actions.is_empty()); - } - _ => panic!("expected clone account instruction"), - } - match bincode::deserialize(instruction_data(&tx, 1)).unwrap() { - PostDelegationActionExecutorInstruction::ScheduleUndelegation { - .. - } => {} - _ => panic!("expected schedule undelegation instruction"), - } - } - - #[test] - fn large_undelegation_clone_schedules_after_final_chunk_tx() { - let pubkey = Pubkey::new_unique(); - let mut request = - request(pubkey, vec![7; MAX_INLINE_DATA_SIZE + 1], vec![action()]); - request.needs_undelegation = true; - let txs = cloner().build_large_account_txs(&request, Hash::default()); - - assert_eq!(txs.len(), 3); - - let final_clone_tx = &txs[1]; - assert_eq!(final_clone_tx.message().instructions.len(), 1); - assert_eq!( - instruction_program_id(final_clone_tx, 0), - magicblock_program::ID - ); - - match bincode::deserialize(instruction_data(final_clone_tx, 0)).unwrap() - { - MagicBlockInstruction::CloneAccountContinue { - pubkey: continue_pubkey, - offset, - data, - is_last, - actions, - needs_undelegation, - } => { - assert_eq!(continue_pubkey, pubkey); - assert_eq!(offset, MAX_INLINE_DATA_SIZE as u32); - assert_eq!(data, vec![7]); - assert!(!is_last); - assert!(actions.is_empty()); - assert!(!needs_undelegation); - } - _ => panic!("expected clone account continue instruction"), - } - - let rescue_tx = txs.last().unwrap(); - assert_eq!(rescue_tx.message().instructions.len(), 2); - - assert_eq!( - instruction_program_id(rescue_tx, 0), - magicblock_program::ID - ); - match bincode::deserialize(instruction_data(rescue_tx, 0)).unwrap() { - MagicBlockInstruction::CloneAccountContinue { - pubkey: continue_pubkey, - offset, - data, - is_last, - actions, - needs_undelegation, - } => { - assert_eq!(continue_pubkey, pubkey); - assert_eq!(offset, (MAX_INLINE_DATA_SIZE + 1) as u32); - assert_eq!(data, Vec::::new()); - assert!(is_last); - assert!(actions.is_empty()); - assert!(needs_undelegation); - } - _ => panic!("expected clone account continue instruction"), - } - - assert_eq!( - instruction_program_id(rescue_tx, 1), - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID - ); - match bincode::deserialize(instruction_data(rescue_tx, 1)).unwrap() { - PostDelegationActionExecutorInstruction::ScheduleUndelegation { - .. - } => {} - _ => panic!("expected schedule undelegation instruction"), - } - } - - #[test] - fn large_undelegation_max_final_chunk_fits() { - let pubkey = Pubkey::new_unique(); - let mut request = - request(pubkey, vec![7; MAX_INLINE_DATA_SIZE * 2], vec![]); - request.needs_undelegation = true; - let txs = cloner().build_large_account_txs(&request, Hash::default()); - - ChainlinkCloner::ensure_transactions_fit(pubkey, &txs).unwrap(); - } - - #[test] - fn chunked_undelegation_short_data_uses_empty_final_continue() { - let pubkey = Pubkey::new_unique(); - let data = vec![1, 2, 3]; - let mut request = request(pubkey, data.clone(), vec![action()]); - request.needs_undelegation = true; - let txs = cloner().build_large_account_txs(&request, Hash::default()); - - assert_eq!(txs.len(), 2); - - let final_clone_tx = &txs[1]; - assert_eq!(final_clone_tx.message().instructions.len(), 2); - match bincode::deserialize(instruction_data(final_clone_tx, 0)).unwrap() - { - MagicBlockInstruction::CloneAccountContinue { - pubkey: continue_pubkey, - offset, - data: continue_data, - is_last, - actions, - needs_undelegation, - } => { - assert_eq!(continue_pubkey, pubkey); - assert_eq!(offset, data.len() as u32); - assert!(continue_data.is_empty()); - assert!(is_last); - assert!(actions.is_empty()); - assert!(needs_undelegation); - } - _ => panic!("expected clone account continue instruction"), - } - - match bincode::deserialize(instruction_data(final_clone_tx, 1)).unwrap() - { - PostDelegationActionExecutorInstruction::ScheduleUndelegation { - .. - } => {} - _ => panic!("expected schedule undelegation instruction"), - } - } - - #[test] - fn small_delegated_clone_with_actions_emits_executor_sibling() { - let pubkey = Pubkey::new_unique(); - let actions = vec![action()]; - let tx = cloner().build_small_account_tx( - &request(pubkey, vec![1, 2, 3], actions.clone()), - Hash::default(), - ); - - assert_eq!(tx.message().instructions.len(), 2); - assert_eq!(instruction_program_id(&tx, 0), magicblock_program::ID); - assert_eq!( - instruction_program_id(&tx, 1), - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID - ); - - match bincode::deserialize(instruction_data(&tx, 0)).unwrap() { - MagicBlockInstruction::CloneAccount { - pubkey: clone_pubkey, - actions: clone_actions, - fields, - .. - } => { - assert_eq!(clone_pubkey, pubkey); - assert!(fields.delegated); - assert_eq!(clone_actions, actions); - } - _ => panic!("expected clone account instruction"), - } - match bincode::deserialize(instruction_data(&tx, 1)).unwrap() { - PostDelegationActionExecutorInstruction::Execute { - cloned_account_pubkey: executor_pubkey, - actions: executor_actions, - } => { - assert_eq!(executor_pubkey, pubkey); - assert_eq!(executor_actions, actions); - } - PostDelegationActionExecutorInstruction::ScheduleUndelegation { - .. - } => { - panic!("expected execute instruction") - } - } - } - - #[test] - fn large_delegated_clone_with_actions_uses_empty_final_continue_then_executor( - ) { - let pubkey = Pubkey::new_unique(); - let actions = vec![action()]; - let data = vec![7; MAX_INLINE_DATA_SIZE + 1]; - let txs = cloner().build_large_account_txs( - &request(pubkey, data, actions.clone()), - Hash::default(), - ); - - let final_tx = txs.last().unwrap(); - assert_eq!(final_tx.message().instructions.len(), 2); - assert_eq!(instruction_program_id(final_tx, 0), magicblock_program::ID); - assert_eq!( - instruction_program_id(final_tx, 1), - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID - ); - - match bincode::deserialize(instruction_data(final_tx, 0)).unwrap() { - MagicBlockInstruction::CloneAccountContinue { - pubkey: continue_pubkey, - offset, - data, - is_last, - actions: continue_actions, - needs_undelegation, - } => { - assert_eq!(continue_pubkey, pubkey); - assert_eq!(offset, (MAX_INLINE_DATA_SIZE + 1) as u32); - assert!(data.is_empty()); - assert!(is_last); - assert_eq!(continue_actions, actions); - assert!(!needs_undelegation); - } - _ => panic!("expected clone account continue instruction"), - } - match bincode::deserialize(instruction_data(final_tx, 1)).unwrap() { - PostDelegationActionExecutorInstruction::Execute { - cloned_account_pubkey: executor_pubkey, - actions: executor_actions, - } => { - assert_eq!(executor_pubkey, pubkey); - assert_eq!(executor_actions, actions); - } - PostDelegationActionExecutorInstruction::ScheduleUndelegation { - .. - } => { - panic!("expected execute instruction") - } - } - } -} diff --git a/magicblock-account-cloner/src/util.rs b/magicblock-account-cloner/src/util.rs deleted file mode 100644 index fcfe876f6..000000000 --- a/magicblock-account-cloner/src/util.rs +++ /dev/null @@ -1,12 +0,0 @@ -use magicblock_program::validator::validator_authority_id; -use solana_pubkey::Pubkey; - -/// Seed for deriving buffer account PDA -const BUFFER_SEED: &[u8] = b"buffer"; - -/// Derives a deterministic buffer account pubkey for program cloning. -/// Uses validator_authority as owner so it works for any loader type. -pub fn derive_buffer_pubkey(program_pubkey: &Pubkey) -> (Pubkey, u8) { - let seeds: &[&[u8]] = &[BUFFER_SEED, program_pubkey.as_ref()]; - Pubkey::find_program_address(seeds, &validator_authority_id()) -} diff --git a/magicblock-accounts-db/Cargo.toml b/magicblock-accounts-db/Cargo.toml deleted file mode 100644 index c07a35413..000000000 --- a/magicblock-accounts-db/Cargo.toml +++ /dev/null @@ -1,41 +0,0 @@ -[package] -name = "magicblock-accounts-db" -version.workspace = true -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[dependencies] -# storage -memmap2 = "0.9" -lmdb = { package = "lmdb-rkv", version = "0.14" } # more up to date fork of lmdb bindings by mozilla, still ancient though :( -reflink = { package = "reflink-copy", version = "0.1" } - -# archival -tar = "0.4" -flate2 = "1.0" - -# solana -solana-pubkey = { workspace = true } -solana-account = { workspace = true } - -# synchronization -parking_lot = { workspace = true } -thiserror = { workspace = true } -tracing = { workspace = true } -magicblock-config = { workspace = true } -magicblock-magic-program-api = { workspace = true } -solana-sdk-ids = { workspace = true } - -# misc -twox-hash = { workspace = true } - -[dev-dependencies] -tempfile = { workspace = true } -tracing-subscriber = { workspace = true } - -[features] -default = [] -dev-tools = [] diff --git a/magicblock-accounts-db/README.md b/magicblock-accounts-db/README.md deleted file mode 100644 index 3529ae76d..000000000 --- a/magicblock-accounts-db/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Accounts Database Manager - -The AccountsDB is a crucial component of the Magicblock Validator. It handles -the storage, retrieval, and management of account data. It's highly optimized -to provide efficient account search and retrieval, using memory mapped storage, -while still being persisted to disk. - -## Features - -- **Account Storage**: Efficient storage of account records, with support for - both owned and borrowed account data. By default accounts are read directly - from memory mapping without any deserialization and/or allocation, as - consequence it allows for direct memory modification in situ in the database, - avoiding read modify write overhead. But this imposes a strict requirement, that - no two threads have access to the same account at the same time for - modification, which is naturally achieved with account locking when executing - transactions. But any other way to hold onto borrowed state of account is - prohibited, as it will inevitably cause an undefined behavior. -- **Index Management**: Fast index-based lookups for account data. - LMDB is used for the purposes of indexing, where several different mappings - are maintained to support database integrity. -- **Snapshot Management**: Automated snapshot creation to enable rollbacks to - previous states of the database if necessary. -- **Concurrency Control**: Uses a "Stop the World" lock to ensure data - consistency during critical operations like snapshotting. This prevents any - writes to the database while it's being manipulated at the OS level. -- **Program Account Scanning**: Retrieve and filter accounts associated with a - specific program, the scanning happens without deserializing anything from the - database. - -## Main API methods - -- **Initialization**: Use `AccountsDb::new` to create or open an accounts - database instance. -- **Account Operations**: Use `get_account`, `insert_account`, - `contains_account` to read, write, or check the existence of accounts. -- **Snapshot Operations**: Schedule and manage snapshots with `set_slot`, which - might trigger snapshot operation, which in turn involves locking everything - down, thus snapshot frequency should be set to sane value. And retrieve - snapshot slots using `get_latest_snapshot_slot` and - `get_oldest_snapshot_slot`. -- **Database Integrity**: Use `ensure_at_most` to ensure database state up to a - specific slot, with rollback to previously taken snapshot if needed. diff --git a/magicblock-accounts-db/src/error.rs b/magicblock-accounts-db/src/error.rs deleted file mode 100644 index ceb9381f6..000000000 --- a/magicblock-accounts-db/src/error.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::io; - -use tracing::error; - -#[derive(Debug, thiserror::Error)] -pub enum AccountsDbError { - #[error("requested account doesn't exist in adb")] - NotFound, - #[error("io error during adb access: {0}")] - Io(#[from] io::Error), - #[error("lmdb index error: {0}")] - Lmdb(lmdb::Error), - #[error("snapshot for slot {0} doesn't exist")] - SnapshotMissing(u64), - #[error("internal accountsdb error: {0}")] - Internal(String), -} - -impl From for AccountsDbError { - fn from(error: lmdb::Error) -> Self { - match error { - lmdb::Error::NotFound => Self::NotFound, - err => Self::Lmdb(err), - } - } -} - -/// Extension trait to easily log errors in Result chains. -pub trait LogErr { - /// Logs the error if the result is `Err`, then returns the result unmodified. - fn log_err(self, msg: F) -> Result - where - F: FnOnce() -> S, - S: std::fmt::Display; -} - -impl LogErr for Result -where - E: std::fmt::Display, -{ - #[track_caller] - fn log_err(self, msg: F) -> Result - where - F: FnOnce() -> S, - S: std::fmt::Display, - { - if let Err(error) = &self { - error!(%error, msg=%msg()); - } - self - } -} diff --git a/magicblock-accounts-db/src/index.rs b/magicblock-accounts-db/src/index.rs deleted file mode 100644 index 96b95f99c..000000000 --- a/magicblock-accounts-db/src/index.rs +++ /dev/null @@ -1,519 +0,0 @@ -use std::path::Path; - -use iterator::OffsetPubkeyIter; -use lmdb::{Cursor, DatabaseFlags, Environment, RwTransaction, Transaction}; -use solana_pubkey::Pubkey; -use table::Table; -use tracing::warn; -use utils::*; - -use crate::{ - error::{AccountsDbError, LogErr}, - storage::{Allocation, ExistingAllocation}, - AccountsDbResult, -}; - -pub type Offset = u32; -pub type Blocks = u32; - -const ACCOUNTS_INDEX: &str = "accounts-idx"; -const PROGRAMS_INDEX: &str = "programs-idx"; -const DEALLOCATIONS_INDEX: &str = "deallocations-idx"; -const OWNERS_INDEX: &str = "owners-idx"; - -#[derive(Clone, Copy)] -pub(crate) struct AccountAllocation { - pub(crate) pubkey: Pubkey, - pub(crate) owner: Pubkey, - pub(crate) offset: Offset, - pub(crate) blocks: Blocks, -} - -#[derive(Clone, Copy)] -pub(crate) struct AccountMove { - pub(crate) pubkey: Pubkey, - pub(crate) owner: Pubkey, - pub(crate) old_offset: Offset, - pub(crate) new_offset: Offset, - pub(crate) blocks: Blocks, -} - -/// LMDB Index manager. -/// -/// Handles secondary indices for mapping Pubkeys to storage offsets, -/// tracking program ownership, and managing deallocated space. -#[cfg_attr(test, derive(Debug))] -pub(crate) struct AccountsDbIndex { - /// Maps Account Pubkey -> (Storage Offset, Block Count). - accounts: Table, - /// Maps Owner Pubkey -> (Storage Offset, Account Pubkey). - /// Used for `get_program_accounts`. - programs: Table, - /// Maps Allocation Size (Blocks) -> (Storage Offset, Block Count). - /// Used for finding recyclable "holes" in storage. - deallocations: Table, - /// Maps Account Pubkey -> Owner Pubkey. - /// Used for cleaning up the `programs` index when an account changes owner. - owners: Table, - /// The LMDB Environment. - env: Environment, -} - -/// Helper macro to pack/unpack types into/from byte buffers. -/// Uses unaligned writes/reads for performance and compactness. -macro_rules! bytes { - (#pack, $val1: expr, $type1: ty, $val2: expr, $type2: ty) => {{ - const S1: usize = std::mem::size_of::<$type1>(); - const S2: usize = std::mem::size_of::<$type2>(); - let mut buffer = [0_u8; S1 + S2]; - let ptr = buffer.as_mut_ptr(); - // SAFETY: Buffer is exactly S1 + S2 bytes. - unsafe { - (ptr as *mut $type1).write_unaligned($val1); - (ptr.add(S1) as *mut $type2).write_unaligned($val2); - } - buffer - }}; - (#unpack, $packed: expr, $type1: ty, $type2: ty) => {{ - let ptr = $packed.as_ptr(); - const S1: usize = std::mem::size_of::<$type1>(); - // SAFETY: Macro caller ensures $packed is valid length. - unsafe { - let v1 = (ptr as *const $type1).read_unaligned(); - let v2 = (ptr.add(S1) as *const $type2).read_unaligned(); - (v1, v2) - } - }}; -} - -impl AccountsDbIndex { - pub(crate) fn new(size: usize, directory: &Path) -> AccountsDbResult { - let env = utils::create_lmdb_env(directory, size).log_err(|| { - format!("main index env creation at {}", directory.display()) - })?; - - let accounts = - Table::new(&env, ACCOUNTS_INDEX, DatabaseFlags::empty())?; - - let programs = Table::new( - &env, - PROGRAMS_INDEX, - DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED, - )?; - - // DEALLOCATIONS: Allow duplicates (multiple holes of same size), - // integer keys (block size) for range searches. - let deallocations = Table::new( - &env, - DEALLOCATIONS_INDEX, - DatabaseFlags::DUP_SORT - | DatabaseFlags::DUP_FIXED - | DatabaseFlags::REVERSE_KEY - | DatabaseFlags::INTEGER_KEY, - )?; - - let owners = Table::new(&env, OWNERS_INDEX, DatabaseFlags::empty())?; - - Ok(Self { - accounts, - programs, - deallocations, - owners, - env, - }) - } - - /// Retrieves the storage offset for a given account. - #[inline(always)] - pub(crate) fn get_offset( - &self, - pubkey: &Pubkey, - ) -> AccountsDbResult { - let txn = self.env.begin_ro_txn()?; - let Some(value_bytes) = self.accounts.get(&txn, pubkey)? else { - return Err(AccountsDbError::NotFound); - }; - - // We only need the first 4 bytes (Offset), ignoring the Blocks count. - // SAFETY: Accounts index values are always created by `bytes!(#pack, ...)` - // which guarantees [Offset(4), Blocks(4)]. - let offset = - unsafe { (value_bytes.as_ptr() as *const Offset).read_unaligned() }; - Ok(offset) - } - - /// Inserts or updates an account's allocation in the indices. - /// - /// If the account already exists, it handles the "move": - /// 1. Marks the old space as deallocated (recyclable). - /// 2. Updates the main index. - /// 3. Updates secondary indices (programs, owners). - pub(crate) fn upsert_account( - &self, - pubkey: &Pubkey, - owner: &Pubkey, - allocation: Allocation, - txn: &mut RwTransaction, - ) -> AccountsDbResult> { - let Allocation { offset, blocks, .. } = allocation; - let mut old_allocation = None; - - let index_value = bytes!(#pack, offset, Offset, blocks, Blocks); - let offset_and_pubkey = bytes!(#pack, offset, Offset, *pubkey, Pubkey); - - // 1. Optimistic insert (returns true if inserted, false if existed) - let inserted = self.accounts.insert(txn, pubkey, index_value)?; - - // 2. Handle update if it already existed - if !inserted { - let previous = - self.reallocate_account(pubkey, txn, &index_value)?; - old_allocation = Some(previous); - } - - // 3. Update secondary indices - self.programs.upsert(txn, owner, offset_and_pubkey)?; - self.owners.upsert(txn, pubkey, owner)?; - - Ok(old_allocation) - } - - /// Handles the logistics of moving an existing account to a new location. - /// Returns the old allocation so it can be added to the free list. - fn reallocate_account( - &self, - pubkey: &Pubkey, - txn: &mut RwTransaction, - new_index_value: &[u8], - ) -> AccountsDbResult { - // Retrieve old location - let old_alloc = self.get_allocation(txn, pubkey)?; - - // Mark old space as free (Deallocations Index) - // Key: Block Count (for size-based lookup), Value: {Offset, Block Count} - let key = old_alloc.blocks.to_le_bytes(); - let value = - bytes!(#pack, old_alloc.offset, Offset, old_alloc.blocks, Blocks); - self.deallocations.upsert(txn, key, value)?; - - // Update Main Index with new location - self.accounts.upsert(txn, pubkey, new_index_value)?; - - // Clean up Programs Index (dangling pointer to old offset) - self.remove_program_index_entry(pubkey, None, txn, old_alloc.offset)?; - - Ok(old_alloc) - } - - /// Removes an account from all indices and marks its space as recyclable. - pub(crate) fn remove( - &self, - pubkey: &Pubkey, - txn: &mut RwTransaction<'_>, - ) -> AccountsDbResult<()> { - // Get allocation to know what to free - let allocation = match self.get_allocation(txn, pubkey) { - Ok(a) => a, - Err(AccountsDbError::NotFound) => return Ok(()), // Idempotent - Err(e) => return Err(e), - }; - - // Remove from Main Index - self.accounts.remove(txn, pubkey, None)?; - - // Add to Deallocations Index - let key = allocation.blocks.to_le_bytes(); - let val = - bytes!(#pack, allocation.offset, Offset, allocation.blocks, Blocks); - self.deallocations.upsert(txn, key, val)?; - - // Remove from Secondary Indices - self.remove_program_index_entry(pubkey, None, txn, allocation.offset)?; - self.owners.remove(txn, pubkey, None)?; - - Ok(()) - } - - /// Reconciles the `owners` and `programs` indices if the account's owner changed. - pub(crate) fn ensure_correct_owner( - &self, - pubkey: &Pubkey, - new_owner: &Pubkey, - txn: &mut RwTransaction, - ) -> AccountsDbResult<()> { - let owner_bytes = self.owners.get(txn, pubkey)?; - - let old_owner = owner_bytes.and_then(|b| Pubkey::try_from(b).ok()); - - let allocation = self.get_allocation(txn, pubkey)?; - - // 1. Clean up old program index entry - self.remove_program_index_entry( - pubkey, - old_owner, - txn, - allocation.offset, - )?; - - // 2. Insert new program index entry - let offset_and_pubkey = - bytes!(#pack, allocation.offset, Offset, *pubkey, Pubkey); - self.programs.upsert(txn, new_owner, offset_and_pubkey)?; - - // 3. Update owners index - self.owners.upsert(txn, pubkey, new_owner)?; - - Ok(()) - } - - /// Internal helper to retrieve full allocation info (Offset + Size). - fn get_allocation( - &self, - txn: &T, - pubkey: &Pubkey, - ) -> AccountsDbResult { - let Some(slice) = self.accounts.get(txn, pubkey)? else { - return Err(AccountsDbError::NotFound); - }; - let (offset, blocks) = bytes!(#unpack, slice, Offset, Blocks); - Ok(ExistingAllocation { offset, blocks }) - } - - /// Finds and removes a `programs` index entry. - /// If `old_owner` is not provided, it is looked up in the `owners` index. - fn remove_program_index_entry( - &self, - pubkey: &Pubkey, - old_owner: Option, - txn: &mut RwTransaction, - offset: Offset, - ) -> lmdb::Result<()> { - // We need the owner to find the key in the programs index (Key=Owner). - let owner = match old_owner { - Some(pk) => pk, - None => match self.owners.get(txn, pubkey)? { - Some(val) => { - Pubkey::try_from(val).map_err(|_| lmdb::Error::Invalid)? - } - None => { - warn!(pubkey = %pubkey, "Account missing from owners index"); - return Ok(()); - } - }, - }; - - // Value in programs index is {Offset, Pubkey}. - // We need exact match to delete from DUPSORT db. - let val = bytes!(#pack, offset, Offset, *pubkey, Pubkey); - self.programs.remove(txn, owner, Some(&val)) - } - - /// Tries to find a free block of `needed` size in the deallocations table. - /// If found, splits it if too large, and returns the recycled allocation. - pub(crate) fn try_recycle_allocation( - &self, - needed: Blocks, - txn: &mut RwTransaction, - ) -> AccountsDbResult { - let mut cursor = self.deallocations.cursor_rw(txn)?; - - // MDB_SET_RANGE: Position cursor at first key >= `needed`. - // This effectively implements a "Best Fit" (or "First Sufficient Fit") strategy. - let (_key_bytes, val_bytes) = cursor.get( - Some(&needed.to_le_bytes()), - None, - utils::MDB_SET_RANGE_OP, - )?; - - let (offset, available_blocks) = - bytes!(#unpack, val_bytes, Offset, Blocks); - - // Remove this allocation from free list - cursor.del(lmdb::WriteFlags::empty())?; - - // If we found a block larger than needed, split it and put the remainder back. - let remainder = available_blocks.saturating_sub(needed); - if remainder > 0 { - let new_hole_offset = offset.saturating_add(needed); - - let new_key = remainder.to_le_bytes(); - let new_val = - bytes!(#pack, new_hole_offset, Offset, remainder, Blocks); - - drop(cursor); - // Insert the remainder back into deallocations - self.deallocations.upsert(txn, new_key, new_val)?; - } - - Ok(ExistingAllocation { - offset, - blocks: needed, - }) - } - - pub(crate) fn accounts_by_offset( - &self, - ) -> AccountsDbResult> { - let txn = self.env.begin_ro_txn()?; - let mut accounts = Vec::new(); - - { - let mut cursor = self.accounts.cursor_ro(&txn)?; - for item in cursor.iter_start() { - let (key_bytes, value_bytes) = item?; - let pubkey = Pubkey::try_from(key_bytes).map_err(|_| { - AccountsDbError::Internal( - "accounts index contains invalid pubkey".into(), - ) - })?; - let (offset, blocks) = - bytes!(#unpack, value_bytes, Offset, Blocks); - accounts.push((pubkey, offset, blocks)); - } - } - - let mut allocations = Vec::with_capacity(accounts.len()); - for (pubkey, offset, blocks) in accounts { - let owner_bytes = self.owners.get(&txn, pubkey)?; - let owner = owner_bytes - .ok_or_else(|| { - AccountsDbError::Internal(format!( - "account {pubkey} missing from owners index" - )) - }) - .and_then(|bytes| { - Pubkey::try_from(bytes).map_err(|_| { - AccountsDbError::Internal(format!( - "account {pubkey} has invalid owner index entry" - )) - }) - })?; - - allocations.push(AccountAllocation { - pubkey, - owner, - offset, - blocks, - }); - } - - allocations.sort_by_key(|allocation| allocation.offset); - Ok(allocations) - } - - pub(crate) fn apply_account_moves( - &self, - moves: &[AccountMove], - ) -> AccountsDbResult<()> { - let mut txn = self.env.begin_rw_txn()?; - - for account_move in moves { - if account_move.old_offset == account_move.new_offset { - continue; - } - - let account_value = bytes!( - #pack, - account_move.new_offset, - Offset, - account_move.blocks, - Blocks - ); - self.accounts.upsert( - &mut txn, - account_move.pubkey, - account_value, - )?; - - let old_program_value = bytes!( - #pack, - account_move.old_offset, - Offset, - account_move.pubkey, - Pubkey - ); - self.programs.remove( - &mut txn, - account_move.owner, - Some(&old_program_value), - )?; - - let new_program_value = bytes!( - #pack, - account_move.new_offset, - Offset, - account_move.pubkey, - Pubkey - ); - self.programs.upsert( - &mut txn, - account_move.owner, - new_program_value, - )?; - } - - self.deallocations.clear(&mut txn)?; - txn.commit()?; - Ok(()) - } - - // --- Iterators & Accessors --- - - pub(crate) fn get_program_accounts_iter( - &self, - program: &Pubkey, - ) -> AccountsDbResult> { - let txn = self.env.begin_ro_txn()?; - OffsetPubkeyIter::new_programs(&self.programs, txn, Some(program)) - } - - pub(crate) fn get_all_accounts( - &self, - ) -> AccountsDbResult> { - let txn = self.env.begin_ro_txn()?; - OffsetPubkeyIter::new_accounts(&self.accounts, txn) - } - - pub(crate) fn offset_finder( - &self, - ) -> AccountsDbResult> { - let txn = self.env.begin_ro_txn()?; - AccountOffsetFinder::new(&self.accounts, txn) - } - - pub(crate) fn get_accounts_count(&self) -> usize { - let Ok(txn) = self.env.begin_ro_txn() else { - return 0; - }; - self.owners.entries(&txn) - } - - #[cfg(test)] - pub(crate) fn get_deallocations_count(&self) -> usize { - let Ok(txn) = self.env.begin_ro_txn() else { - return 0; - }; - self.deallocations.entries(&txn) - } - - pub(crate) fn flush(&self) -> AccountsDbResult<()> { - self.env.sync(true).log_err(|| "main index flushing")?; - Ok(()) - } - - pub(crate) fn reload(&mut self, dbpath: &Path) -> AccountsDbResult<()> { - const DEFAULT_SIZE: usize = 1024 * 1024; - *self = Self::new(DEFAULT_SIZE, dbpath)?; - Ok(()) - } - - pub(crate) fn rwtxn(&self) -> lmdb::Result> { - self.env.begin_rw_txn() - } -} - -pub(crate) mod iterator; -mod table; -#[cfg(test)] -mod tests; -pub(super) mod utils; diff --git a/magicblock-accounts-db/src/index/iterator.rs b/magicblock-accounts-db/src/index/iterator.rs deleted file mode 100644 index 8ed3a6d37..000000000 --- a/magicblock-accounts-db/src/index/iterator.rs +++ /dev/null @@ -1,119 +0,0 @@ -use lmdb::{Cursor, Error as LmdbError, Iter, RoCursor, RoTransaction}; -use solana_pubkey::Pubkey; - -use super::{table::Table, MDB_SET_OP}; -use crate::{index::Offset, AccountsDbResult}; - -/// Iterator over pubkeys and offsets. -/// -/// This struct bundles the transaction, cursor, and iterator to ensure -/// correct lifetime management for iterating over the index tables. -pub struct OffsetPubkeyIter<'env> { - /// The underlying LMDB iterator. - /// Note: Must be dropped before `_cursor` and `_txn`. - iter: Iter<'env>, - /// The cursor used by the iterator. - /// Kept alive to maintain the validity of the iterator. - _cursor: RoCursor<'env>, - /// The read-only transaction. - /// Kept alive to maintain the validity of the cursor. - _txn: RoTransaction<'env>, - layout: Layout, -} - -#[derive(Clone, Copy)] -enum Layout { - ProgramValue, - AccountKey, -} - -impl<'env> OffsetPubkeyIter<'env> { - fn new( - table: &Table, - txn: RoTransaction<'env>, - pubkey: Option<&Pubkey>, - layout: Layout, - ) -> AccountsDbResult { - let cursor = table.cursor_ro(&txn)?; - - // SAFETY: - // We are constructing a self-referential struct here: - // 1. `txn` is moved into the struct. - // 2. `cursor` borrows `txn`. - // 3. `iter` borrows `cursor`. - // - // Normally, moving `txn` would invalidate the `cursor`. However, LMDB handles - // (internal C pointers) are stable in memory. We transmute the cursor to - // extend its lifetime to `'env`, treating it as if it borrows from the - // environment (which outlives this struct) rather than the local `txn`. - // - // Correctness relies on the struct drop order: `iter` -> `_cursor` -> `_txn`. - let mut cursor: RoCursor<'env> = unsafe { std::mem::transmute(cursor) }; - - let iter = if let Some(pubkey) = pubkey { - // Workaround for LMDB bug with DUPSORT databases where NotFound is ignored - // during `iter_dup_of`. We explicitly check existence first. - let key_bytes = pubkey.as_ref(); - match cursor.get(Some(key_bytes), None, MDB_SET_OP) { - Ok(_) => cursor.iter_dup_of(pubkey), - Err(LmdbError::NotFound) => Iter::Err(LmdbError::NotFound), - Err(e) => return Err(e.into()), - } - } else { - cursor.iter_start() - }; - - Ok(Self { - iter, - _cursor: cursor, - _txn: txn, - layout, - }) - } - - pub(super) fn new_programs( - table: &Table, - txn: RoTransaction<'env>, - pubkey: Option<&Pubkey>, - ) -> AccountsDbResult { - Self::new(table, txn, pubkey, Layout::ProgramValue) - } - - pub(super) fn new_accounts( - table: &Table, - txn: RoTransaction<'env>, - ) -> AccountsDbResult { - Self::new(table, txn, None, Layout::AccountKey) - } -} - -impl Iterator for OffsetPubkeyIter<'_> { - type Item = (Offset, Pubkey); - - fn next(&mut self) -> Option { - // Retrieve the next record from the LMDB iterator. - // The iterator returns `(&[u8], &[u8])` representing (key, value). - let (key_bytes, value_bytes) = self.iter.next()?.ok()?; - - match self.layout { - Layout::ProgramValue => { - // `programs` values are `[offset | pubkey]`. - Some(bytes!(#unpack, value_bytes, Offset, Pubkey)) - } - Layout::AccountKey => { - // `accounts` keys are the pubkey; values are `[offset | blocks]`. - let pubkey = Pubkey::try_from(key_bytes).ok()?; - // SAFETY: In the `Layout::AccountKey` path, `value_bytes` comes - // from the `accounts` index, whose values are always encoded as - // `[Offset | Blocks]`. That guarantees `value_bytes` has at least - // `size_of::()` bytes, and `read_unaligned` does not - // require aligned input. The first bytes therefore represent a - // valid `Offset` for the `Pubkey::try_from(key_bytes)` entry. - let offset = unsafe { - (value_bytes.as_ptr() as *const Offset).read_unaligned() - }; - Some((offset, pubkey)) - } - } - } -} diff --git a/magicblock-accounts-db/src/index/table.rs b/magicblock-accounts-db/src/index/table.rs deleted file mode 100644 index a97df31d3..000000000 --- a/magicblock-accounts-db/src/index/table.rs +++ /dev/null @@ -1,108 +0,0 @@ -use lmdb::{ - Database, DatabaseFlags, Environment, RoCursor, RwCursor, RwTransaction, - Transaction, WriteFlags, -}; - -/// Wrapper around LMDB Database providing a safe, typed interface. -#[cfg_attr(test, derive(Debug))] -pub(super) struct Table { - db: Database, -} - -impl Table { - /// Opens or creates a database within the given environment. - pub(super) fn new( - env: &Environment, - name: &str, - flags: DatabaseFlags, - ) -> lmdb::Result { - let db = env.create_db(Some(name), flags)?; - Ok(Self { db }) - } - - /// Retrieves a value by key. Returns `None` if the key does not exist. - #[inline] - pub(super) fn get<'txn, T: Transaction, K: AsRef<[u8]>>( - &self, - txn: &'txn T, - key: K, - ) -> lmdb::Result> { - match txn.get(self.db, &key) { - Ok(bytes) => Ok(Some(bytes)), - Err(lmdb::Error::NotFound) => Ok(None), - Err(e) => Err(e), - } - } - - /// Inserts a key-value pair, **overwriting** any existing value. - #[inline] - pub(super) fn upsert, V: AsRef<[u8]>>( - &self, - txn: &mut RwTransaction, - key: K, - value: V, - ) -> lmdb::Result<()> { - txn.put(self.db, &key, &value, WriteFlags::empty()) - } - - /// Inserts a key-value pair **only if the key does not already exist**. - /// Returns `true` if inserted, `false` if the key already existed. - #[inline] - pub(super) fn insert, V: AsRef<[u8]>>( - &self, - txn: &mut RwTransaction, - key: K, - value: V, - ) -> lmdb::Result { - match txn.put(self.db, &key, &value, WriteFlags::NO_OVERWRITE) { - Ok(_) => Ok(true), - Err(lmdb::Error::KeyExist) => Ok(false), - Err(e) => Err(e), - } - } - - /// Removes a key. Returns `Ok(())` even if the key was not found (idempotent). - /// - /// If `value` is provided, the deletion only occurs if the data in the DB matches. - #[inline] - pub(super) fn remove>( - &self, - txn: &mut RwTransaction, - key: K, - value: Option<&[u8]>, - ) -> lmdb::Result<()> { - match txn.del(self.db, &key, value) { - Ok(_) | Err(lmdb::Error::NotFound) => Ok(()), - Err(e) => Err(e), - } - } - - /// Removes all entries from the table. - #[inline] - pub(super) fn clear(&self, txn: &mut RwTransaction) -> lmdb::Result<()> { - txn.clear_db(self.db) - } - - /// Opens a read-only cursor. - #[inline] - pub(super) fn cursor_ro<'txn, T: Transaction>( - &self, - txn: &'txn T, - ) -> lmdb::Result> { - txn.open_ro_cursor(self.db) - } - - /// Opens a read-write cursor. - #[inline] - pub(super) fn cursor_rw<'txn>( - &self, - txn: &'txn mut RwTransaction, - ) -> lmdb::Result> { - txn.open_rw_cursor(self.db) - } - - /// Returns the number of entries in the database. - pub(super) fn entries(&self, txn: &T) -> usize { - txn.stat(self.db).map(|s| s.entries()).unwrap_or_default() - } -} diff --git a/magicblock-accounts-db/src/index/tests.rs b/magicblock-accounts-db/src/index/tests.rs deleted file mode 100644 index b2c184e55..000000000 --- a/magicblock-accounts-db/src/index/tests.rs +++ /dev/null @@ -1,478 +0,0 @@ -use std::{ - ops::Deref, - ptr::NonNull, - sync::atomic::{AtomicU32, Ordering}, -}; - -use lmdb::Transaction; -use magicblock_config::config::AccountsDbConfig; -use solana_pubkey::Pubkey; -use tempfile::TempDir; - -use super::{AccountMove, AccountsDbIndex, Allocation}; -use crate::error::AccountsDbError; - -/// Verifies that `upsert_account` correctly handles both new insertions -/// and updates to existing accounts, returning the old allocation when applicable. -#[test] -fn test_upsert_account() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - - // 1. First Insertion (New Account) - let mut txn = env.rw_txn(); - let result = env.upsert_account(&account, &mut txn); - txn.commit().expect("commit failed"); - - assert!(result.is_ok(), "failed to insert account"); - assert!( - result.unwrap().is_none(), - "new account should not have a previous allocation" - ); - - // 2. Re-insertion (Update Account) - let mut txn = env.rw_txn(); - let new_allocation = env.new_allocation(); - - // We manually call the internal method to inject a specific new allocation - let result = env.index.upsert_account( - &account.pubkey, - &account.owner, - new_allocation, - &mut txn, - ); - txn.commit().expect("commit failed"); - - assert!(result.is_ok(), "failed to update account"); - let previous = result.unwrap(); - assert_eq!( - previous, - Some(account.allocation.into()), - "update should return the old allocation for recycling" - ); -} - -/// Verifies we can retrieve the correct storage offset for an account -/// using both the convenience method and raw transaction lookup. -#[test] -fn test_get_offset() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - - env.persist_account(&account); - - // Test high-level convenience method - let offset = env.get_offset(&account.pubkey); - assert_eq!(offset.unwrap(), account.allocation.offset); - - // Test low-level internal retrieval - let txn = env.rw_txn(); - let allocation = env.get_allocation(&txn, &account.pubkey); - assert_eq!(allocation.unwrap(), account.allocation.into()); -} - -/// Ensures that `reallocate_account` moves the account record, updates the index, -/// and marks the old space as deallocated. -#[test] -fn test_reallocate_account() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - env.persist_account(&account); - - let mut txn = env.rw_txn(); - let new_allocation = env.new_allocation(); - - let new_index_value = bytes!( - #pack, - new_allocation.offset, u32, - new_allocation.blocks, u32 - ); - - let result = - env.reallocate_account(&account.pubkey, &mut txn, &new_index_value); - txn.commit().expect("commit failed"); - - assert!(result.is_ok()); - assert_eq!( - result.unwrap(), - account.allocation.into(), - "should return old allocation" - ); - - // Verify persistence of new offset - let stored_offset = env.get_offset(&account.pubkey).unwrap(); - assert_eq!(stored_offset, new_allocation.offset); -} - -#[test] -fn test_remove_account() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - env.persist_account(&account); - - // Perform removal - let mut txn = env.rw_txn(); - let result = env.remove(&account.pubkey, &mut txn); - txn.commit().unwrap(); - assert!(result.is_ok()); - - // Verify Index Entry is gone - let offset = env.get_offset(&account.pubkey); - assert!(matches!(offset, Err(AccountsDbError::NotFound))); - - // Verify Space was Deallocated - assert_eq!(env.count_deallocations(), 1, "deallocations count mismatch"); -} - -/// Tests the owner consistency check. -/// If an account's owner changes, the `programs` index must be updated -/// to reflect the move from the old owner to the new owner. -#[test] -fn test_ensure_correct_owner() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - env.persist_account(&account); - - // Verify initial state - let programs: Vec<_> = env - .get_program_accounts_iter(&account.owner) - .unwrap() - .collect(); - assert_eq!(programs.len(), 1); - assert_eq!(programs[0], (account.allocation.offset, account.pubkey)); - - // Change Owner - let new_owner = Pubkey::new_unique(); - let mut txn = env.rw_txn(); - let result = - env.ensure_correct_owner(&account.pubkey, &new_owner, &mut txn); - txn.commit().expect("commit failed"); - assert!(result.is_ok()); - - // Verify old owner is empty - let old_programs_count = env - .get_program_accounts_iter(&account.owner) - .unwrap() - .count(); - assert_eq!(old_programs_count, 0); - - // Verify new owner has account - let new_programs: Vec<_> = - env.get_program_accounts_iter(&new_owner).unwrap().collect(); - assert_eq!(new_programs.len(), 1); - assert_eq!(new_programs[0], (account.allocation.offset, account.pubkey)); -} - -#[test] -fn test_program_index_cleanup() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - env.persist_account(&account); - - let mut txn = env.rw_txn(); - let result = env.remove_program_index_entry( - &account.pubkey, - None, - &mut txn, - account.allocation.offset, - ); - txn.commit().expect("commit failed"); - assert!(result.is_ok()); - - // Verify cleanup - let count = env - .get_program_accounts_iter(&account.owner) - .unwrap() - .count(); - assert_eq!(count, 0); -} - -/// Verifies the full cycle of: -/// Insert -> Reallocate (creates hole) -> Insert New (fills hole). -#[test] -fn test_recycle_allocation_flow() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - - // 1. Insert - env.persist_account(&account); - - // 2. Reallocate (moves account, creating a hole at `account.allocation`) - let mut txn = env.rw_txn(); - let new_allocation = env.new_allocation(); - let index_val = - bytes!(#pack, new_allocation.offset, u32, new_allocation.blocks, u32); - env.reallocate_account(&account.pubkey, &mut txn, &index_val) - .unwrap(); - txn.commit().expect("commit failed"); - - assert_eq!(env.count_deallocations(), 1, "hole created"); - - // 3. Recycle exact fit - // We request a block size exactly matching the hole we just made. - let mut txn = env.rw_txn(); - let recycled = - env.try_recycle_allocation(account.allocation.blocks, &mut txn); - txn.commit().expect("commit failed"); - - assert!(recycled.is_ok()); - assert_eq!( - recycled.unwrap(), - account.allocation.into(), - "should reuse the exact hole" - ); - assert_eq!(env.count_deallocations(), 0, "hole consumed"); - - // 4. Recycle missing (should fail) - let mut txn = env.rw_txn(); - let missing = env.try_recycle_allocation(100, &mut txn); - assert!(matches!(missing, Err(AccountsDbError::NotFound))); -} - -/// Verifies that requesting a smaller size than an available hole -/// correctly splits the hole and returns the remainder to the free list. -#[test] -fn test_recycle_allocation_split() { - let env = IndexTestEnv::new(); - let account = env.new_account(); - env.persist_account(&account); - - let mut txn = env.rw_txn(); - env.remove(&account.pubkey, &mut txn).unwrap(); - txn.commit().unwrap(); - - assert_eq!(env.count_deallocations(), 1); - - let half_size = account.allocation.blocks / 2; - - // 1. Recycle first half - let mut txn = env.rw_txn(); - let part1 = env.try_recycle_allocation(half_size, &mut txn).unwrap(); - txn.commit().expect("commit failed"); - - assert_eq!(part1.blocks, half_size); - assert_eq!(part1.offset, account.allocation.offset); - - // Should still have one hole (the remainder) - assert_eq!(env.count_deallocations(), 1); - - // 2. Recycle second half (remainder) - let mut txn = env.rw_txn(); - let part2 = env.try_recycle_allocation(half_size, &mut txn).unwrap(); - txn.commit().expect("commit failed"); - - assert_eq!(part2.blocks, half_size); - // The new offset should be shifted by the size of the first part - assert!(part2.offset > account.allocation.offset); - - // Empty now - assert_eq!(env.count_deallocations(), 0); -} - -#[test] -fn test_accounts_by_offset_sorts_live_allocations() { - let env = IndexTestEnv::new(); - let acc1 = env.new_account_at(8, 2); - let acc2 = env.new_account_at(0, 1); - let acc3 = env.new_account_at(4, 3); - - env.persist_account(&acc1); - env.persist_account(&acc2); - env.persist_account(&acc3); - - let offsets = env - .accounts_by_offset() - .unwrap() - .into_iter() - .map(|allocation| allocation.offset) - .collect::>(); - - assert_eq!(offsets, [0, 4, 8]); -} - -#[test] -fn test_apply_account_moves_updates_offsets_and_clears_holes() { - let env = IndexTestEnv::new(); - let dead = env.new_account_at(0, 1); - let acc1 = env.new_account_at(4, 2); - let acc2 = env.new_account_at(12, 3); - - env.persist_account(&dead); - env.persist_account(&acc1); - env.persist_account(&acc2); - - let mut txn = env.rw_txn(); - env.remove(&dead.pubkey, &mut txn).unwrap(); - txn.commit().unwrap(); - assert_eq!(env.count_deallocations(), 1); - - env.apply_account_moves(&[ - AccountMove { - pubkey: acc1.pubkey, - owner: acc1.owner, - old_offset: acc1.allocation.offset, - new_offset: 0, - blocks: acc1.allocation.blocks, - }, - AccountMove { - pubkey: acc2.pubkey, - owner: acc2.owner, - old_offset: acc2.allocation.offset, - new_offset: acc1.allocation.blocks, - blocks: acc2.allocation.blocks, - }, - ]) - .unwrap(); - - assert_eq!(env.get_offset(&acc1.pubkey).unwrap(), 0); - assert_eq!( - env.get_offset(&acc2.pubkey).unwrap(), - acc1.allocation.blocks - ); - assert_eq!(env.count_deallocations(), 0); - - let acc1_programs = env - .get_program_accounts_iter(&acc1.owner) - .unwrap() - .collect::>(); - assert_eq!(acc1_programs, [(0, acc1.pubkey)]); - - let acc2_programs = env - .get_program_accounts_iter(&acc2.owner) - .unwrap() - .collect::>(); - assert_eq!(acc2_programs, [(acc1.allocation.blocks, acc2.pubkey)]); -} - -#[test] -fn test_byte_pack_unpack_macro() { - macro_rules! check_pack { - ($v1: expr, $t1: ty, $v2: expr, $t2: ty) => {{ - let packed = bytes!(#pack, $v1, $t1, $v2, $t2); - let (u1, u2) = bytes!(#unpack, packed, $t1, $t2); - assert_eq!($v1, u1); - assert_eq!($v2, u2); - }}; - } - - check_pack!(13u8, u8, 42u64, u64); - check_pack!(12345u32, u32, 67890u32, u32); - - let pubkey = Pubkey::new_unique(); - check_pack!(100u32, u32, pubkey, Pubkey); - check_pack!(pubkey, Pubkey, 255u8, u8); -} - -// ============================================================== -// TEST UTILITIES -// ============================================================== - -struct IndexTestEnv { - index: AccountsDbIndex, - // Kept to ensure directory deletion on drop - _temp_dir: TempDir, -} - -struct IndexAccount { - pubkey: Pubkey, - owner: Pubkey, - allocation: Allocation, -} - -impl IndexTestEnv { - fn new() -> Self { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .try_init(); - let _temp_dir = tempfile::tempdir().expect("failed to create temp dir"); - let index = AccountsDbIndex::new( - AccountsDbConfig::default().index_size, - _temp_dir.path(), - ) - .expect("failed to create index"); - - Self { index, _temp_dir } - } - - /// Opens a read-write transaction. Panics on failure. - fn rw_txn(&self) -> lmdb::RwTransaction<'_> { - self.index - .env - .begin_rw_txn() - .expect("failed to begin rw txn") - } - - /// Generates a new dummy allocation with a unique offset. - fn new_allocation(&self) -> Allocation { - static OFFSET_COUNTER: AtomicU32 = AtomicU32::new(0); - let blocks = 4; - // Mocking an offset allocator - let offset = OFFSET_COUNTER.fetch_add(blocks, Ordering::Relaxed); - - Allocation { - ptr: NonNull::dangling(), - offset, - blocks, - } - } - - fn allocation_at(&self, offset: u32, blocks: u32) -> Allocation { - Allocation { - ptr: NonNull::dangling(), - offset, - blocks, - } - } - - /// Generates a random account with a dummy allocation. - fn new_account(&self) -> IndexAccount { - IndexAccount { - pubkey: Pubkey::new_unique(), - owner: Pubkey::new_unique(), - allocation: self.new_allocation(), - } - } - - fn new_account_at(&self, offset: u32, blocks: u32) -> IndexAccount { - IndexAccount { - pubkey: Pubkey::new_unique(), - owner: Pubkey::new_unique(), - allocation: self.allocation_at(offset, blocks), - } - } - - /// Helper to insert an account into the index immediately. - /// Useful for setting up test state. - fn persist_account(&self, acc: &IndexAccount) { - let mut txn = self.rw_txn(); - self.index - .upsert_account(&acc.pubkey, &acc.owner, acc.allocation, &mut txn) - .expect("persist failed"); - txn.commit().expect("persist commit failed"); - } - - /// Helper to count entries in the private deallocations table. - fn count_deallocations(&self) -> usize { - let txn = self.index.env.begin_ro_txn().unwrap(); - self.index.deallocations.entries(&txn) - } - - /// Helper wrapper to call upsert on the internal account struct. - fn upsert_account( - &self, - acc: &IndexAccount, - txn: &mut lmdb::RwTransaction, - ) -> crate::AccountsDbResult> - { - self.index - .upsert_account(&acc.pubkey, &acc.owner, acc.allocation, txn) - } -} - -// Enable calling Index methods directly on Env -impl Deref for IndexTestEnv { - type Target = AccountsDbIndex; - fn deref(&self) -> &Self::Target { - &self.index - } -} diff --git a/magicblock-accounts-db/src/index/utils.rs b/magicblock-accounts-db/src/index/utils.rs deleted file mode 100644 index 066ee6db7..000000000 --- a/magicblock-accounts-db/src/index/utils.rs +++ /dev/null @@ -1,93 +0,0 @@ -use std::{fs, path::Path}; - -use lmdb::{ - Cursor, Environment, EnvironmentFlags, Error as LmdbError, RoCursor, - RoTransaction, -}; -use solana_pubkey::Pubkey; - -use super::{table::Table, Offset}; -use crate::{index::Blocks, AccountsDbResult}; - -// Re-exporting LMDB constants that are missing from the safe wrapper crate. -// Values correspond to MDB_SET_RANGE and MDB_SET in lmdb.h -pub(super) const MDB_SET_RANGE_OP: u32 = 17; -pub(super) const MDB_SET_OP: u32 = 15; - -const MAX_DBS: u32 = 4; - -/// Configures and opens the LMDB environment. -/// -/// Renamed from `lmdb_env` for clarity. -pub(super) fn create_lmdb_env( - dir: &Path, - map_size: usize, -) -> lmdb::Result { - // Flags optimization: - // - NO_SYNC: Let OS manage flushing (we handle durability via snapshots/other means). - // - WRITE_MAP: Use mmap for writes (avoids double buffering). - // - NO_MEM_INIT: Don't zero-out memory (we overwrite anyway). - // - NO_READAHEAD: Random access pattern optimization. - let flags = EnvironmentFlags::NO_SYNC - | EnvironmentFlags::WRITE_MAP - | EnvironmentFlags::NO_MEM_INIT - | EnvironmentFlags::NO_READAHEAD; - - let index_dir = dir.join("index"); - if !index_dir.exists() { - fs::create_dir_all(&index_dir) - .map_err(|e| LmdbError::Other(e.raw_os_error().unwrap_or(0)))?; - } - - Environment::new() - .set_map_size(map_size) - .set_max_dbs(MAX_DBS) - .set_flags(flags) - .open_with_permissions(&index_dir, 0o644) -} - -/// A self-contained cursor wrapper for efficient account lookups. -/// -/// Holds both the transaction and the cursor to ensure safe lifetime management -/// while exposing a simple API. -pub struct AccountOffsetFinder<'env> { - cursor: RoCursor<'env>, - // Kept alive to support the cursor's lifetime. - // Note: Field order matters for Drop! `cursor` depends on `_txn`, so `cursor` must be dropped first. - // Rust drops fields in declaration order (top to bottom). - _txn: RoTransaction<'env>, -} - -impl<'env> AccountOffsetFinder<'env> { - /// Creates a new finder instance. - pub(super) fn new( - table: &Table, - txn: RoTransaction<'env>, - ) -> AccountsDbResult { - let cursor = table.cursor_ro(&txn)?; - - // SAFETY: - // We are constructing a self-referential struct. - // 1. `txn` is moved into the struct. - // 2. `cursor` is created from `txn` but we must extend its lifetime to `'env` - // to store it alongside `txn`. - // - // This is safe because: - // - LMDB cursors are valid as long as the transaction is open. - // - We ensure `cursor` is dropped before `_txn` via struct field order. - let cursor: RoCursor<'env> = unsafe { std::mem::transmute(cursor) }; - - Ok(Self { cursor, _txn: txn }) - } - - /// Finds the storage offset for a given account public key. - pub(crate) fn find(&self, pubkey: &Pubkey) -> Option { - let key_bytes = pubkey.as_ref(); - - // MDB_SET_OP: Position at specified key - self.cursor - .get(Some(key_bytes), None, MDB_SET_OP) - .ok() - .map(|(_, val_bytes)| bytes!(#unpack, val_bytes, Offset, Blocks).0) - } -} diff --git a/magicblock-accounts-db/src/lib.rs b/magicblock-accounts-db/src/lib.rs deleted file mode 100644 index 82f147c3c..000000000 --- a/magicblock-accounts-db/src/lib.rs +++ /dev/null @@ -1,650 +0,0 @@ -use std::{fs, hash::Hasher, path::Path, sync::Arc, thread}; - -use error::{AccountsDbError, LogErr}; -use index::{ - iterator::OffsetPubkeyIter, utils::AccountOffsetFinder, AccountMove, - AccountsDbIndex, -}; -use lmdb::{RwTransaction, Transaction}; -use magicblock_config::config::AccountsDbConfig; -use solana_account::{ - cow::AccountBorrowed, AccountSharedData, ReadableAccount, -}; -use solana_pubkey::Pubkey; -use solana_sdk_ids::feature; -use storage::AccountsStorage; -use tracing::{error, info, warn}; -use twox_hash::xxhash3_64; - -use crate::{snapshot::SnapshotManager, traits::AccountsBank}; - -pub type AccountsDbResult = Result; - -pub const ACCOUNTSDB_DIR: &str = "accountsdb"; - -/// The main Accounts Database. -/// -/// Coordinates: -/// 1. **Storage**: Append-only memory-mapped log (`AccountsStorage`). -/// 2. **Indexing**: LMDB-based key-value store (`AccountsDbIndex`). -/// 3. **Persistence**: Snapshot management (`SnapshotManager`). -#[cfg_attr(test, derive(Debug))] -pub struct AccountsDb { - /// Underlying append-only storage for account data. - storage: AccountsStorage, - /// Fast index for account lookups (Pubkey -> Offset). - index: AccountsDbIndex, - /// Manages snapshots and state restoration. - snapshot_manager: Arc, -} - -impl AccountsDb { - /// Initializes the Accounts Database. - /// - /// This handles directory creation and potentially resets the state - /// if `config.reset` is true. - pub fn new( - config: &AccountsDbConfig, - root_dir: &Path, - max_slot: u64, - ) -> AccountsDbResult { - let db_dir = root_dir.join(ACCOUNTSDB_DIR).join("main"); - - if config.reset && db_dir.exists() { - info!(db_path = %db_dir.display(), "Resetting AccountsDb"); - fs::remove_dir_all(&db_dir) - .log_err(|| "Failed to reset accountsdb directory")?; - } - fs::create_dir_all(&db_dir) - .log_err(|| "Failed to create accountsdb directory")?; - - let storage = AccountsStorage::new(config, &db_dir) - .log_err(|| "Failed to initialize storage")?; - - let index = AccountsDbIndex::new(config.index_size, &db_dir) - .log_err(|| "Failed to initialize index")?; - - let snapshot_manager = - SnapshotManager::new(db_dir.clone(), config.max_snapshots as usize) - .log_err(|| "Failed to initialize snapshot manager")?; - - let mut this = Self { - storage, - index, - snapshot_manager, - }; - - // Recover state if the requested slot is older than our current state - this.restore_state_if_needed(max_slot)?; - - Ok(this) - } - - /// Opens an existing database (helper for tooling/tests). - pub fn open(directory: &Path) -> AccountsDbResult { - let config = AccountsDbConfig::default(); - Self::new(&config, directory, 0) - } - - /// Inserts or updates a single account. - pub fn insert_account( - &self, - pubkey: &Pubkey, - account: &AccountSharedData, - ) -> AccountsDbResult<()> { - let mut txn = None; - self.upsert(pubkey, account, &mut txn)?; - if let Some(t) = txn { - t.commit()?; - } - Ok(()) - } - - /// Inserts multiple accounts atomically. - pub fn insert_batch<'a>( - &self, - accounts: impl Iterator + Clone, - ) -> AccountsDbResult<()> { - let (mut txn, mut count) = (None, 0usize); - - for (pubkey, account) in accounts.clone() { - match self.upsert(pubkey, account, &mut txn) { - Ok(()) => count += 1, - Err(e) => { - self.rollback(accounts, count); - return Err(e); - } - } - } - - if let Some(Err(error)) = txn.map(|t| t.commit()) { - error!(%error, "Batch insert commit failed"); - self.rollback(accounts, count); - Err(error)?; - } - Ok(()) - } - - fn rollback<'a>( - &self, - accounts: impl Iterator, - count: usize, - ) { - for acc in accounts.take(count) { - // SAFETY: - // we are rolling back committed accounts - unsafe { acc.1.rollback() }; - } - } - - fn upsert<'a>( - &'a self, - pubkey: &Pubkey, - account: &AccountSharedData, - txn: &mut Option>, - ) -> AccountsDbResult<()> { - /// Get or create LMDB transaction for index writes. - macro_rules! txn { - () => { - match txn.as_mut() { - Some(t) => t, - None => txn.get_or_insert(self.index.rwtxn()?), - } - }; - } - // The ephemeral account has been closed, remove it from DB - if account.ephemeral() && account.owner() == &Pubkey::default() { - self.index.remove(pubkey, txn!())?; - return Ok(()); - } - let owner_changed = account.owner_changed(); - match account { - AccountSharedData::Borrowed(acc) => { - if owner_changed { - self.index - .ensure_correct_owner(pubkey, account.owner(), txn!()) - .log_err(|| "Failed to update owner index")?; - } - acc.commit(); - } - AccountSharedData::Owned(acc) => { - let data_len = account.data().len() as u32; - let block_size = self.storage.block_size() as u32; - let size_bytes = AccountSharedData::serialized_size_aligned( - data_len, block_size, - ) as usize; - let blocks_needed = self.storage.blocks_required(size_bytes); - - let result = - self.index.try_recycle_allocation(blocks_needed, txn!()); - let allocation = match result { - Ok(recycled) => { - self.storage.dec_recycled_count(recycled.blocks); - self.storage.recycle(recycled) - } - Err(AccountsDbError::NotFound) => { - self.storage.allocate(size_bytes)? - } - Err(e) => return Err(e), - }; - - let old_allocation = self - .index - .upsert_account(pubkey, account.owner(), allocation, txn!()) - .log_err(|| "Index update failed")?; - - if let Some(old) = old_allocation { - self.storage.inc_recycled_count(old.blocks); - } - - let ptr = allocation.ptr.as_ptr(); - let size = block_size * allocation.blocks; - // SAFETY: - // `allocation` provides a valid, exclusive pointer - // range managed by `AccountsStorage`. - unsafe { AccountSharedData::serialize_to_mmap(acc, ptr, size) }; - } - } - Ok(()) - } - - /// Checks if any of the `owners` own the `account`. - pub fn account_matches_owners( - &self, - account: &Pubkey, - owners: &[Pubkey], - ) -> Option { - let offset = self.index.get_offset(account).ok()?; - let ptr = self.storage.resolve_ptr(offset); - // SAFETY: `ptr` is guaranteed valid by `AccountsStorage` for the duration of the map. - unsafe { AccountBorrowed::any_owner_matches(ptr.as_ptr(), owners) } - } - - /// Returns a filterable iterator over accounts owned by `program`. - pub fn get_program_accounts( - &self, - program: &Pubkey, - filter: F, - ) -> AccountsDbResult> - where - F: Fn(&AccountSharedData) -> bool + 'static, - { - let iterator = self - .index - .get_program_accounts_iter(program) - .log_err(|| "Failed to create program accounts iterator")?; - - Ok(AccountsScanner { - iterator, - storage: &self.storage, - filter, - }) - } - - /// An optimized accountsdb accessor, which can be used for multiple reads, - /// without incurring the overhead of repeated creation of index transaction - pub fn reader(&self) -> AccountsDbResult> { - let offset = self - .index - .offset_finder() - .log_err(|| "Failed to create offset iterator")?; - Ok(AccountsReader { - offset, - storage: &self.storage, - }) - } - - /// Check whether pubkey is present in the AccountsDB - pub fn contains_account(&self, pubkey: &Pubkey) -> bool { - self.index.get_offset(pubkey).is_ok() - } - - /// Return total number of accounts present in the database - pub fn account_count(&self) -> usize { - self.index.get_accounts_count() - } - - /// Return the last slot written to AccountsDB - #[inline(always)] - pub fn slot(&self) -> u64 { - self.storage.slot() - } - - /// Updates the current slot. - pub fn set_slot(&self, slot: u64) { - self.storage.update_slot(slot); - } - - /// Takes a database snapshot for the given slot. - /// - /// The snapshot is created in two phases: - /// 1. **Synchronous**: Flush data, compute checksum, create snapshot directory - /// (with write lock held to ensure consistency) - /// 2. **Background**: Archive directory to tar.gz and register - /// - /// Returns the state checksum computed at snapshot time. - /// The checksum can be used to verify state consistency across nodes. - /// - /// - /// # Safety - /// the caller must ensure that no state transitions are taking - /// place concurrently when this operation is in progress - pub unsafe fn take_snapshot(&self, slot: u64) -> AccountsDbResult { - // Create snapshot directory (potential deep copy) - self.flush(); - let checksum = self.checksum(); - let used_storage = self.storage.active_segment(); - - let manager = self.snapshot_manager.clone(); - let dir = manager.create_snapshot_dir(slot, used_storage)?; - // Archive directory in background to avoid blocking the caller - thread::spawn(move || manager.archive_and_register(&dir)); - Ok(checksum) - } - - /// Ensures the database state is at most `slot`. - /// - /// If the current state is newer than `slot`, this performs a **rollback** - /// to the nearest valid snapshot. - pub fn restore_state_if_needed( - &mut self, - target_slot: u64, - ) -> AccountsDbResult { - // Allow slot-1 because we might be in the middle of processing the current slot - if target_slot >= self.slot().saturating_sub(1) || target_slot == 0 { - return Ok(self.slot()); - } - - warn!( - current_slot = self.slot(), - target_slot = target_slot, - "Current slot ahead of target, rolling back" - ); - - let restored_slot = self - .snapshot_manager - .restore_from_snapshot(target_slot) - .log_err(|| { - format!("Snapshot restoration failed for target {target_slot}",) - })?; - - // Reload components to reflect new FS state - let path = self.snapshot_manager.database_path(); - self.storage.reload(path)?; - self.index.reload(path)?; - - info!(restored_slot, "Successfully rolled back"); - Ok(restored_slot) - } - - /// Removes stale non-delegated accounts from the bank after startup. - pub fn reset_bank(&self, validator_id: &Pubkey) -> AccountsDbResult<()> { - let protected_accounts = reset::protected_accounts(validator_id); - - let mut delegated_only = 0; - let mut kept_ephemeral = 0; - let mut undelegating = 0; - let mut confined = 0; - let mut protected = 0; - let mut remaining = 0u32; - - let removed = self.remove_where(|pubkey, account| { - if protected_accounts.contains(pubkey) { - protected += 1; - return false; - } - // Undelegating accounts are normally also delegated, but if that ever - // changes we should keep the account until chain state is clear. - let should_remove = if account.undelegating() { - undelegating += 1; - false - } else if account.ephemeral() { - kept_ephemeral += 1; - false - } else if account.confined() { - confined += 1; - false - } else if account.delegated() { - delegated_only += 1; - false - } else { - *account.owner() != feature::ID - }; - if should_remove { - tracing::trace!( - pubkey = %pubkey, - account = %format!("{account:#?}"), - "Removing non-delegated account during accountsdb reset" - ); - } else { - remaining += 1; - } - should_remove - })?; - - info!( - total_removed = removed, - delegated_not_undelegating = delegated_only, - delegated_and_undelegating = undelegating, - kept_confined = confined, - kept_delegated = delegated_only, - kept_protected = protected, - kept_ephemeral, - remaining, - "Removed accounts from bank" - ); - Ok(()) - } - - pub fn storage_size(&self) -> u64 { - self.storage.size_bytes() - } - - /// Moves live account records left in storage and removes all tracked holes. - /// - /// This is a best-effort maintenance operation, not a crash-recoverable - /// migration. The index is updated before bytes are moved, so an interrupted - /// defragmentation can leave the database inconsistent. A successful return - /// means storage and index changes were flushed. - /// - /// # Safety - /// The caller must stop all concurrent database reads and writes and ensure - /// there are no live borrowed account references into the mmap. - pub unsafe fn defragment(&self) -> AccountsDbResult<()> { - let allocations = self.index.accounts_by_offset()?; - let mut next_offset = 0; - let mut moves = Vec::with_capacity(allocations.len()); - - for allocation in allocations { - self.storage - .validate_allocation(allocation.offset, allocation.blocks)?; - self.storage - .validate_allocation(next_offset, allocation.blocks)?; - - moves.push(AccountMove { - pubkey: allocation.pubkey, - owner: allocation.owner, - old_offset: allocation.offset, - new_offset: next_offset, - blocks: allocation.blocks, - }); - - next_offset = next_offset - .checked_add(allocation.blocks) - .ok_or_else(|| { - AccountsDbError::Internal( - "accountsdb compacted cursor overflow".into(), - ) - })?; - } - - self.storage.validate_cursor(next_offset)?; - self.index.apply_account_moves(&moves)?; - - let mut scratch = Vec::new(); - for account_move in &moves { - // SAFETY: all source and destination ranges were validated above, - // and the caller guarantees no account references are live. - unsafe { - self.storage.move_allocation( - account_move.old_offset, - account_move.new_offset, - account_move.blocks, - &mut scratch, - ) - }; - } - - // SAFETY: every live allocation has been moved into the prefix ending at - // `next_offset`, and the caller guarantees exclusive database access. - unsafe { self.storage.finish_defragment(next_offset) }; - self.storage.flush()?; - self.index.flush()?; - Ok(()) - } - - pub fn iter_all( - &self, - ) -> impl Iterator + '_ { - let result = self.index.get_all_accounts().ok(); - let accounts = result.into_iter().flatten(); - accounts.map(|(offset, pk)| (pk, self.storage.read_account(offset))) - } - - pub fn flush(&self) { - let _ = self.storage.flush(); - let _ = self.index.flush(); - } - - /// Inserts an external snapshot archive received over the network. - /// - /// If the snapshot slot is newer than the current DB slot, immediately - /// fast-forwards to it (bringing state forward in time). - /// - /// Returns `true` if fast-forward was performed, `false` if just registered. - /// NOTE: - /// This method is only ever used to initialize accountsdb from external source - /// It's not used to overwrite existing instance - pub fn insert_external_snapshot( - &mut self, - slot: u64, - archive_bytes: &[u8], - ) -> AccountsDbResult { - let current_slot = self.slot(); - // we do not overwrite accountsdb which already has been initialized - if current_slot > 0 { - return Ok(false); - } - let fast_forwarded = self.snapshot_manager.insert_external_snapshot( - slot, - archive_bytes, - current_slot, - )?; - - if fast_forwarded { - // Reload components to reflect new state - let path = self.snapshot_manager.database_path(); - self.storage.reload(path)?; - self.index.reload(path)?; - } - - Ok(fast_forwarded) - } - - /// Computes a deterministic checksum of all active accounts. - /// - /// Iterates all accounts in key-sorted order (via LMDB) and hashes both - /// pubkey and serialized account data using xxHash3. Returns a 64-bit hash - /// suitable for verifying state consistency across nodes. - /// - /// # Safety - /// the caller must ensure that no concurrent write access is being performed on - /// accountsdb, so that the state doesn't change during checksum computation - pub unsafe fn checksum(&self) -> u64 { - let mut hasher = xxhash3_64::Hasher::new(); - for (pubkey, acc) in self.iter_all() { - if !(acc.delegated() - || acc.ephemeral() - || acc.undelegating() - || acc.confined()) - { - continue; - } - let AccountSharedData::Borrowed(borrowed) = &acc else { - continue; - }; - hasher.write(pubkey.as_ref()); - hasher.write(borrowed.buffer()); - } - hasher.finish() - } - - pub fn database_directory(&self) -> &Path { - &self.snapshot_manager.snapshots_dir - } -} - -impl AccountsBank for AccountsDb { - #[inline(always)] - fn get_account(&self, pubkey: &Pubkey) -> Option { - let offset = self.index.get_offset(pubkey).ok()?; - Some(self.storage.read_account(offset)) - } - - fn remove_account(&self, pubkey: &Pubkey) { - let Ok(mut txn) = self.index.rwtxn() else { - error!("accountsdb: couldn't create rw index transaction"); - return; - }; - let _ = self - .index - .remove(pubkey, &mut txn) - .and_then(|_| txn.commit().map_err(Into::into)) - .log_err(|| format!("Failed to remove account {pubkey}")); - } - - /// Removes accounts matching a predicate. - fn remove_where( - &self, - mut predicate: impl FnMut(&Pubkey, &AccountSharedData) -> bool, - ) -> AccountsDbResult { - let to_remove = self - .iter_all() - .filter_map(|(pk, acc)| predicate(&pk, &acc).then_some(pk)) - .collect::>(); - - let count = to_remove.len(); - let mut txn = self.index.rwtxn()?; - for pubkey in to_remove { - self.index.remove(&pubkey, &mut txn)?; - } - txn.commit()?; - - Ok(count) - } -} - -// SAFETY: AccountsDb uses internal locking (LMDB + RwLock) to ensure thread safety. -unsafe impl Sync for AccountsDb {} -unsafe impl Send for AccountsDb {} - -pub struct AccountsScanner<'db, F> { - storage: &'db AccountsStorage, - filter: F, - iterator: OffsetPubkeyIter<'db>, -} - -impl Iterator for AccountsScanner<'_, F> -where - F: Fn(&AccountSharedData) -> bool, -{ - type Item = (Pubkey, AccountSharedData); - fn next(&mut self) -> Option { - for (offset, pubkey) in self.iterator.by_ref() { - let account = self.storage.read_account(offset); - if (self.filter)(&account) { - return Some((pubkey, account)); - } - } - None - } -} - -pub struct AccountsReader<'db> { - offset: AccountOffsetFinder<'db>, - storage: &'db AccountsStorage, -} - -unsafe impl Send for AccountsReader<'_> {} -unsafe impl Sync for AccountsReader<'_> {} - -impl AccountsReader<'_> { - pub fn read(&self, pubkey: &Pubkey, reader: F) -> Option - where - F: Fn(AccountSharedData) -> R, - { - let offset = self.offset.find(pubkey)?; - let account = self.storage.read_account(offset); - Some(reader(account)) - } - - pub fn contains(&self, pubkey: &Pubkey) -> bool { - self.offset.find(pubkey).is_some() - } -} - -#[cfg(test)] -impl AccountsDb { - pub fn snapshot_exists(&self, slot: u64) -> bool { - self.snapshot_manager.snapshot_exists(slot) - } -} - -pub mod error; -mod index; -mod reset; -mod snapshot; -mod storage; -#[cfg(test)] -mod tests; -pub mod traits; diff --git a/magicblock-accounts-db/src/reset.rs b/magicblock-accounts-db/src/reset.rs deleted file mode 100644 index 3c4766587..000000000 --- a/magicblock-accounts-db/src/reset.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::collections::HashSet; - -use magicblock_magic_program_api as magic_program; -use solana_pubkey::Pubkey; -use solana_sdk_ids::{ - address_lookup_table, bpf_loader, bpf_loader_deprecated, - bpf_loader_upgradeable, compute_budget, config, ed25519_program, - incinerator, loader_v4, native_loader, secp256k1_program, - secp256r1_program, stake, system_program, sysvar, vote, - zk_elgamal_proof_program, -}; - -pub(crate) fn protected_accounts(validator_id: &Pubkey) -> HashSet { - // This is buried in the accounts_db::native_mint module and we do not - // want to take a dependency on that crate just for this stable ID. - const NATIVE_SOL_ID: Pubkey = - Pubkey::from_str_const("So11111111111111111111111111111111111111112"); - - let mut accounts = sysvar_accounts() - .into_iter() - .chain(native_program_accounts()) - .collect::>(); - - accounts.insert(stake::config::ID); - accounts.insert(NATIVE_SOL_ID); - accounts.insert(magic_program::ID); - accounts.insert(magic_program::CRANK_PROGRAM_ID); - accounts.insert(magic_program::CALLBACK_PROGRAM_ID); - accounts.insert(magic_program::POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID); - accounts.insert(magic_program::MAGIC_CONTEXT_PUBKEY); - accounts.insert(magic_program::EPHEMERAL_VAULT_PUBKEY); - accounts.insert(*validator_id); - accounts -} - -fn sysvar_accounts() -> HashSet { - let mut accounts = HashSet::new(); - accounts.insert(sysvar::ID); - accounts.insert(sysvar::clock::ID); - accounts.insert(sysvar::epoch_rewards::ID); - accounts.insert(sysvar::epoch_schedule::ID); - accounts.insert(sysvar::instructions::ID); - accounts.insert(sysvar::fees::ID); - accounts.insert(sysvar::last_restart_slot::ID); - accounts.insert(sysvar::recent_blockhashes::ID); - accounts.insert(sysvar::rent::ID); - accounts.insert(sysvar::rewards::ID); - accounts.insert(sysvar::slot_hashes::ID); - accounts.insert(sysvar::slot_history::ID); - accounts.insert(sysvar::stake_history::ID); - accounts -} - -fn native_program_accounts() -> HashSet { - let mut accounts = HashSet::new(); - accounts.insert(address_lookup_table::ID); - accounts.insert(bpf_loader::ID); - accounts.insert(bpf_loader_upgradeable::ID); - accounts.insert(bpf_loader_deprecated::ID); - accounts.insert(compute_budget::ID); - accounts.insert(config::ID); - accounts.insert(ed25519_program::ID); - accounts.insert(incinerator::ID); - accounts.insert(loader_v4::ID); - accounts.insert(native_loader::ID); - accounts.insert(secp256k1_program::ID); - accounts.insert(secp256r1_program::ID); - accounts.insert(stake::ID); - accounts.insert(system_program::ID); - accounts.insert(vote::ID); - accounts.insert(zk_elgamal_proof_program::ID); - accounts -} diff --git a/magicblock-accounts-db/src/snapshot.rs b/magicblock-accounts-db/src/snapshot.rs deleted file mode 100644 index 093a70731..000000000 --- a/magicblock-accounts-db/src/snapshot.rs +++ /dev/null @@ -1,500 +0,0 @@ -use std::{ - collections::VecDeque, - ffi::OsStr, - fs::{self, File}, - io::{self, BufWriter, Cursor, Write}, - path::{Path, PathBuf}, - sync::Arc, -}; - -use flate2::{read::GzDecoder, write::GzEncoder, Compression}; -use parking_lot::Mutex; -use tar::{Archive, Builder}; -use tracing::{error, info, warn}; - -use crate::{ - error::{AccountsDbError, LogErr}, - storage::ACCOUNTS_DB_FILENAME, - AccountsDbResult, -}; - -const SNAPSHOT_PREFIX: &str = "snapshot-"; -const ARCHIVE_EXT: &str = "tar.gz"; - -/// Snapshot persistence strategy. -#[derive(Debug, Clone, Copy)] -enum SnapshotStrategy { - /// CoW reflink - O(1) metadata operation when filesystem supports it. - Reflink, - /// Deep copy with memory capture for consistency. - LegacyCopy, -} - -impl SnapshotStrategy { - fn detect(dir: &Path) -> AccountsDbResult { - let supports_cow = - fs_backend::supports_reflink(dir).log_err(|| "CoW check failed")?; - if supports_cow { - info!("Snapshot Strategy: Reflink (Fast/CoW)"); - Ok(Self::Reflink) - } else { - warn!("Snapshot Strategy: Deep Copy (Slow)"); - Ok(Self::LegacyCopy) - } - } - - fn execute( - &self, - src: &Path, - dst: &Path, - memory_state: Vec, - ) -> io::Result<()> { - match self { - Self::Reflink => fs_backend::reflink_dir(src, dst), - Self::LegacyCopy => { - fs_backend::deep_copy_dir(src, dst, &memory_state) - } - } - } -} - -/// Manages snapshot lifecycle: creation, archival, and restoration. -/// -/// Snapshots are stored as compressed tar archives (`.tar.gz`). During creation, -/// a directory is first created, then archived asynchronously after releasing -/// the write lock to minimize blocking time. -#[derive(Debug)] -pub struct SnapshotManager { - db_path: PathBuf, - pub(crate) snapshots_dir: PathBuf, - strategy: SnapshotStrategy, - /// Ordered registry of archive paths (oldest to newest). - registry: Mutex>, - max_snapshots: usize, -} - -impl SnapshotManager { - pub fn new( - db_path: PathBuf, - max_snapshots: usize, - ) -> AccountsDbResult> { - let snapshots_dir = db_path - .parent() - .ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "Invalid db_path") - }) - .log_err(|| "Failed to resolve snapshots directory")? - .to_path_buf(); - - let strategy = SnapshotStrategy::detect(&snapshots_dir)?; - let registry = Self::recover_registry(&snapshots_dir, max_snapshots) - .log_err(|| "Failed to load snapshot registry")?; - - Ok(Arc::new(Self { - db_path, - snapshots_dir, - strategy, - registry: Mutex::new(registry), - max_snapshots, - })) - } - - /// Creates a snapshot directory. Returns the path for later archiving. - pub fn create_snapshot_dir( - &self, - slot: u64, - active_mem: &[u8], - ) -> AccountsDbResult { - let snap_path = self.slot_to_dir_path(slot); - let memory_capture = - matches!(self.strategy, SnapshotStrategy::LegacyCopy) - .then(|| active_mem.to_vec()) - .unwrap_or_default(); - - self.strategy - .execute(&self.db_path, &snap_path, memory_capture) - .log_err(|| "Snapshot failed")?; - self.prune_registry(); - - Ok(snap_path) - } - - /// Archives the snapshot directory to `.tar.gz` and removes the directory. - /// - /// Uses atomic write: writes to `.tmp` first, then renames to final path. - pub fn archive_and_register( - &self, - snapshot_dir: &Path, - ) -> AccountsDbResult { - let archive_path = snapshot_dir.with_extension(ARCHIVE_EXT); - let tmp_path = archive_path.with_extension("tmp"); - - if let Some(archive) = archive_path.file_name() { - info!(?archive, "Archiving snapshot"); - } - - // Write to temporary file first - let file = File::create(&tmp_path).log_err(|| { - format!("Failed to create temp archive at {}", tmp_path.display()) - })?; - let enc = GzEncoder::new(file, Compression::fast()); - let mut tar = Builder::new(enc); - tar.append_dir_all(".", snapshot_dir) - .log_err(|| "Failed to append directory to tar")?; - tar.finish().log_err(|| "Failed to finalize tar archive")?; - let enc = tar - .into_inner() - .log_err(|| "Failed to recover gzip encoder from tar builder")?; - let file = enc.finish().log_err(|| "Failed to finish gzip archive")?; - file.sync_all() - .log_err(|| "Failed to sync archive to disk")?; - drop(file); - - // Atomically rename to final path - fs::rename(&tmp_path, &archive_path).log_err(|| { - format!( - "Failed to rename {} to {}", - tmp_path.display(), - archive_path.display() - ) - })?; - - fs::remove_dir_all(snapshot_dir).log_err(|| { - format!( - "Failed to remove snapshot directory {}", - snapshot_dir.display() - ) - })?; - - self.register_archive(archive_path.clone()); - Ok(archive_path) - } - - /// Inserts an external snapshot archive received from network. - /// - /// If `slot > current_slot`, immediately fast-forwards to the snapshot. - /// Returns `true` if fast-forward was performed. - pub fn insert_external_snapshot( - &self, - slot: u64, - archive_bytes: &[u8], - current_slot: u64, - ) -> AccountsDbResult { - // Validate archive structure - Self::validate_archive(archive_bytes).log_err(|| { - format!("snapshot archive bytes are corrupted, slot: {slot}") - })?; - - let archive_path = self.slot_to_archive_path(slot); - if archive_path.exists() { - return Err(AccountsDbError::Internal(format!( - "Snapshot for slot {} already exists", - slot - ))); - } - - info!(slot, "Inserting external snapshot"); - - // Write to temporary file first, then atomically rename - let tmp_path = archive_path.with_extension("tmp"); - let mut file = File::create(&tmp_path).log_err(|| { - format!("Failed to create temp archive at {}", tmp_path.display()) - })?; - file.write_all(archive_bytes) - .log_err(|| "Failed to write archive bytes")?; - file.sync_all() - .log_err(|| "Failed to sync archive to disk")?; - drop(file); - - fs::rename(&tmp_path, &archive_path).log_err(|| { - format!( - "Failed to rename {} to {}", - tmp_path.display(), - archive_path.display() - ) - })?; - - // Fast-forward if snapshot is newer than current state - if slot > current_slot { - info!(slot, current_slot, "Fast-forwarding to external snapshot"); - self.fast_forward(slot, &archive_path)?; - return Ok(true); - } - - // Otherwise just register for later use - self.prune_registry(); - self.register_archive(archive_path); - Ok(false) - } - - /// Restores database to the snapshot nearest to `target_slot`. - pub fn restore_from_snapshot( - &self, - target_slot: u64, - ) -> AccountsDbResult { - let (chosen_archive, chosen_slot, index) = - self.find_and_remove_snapshot(target_slot)?; - - let extracted_dir = self.extract_archive(&chosen_archive)?; - self.atomic_swap(&extracted_dir)?; - self.prune_invalidated_snapshots(index); - let _ = fs::remove_file(&chosen_archive); - - Ok(chosen_slot) - } - - /// Validates that bytes represent a valid gzip tar archive. - /// NOTE: - /// we only validate that the archive is valid, - /// not the contents of accountsdb, which is not possible at the moment - fn validate_archive(bytes: &[u8]) -> AccountsDbResult<()> { - let cursor = Cursor::new(bytes); - let dec = GzDecoder::new(cursor); - let mut tar = Archive::new(dec); - tar.entries() - .log_err(|| "Invalid snapshot archive: not a valid gzip tar")?; - Ok(()) - } - - /// Finds the best snapshot for target_slot, removes from registry. - /// Returns (archive_path, slot, index). - fn find_and_remove_snapshot( - &self, - target_slot: u64, - ) -> AccountsDbResult<(PathBuf, u64, usize)> { - let mut registry = self.registry.lock(); - let search_key = self.slot_to_archive_path(target_slot); - let index = match registry.binary_search(&search_key) { - Ok(i) => i, - Err(i) if i > 0 => i - 1, - _ => return Err(AccountsDbError::SnapshotMissing(target_slot)), - }; - - let chosen_archive = registry.remove(index).unwrap(); - let chosen_slot = Self::parse_slot(&chosen_archive) - .ok_or(AccountsDbError::SnapshotMissing(target_slot))?; - - info!(chosen_slot, target_slot, "Restoring snapshot"); - Ok((chosen_archive, chosen_slot, index)) - } - - /// Extracts a tar.gz archive to a temporary directory. - fn extract_archive( - &self, - archive_path: &Path, - ) -> AccountsDbResult { - let extract_dir = archive_path.with_extension("extract"); - - info!(archive_path = %archive_path.display(), "Extracting snapshot archive"); - - let file = File::open(archive_path).log_err(|| { - format!("Failed to open archive {}", archive_path.display()) - })?; - let mut tar = Archive::new(GzDecoder::new(file)); - tar.unpack(&extract_dir).log_err(|| { - format!("Failed to extract archive to {}", extract_dir.display()) - })?; - - Ok(extract_dir) - } - - /// Performs atomic swap: current db -> backup, extracted -> current db. - /// On failure, rolls back to original state. - fn atomic_swap(&self, extracted_dir: &Path) -> AccountsDbResult<()> { - let backup = self.db_path.with_extension("bak"); - - if self.db_path.exists() { - fs::rename(&self.db_path, &backup) - .log_err(|| "Failed to stage backup")?; - } - - if let Err(e) = fs::rename(extracted_dir, &self.db_path) { - error!(error = ?e, "Atomic swap failed during promote"); - if backup.exists() { - let _ = fs::rename(&backup, &self.db_path); - } - let _ = fs::remove_dir_all(extracted_dir); - return Err(e.into()); - } - - let _ = fs::remove_dir_all(&backup); - Ok(()) - } - - /// Fast-forwards to a snapshot that's newer than current state. - fn fast_forward( - &self, - slot: u64, - archive_path: &Path, - ) -> AccountsDbResult<()> { - let extracted_dir = self.extract_archive(archive_path)?; - self.atomic_swap(&extracted_dir)?; - self.registry.lock().push_back(archive_path.to_path_buf()); - info!(slot, "Fast-forward complete"); - Ok(()) - } - - /// Registers an archive in the registry. - fn register_archive(&self, archive_path: PathBuf) { - self.registry.lock().push_back(archive_path); - } - - /// Removes snapshots newer than the chosen one (invalidated by rollback). - fn prune_invalidated_snapshots(&self, from_index: usize) { - let mut registry = self.registry.lock(); - for invalidated in registry.drain(from_index..) { - warn!(invalidated_path = %invalidated.display(), "Pruning invalidated snapshot"); - let _ = fs::remove_file(&invalidated); - } - } - - /// Removes oldest snapshots until under the limit. - fn prune_registry(&self) { - let mut registry = self.registry.lock(); - while registry.len() >= self.max_snapshots { - let Some(path) = registry.pop_front() else { - break; - }; - if let Err(e) = fs::remove_file(&path) { - warn!(path = %path.display(), error = ?e, "Failed to prune snapshot archive"); - } - } - } - - /// Scans disk for existing snapshot archives. - fn recover_registry( - dir: &Path, - max: usize, - ) -> io::Result> { - if !dir.exists() { - fs::create_dir_all(dir)?; - return Ok(VecDeque::new()); - } - - let mut paths: Vec<_> = fs::read_dir(dir)? - .filter_map(|e| e.ok()) - .map(|e| e.path()) - .filter(|p| { - p.is_file() - && p.extension().is_some_and(|e| e == "gz") - && Self::parse_slot(p).is_some() - }) - .collect(); - paths.sort(); - - // Clean orphan directories (interrupted archiving) - for entry in fs::read_dir(dir)?.filter_map(|e| e.ok()) { - let path = entry.path(); - let name = path.file_name().and_then(OsStr::to_str); - if path.is_dir() - && name.is_some_and(|n| { - n.starts_with(SNAPSHOT_PREFIX) && !n.contains('.') - }) - { - warn!(path = %path.display(), "Cleaning up orphan snapshot directory"); - let _ = fs::remove_dir_all(&path); - } - } - - let offset = paths.len().saturating_sub(max); - Ok(paths.into_iter().skip(offset).collect()) - } - - fn slot_to_dir_path(&self, slot: u64) -> PathBuf { - self.snapshots_dir - .join(format!("{SNAPSHOT_PREFIX}{slot:0>12}")) - } - - fn slot_to_archive_path(&self, slot: u64) -> PathBuf { - self.snapshots_dir - .join(format!("{SNAPSHOT_PREFIX}{slot:0>12}.{ARCHIVE_EXT}")) - } - - fn parse_slot(path: &Path) -> Option { - path.file_name()? - .to_str()? - .strip_prefix(SNAPSHOT_PREFIX)? - .strip_suffix(&format!(".{ARCHIVE_EXT}"))? - .parse() - .ok() - } - - pub(crate) fn database_path(&self) -> &Path { - &self.db_path - } - - #[cfg(test)] - pub fn snapshot_exists(&self, slot: u64) -> bool { - self.registry - .lock() - .binary_search(&self.slot_to_archive_path(slot)) - .is_ok() - } -} - -mod fs_backend { - use super::*; - - pub(crate) fn supports_reflink(dir: &Path) -> io::Result { - let src = dir.join(".tmp_cow_probe_src"); - let dst = dir.join(".tmp_cow_probe_dst"); - let _ = (fs::remove_file(&src), fs::remove_file(&dst)); - - File::create(&src)?.write_all(&[0u8; 64])?; - let result = reflink::reflink(&src, &dst).is_ok(); - let _ = (fs::remove_file(src), fs::remove_file(dst)); - Ok(result) - } - - pub(crate) fn reflink_dir(src: &Path, dst: &Path) -> io::Result<()> { - if !src.is_dir() { - return reflink::reflink(src, dst); - } - fs::create_dir_all(dst)?; - for entry in fs::read_dir(src)? { - let entry = entry?; - let (src_path, dst_path) = - (entry.path(), dst.join(entry.file_name())); - if entry.file_type()?.is_dir() { - reflink_dir(&src_path, &dst_path)?; - } else { - reflink::reflink(&src_path, &dst_path)?; - } - } - Ok(()) - } - - /// Deep copy with special handling: writes `mem_dump` for `accounts.db` - /// instead of copying from disk to ensure consistency. - pub(crate) fn deep_copy_dir( - src: &Path, - dst: &Path, - mem_dump: &[u8], - ) -> io::Result<()> { - fs::create_dir_all(dst)?; - for entry in fs::read_dir(src)? { - let entry = entry?; - let (src_path, dst_path) = - (entry.path(), dst.join(entry.file_name())); - if entry.file_type()?.is_dir() { - deep_copy_dir(&src_path, &dst_path, mem_dump)?; - } else if src_path.file_name().and_then(OsStr::to_str) - == Some(ACCOUNTS_DB_FILENAME) - { - write_dump_file(&dst_path, mem_dump)?; - } else { - fs::copy(&src_path, &dst_path)?; - } - } - Ok(()) - } - - fn write_dump_file(path: &Path, data: &[u8]) -> io::Result<()> { - let f = File::create(path)?; - f.set_len(data.len() as u64)?; - let mut writer = BufWriter::new(f); - writer.write_all(data)?; - writer.flush()?; - writer.get_mut().sync_all() - } -} diff --git a/magicblock-accounts-db/src/storage.rs b/magicblock-accounts-db/src/storage.rs deleted file mode 100644 index 82eaf59ed..000000000 --- a/magicblock-accounts-db/src/storage.rs +++ /dev/null @@ -1,642 +0,0 @@ -use std::{ - fs::File, - io::{self, Read, Seek, SeekFrom, Write}, - mem::size_of, - path::Path, - ptr::{self, NonNull}, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, -}; - -use magicblock_config::config::{AccountsDbConfig, BlockSize}; -use memmap2::MmapMut; -use solana_account::AccountSharedData; -use tracing::error; - -use crate::{ - error::{AccountsDbError, LogErr}, - index::{Blocks, Offset}, - AccountsDbResult, -}; - -/// The reserved size in bytes at the beginning of the file for the `StorageHeader`. -/// This area is excluded from the data region used for account storage. -const METADATA_STORAGE_SIZE: usize = 256; - -// Size calculation: -// write_cursor(8) + slot(8) + block_size(4) + capacity_blocks(4) + recycled_count(4) = 28 bytes used. -const METADATA_HEADER_USED: usize = 28; -const METADATA_PADDING_SIZE: usize = - METADATA_STORAGE_SIZE - METADATA_HEADER_USED; - -/// The standard filename for the accounts database file. -pub(crate) const ACCOUNTS_DB_FILENAME: &str = "accounts.db"; - -/// The persistent memory layout of the storage metadata. -/// -/// This struct maps directly to the first `METADATA_STORAGE_SIZE` bytes of the -/// memory-mapped file. It uses atomic types to allow concurrent access/updates -/// from multiple threads without external locking. -#[repr(C)] -struct StorageHeader { - /// The current write position (high-water mark) in blocks. - /// This is an index: `actual_offset = write_cursor * block_size`. - /// - /// This acts as the "head" of the append-only log. Atomic fetch_add is used - /// here to reserve space for new allocations. - write_cursor: AtomicU64, - - /// The latest slot number that has been written to this storage. - /// Used for snapshotting and recovery to know the age of the data. - slot: AtomicU64, - - /// The size of a single block in bytes (e.g., 128, 256, 512). - /// This is immutable once the database is created. - block_size: u32, - - /// The total capacity of the database in blocks. - /// derived from: `(file_size - METADATA_STORAGE_SIZE) / block_size`. - capacity_blocks: u32, - - /// A counter of blocks that have been marked as dead/recycled. - /// This is purely for metrics or fragmentation estimation; it does not - /// automatically reclaim space (this is an append-only structure). - recycled_count: AtomicU32, - - /// Padding to ensure the struct size exactly matches `METADATA_STORAGE_SIZE` - /// and maintains alignment for future extensions. - _padding: [u8; METADATA_PADDING_SIZE], -} - -// Compile-time assertion to ensure the header definition matches the reserved space. -const _: () = assert!(size_of::() == METADATA_STORAGE_SIZE); -// Ensure 8-byte alignment for 64-bit atomics. -const _: () = assert!(size_of::().is_multiple_of(8)); - -/// A handle to the memory-mapped accounts database. -/// -/// This struct provides safe wrappers around the raw memory map, handling -/// atomic allocation, bounds checking, and pointer arithmetic. -#[cfg_attr(test, derive(Debug))] -pub(crate) struct AccountsStorage { - /// The underlying memory-mapped file. - /// Kept alive here to ensure the memory region remains valid. - mmap: MmapMut, - - /// A raw pointer to the start of the *data* segment (skipping the header). - /// - /// # Safety - /// This pointer is derived from `mmap` and is guaranteed to be valid - /// as long as `mmap` is alive. - data_region: NonNull, - - /// A cached copy of `header.block_size` as a `usize`. - /// Optimization to avoid reading from the mmap (potential cache miss/atomic read) - /// on every allocation or read. - block_size: usize, -} - -impl AccountsStorage { - /// Opens an existing accounts database or creates a new one if it doesn't exist. - /// - /// # Arguments - /// * `config` - Configuration defining block size and initial file size. - /// * `directory` - The directory path where `accounts.db` will be located. - /// - /// # Returns - /// * `AccountsDbResult` - The initialized storage handle. - pub(crate) fn new( - config: &AccountsDbConfig, - directory: &Path, - ) -> AccountsDbResult { - let db_path = directory.join(ACCOUNTS_DB_FILENAME); - let exists = db_path.exists(); - - let mut file = File::options() - .create(true) - .truncate(false) - .write(true) - .read(true) - .open(&db_path) - .log_err(|| { - format!("opening accounts db file at {}", db_path.display()) - })?; - - // If the file is new or empty, we must initialize the header and resize it. - if !exists || file.metadata()?.len() == 0 { - initialize_db_file(&mut file, config).log_err(|| { - format!("initializing new accounts db at {}", db_path.display()) - })?; - } else { - // If it exists, ensure it is at least the expected size from config. - let target_size = calculate_file_size(config); - ensure_file_size(&mut file, target_size as u64)?; - } - - Self::map_file(file) - } - - /// Internal helper to memory-map the file and validate the header. - fn map_file(file: File) -> AccountsDbResult { - // SAFETY: - // We are the exclusive owner of the File object here. - // We rely on the OS to ensure the mmap is valid. - let mut mmap = unsafe { MmapMut::map_mut(&file) }?; - - if mmap.len() < METADATA_STORAGE_SIZE { - return Err(AccountsDbError::Internal( - "memory map length is less than metadata requirement".into(), - )); - } - - // Validate and fixup metadata. - // We scope the mutable borrow of the header to this block. - let block_size = { - // SAFETY: We verified mmap.len() >= METADATA_STORAGE_SIZE above. - // Casting the first bytes to StorageHeader is safe because StorageHeader is #[repr(C)]. - let header = - unsafe { &mut *(mmap.as_mut_ptr() as *mut StorageHeader) }; - Self::validate_header(header)?; - - // Check if the file was resized (e.g. by external tool or config change) - // and update capacity_blocks to reflect reality. - let actual_capacity = (mmap.len() - METADATA_STORAGE_SIZE) - / header.block_size as usize; - - if actual_capacity as u32 != header.capacity_blocks { - header.capacity_blocks = actual_capacity as u32; - } - - header.block_size as usize - }; - - // Calculate the pointer to the data region (just past the header). - // SAFETY: We verified mmap.len() >= METADATA_STORAGE_SIZE. - // The pointer arithmetic remains within the valid mmap region. - let data_region = unsafe { - NonNull::new_unchecked(mmap.as_mut_ptr().add(METADATA_STORAGE_SIZE)) - }; - - Ok(Self { - mmap, - data_region, - block_size, - }) - } - - /// Validates the consistency of the storage header. - fn validate_header(header: &StorageHeader) -> AccountsDbResult<()> { - let block_size_valid = [ - BlockSize::Block128, - BlockSize::Block256, - BlockSize::Block512, - ] - .iter() - .any(|&bs| bs as u32 == header.block_size); - - if header.capacity_blocks == 0 || !block_size_valid { - error!( - block_size = header.block_size, - capacity_blocks = header.capacity_blocks, - "AccountsDB corruption detected" - ); - return Err(AccountsDbError::Internal( - "AccountsDB file corrupted".into(), - )); - } - Ok(()) - } - - /// Reserves space for a new allocation in the storage. - /// - /// This method is thread-safe and lock-free. It uses atomic arithmetic to - /// advance the write cursor. - /// - /// # Arguments - /// * `size_bytes` - The number of bytes required for the account data. - /// - /// # Returns - /// * `Ok(Allocation)` - A handle containing the raw pointer and offset. - /// * `Err` - If the database is full. - pub(crate) fn allocate( - &self, - size_bytes: usize, - ) -> AccountsDbResult { - let blocks_needed = self.blocks_required(size_bytes) as u64; - let header = self.header(); - - // Atomic fetch_add reserves a range of blocks for this thread. - // `Relaxed` ordering is sufficient because we only care about uniqueness - // of the index, not synchronization with other memory operations yet. - let start_index = header - .write_cursor - .fetch_add(blocks_needed, Ordering::Relaxed); - - let end_index = start_index as usize + blocks_needed as usize; - let capacity = header.capacity_blocks as usize; - - // Check for overflow (Database Full). - if end_index > capacity { - let _ = header.write_cursor.compare_exchange( - end_index as u64, - start_index, - Ordering::Release, - Ordering::Acquire, - ); - return Err(AccountsDbError::Internal(format!( - "Database full: required {} blocks, available {}", - blocks_needed, - capacity.saturating_sub(start_index as usize) - ))); - } - - // Calculate the raw memory address for this allocation. - // SAFETY: We validated `end_index <= capacity` above, so this pointer - // is guaranteed to be within the mapped data region. - let ptr = unsafe { - self.data_region.add(start_index as usize * self.block_size) - }; - - Ok(Allocation { - ptr, - offset: start_index as Offset, - blocks: blocks_needed as Blocks, - }) - } - - /// Reads an account from the storage at the given offset. - /// - /// # Arguments - /// * `offset` - The block index where the account starts. - /// - /// # Safety - /// This assumes `offset` points to a valid, previously allocated account. - /// The Index system ensures we only request valid offsets. - #[inline(always)] - pub(crate) fn read_account(&self, offset: Offset) -> AccountSharedData { - let ptr = self.resolve_ptr(offset).as_ptr(); - // SAFETY: - // 1. `resolve_ptr` ensures the pointer is within bounds if `offset` is valid. - // 2. `deserialize_from_mmap` must handle potentially untrusted bytes safely. - unsafe { AccountSharedData::deserialize_from_mmap(ptr) }.into() - } - - /// Reconstructs an `Allocation` handle from a previously stored offset. - /// - /// Used when reusing an existing slot in the index (though this append-only - /// implementation generally doesn't overwrite data in place, this supports - /// designs that might). - pub(crate) fn recycle(&self, recycled: ExistingAllocation) -> Allocation { - let ptr = self.resolve_ptr(recycled.offset); - Allocation { - offset: recycled.offset, - blocks: recycled.blocks, - ptr, - } - } - - /// Translates an abstract `Offset` (block index) to a raw memory pointer. - #[inline] - pub(crate) fn resolve_ptr(&self, offset: Offset) -> NonNull { - let offset_bytes = offset as usize * self.block_size; - // SAFETY: - // The caller is responsible for ensuring `offset` is within valid bounds. - // In the context of `AccountsStorage`, offsets come from the Index which - // tracks valid allocations. - unsafe { self.data_region.add(offset_bytes) } - } - - // --- Metadata Accessors --- - - /// Helper to get a reference to the header structure. - #[inline(always)] - fn header(&self) -> &StorageHeader { - // SAFETY: `mmap` is guaranteed to be at least METADATA_STORAGE_SIZE large - // and properly aligned for `StorageHeader`. - unsafe { &*(self.mmap.as_ptr() as *const StorageHeader) } - } - - pub(crate) fn slot(&self) -> u64 { - self.header().slot.load(Ordering::Relaxed) - } - - pub(crate) fn update_slot(&self, val: u64) { - self.header().slot.store(val, Ordering::Relaxed) - } - - pub(crate) fn inc_recycled_count(&self, val: Blocks) { - self.header() - .recycled_count - .fetch_add(val, Ordering::Relaxed); - } - - pub(crate) fn dec_recycled_count(&self, val: Blocks) { - self.header() - .recycled_count - .fetch_sub(val, Ordering::Relaxed); - } - - pub(crate) fn validate_allocation( - &self, - offset: Offset, - blocks: Blocks, - ) -> AccountsDbResult<()> { - if blocks == 0 { - return Err(AccountsDbError::Internal( - "accountsdb allocation has zero blocks".into(), - )); - } - - let end = offset.checked_add(blocks).ok_or_else(|| { - AccountsDbError::Internal( - "accountsdb allocation offset overflow".into(), - ) - })? as u64; - let cursor = self.header().write_cursor.load(Ordering::Relaxed); - let capacity = self.header().capacity_blocks as u64; - - if end > cursor || end > capacity { - return Err(AccountsDbError::Internal(format!( - "accountsdb allocation out of bounds: offset={offset}, blocks={blocks}, cursor={cursor}, capacity={capacity}" - ))); - } - - Ok(()) - } - - pub(crate) fn validate_cursor( - &self, - cursor: Offset, - ) -> AccountsDbResult<()> { - let current = self.header().write_cursor.load(Ordering::Relaxed); - let capacity = self.header().capacity_blocks as u64; - let cursor = cursor as u64; - - if cursor > current || cursor > capacity { - return Err(AccountsDbError::Internal(format!( - "accountsdb compacted cursor out of bounds: cursor={cursor}, current={current}, capacity={capacity}" - ))); - } - - Ok(()) - } - - /// Moves an allocated block range inside the mmap. - /// - /// # Safety - /// The caller must ensure no account references into the storage are live and - /// that `old_offset..old_offset + blocks` and - /// `new_offset..new_offset + blocks` are valid mapped block ranges. - pub(crate) unsafe fn move_allocation( - &self, - old_offset: Offset, - new_offset: Offset, - blocks: Blocks, - scratch: &mut Vec, - ) { - if old_offset == new_offset { - return; - } - - let len = blocks as usize * self.block_size; - let src = self.resolve_ptr(old_offset).as_ptr(); - let dst = self.resolve_ptr(new_offset).as_ptr(); - scratch.resize(len, 0); - // SAFETY: The caller validated both block ranges. The temporary buffer - // prevents an earlier move from overwriting a later move's source. - unsafe { - ptr::copy_nonoverlapping(src, scratch.as_mut_ptr(), len); - ptr::copy_nonoverlapping(scratch.as_ptr(), dst, len); - }; - } - - /// Finalizes compaction metadata and clears bytes after the new cursor. - /// - /// # Safety - /// The caller must ensure all live allocations have already been moved below - /// `new_cursor` and no account references into the old tail are live. - pub(crate) unsafe fn finish_defragment(&self, new_cursor: Offset) { - let header = self.header(); - let old_cursor = header - .write_cursor - .swap(new_cursor as u64, Ordering::Relaxed); - header.recycled_count.store(0, Ordering::Relaxed); - - let new_cursor = new_cursor as u64; - if old_cursor <= new_cursor { - return; - } - - let tail_blocks = old_cursor - new_cursor; - let tail_offset = new_cursor as usize * self.block_size; - let tail_len = tail_blocks as usize * self.block_size; - let tail_ptr = unsafe { self.data_region.as_ptr().add(tail_offset) }; - // SAFETY: `old_cursor` was the previous active cursor, so the tail lies - // within the mapped data region. - unsafe { tail_ptr.write_bytes(0, tail_len) }; - } - - /// Calculates how many blocks are needed to store `size_bytes`. - pub(crate) fn blocks_required(&self, size_bytes: usize) -> Blocks { - size_bytes.div_ceil(self.block_size) as Blocks - } - - /// Returns the slice of memory currently containing valid data (up to the write cursor). - pub(crate) fn active_segment(&self) -> &[u8] { - let cursor = - self.header().write_cursor.load(Ordering::Relaxed) as usize; - // Calculate length: Header Size + (Written Blocks * Block Size) - let len = (cursor * self.block_size + METADATA_STORAGE_SIZE) - .min(self.mmap.len()); - &self.mmap[..len] - } - - /// Flushes changes to disk. - pub(crate) fn flush(&self) -> AccountsDbResult<()> { - self.mmap - .flush() - .log_err(|| "failed to sync flush the mmap")?; - Ok(()) - } - - /// Reloads the database from a different path (used for snapshots). - /// - /// This drops the current mmap and opens a new one at `db_path`. - pub(crate) fn reload(&mut self, db_path: &Path) -> AccountsDbResult<()> { - let mut file = File::options() - .write(true) - .read(true) - .open(db_path.join(ACCOUNTS_DB_FILENAME)) - .log_err(|| { - format!("opening adb file for reload at {}", db_path.display()) - })?; - - ensure_file_size(&mut file, self.size_bytes())?; - *self = Self::map_file(file)?; - Ok(()) - } - - /// Returns the total occupied size of the storage file in bytes. - pub(crate) fn size_bytes(&self) -> u64 { - self.active_segment().len() as u64 - } - - pub(crate) fn block_size(&self) -> usize { - self.block_size - } -} - -/// Initializes a fresh accounts database file with the header. -fn initialize_db_file( - file: &mut File, - config: &AccountsDbConfig, -) -> AccountsDbResult<()> { - const MIN_DB_SIZE: usize = 16 * 1024 * 1024; - if config.database_size < MIN_DB_SIZE { - return Err(AccountsDbError::Internal(format!( - "database file should be larger than {} bytes", - MIN_DB_SIZE - ))); - } - - let target_size = calculate_file_size(config); - // Determine capacity based on file size minus header. - let total_blocks = - (target_size - METADATA_STORAGE_SIZE) / config.block_size as usize; - - ensure_file_size(file, target_size as u64)?; - - // Prepare the initial header state. - let header = StorageHeader { - write_cursor: AtomicU64::new(0), - slot: AtomicU64::new(0), - block_size: config.block_size as u32, - capacity_blocks: total_blocks as u32, - recycled_count: AtomicU32::new(0), - _padding: [0u8; METADATA_PADDING_SIZE], - }; - - // Serialize the header directly to bytes. - // SAFETY: StorageHeader is `repr(C)` and contains only POD (Plain Old Data) types. - let header_bytes = unsafe { - std::slice::from_raw_parts( - &header as *const StorageHeader as *const u8, - size_of::(), - ) - }; - - file.write_all(header_bytes)?; - Ok(file.flush()?) -} - -struct StorageHeaderSnapshot { - write_cursor: u64, - block_size: u64, -} - -/// Resizes the file toward `size` without truncating live storage bytes. -fn ensure_file_size(file: &mut File, size: u64) -> io::Result<()> { - let current_size = file.metadata()?.len(); - if current_size < size { - file.set_len(size)?; - return Ok(()); - } - - if current_size == size { - return Ok(()); - } - - let Some(header) = read_storage_header(file)? else { - return Ok(()); - }; - let min_size = METADATA_STORAGE_SIZE as u64 + header.block_size; - let target_size = size.max(min_size); - let Some(live_size) = header - .write_cursor - .checked_mul(header.block_size) - .and_then(|size| size.checked_add(METADATA_STORAGE_SIZE as u64)) - else { - return Ok(()); - }; - - if target_size >= live_size { - file.set_len(target_size)?; - } - - Ok(()) -} - -fn read_storage_header( - file: &mut File, -) -> io::Result> { - if file.metadata()?.len() < METADATA_STORAGE_SIZE as u64 { - return Ok(None); - } - - let mut header = std::mem::MaybeUninit::::uninit(); - let bytes = unsafe { - std::slice::from_raw_parts_mut( - header.as_mut_ptr() as *mut u8, - size_of::(), - ) - }; - - file.seek(SeekFrom::Start(0))?; - file.read_exact(bytes)?; - - let header = unsafe { header.assume_init() }; - let block_size_valid = [ - BlockSize::Block128, - BlockSize::Block256, - BlockSize::Block512, - ] - .iter() - .any(|&bs| bs as u32 == header.block_size); - - if header.capacity_blocks == 0 || !block_size_valid { - return Ok(None); - } - - Ok(Some(StorageHeaderSnapshot { - write_cursor: header.write_cursor.load(Ordering::Relaxed), - block_size: header.block_size as u64, - })) -} - -/// Calculates the target file size, ensuring alignment with block size. -fn calculate_file_size(config: &AccountsDbConfig) -> usize { - let block_size = config.block_size as usize; - // Align total size to block size + metadata - let blocks = config.database_size.div_ceil(block_size); - blocks * block_size + METADATA_STORAGE_SIZE -} - -/// Represents a successful allocation within the storage. -#[derive(Clone, Copy)] -pub(crate) struct Allocation { - /// Raw pointer to the start of the allocated memory. - pub(crate) ptr: NonNull, - /// The block index (offset) of this allocation. - pub(crate) offset: Offset, - /// The number of blocks reserved. - pub(crate) blocks: Blocks, -} - -/// A struct representing a previously known allocation. -/// Used for recycling or testing equality. -#[cfg_attr(test, derive(Debug, Eq, PartialEq))] -pub(crate) struct ExistingAllocation { - /// The block index (offset) of this allocation. - pub(crate) offset: Offset, - /// The number of blocks reserved. - pub(crate) blocks: Blocks, -} - -#[cfg(test)] -impl From for ExistingAllocation { - fn from(value: Allocation) -> Self { - Self { - offset: value.offset, - blocks: value.blocks, - } - } -} diff --git a/magicblock-accounts-db/src/tests.rs b/magicblock-accounts-db/src/tests.rs deleted file mode 100644 index 2a0a7bd75..000000000 --- a/magicblock-accounts-db/src/tests.rs +++ /dev/null @@ -1,911 +0,0 @@ -use std::{collections::HashSet, ops::Deref, sync::Arc}; - -use magicblock_config::config::{AccountsDbConfig, BlockSize}; -use magicblock_magic_program_api as magic_program; -use solana_account::{AccountSharedData, ReadableAccount, WritableAccount}; -use solana_pubkey::Pubkey; -use solana_sdk_ids::feature; -use tempfile::TempDir; - -use crate::{storage::ACCOUNTS_DB_FILENAME, traits::AccountsBank, AccountsDb}; - -const LAMPORTS: u64 = 4425; -const SPACE: usize = 73; -const OWNER: Pubkey = Pubkey::new_from_array([23; 32]); -const ACCOUNT_DATA: &[u8] = b"hello world?"; -const INIT_DATA_LEN: usize = ACCOUNT_DATA.len(); -/// Default slot used for snapshots in tests. -const SNAPSHOT_SLOT: u64 = 1024; - -/// Verifies basic account insertion and retrieval. -#[test] -fn test_get_account() { - let env = TestEnv::new(); - let AccountWithPubkey { pubkey, .. } = env.create_and_insert_account(); - - let acc = env.get_account(&pubkey).expect("account should exist"); - - assert_eq!(acc.lamports(), LAMPORTS); - assert_eq!(acc.owner(), &OWNER); - assert_eq!(&acc.data()[..INIT_DATA_LEN], ACCOUNT_DATA); - assert_eq!(acc.data().len(), SPACE); -} - -/// Verifies Copy-on-Write semantics. -/// Modifying an account in memory should not affect the persistent store -/// until `upsert_account` is called. -#[test] -fn test_modify_account() { - let env = TestEnv::new(); - let AccountWithPubkey { - pubkey, - account: mut uncommitted, - } = env.create_and_insert_account(); - - let new_lamports = 42; - - // Modify in memory - assert_eq!(uncommitted.lamports(), LAMPORTS); - uncommitted.set_lamports(new_lamports); - assert_eq!(uncommitted.lamports(), new_lamports); - - // Verify DB is unchanged - let committed_before = env.get_account(&pubkey).unwrap(); - assert_eq!( - committed_before.lamports(), - LAMPORTS, - "database should retain old state before commit" - ); - - // Commit changes - env.insert_account(&pubkey, &uncommitted).unwrap(); - - // Verify DB is updated - let committed_after = env.get_account(&pubkey).unwrap(); - assert_eq!( - committed_after.lamports(), - new_lamports, - "database should reflect updates after commit" - ); -} - -/// Verifies that accounts are correctly reallocated when their data size increases. -#[test] -fn test_account_resize() { - let env = TestEnv::new(); - let huge_data = [42; SPACE * 4]; - let AccountWithPubkey { - pubkey, - account: mut uncommitted, - } = env.create_and_insert_account(); - - // Resize in memory - uncommitted.set_data_from_slice(&huge_data); - assert_eq!(uncommitted.data().len(), SPACE * 4); - - // Verify DB still has old size - let committed_before = env.get_account(&pubkey).unwrap(); - assert_eq!(committed_before.data().len(), SPACE); - - // Update DB - env.insert_account(&pubkey, &uncommitted).unwrap(); - - // Verify DB has new size and data - let committed_after = env.get_account(&pubkey).unwrap(); - assert_eq!( - committed_after.data(), - huge_data, - "account data should match resized buffer" - ); -} - -/// Verifies that the storage allocator reuses space (holes) created by updates. -#[test] -fn test_alloc_reuse() { - let env = TestEnv::new(); - let AccountWithPubkey { - pubkey: pk1, - account: mut acc1, - } = env.create_and_insert_account(); - - // Capture the pointer address of the first allocation - let old_ptr = env.get_account(&pk1).unwrap().data().as_ptr(); - - // Resize acc1 significantly to force a move, freeing the old slot - let huge_data = [42; SPACE * 4]; - acc1.set_data_from_slice(&huge_data); - env.insert_account(&pk1, &acc1).unwrap(); - - // Insert a new account that fits in the old slot - let AccountWithPubkey { pubkey: pk2, .. } = env.create_and_insert_account(); - - let new_ptr = env.get_account(&pk2).unwrap().data().as_ptr(); - - assert_eq!( - new_ptr, old_ptr, - "allocator should recycle the freed slot for the new account" - ); -} - -/// Verifies complex reallocation reuse logic (holes split/merge behavior). -#[test] -fn test_larger_alloc_reuse() { - let env = TestEnv::new(); - - // 1. Insert Account 1 - let mut acc1 = env.new_account_obj(SPACE); - let huge_data_2x = vec![42; SPACE * 2]; - acc1.account.set_data_from_slice(&huge_data_2x); - env.insert_account(&acc1.pubkey, &acc1.account).unwrap(); - - // 2. Insert Account 2 (same size) - let mut acc2 = env.new_account_obj(SPACE); - acc2.account.set_data_from_slice(&huge_data_2x); - env.insert_account(&acc2.pubkey, &acc2.account).unwrap(); - - // 3. Insert Account 3 (4x size) - let mut acc3 = env.new_account_obj(SPACE); - let huge_data_4x = vec![42; SPACE * 4]; - acc3.account.set_data_from_slice(&huge_data_4x); - env.insert_account(&acc3.pubkey, &acc3.account).unwrap(); - - // Read back Account 3 to get its pointer - let acc3_stored = env.get_account(&acc3.pubkey).unwrap(); - let acc3_ptr = acc3_stored.data().as_ptr(); - - // 4. Resize Account 3 to 5x (forces move, freeing the 4x slot) - let huge_data_5x = vec![42; SPACE * 5]; - acc3.account.set_data_from_slice(&huge_data_5x); - env.insert_account(&acc3.pubkey, &acc3.account).unwrap(); - - // 5. Insert Account 4 (3x size - fits in the 4x hole) - let mut acc4 = env.new_account_obj(SPACE); - let huge_data_3x = vec![42; SPACE * 3]; - acc4.account.set_data_from_slice(&huge_data_3x); - env.insert_account(&acc4.pubkey, &acc4.account).unwrap(); - - let acc4_stored = env.get_account(&acc4.pubkey).unwrap(); - - assert_eq!( - acc4_stored.data().as_ptr(), - acc3_ptr, - "account 4 should have reused account 3's old allocation" - ); -} - -#[test] -fn test_get_program_accounts() { - let env = TestEnv::new(); - let acc = env.create_and_insert_account(); - - let accounts = env.get_program_accounts(&OWNER, |_| true); - assert!(accounts.is_ok()); - - let mut iter = accounts.unwrap(); - let (pk, data) = iter.next().unwrap(); - - assert_eq!(pk, acc.pubkey); - assert_eq!(data, acc.account); - assert!(iter.next().is_none()); -} - -#[test] -fn test_get_all_accounts() { - let env = TestEnv::new(); - let acc1 = env.create_and_insert_account(); - let acc2 = env.create_and_insert_account(); - let acc3 = env.create_and_insert_account(); - - let expected_pks: HashSet<_> = - [acc1.pubkey, acc2.pubkey, acc3.pubkey].into(); - - let stored_pks: HashSet<_> = env.iter_all().map(|(pk, _)| pk).collect(); - - assert_eq!(stored_pks, expected_pks); -} - -#[test] -fn test_take_snapshot() { - let env = TestEnv::new(); - let mut acc = env.create_and_insert_account(); - - assert_eq!(env.slot(), 0); - - // Trigger Snapshot 1 - env.set_slot(SNAPSHOT_SLOT); - env.take_snapshot_and_wait(SNAPSHOT_SLOT); - assert_eq!(env.slot(), SNAPSHOT_SLOT); - assert!(env.snapshot_exists(SNAPSHOT_SLOT)); - - // Verify archive file exists (not directory) - let archive_path = env - .snapshot_manager - .database_path() - .parent() - .unwrap() - .join(format!("snapshot-{:0>12}.tar.gz", SNAPSHOT_SLOT)); - assert!(archive_path.exists(), "Archive file should exist"); - assert!( - archive_path.is_file(), - "Snapshot should be a file, not directory" - ); - - // Verify archive file exists (not directory) - let archive_path = env - .snapshot_manager - .database_path() - .parent() - .unwrap() - .join(format!("snapshot-{:0>12}.tar.gz", SNAPSHOT_SLOT)); - assert!(archive_path.exists(), "Archive file should exist"); - assert!( - archive_path.is_file(), - "Snapshot should be a file, not directory" - ); - - // Update Account - acc.account.set_data(ACCOUNT_DATA.to_vec()); - env.insert_account(&acc.pubkey, &acc.account).unwrap(); - - // Trigger Snapshot 2 - let slot2 = SNAPSHOT_SLOT * 2; - env.set_slot(slot2); - env.take_snapshot_and_wait(slot2); - assert!(env.snapshot_exists(slot2)); -} - -/// Verifies that orphan snapshot directories are cleaned up on startup. -#[test] -fn test_orphan_directory_cleanup() { - let (adb, temp_dir) = TestEnv::init_raw_db(); - - // Create an orphan directory (simulating interrupted archiving) - let orphan_dir = temp_dir.path().join("accountsdb/snapshot-000000000512"); - std::fs::create_dir_all(&orphan_dir).unwrap(); - std::fs::write(orphan_dir.join("test.txt"), "orphan data").unwrap(); - - // Drop and reopen - orphan should be cleaned up - drop(adb); - let config = AccountsDbConfig::default(); - let _adb = AccountsDb::new(&config, temp_dir.path(), 0).unwrap(); - - assert!( - !orphan_dir.exists(), - "Orphan directory should be cleaned up on startup" - ); -} - -/// Verifies external snapshot fast-forward when snapshot is newer than current state. -#[test] -fn test_external_snapshot_fast_forward() { - let env = TestEnv::new(); - - // Create an account and take a local snapshot - let acc = env.create_and_insert_account(); - env.set_slot(SNAPSHOT_SLOT); - env.take_snapshot_and_wait(SNAPSHOT_SLOT); - - // Read the archive bytes - let archive_path = env - .snapshot_manager - .database_path() - .parent() - .unwrap() - .join(format!("snapshot-{:0>12}.tar.gz", SNAPSHOT_SLOT)); - let archive_bytes = - std::fs::read(&archive_path).expect("Failed to read archive"); - let pubkey = acc.pubkey; - - // Drop current DB and create new one at slot 0 - drop(env); - let temp_dir = tempfile::tempdir().unwrap(); - let config = AccountsDbConfig { - reset: true, - ..Default::default() - }; - let mut new_db = AccountsDb::new(&config, temp_dir.path(), 0).unwrap(); - assert_eq!(new_db.slot(), 0, "New DB should start at slot 0"); - - // Insert external snapshot (snapshot slot > current slot 0, should fast-forward) - let fast_forwarded = new_db - .insert_external_snapshot(SNAPSHOT_SLOT, &archive_bytes) - .unwrap(); - assert!(fast_forwarded, "Should fast-forward when snapshot is newer"); - - // Verify the account exists immediately after fast-forward - let restored = new_db.get_account(&pubkey); - assert!( - restored.is_some(), - "Account should exist after fast-forward" - ); - assert_eq!(restored.unwrap().lamports(), LAMPORTS); -} - -#[test] -fn test_restore_from_snapshot() { - let mut env = TestEnv::new(); - let mut acc = env.create_and_insert_account(); - - // Create Base Snapshot - env.set_slot(SNAPSHOT_SLOT); - env.take_snapshot_and_wait(SNAPSHOT_SLOT); - - // Make changes after snapshot - env.set_slot(SNAPSHOT_SLOT + 3); - let new_lamports = 999; - acc.account.set_lamports(new_lamports); - env.insert_account(&acc.pubkey, &acc.account).unwrap(); - env.set_slot(SNAPSHOT_SLOT + 6); - - // Verify update persisted in current state - assert_eq!( - env.get_account(&acc.pubkey).unwrap().lamports(), - new_lamports - ); - - // Rollback to before the update - env = env.restore_to_slot(SNAPSHOT_SLOT); - - let restored_acc = env.get_account(&acc.pubkey).unwrap(); - assert_eq!( - restored_acc.lamports(), - LAMPORTS, - "account should be restored to state at snapshot" - ); -} - -#[test] -fn test_get_all_accounts_after_rollback() { - let mut env = TestEnv::new(); - let acc = env.create_and_insert_account(); - let mut pks = vec![acc.pubkey]; - const ITERS: u64 = 1024; - - // Create initial state - for i in 0..=ITERS { - let acc = env.create_and_insert_account(); - pks.push(acc.pubkey); - env.set_slot(i); - } - - // Take a snapshot at ITERS - env.take_snapshot_and_wait(ITERS); - - // Add accounts after the restore point - let mut post_snap_pks = vec![]; - for i in ITERS..ITERS + 100 { - let acc = env.create_and_insert_account(); - env.set_slot(i + 1); - post_snap_pks.push(acc.pubkey); - } - - // Rollback - env = env.restore_to_slot(ITERS); - assert_eq!(env.slot(), ITERS); - - // Verify State - let pubkeys: HashSet<_> = env.iter_all().map(|(pk, _)| pk).collect(); - - assert_eq!(pubkeys.len(), pks.len()); - - for pk in pks { - assert!(pubkeys.contains(&pk), "Missing account {}", pk); - } - for pk in post_snap_pks { - assert!( - !pubkeys.contains(&pk), - "Account {} should have been rolled back", - pk - ); - } -} - -#[test] -fn test_db_size_after_rollback() { - let mut env = TestEnv::new(); - let last_slot = 512; - for i in 0..=last_slot { - env.create_and_insert_account(); - env.advance_slot(i); - } - - let pre_rollback_db_size = env.storage_size(); - let file_path = env - .snapshot_manager - .database_path() - .join(ACCOUNTS_DB_FILENAME); - let pre_rollback_file_size = file_path.metadata().unwrap().len(); - - env = env.restore_to_slot(last_slot); - - assert_eq!( - env.storage_size(), - pre_rollback_db_size, - "database size mismatch after rollback" - ); - - let post_rollback_file_size = file_path.metadata().unwrap().len(); - assert_eq!( - post_rollback_file_size, pre_rollback_file_size, - "adb file size mismatch after rollback" - ); -} - -#[test] -fn test_zero_lamports_account() { - let env = TestEnv::new(); - let mut acc = env.create_and_insert_account(); - - // Explicitly set 0 lamports (simulating escrow or marker account) - acc.account.set_lamports(0); - env.insert_account(&acc.pubkey, &acc.account).unwrap(); - - let stored = env.get_account(&acc.pubkey); - assert!(stored.is_some(), "zero lamport account should be retained"); - assert_eq!(stored.unwrap().lamports(), 0); -} - -#[test] -fn test_owner_change() { - let env = TestEnv::new(); - let mut acc = env.create_and_insert_account(); - - // Verify index before change - assert!(matches!( - env.account_matches_owners(&acc.pubkey, &[OWNER]), - Some(0) - )); - - // Change owner - let new_owner = Pubkey::new_unique(); - acc.account.set_owner(new_owner); - env.insert_account(&acc.pubkey, &acc.account).unwrap(); - - // Verify index after change - // Old owner should return nothing - assert!(env.account_matches_owners(&acc.pubkey, &[OWNER]).is_none()); - assert_eq!( - env.get_program_accounts(&OWNER, |_| true).unwrap().count(), - 0 - ); - - // New owner should match - assert!(matches!( - env.account_matches_owners(&acc.pubkey, &[new_owner]), - Some(0) - )); - assert_eq!( - env.get_program_accounts(&new_owner, |_| true) - .unwrap() - .count(), - 1 - ); -} - -/// Verifies that we eventually hit a limit or handle capacity gracefully. -#[test] -fn test_database_full_error() { - let env = TestEnv::new(); - - // Fill DB with huge accounts - let huge_data = vec![42; 9_000_000]; // 9MB - let mut hit_limit = false; - - // Try to insert until failure - for _ in 0..50 { - let mut acc = env.new_account_obj(SPACE); - acc.account.set_data_from_slice(&huge_data); - - if env.insert_account(&acc.pubkey, &acc.account).is_err() { - hit_limit = true; - break; - } - } - - assert!( - hit_limit, - "Database should eventually return error when full" - ); -} - -#[test] -fn test_account_shrinking() { - let env = TestEnv::new(); - let mut acc = env.create_and_insert_account(); - - // Shrink via set_data - acc.account.set_data(vec![]); - env.insert_account(&acc.pubkey, &acc.account).unwrap(); - - let stored = env.get_account(&acc.pubkey).unwrap(); - assert_eq!(stored.data().len(), 0); -} - -#[test] -fn test_reallocation_split() { - let env = TestEnv::new(); - const SIZE: usize = 1024; - - // Create a hole of size 2048 - let acc1 = env.create_account_with_size(SIZE * 2); - let ptr1 = env.get_account(&acc1.pubkey).unwrap().data().as_ptr(); - env.remove_account(&acc1.pubkey); // Creates hole - - // Create 2 accounts of size 256 - let acc2 = env.create_account_with_size(SIZE / 4); - let acc3 = env.create_account_with_size(SIZE / 4); - - let ptr2 = env.get_account(&acc2.pubkey).unwrap().data().as_ptr(); - let ptr3 = env.get_account(&acc3.pubkey).unwrap().data().as_ptr(); - - // Verify they reused the space (ptr2 should be exactly at ptr1) - assert_eq!(ptr2, ptr1, "First small account should take start of hole"); - assert!(ptr3 > ptr2, "Second small account should follow first"); -} - -#[test] -fn test_defragment_moves_accounts_left_and_clears_tail() { - let env = TestEnv::new(); - const SIZE: usize = 1024; - - let removed = env.create_account_with_size(SIZE); - let acc2 = env.create_account_with_size(SIZE * 2); - let acc3 = env.create_account_with_size(SIZE * 3); - - let mut expected2 = env.get_account(&acc2.pubkey).unwrap(); - let mut expected3 = env.get_account(&acc3.pubkey).unwrap(); - expected2.ensure_owned(); - expected3.ensure_owned(); - - env.remove_account(&removed.pubkey); - assert_eq!(env.index.get_deallocations_count(), 1); - - let checksum_before = unsafe { env.checksum() }; - let size_before = env.storage_size() as usize; - let file_path = env - .snapshot_manager - .database_path() - .join(ACCOUNTS_DB_FILENAME); - - unsafe { env.defragment() }.unwrap(); - - assert_eq!(env.index.get_deallocations_count(), 0); - assert!(env.storage_size() < size_before as u64); - assert_eq!(unsafe { env.checksum() }, checksum_before); - - let mut actual2 = env.get_account(&acc2.pubkey).unwrap(); - let mut actual3 = env.get_account(&acc3.pubkey).unwrap(); - actual2.ensure_owned(); - actual3.ensure_owned(); - assert_eq!(actual2, expected2); - assert_eq!(actual3, expected3); - assert!(env.get_account(&removed.pubkey).is_none()); - assert!(env.account_matches_owners(&acc2.pubkey, &[OWNER]).is_some()); - assert!(env.account_matches_owners(&acc3.pubkey, &[OWNER]).is_some()); - assert_eq!( - env.get_program_accounts(&OWNER, |_| true).unwrap().count(), - 2 - ); - - let size_after = env.storage_size() as usize; - let file_bytes = std::fs::read(file_path).unwrap(); - assert!( - file_bytes[size_after..size_before] - .iter() - .all(|byte| *byte == 0), - "defragmentation should zero the old active tail" - ); -} - -#[test] -fn test_defragment_empty_database_is_noop() { - let env = TestEnv::new(); - let size_before = env.storage_size(); - - unsafe { env.defragment() }.unwrap(); - - assert_eq!(env.storage_size(), size_before); - assert_eq!(env.index.get_deallocations_count(), 0); - assert_eq!(env.account_count(), 0); -} - -#[test] -fn test_reopen_shrinks_storage_when_cursor_fits_config_size() { - let temp_dir = tempfile::tempdir().unwrap(); - let config = AccountsDbConfig::default(); - let adb = AccountsDb::new(&config, temp_dir.path(), 0).unwrap(); - let pubkey = Pubkey::new_unique(); - let account = AccountSharedData::new(LAMPORTS, SPACE, &OWNER); - let storage_path = temp_dir - .path() - .join("accountsdb") - .join("main") - .join(ACCOUNTS_DB_FILENAME); - - adb.insert_account(&pubkey, &account).unwrap(); - let initial_size = std::fs::metadata(&storage_path).unwrap().len(); - drop(adb); - - let database_size = 16 * 1024 * 1024; - let smaller_config = AccountsDbConfig { - database_size, - ..Default::default() - }; - let reopened = AccountsDb::new(&smaller_config, temp_dir.path(), 0) - .expect("reopen with smaller storage config"); - - let block_size = BlockSize::Block256 as usize; - let expected_size = database_size.div_ceil(block_size) * block_size + 256; - assert!(expected_size as u64 > reopened.storage_size()); - assert_eq!( - std::fs::metadata(storage_path).unwrap().len(), - expected_size as u64 - ); - assert!((expected_size as u64) < initial_size); - - let restored = reopened.get_account(&pubkey).unwrap(); - assert_eq!(restored.lamports(), account.lamports()); - assert_eq!(restored.owner(), account.owner()); - assert_eq!(restored.data(), account.data()); -} - -#[test] -fn test_database_reset() { - let (adb, temp_dir) = TestEnv::init_raw_db(); - let pubkey = Pubkey::new_unique(); - let account = AccountSharedData::new(LAMPORTS, SPACE, &OWNER); - - adb.insert_account(&pubkey, &account).unwrap(); - assert!(adb.get_account(&pubkey).is_some()); - - // Explicitly drop to release locks - drop(adb); - - // Re-open with reset=true - let config = AccountsDbConfig { - reset: true, - ..Default::default() - }; - - let adb_reset = AccountsDb::new(&config, temp_dir.path(), 0).unwrap(); - - assert!(adb_reset.get_account(&pubkey).is_none()); - assert_eq!(adb_reset.account_count(), 0); -} - -#[test] -fn test_checksum_deterministic_across_dbs() { - // Two independent DBs with identical accounts must produce identical checksums - let dir1 = tempfile::tempdir().unwrap(); - let dir2 = tempfile::tempdir().unwrap(); - let config = AccountsDbConfig::default(); - - let db1 = AccountsDb::new(&config, dir1.path(), 0).unwrap(); - let db2 = AccountsDb::new(&config, dir2.path(), 0).unwrap(); - - // Insert same accounts into both DBs - for i in 0..50 { - let pubkey = Pubkey::new_unique(); - let mut account = AccountSharedData::new(LAMPORTS, SPACE, &OWNER); - account.data_as_mut_slice()[..8] - .copy_from_slice(&(i as u64).to_le_bytes()); - db1.insert_account(&pubkey, &account).unwrap(); - db2.insert_account(&pubkey, &account).unwrap(); - } - - assert_eq!( - unsafe { db1.checksum() }, - unsafe { db2.checksum() }, - "checksums must match for identical state" - ); -} - -#[test] -fn test_checksum_detects_state_change() { - let env = TestEnv::new(); - - // Create initial state - let mut accounts: Vec<_> = (0..20) - .map(|_| { - let acc = env.create_and_insert_account(); - (acc.pubkey, acc.account) - }) - .collect(); - - let original_checksum = unsafe { env.checksum() }; - - // Modify a single account's data - accounts[5].1.data_as_mut_slice()[0] ^= 0xFF; - env.insert_account(&accounts[5].0, &accounts[5].1).unwrap(); - - { - assert_ne!( - unsafe { env.checksum() }, - original_checksum, - "checksum must detect single account modification" - ); - } - - // Modify lamports on a different account - accounts[10].1.set_lamports(1_000_000); - env.insert_account(&accounts[10].0, &accounts[10].1) - .unwrap(); - - { - assert_ne!( - unsafe { env.checksum() }, - original_checksum, - "checksum must detect lamport change" - ); - } -} - -#[test] -fn test_reset_bank_removes_only_stale_accounts() { - let env = TestEnv::new(); - let validator_id = Pubkey::new_unique(); - - let removable = Pubkey::new_unique(); - env.insert_account(&removable, &AccountSharedData::default()) - .unwrap(); - - let feature_owned = Pubkey::new_unique(); - env.insert_account( - &feature_owned, - &AccountSharedData::new(1, 0, &feature::ID), - ) - .unwrap(); - - let mut delegated_account = AccountSharedData::default(); - delegated_account.set_delegated(true); - let delegated = Pubkey::new_unique(); - env.insert_account(&delegated, &delegated_account).unwrap(); - - let mut undelegating_account = AccountSharedData::default(); - undelegating_account.set_undelegating(true); - let undelegating = Pubkey::new_unique(); - env.insert_account(&undelegating, &undelegating_account) - .unwrap(); - - let mut ephemeral_account = AccountSharedData::new(1, 0, &OWNER); - ephemeral_account.set_ephemeral(true); - let ephemeral = Pubkey::new_unique(); - env.insert_account(&ephemeral, &ephemeral_account).unwrap(); - - env.insert_account(&validator_id, &AccountSharedData::default()) - .unwrap(); - env.insert_account( - &magic_program::POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, - &AccountSharedData::default(), - ) - .unwrap(); - - env.reset_bank(&validator_id) - .expect("bank reset should succeed"); - - assert!(env.get_account(&removable).is_none()); - assert!(env.get_account(&feature_owned).is_some()); - assert!(env.get_account(&delegated).is_some()); - assert!(env.get_account(&undelegating).is_some()); - assert!(env.get_account(&ephemeral).is_some()); - assert!(env.get_account(&validator_id).is_some()); - assert!(env - .get_account(&magic_program::POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID) - .is_some()); -} -// ============================================================== -// TEST UTILITIES -// ============================================================== - -struct AccountWithPubkey { - pubkey: Pubkey, - account: AccountSharedData, -} - -struct TestEnv { - adb: Arc, - // Kept to ensure temp dir is cleaned up on drop - _directory: TempDir, -} - -impl TestEnv { - fn new() -> Self { - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .try_init(); - let (adb, _directory) = Self::init_raw_db(); - Self { adb, _directory } - } - - fn init_raw_db() -> (Arc, TempDir) { - let dir = tempfile::tempdir().expect("temp dir creation failed"); - let config = AccountsDbConfig::default(); - - let adb = AccountsDb::new(&config, dir.path(), 0) - .expect("ADB init failed") - .into(); - - (adb, dir) - } - - fn new_account_obj(&self, size: usize) -> AccountWithPubkey { - let pubkey = Pubkey::new_unique(); - let mut account = AccountSharedData::new(LAMPORTS, size, &OWNER); - account.set_delegated(true); - // Fill with some data - if size >= INIT_DATA_LEN { - account.data_as_mut_slice()[..INIT_DATA_LEN] - .copy_from_slice(ACCOUNT_DATA); - } - AccountWithPubkey { pubkey, account } - } - - fn create_and_insert_account(&self) -> AccountWithPubkey { - let acc = self.new_account_obj(SPACE); - self.adb.insert_account(&acc.pubkey, &acc.account).unwrap(); - // Re-fetch to ensure we have the stored state - let stored = self.adb.get_account(&acc.pubkey).unwrap(); - AccountWithPubkey { - pubkey: acc.pubkey, - account: stored, - } - } - - fn create_account_with_size(&self, size: usize) -> AccountWithPubkey { - let acc = self.new_account_obj(size); - self.adb.insert_account(&acc.pubkey, &acc.account).unwrap(); - AccountWithPubkey { - pubkey: acc.pubkey, - account: acc.account, - } - } - - fn advance_slot(&self, target_slot: u64) { - self.adb.set_slot(target_slot); - } - - /// Takes a snapshot and waits for archiving to complete. - fn take_snapshot_and_wait(&self, slot: u64) -> u64 { - let checksum = unsafe { self.adb.take_snapshot(slot) }; - // Wait for background archiving to complete - let mut retries = 0; - while !self.adb.snapshot_exists(slot) && retries < 200 { - std::thread::sleep(std::time::Duration::from_millis(50)); - retries += 1; - } - assert!(self.adb.snapshot_exists(slot), "Snapshot should exist"); - checksum.expect("failed to take accountsdb snapshot") - } - - fn restore_to_slot(mut self, slot: u64) -> Self { - // Robustly wait for background threads (snapshots) to release the Arc. - // Archiving can take several seconds for large databases. - let mut retries = 0; - let mut inner = loop { - match Arc::try_unwrap(self.adb) { - Ok(inner) => break inner, - Err(adb) => { - if retries > 200 { - // Panic if still shared after ~10 seconds - panic!("Cannot restore: DB is shared (background snapshot thread likely still running)"); - } - self.adb = adb; // Put it back to retry - std::thread::sleep(std::time::Duration::from_millis(50)); - retries += 1; - } - } - }; - - inner.restore_state_if_needed(slot).unwrap(); - self.adb = Arc::new(inner); - self - } -} - -// Allow calling AccountsDb methods directly on TestEnv -impl Deref for TestEnv { - type Target = Arc; - fn deref(&self) -> &Self::Target { - &self.adb - } -} diff --git a/magicblock-accounts-db/src/traits.rs b/magicblock-accounts-db/src/traits.rs deleted file mode 100644 index 6531fb925..000000000 --- a/magicblock-accounts-db/src/traits.rs +++ /dev/null @@ -1,25 +0,0 @@ -use solana_account::AccountSharedData; -use solana_pubkey::Pubkey; - -use crate::AccountsDbResult; - -pub trait AccountsBank: Send + Sync + 'static { - fn get_account(&self, pubkey: &Pubkey) -> Option; - fn remove_account(&self, pubkey: &Pubkey); - fn remove_where( - &self, - predicate: impl FnMut(&Pubkey, &AccountSharedData) -> bool, - ) -> AccountsDbResult; - - fn remove_account_conditionally( - &self, - pubkey: &Pubkey, - predicate: impl Fn(&AccountSharedData) -> bool, - ) { - if let Some(acc) = self.get_account(pubkey) { - if predicate(&acc) { - self.remove_account(pubkey); - } - } - } -} diff --git a/magicblock-accounts/Cargo.toml b/magicblock-accounts/Cargo.toml deleted file mode 100644 index 5b48785a5..000000000 --- a/magicblock-accounts/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "magicblock-accounts" -version.workspace = true -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[dependencies] -async-trait = { workspace = true } - -magicblock-committor-service = { workspace = true } -solana-pubkey = { workspace = true } -solana-transaction-error = { workspace = true } -tokio = { workspace = true } -thiserror = { workspace = true } -url = { workspace = true } diff --git a/magicblock-accounts/README.md b/magicblock-accounts/README.md deleted file mode 100644 index 18ce5d972..000000000 --- a/magicblock-accounts/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -# Summary - -Implements a `AccountsManager`, which is reponsible for: - -- fetching chain accounts content -- commiting content back to chain - -# Details - -*Important symbols:* - -- `AccountsManager` type - - Implemented by a `ExternalAccountsManager` - - depends on an `InternalAccountProvider` (implemented by `BankAccountProvider`) - - depends on an `AccountCloner` (implemented by `RemoteAccountCloner`) - - depends on a `Transwise` - - denepds on a `CommittorServiceExt` that is used for Manual commits - - Implements `ensure_accounts` function - - Maintains a local cache of accounts already validated and cloned - -- `BankAccountProvider` - - depends on a `Bank` - -- `RemoteAccountCloner` - - depends on a `Bank` - -# Notes - -*How does `ensure_accounts` work:* - -- Collect readonly and writable accounts that we haven't already cloned in the validator -- Those accounts we haven't seen yet we "validate" using `Transwise.validate_accounts` -- We need all accounts to be cloned for the transaction to run, so we clone accounts after the validation -- We also set the correct owners on cloned delegated accounts so that the smart contracts can use them -- Also fund the payer lamports so that it can pay for transactions costs -- Also modify the delegated accounts to have the original owner inside the validator - -*Important dependencies:* - -- Provides `Transwise`: the conjuncto repository diff --git a/magicblock-accounts/src/config.rs b/magicblock-accounts/src/config.rs deleted file mode 100644 index 70e3995d2..000000000 --- a/magicblock-accounts/src/config.rs +++ /dev/null @@ -1,18 +0,0 @@ -#[derive(Debug, PartialEq, Eq)] -pub enum LifecycleMode { - Replica, - ProgramsReplica, - Ephemeral, - Offline, -} - -impl LifecycleMode { - pub fn requires_ephemeral_validation(&self) -> bool { - match self { - LifecycleMode::Replica => false, - LifecycleMode::ProgramsReplica => false, - LifecycleMode::Ephemeral => true, - LifecycleMode::Offline => false, - } - } -} diff --git a/magicblock-accounts/src/errors.rs b/magicblock-accounts/src/errors.rs deleted file mode 100644 index d4a7b5980..000000000 --- a/magicblock-accounts/src/errors.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::collections::HashSet; - -use magicblock_committor_service::{ - error::CommittorServiceError, ChangesetMeta, -}; -use solana_pubkey::Pubkey; -use solana_transaction_error::TransactionError; -use thiserror::Error; -use tokio::sync::oneshot::error::RecvError; - -pub type AccountsResult = std::result::Result; - -#[derive(Error, Debug)] -pub enum AccountsError { - #[error("UrlParseError: {0}")] - UrlParseError(#[from] Box), - - #[error("TransactionError: {0}")] - TransactionError(#[from] Box), - - #[error("CommittorSerivceError: {0}")] - CommittorSerivceError(#[from] CommittorServiceError), - - #[error("TokioOneshotRecvError")] - TokioOneshotRecvError(#[from] Box), - - #[error("InvalidRpcUrl '{0}'")] - InvalidRpcUrl(String), - - #[error("FailedToUpdateUrlScheme")] - FailedToUpdateUrlScheme, - - #[error("FailedToUpdateUrlPort")] - FailedToUpdateUrlPort, - - #[error("FailedToGetLatestBlockhash '{0}'")] - FailedToGetLatestBlockhash(String), - - #[error("FailedToGetReimbursementAddress '{0}'")] - FailedToGetReimbursementAddress(String), - - #[error("FailedToSendCommitTransaction '{0}'")] - FailedToSendCommitTransaction(String, HashSet, HashSet), - - #[error("Too many committees: {0}")] - TooManyCommittees(usize), - - #[error("FailedToObtainReqidForCommittedChangeset {0:?}")] - FailedToObtainReqidForCommittedChangeset(Box), -} - -#[derive(Error, Debug)] -pub enum ScheduledCommitsProcessorError { - #[error("RecvError: {0}")] - RecvError(#[from] RecvError), - #[error("CommittorSerivceError: {0}")] - CommittorSerivceError(Box), -} - -impl From for ScheduledCommitsProcessorError { - fn from(e: CommittorServiceError) -> Self { - Self::CommittorSerivceError(Box::new(e)) - } -} - -pub type ScheduledCommitsProcessorResult< - T, - E = ScheduledCommitsProcessorError, -> = Result; diff --git a/magicblock-accounts/src/lib.rs b/magicblock-accounts/src/lib.rs deleted file mode 100644 index c7eac2118..000000000 --- a/magicblock-accounts/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod config; -pub mod errors; -mod traits; - -pub use config::*; -pub use traits::*; diff --git a/magicblock-accounts/src/traits.rs b/magicblock-accounts/src/traits.rs deleted file mode 100644 index c039558be..000000000 --- a/magicblock-accounts/src/traits.rs +++ /dev/null @@ -1,18 +0,0 @@ -use async_trait::async_trait; - -use crate::errors::ScheduledCommitsProcessorResult; - -#[async_trait] -pub trait ScheduledCommitsProcessor: Send + Sync + 'static { - /// Processes all commits that were scheduled and accepted - async fn process(&self) -> ScheduledCommitsProcessorResult<()>; - - /// Returns the number of commits that were scheduled and accepted - fn scheduled_commits_len(&self) -> usize; - - /// Clears all scheduled commits - fn clear_scheduled_commits(&self); - - /// Stop processor - fn stop(&self); -} diff --git a/magicblock-processor/Cargo.toml b/magicblock-processor/Cargo.toml deleted file mode 100644 index 60d25ce7b..000000000 --- a/magicblock-processor/Cargo.toml +++ /dev/null @@ -1,62 +0,0 @@ -[package] -name = "magicblock-processor" -version.workspace = true -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[dependencies] -agave-precompiles = { workspace = true, features = ["agave-unstable-api"] } -agave-syscalls = { workspace = true } -bincode = { workspace = true } -blake3 = { workspace = true } -tracing = { workspace = true } -parking_lot = { workspace = true } -serde = { workspace = true } -tokio = { workspace = true } -tokio-util = { workspace = true } - -magicblock-accounts-db = { workspace = true } -magicblock-core = { workspace = true } -magicblock-ledger = { workspace = true } -magicblock-metrics = { workspace = true } -magicblock-program = { workspace = true } - -solana-account = { workspace = true } -solana-bpf-loader-program = { workspace = true } -solana-compute-budget-program = { workspace = true } -solana-compute-budget = { workspace = true } -solana-compute-budget-instruction = { workspace = true } -solana-feature-gate-interface = { workspace = true, features = ["bincode"] } -solana-feature-set = { workspace = true } -solana-fee-structure = { workspace = true } -solana-instruction = { workspace = true } -solana-loader-v3-interface = { workspace = true } -solana-loader-v4-program = { workspace = true } -solana-program = { workspace = true } -solana-program-runtime = { workspace = true } -solana-precompile-error = { workspace = true } -solana-pubkey = { workspace = true } -solana-sdk-ids = { workspace = true } -solana-svm = { workspace = true } -solana-svm-callback = { workspace = true } -solana-svm-transaction = { workspace = true } -solana-system-program = { workspace = true } -solana-transaction = { workspace = true } -solana-transaction-context = { workspace = true } -solana-transaction-error = { workspace = true } -solana-transaction-status = { workspace = true } -solana-zk-elgamal-proof-program = { workspace = true } - -rustc-hash = { workspace = true } - -[dev-dependencies] -guinea = { workspace = true } -magicblock-magic-program-api = { workspace = true } -solana-compute-budget-interface = { workspace = true } -solana-keypair = { workspace = true } -solana-signature = { workspace = true } -solana-signer = { workspace = true } -test-kit = { workspace = true } diff --git a/magicblock-processor/README.md b/magicblock-processor/README.md deleted file mode 100644 index 32ccfc1fd..000000000 --- a/magicblock-processor/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Magicblock Processor - -Core transaction processing engine for the Magicblock validator. - -## Overview - -This crate is the heart of the validator's execution layer. It provides a high-performance, parallel transaction processing pipeline built around the Solana Virtual Machine (SVM). Its primary responsibility is to take sanitized transactions from the rest of the system, execute or simulate them, commit the resulting state changes, and broadcast the outcomes. - -The design is centered around a central **Scheduler** that distributes work to a pool of isolated **Executor** workers, enabling concurrent transaction processing. - -## Architecture - -``` - ┌─────────────────────────────────────────┐ - │ TransactionScheduler │ - │ ┌───────────────────────────────────┐ │ - Transactions ───▶ │ │ ExecutionCoordinator │ │ - │ │ ┌─────────┐ ┌───────────────┐ │ │ - │ │ │ Locks │ │ Blocked Queues│ │ │ - │ │ │ (u64/ │ │ (BinaryHeap) │ │ │ - │ │ │ account)│ │ │ │ │ - │ │ └─────────┘ └───────────────┘ │ │ - │ └───────────────────────────────────┘ │ - └──────────────┬──────────────────────────┘ - │ - ┌────────────────────┼────────────────────┐ - ▼ ▼ ▼ - ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ - │ Executor 0 │ │ Executor 1 │ │ Executor N │ - │ (thread) │ │ (thread) │ │ (thread) │ - └─────────────┘ └─────────────┘ └─────────────┘ -``` - -## Core Components - -### TransactionScheduler - -The central coordinator and single entry point for all transactions. Runs in a dedicated thread with its own Tokio runtime. - -- Receives transactions from external components via MPSC channel -- Dispatches transactions to available executors -- Handles executor readiness notifications -- Manages slot transitions (sysvar updates, program cache pruning) - -### ExecutionCoordinator - -Manages transaction scheduling and account locking: - -- **Locking**: Bitmask-based read/write locks per account (single `u64` per account) - - Supports up to 63 concurrent executors - - Multiple readers OR single writer per account -- **Queuing**: Blocked transactions queued behind the blocking executor - - Min-heap ordering by transaction ID (FIFO) - - Transactions keep their ID when requeued (older transactions get priority) -- **No fairness blocking**: Transactions only block on actual lock conflicts, not on queued transactions - -### TransactionExecutor - -The workhorse of the system. Each executor runs in its own dedicated OS thread: - -- Loads accounts from AccountsDb -- Executes transactions via SVM -- Commits state changes -- Writes to ledger -- Broadcasts status updates - -## Scheduling Strategy - -The scheduler uses a simple, deadlock-free approach: - -1. **Try all locks**: Attempt to acquire all account locks for a transaction -2. **On conflict**: Release any partial locks, queue behind the blocking executor -3. **On executor ready**: Drain its blocked queue, retry transactions (oldest first) - -This design: -- **Prevents deadlocks**: No circular wait conditions possible -- **Allows livelocks**: A transaction may be repeatedly requeued (acceptable trade-off) -- **Maintains FIFO ordering**: Within each executor's queue via min-heap - -## Transaction Workflow - -1. External component sends `ProcessableTransaction` to scheduler -2. Scheduler assigns to a ready executor -3. Coordinator attempts to acquire account locks: - - **Success**: Transaction sent to executor for processing - - **Conflict**: Transaction queued behind blocking executor, original executor released -4. Executor processes transaction via SVM -5. On completion: commits state, writes to ledger, broadcasts status -6. Executor signals ready, scheduler drains its blocked queue - -## Performance Considerations - -- **Thread isolation**: Scheduler and each executor run in dedicated OS threads -- **Lock efficiency**: Single `u64` bitmask per account (no heap allocations for locks) -- **Shared program cache**: BPF programs compiled once, shared across all executors -- **No contention tracking overhead**: Simplified scheduler removes fairness bookkeeping diff --git a/magicblock-processor/src/builtins.rs b/magicblock-processor/src/builtins.rs deleted file mode 100644 index c7a500e05..000000000 --- a/magicblock-processor/src/builtins.rs +++ /dev/null @@ -1,78 +0,0 @@ -use magicblock_program::magicblock_processor; -use solana_program_runtime::invoke_context::BuiltinFunctionWithContext; -use solana_pubkey::Pubkey; -use solana_sdk_ids::{ - bpf_loader_upgradeable, compute_budget, loader_v4, system_program, - zk_elgamal_proof_program, -}; - -/// Represents a builtin program to be registered with the SVM. -pub struct Builtin { - pub program_id: Pubkey, - pub name: &'static str, - pub entrypoint: BuiltinFunctionWithContext, -} - -/// The set of builtin programs loaded at startup. -/// -/// **Supported:** -/// - `system_program`: Core system account management. -/// - `bpf_loader_upgradeable`: Loads upgradeable BPF programs. -/// - `loader_v4`: Loads V4 programs. -/// - `compute_budget`: Manages transaction compute units. -/// - `zk_elgamal_proof_program`: Verifies native ZK ElGamal proofs. -/// - `magicblock_program`: Validator-specific logic (e.g., account mutations). -/// -/// **Explicitly Unsupported:** -/// - `vote_program`, `stake_program`: Consensus and staking are not supported. -/// - `config_program`: On-chain configuration is not supported. -/// - `bpf_loader_deprecated`, `bpf_loader`: Superseded by `upgradeable`. -/// - `zk_token_proof_program`: Legacy ZK Token proof program is not enabled. -pub static BUILTINS: &[Builtin] = &[ - Builtin { - program_id: system_program::ID, - name: "system_program", - entrypoint: solana_system_program::system_processor::Entrypoint::vm, - }, - Builtin { - program_id: bpf_loader_upgradeable::ID, - name: "solana_bpf_loader_upgradeable_program", - entrypoint: solana_bpf_loader_program::Entrypoint::vm, - }, - Builtin { - program_id: loader_v4::ID, - name: "solana_loader_v4_program", - entrypoint: solana_loader_v4_program::Entrypoint::vm, - }, - Builtin { - program_id: magicblock_program::ID, - name: "magicblock_program", - entrypoint: magicblock_processor::Entrypoint::vm, - }, - Builtin { - program_id: magicblock_program::CRANK_PROGRAM_ID, - name: "magicblock_crank_program", - entrypoint: magicblock_processor::CrankEntrypoint::vm, - }, - Builtin { - program_id: magicblock_program::CALLBACK_PROGRAM_ID, - name: "magicblock_callback_program", - entrypoint: magicblock_processor::CallbackEntrypoint::vm, - }, - Builtin { - program_id: - magicblock_program::POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, - name: "magicblock_post_delegation_action_executor_program", - entrypoint: magicblock_processor::PostDelegationActionEntrypoint::vm, - }, - Builtin { - program_id: compute_budget::ID, - name: "compute_budget_program", - entrypoint: solana_compute_budget_program::Entrypoint::vm, - }, - Builtin { - program_id: zk_elgamal_proof_program::ID, - name: "solana_zk_elgamal_proof_program", - entrypoint: solana_zk_elgamal_proof_program::Entrypoint::vm, - }, -]; diff --git a/magicblock-processor/src/executor/callback.rs b/magicblock-processor/src/executor/callback.rs deleted file mode 100644 index 19cd95e76..000000000 --- a/magicblock-processor/src/executor/callback.rs +++ /dev/null @@ -1,44 +0,0 @@ -use magicblock_accounts_db::traits::AccountsBank; -use solana_account::AccountSharedData; -use solana_precompile_error::PrecompileError; -use solana_pubkey::Pubkey; -use solana_svm::transaction_processing_callback::TransactionProcessingCallback; -use solana_svm_callback::InvokeContextCallback; - -use super::TransactionExecutor; - -impl InvokeContextCallback for TransactionExecutor { - fn is_precompile(&self, program_id: &Pubkey) -> bool { - agave_precompiles::is_precompile(program_id, |feature_id| { - self.feature_set.is_active(feature_id) - }) - } - - fn process_precompile( - &self, - program_id: &Pubkey, - data: &[u8], - instruction_datas: Vec<&[u8]>, - ) -> Result<(), PrecompileError> { - let Some(precompile) = - agave_precompiles::get_precompile(program_id, |feature_id| { - self.feature_set.is_active(feature_id) - }) - else { - return Err(PrecompileError::InvalidPublicKey); - }; - - precompile.verify(data, &instruction_datas, &self.feature_set) - } -} - -impl TransactionProcessingCallback for TransactionExecutor { - fn get_account_shared_data( - &self, - pubkey: &Pubkey, - ) -> Option<(AccountSharedData, u64)> { - self.accountsdb - .get_account(pubkey) - .map(|account| (account, self.accountsdb.slot())) - } -} diff --git a/magicblock-processor/src/executor/mod.rs b/magicblock-processor/src/executor/mod.rs deleted file mode 100644 index 0a2edab46..000000000 --- a/magicblock-processor/src/executor/mod.rs +++ /dev/null @@ -1,301 +0,0 @@ -use std::{ - cmp::Ordering, - collections::BTreeMap, - ops::Deref, - sync::{Arc, RwLock}, -}; - -use magicblock_accounts_db::AccountsDb; -use magicblock_core::{ - link::{ - accounts::AccountUpdateTx, - transactions::{ - ProcessableTransaction, ScheduledTasksTx, - TransactionProcessingMode, TransactionStatusTx, - }, - }, - Slot, -}; -use magicblock_ledger::{LatestBlock, LatestBlockInner, Ledger}; -use solana_feature_set::FeatureSet; -use solana_program::slot_hashes::SlotHashes; -use solana_program_runtime::loaded_programs::{ - BlockRelation, ForkGraph, ProgramCache, ProgramCacheEntry, -}; -use solana_svm::transaction_processor::{ - ExecutionRecordingConfig, TransactionBatchProcessor, - TransactionProcessingConfig, TransactionProcessingEnvironment, -}; -use solana_transaction::sanitized::SanitizedTransaction; -use tokio::{ - runtime::Builder, - sync::{ - broadcast, - mpsc::{Receiver, Sender}, - oneshot, Semaphore, - }, -}; -use tracing::{info, instrument, warn}; - -use crate::{ - builtins::BUILTINS, - scheduler::{locks::ExecutorId, state::TransactionSchedulerState}, -}; - -const BLOCK_HISTORY_SIZE: usize = 32; - -pub(crate) struct IndexedTransaction { - pub(crate) slot: Slot, - pub(crate) index: u32, - pub(crate) txn: ProcessableTransaction, -} - -#[allow(clippy::large_enum_variant)] -pub(crate) enum ExecutorCommand { - Transaction(IndexedTransaction), - Block { - block: LatestBlockInner, - ack: oneshot::Sender<()>, - }, -} - -impl Deref for IndexedTransaction { - type Target = SanitizedTransaction; - fn deref(&self) -> &Self::Target { - &self.txn.transaction - } -} - -/// A dedicated, single-threaded worker responsible for processing transactions. -pub(super) struct TransactionExecutor { - id: ExecutorId, - - // State Handles - accountsdb: Arc, - ledger: Arc, - block: LatestBlock, - block_history: BTreeMap, - execution_permits: Arc, - - // SVM Components - processor: TransactionBatchProcessor, - config: Box>, - environment: TransactionProcessingEnvironment, - feature_set: FeatureSet, - - // Channels - rx: Receiver, - transaction_tx: TransactionStatusTx, - accounts_tx: AccountUpdateTx, - tasks_tx: ScheduledTasksTx, - ready_tx: Sender, -} - -impl TransactionExecutor { - pub(super) fn new( - id: ExecutorId, - state: &TransactionSchedulerState, - rx: Receiver, - ready_tx: Sender, - execution_permits: Arc, - programs_cache: Arc>>, - ) -> Self { - let slot = state.accountsdb.slot(); - let mut processor = - TransactionBatchProcessor::new_uninitialized(slot, 0); - - // Use global program cache to share compilation results across executors - processor.global_program_cache = programs_cache; - processor.environments = state - .environment - .program_runtime_environments_for_execution - .clone(); - - // Enable recording for accurate fee/unit usage tracking - let config = Box::new(TransactionProcessingConfig { - recording_config: ExecutionRecordingConfig::new_single_setting( - true, - ), - limit_to_load_programs: true, - ..Default::default() - }); - - let block = state.ledger.latest_block(); - let initial_block = LatestBlockInner::clone(&*block.load()); - - let mut block_history = BTreeMap::new(); - block_history.insert(initial_block.slot, initial_block.clone()); - - let this = Self { - id, - processor, - accountsdb: state.accountsdb.clone(), - ledger: state.ledger.clone(), - config, - block: block.clone(), - environment: copy_env(&state.environment), - feature_set: state.feature_set.clone(), - execution_permits, - rx, - ready_tx, - accounts_tx: state.account_update_tx.clone(), - transaction_tx: state.transaction_status_tx.clone(), - tasks_tx: state.tasks_tx.clone(), - block_history, - }; - - this.processor.fill_missing_sysvar_cache_entries(&this); - this - } - - pub(super) fn populate_builtins(&self) { - for builtin in BUILTINS { - let entry = ProgramCacheEntry::new_builtin( - 0, - builtin.name.len(), - builtin.entrypoint, - ); - self.processor.add_builtin(builtin.program_id, entry); - } - } - - pub(super) fn spawn(self) { - std::thread::spawn(move || { - let runtime = Builder::new_current_thread() - .thread_name(format!("txn-executor-{}", self.id)) - .build() - .expect("Failed to build executor runtime"); - - runtime.block_on(tokio::task::unconstrained(self.run())); - }); - } - - #[allow(clippy::await_holding_lock)] - #[instrument(skip(self), fields(executor_id = self.id))] - async fn run(mut self) { - let mut block_updated = self.block.subscribe(); - - loop { - tokio::select! { - biased; - result = block_updated.recv() => { - match result { - Ok(latest) => self.register_new_block(latest), - Err(broadcast::error::RecvError::Lagged(_)) => { - self.register_new_block(self.block.load().as_ref().clone()); - } - Err(broadcast::error::RecvError::Closed) => break, - } - } - command = self.rx.recv() => { - let Some(command) = command else { break }; - match command { - ExecutorCommand::Transaction(transaction) => { - if transaction.slot != self.processor.slot { - self.transition_to_slot(transaction.slot); - } - let _permit = self.execution_permits.acquire().await; - match transaction.txn.mode { - TransactionProcessingMode::Execution(_) => { - self.execute(transaction, None); - } - TransactionProcessingMode::Simulation(tx) => { - self.simulate([transaction.txn.transaction], tx); - } - TransactionProcessingMode::Replay(ctx) => { - self.execute(transaction, Some(ctx.persist)); - } - } - let _ = self.ready_tx.try_send(self.id); - } - ExecutorCommand::Block { block, ack } => { - self.apply_block(block); - let _ = ack.send(()); - } - } - } - else => break, - } - } - info!("Executor terminated"); - } - - fn register_new_block(&mut self, block: LatestBlockInner) { - while self.block_history.len() >= BLOCK_HISTORY_SIZE { - self.block_history.pop_first(); - } - self.block_history.insert(block.slot, block); - } - - fn apply_block(&mut self, block: LatestBlockInner) { - let slot = block.clock.slot; - self.register_new_block(block.clone()); - self.environment.blockhash = block.blockhash; - self.processor.slot = slot; - self.set_sysvars(&block); - } - - fn transition_to_slot(&mut self, slot: Slot) { - // transactions execute in the latest finalized block + 1 - let prev_slot = slot.saturating_sub(1); - let Some(block) = self.block_history.get(&prev_slot) else { - // should never happen in practice - warn!(slot, "tried to transition to slot which wasn't registered"); - return; - }; - self.environment.blockhash = block.blockhash; - self.processor.slot = slot; - self.set_sysvars(block); - } - - /// Updates cache and persists slot hashes. - fn set_sysvars(&self, block: &LatestBlockInner) { - let mut cache = self.processor.writable_sysvar_cache().write().unwrap(); - cache.set_sysvar_for_tests(&block.clock); - - if let Ok(hashes) = cache.get_slot_hashes() { - let mut hashes = SlotHashes::new(hashes.slot_hashes()); - hashes.add(block.slot, block.blockhash); - cache.set_sysvar_for_tests(&hashes); - } - } -} - -/// A dummy, low-overhead ForkGraph for a linear (forkless) chain. -#[derive(Default)] -pub(super) struct SimpleForkGraph; - -impl ForkGraph for SimpleForkGraph { - fn relationship(&self, a: u64, b: u64) -> BlockRelation { - match a.cmp(&b) { - Ordering::Less => BlockRelation::Ancestor, - Ordering::Greater => BlockRelation::Descendant, - Ordering::Equal => BlockRelation::Equal, - } - } -} - -// SAFETY: Required for SVM internals (`dyn SVMRentCollector` interactions). -// Concrete types used here are Send. -unsafe impl Send for TransactionExecutor {} - -fn copy_env( - env: &TransactionProcessingEnvironment, -) -> TransactionProcessingEnvironment { - TransactionProcessingEnvironment { - blockhash: env.blockhash, - blockhash_lamports_per_signature: env.blockhash_lamports_per_signature, - epoch_total_stake: env.epoch_total_stake, - feature_set: env.feature_set, - program_runtime_environments_for_execution: env - .program_runtime_environments_for_execution - .clone(), - program_runtime_environments_for_deployment: env - .program_runtime_environments_for_deployment - .clone(), - rent: env.rent.clone(), - } -} - -mod callback; -mod processing; diff --git a/magicblock-processor/src/executor/processing.rs b/magicblock-processor/src/executor/processing.rs deleted file mode 100644 index a75315637..000000000 --- a/magicblock-processor/src/executor/processing.rs +++ /dev/null @@ -1,511 +0,0 @@ -use std::sync::Arc; - -use magicblock_accounts_db::AccountsDbResult; -use magicblock_core::{ - link::{ - accounts::{AccountWithSlot, LockedAccount}, - transactions::{ - ProcessableTransaction, TransactionProcessingMode, - TransactionSimulationResult, TransactionStatus, - TxnSimulationResultTx, - }, - }, - tls::ExecutionTlsStash, -}; -use magicblock_metrics::metrics::{ - FAILED_TRANSACTIONS_COUNT, TRANSACTION_COUNT, -}; -use solana_account::AccountSharedData; -use solana_compute_budget_instruction::instructions_processor::process_compute_budget_instructions; -use solana_feature_set::raise_cpi_nesting_limit_to_8; -use solana_fee_structure::FeeDetails; -use solana_program_runtime::execution_budget::SVMTransactionExecutionAndFeeBudgetLimits; -use solana_pubkey::Pubkey; -use solana_svm::{ - account_loader::CheckedTransactionDetails, - rollback_accounts::RollbackAccounts, - transaction_balances::BalanceCollector, - transaction_processing_result::{ - ProcessedTransaction, TransactionProcessingResult, - }, -}; -use solana_svm_transaction::svm_message::SVMStaticMessage; -use solana_transaction::sanitized::SanitizedTransaction; -use solana_transaction_error::{TransactionError, TransactionResult}; -use solana_transaction_status::{ - map_inner_instructions, TransactionStatusMeta, -}; -use tracing::*; - -use crate::executor::IndexedTransaction; - -impl super::TransactionExecutor { - /// Executes a transaction and conditionally commits its results. - /// - /// # Arguments - /// * `transaction` - The transaction to execute - /// * `tx` - Channel to send the execution result (None for replay) - /// * `persist` - Controls persistence behavior: - /// - `None`: Execution mode - notify subscribers, record to ledger, process tasks - /// - `Some(true)`: Replay with persist - record to ledger, no notifications - /// - `Some(false)`: Replay without persist - no side effects - pub(super) fn execute( - &self, - mut transaction: IndexedTransaction, - persist: Option, - ) { - TRANSACTION_COUNT.inc(); - let (result, balances) = { - let txn = [transaction.txn.transaction]; - let result = self.process(&txn); - let [txn] = txn; - transaction.txn.transaction = txn; - result - }; - - // 1. Handle Loading/Processing Failures - let processed = match result { - Ok(processed) => processed, - Err(err) => { - return self.handle_failure(transaction, err, None); - } - }; - - // 2. Commit Account State (DB Update) - // Note: Failed transactions still pay fees, so we attempt commit even on execution failure. - let fee_payer = *transaction.fee_payer(); - // Only send account updates for Execution mode (persist is None). - // Wrap the causing transaction in an `Arc` so it can be shared cheaply - // across every account it mutated and across the notification channel. - let notify = persist.is_none(); - let txn = notify.then(|| Arc::new(transaction.txn.transaction.clone())); - if let Err(err) = self.commit_accounts(fee_payer, &processed, txn) { - return self.handle_failure( - transaction, - TransactionError::CommitCancelled, - Some(vec![err.to_string()]), - ); - } - - let status = processed.status(); - - // 3. Post-Processing (Tasks & Ledger) - // Only process scheduled tasks for successful transactions in Execution mode - if status.is_ok() && persist.is_none() { - self.process_scheduled_tasks(); - } - let tx = if let TransactionProcessingMode::Execution(ref mut tx) = - transaction.txn.mode - { - tx.take() - } else { - None - }; - // Record to ledger for Execution mode (persist is None) or Replay with persist=true - if persist.unwrap_or(true) { - if let Err(err) = - self.record_transaction(transaction, processed, balances) - { - error!(error = ?err, "Failed to record transaction to ledger"); - } - } - - ExecutionTlsStash::clear(); - if let Some(tx) = tx { - let _ = tx.send(status); - } - } - - /// Executes a transaction in simulation mode (no state persistence). - pub(super) fn simulate( - &self, - transaction: [SanitizedTransaction; 1], - tx: TxnSimulationResultTx, - ) { - let number_of_accounts = transaction[0].message().account_keys().len(); - let (result, _) = self.process(&transaction); - let simulation_result = match result { - Ok(processed) => { - let status = processed.status(); - let units_consumed = processed.executed_units(); - let ( - logs, - post_simulation_accounts, - return_data, - inner_instructions, - ) = match processed { - ProcessedTransaction::Executed(executed) => { - let execution_details = executed.execution_details; - let post_simulation_accounts = executed - .loaded_transaction - .accounts - .into_iter() - .take(number_of_accounts) - .collect(); - ( - execution_details.log_messages, - post_simulation_accounts, - execution_details.return_data, - execution_details.inner_instructions, - ) - } - ProcessedTransaction::FeesOnly(_) => { - (None, vec![], None, None) - } - }; - TransactionSimulationResult { - result: status, - units_consumed, - logs, - post_simulation_accounts, - return_data, - inner_instructions, - } - } - Err(error) => TransactionSimulationResult { - result: Err(error), - units_consumed: 0, - logs: Default::default(), - post_simulation_accounts: vec![], - return_data: None, - inner_instructions: None, - }, - }; - - ExecutionTlsStash::clear(); - let _ = tx.send(simulation_result); - } - - /// Wraps the SVM load_and_execute logic. - fn process( - &self, - txn: &[SanitizedTransaction; 1], - ) -> (TransactionProcessingResult, Option) { - let limits = match self.compute_budget_limits(&txn[0]) { - Ok(limits) => limits, - Err(err) => return (Err(err), None), - }; - let checked = CheckedTransactionDetails::new(None, limits); - let mut output = - self.processor.load_and_execute_sanitized_transactions( - self, - txn, - vec![Ok(checked); 1], - &self.environment, - &self.config, - ); - - let mut result = output - .processing_results - .pop() - .expect("single transaction result is guaranteed"); - - if let Ok(ref mut processed) = result { - self.verify_account_states(processed); - } - - (result, output.balance_collector) - } - - /// Common handler for transaction failures (load error or commit error). - fn handle_failure( - &self, - mut txn: IndexedTransaction, - err: TransactionError, - logs: Option>, - ) { - FAILED_TRANSACTIONS_COUNT.inc(); - - // Even on failure, ensure stash is clear (though likely empty if load failed). - ExecutionTlsStash::clear(); - - if let TransactionProcessingMode::Execution(ref mut tx) = txn.txn.mode { - if let Some(tx) = tx.take() { - let _ = tx.send(Err(err.clone())); - } - } - self.record_failure(txn, Err(err), logs); - } - - fn process_scheduled_tasks(&self) { - while let Some(task) = ExecutionTlsStash::next_task() { - if let Err(e) = self.tasks_tx.send(task) { - error!(error = ?e, "Scheduled tasks service disconnected"); - } - } - } - - /// Writes a fully processed transaction to the Ledger. - fn record_transaction( - &self, - txn: IndexedTransaction, - result: ProcessedTransaction, - balances: Option, - ) -> Result<(), Box> { - let (pre_balances, post_balances) = - transaction_balances(&txn, balances); - let meta = match result { - ProcessedTransaction::Executed(executed) => TransactionStatusMeta { - fee: executed.loaded_transaction.fee_details.total_fee(), - compute_units_consumed: Some( - executed.execution_details.executed_units, - ), - status: executed.execution_details.status, - pre_balances, - post_balances, - log_messages: executed.execution_details.log_messages, - loaded_addresses: txn.get_loaded_addresses(), - return_data: executed.execution_details.return_data, - inner_instructions: executed - .execution_details - .inner_instructions - .map(map_inner_instructions) - .map(|i| i.collect()), - ..Default::default() - }, - ProcessedTransaction::FeesOnly(fo) => TransactionStatusMeta { - fee: fo.fee_details.total_fee(), - status: Err(fo.load_error), - pre_balances, - post_balances, - loaded_addresses: txn.get_loaded_addresses(), - ..Default::default() - }, - }; - - self.write_to_ledger(txn, meta) - } - - /// Writes a failed transaction (load or commit error) to the Ledger. - fn record_failure( - &self, - txn: IndexedTransaction, - status: TransactionResult<()>, - logs: Option>, - ) { - let count = txn.message().account_keys().len(); - let meta = TransactionStatusMeta { - status, - pre_balances: vec![0; count], - post_balances: vec![0; count], - log_messages: logs, - ..Default::default() - }; - if let Err(err) = self.write_to_ledger(txn, meta) { - error!(error = ?err, "Failed to record failed transaction to ledger"); - } - } - - fn write_to_ledger( - &self, - txn: IndexedTransaction, - meta: TransactionStatusMeta, - ) -> Result<(), Box> { - let signature = *txn.signature(); - let slot = txn.slot; - let index = txn.index; - - let ProcessableTransaction { - transaction, - encoded, - .. - } = txn.txn; - - // Use pre-encoded bytes or serialize on the spot - let encoded = match encoded { - Some(bytes) => bytes, - None => { - let versioned = transaction.to_versioned_transaction(); - bincode::serialize(&versioned) - .map_err(|e| Box::new(e) as Box)? - .into() - } - }; - - let tx_account_locks = transaction.get_account_locks_unchecked(); - - let result = self.ledger.write_transaction( - signature, - slot, - index, - tx_account_locks.writable, - tx_account_locks.readonly, - &encoded, - meta.clone(), - ); - if let Err(error) = result { - error!(error = ?error, "Failed to commit transaction to ledger"); - return Err(error.into()); - } - - let status = TransactionStatus { - slot, - index, - txn: transaction, - meta, - }; - - // Notify listeners - let _ = self.transaction_tx.send(status); - Ok(()) - } - - /// Persists account changes to AccountsDb and notifies listeners. - fn commit_accounts( - &self, - fee_payer: Pubkey, - result: &ProcessedTransaction, - txn: Option>, - ) -> AccountsDbResult<()> { - let succeeded = result.status().is_ok(); - let accounts = match result { - ProcessedTransaction::Executed(executed) => { - if succeeded && !executed.programs_modified_by_tx.is_empty() { - self.processor.global_program_cache.write().unwrap().merge( - &self.processor.environments, - &executed.programs_modified_by_tx, - ); - } - - if !succeeded { - &executed.loaded_transaction.accounts[..1] - } else { - &executed.loaded_transaction.accounts - } - } - ProcessedTransaction::FeesOnly(fo) => { - if let RollbackAccounts::FeePayerOnly { fee_payer: account } = - &fo.rollback_accounts - { - return self.insert_and_notify( - &[(fee_payer, account.1.clone())], - txn, - false, - ); - } - return Ok(()); - } - }; - - let privileged = accounts - .first() - .map(|(_, acc)| acc.privileged()) - .unwrap_or(false); - - self.insert_and_notify(accounts, txn, privileged) - } - - fn insert_and_notify( - &self, - accounts: &[(Pubkey, AccountSharedData)], - txn: Option>, - privileged: bool, - ) -> AccountsDbResult<()> { - // Filter: Persist only dirty or privileged accounts - let to_commit = accounts - .iter() - .filter(|(_, acc)| privileged || acc.is_dirty()); - - self.accountsdb.insert_batch(to_commit)?; - - // `txn` is `Some` only in Execution mode, where subscribers are notified. - let Some(txn) = txn else { - return Ok(()); - }; - - for (pubkey, account) in accounts { - let update = AccountWithSlot { - slot: self.processor.slot, - account: LockedAccount::new(*pubkey, account.clone()), - transaction: txn.clone(), - }; - let _ = self.accounts_tx.send(update); - } - Ok(()) - } - - fn verify_account_states(&self, processed: &mut ProcessedTransaction) { - let ProcessedTransaction::Executed(executed) = processed else { - return; - }; - let txn = &executed.loaded_transaction; - let Some((_, fee_payer_acc)) = txn.accounts.first() else { - return; - }; - - // Privileged fee payers bypass all checks - if fee_payer_acc.privileged() { - return; - } - - let logs = executed - .execution_details - .log_messages - .get_or_insert_default(); - - // Confined Account Integrity Check - // Confined accounts must not have their lamport balance changed. - for (pubkey, acc) in &txn.accounts { - if !acc.confined() { - continue; - } - if acc.lamports_changed() { - executed.execution_details.status = - Err(TransactionError::UnbalancedTransaction); - logs.push(format!( - "Confined account {pubkey} has been illegally modified" - )); - break; - } - } - } - - fn compute_budget_limits( - &self, - txn: &SanitizedTransaction, - ) -> TransactionResult { - let limits = process_compute_budget_instructions( - txn.program_instructions_iter(), - &self.feature_set, - )?; - let signature_fee = signature_fee( - txn, - self.environment.blockhash_lamports_per_signature, - ); - let fee_details = FeeDetails::new(signature_fee, 0); - let raise_cpi_limit = self - .feature_set - .is_active(&raise_cpi_nesting_limit_to_8::id()); - - Ok(limits.get_compute_budget_and_limits( - limits.loaded_accounts_bytes, - fee_details, - raise_cpi_limit, - )) - } -} - -fn transaction_balances( - txn: &IndexedTransaction, - balances: Option, -) -> (Vec, Vec) { - let count = txn.message().account_keys().len(); - let Some(balances) = balances else { - return (vec![0; count], vec![0; count]); - }; - - let (mut pre, mut post, _, _) = balances.into_vecs(); - ( - pre.pop().unwrap_or_else(|| vec![0; count]), - post.pop().unwrap_or_else(|| vec![0; count]), - ) -} - -fn signature_fee( - txn: &SanitizedTransaction, - lamports_per_signature: u64, -) -> u64 { - txn.message() - .num_total_signatures() - .saturating_mul(lamports_per_signature) -} diff --git a/magicblock-processor/src/lib.rs b/magicblock-processor/src/lib.rs deleted file mode 100644 index 03a2c145c..000000000 --- a/magicblock-processor/src/lib.rs +++ /dev/null @@ -1,273 +0,0 @@ -#![allow(deprecated)] - -use std::sync::Arc; - -use agave_syscalls::{ - create_program_runtime_environment_v1, - create_program_runtime_environment_v2, -}; -use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; -use magicblock_core::link::blocks::BlockHash; -use solana_account::{AccountSharedData, WritableAccount}; -use solana_feature_gate_interface::state::Feature; -use solana_feature_set::{ - curve25519_restrict_msm_length, curve25519_syscall_enabled, - disable_rent_fees_collection, ed25519_program_enabled, - enable_poseidon_syscall, enable_sbpf_v3_deployment_and_execution, - enable_secp256r1_precompile, enable_transaction_loading_failure_fees, - get_sysvar_syscall_enabled, secp256k1_program_enabled, FeatureSet, -}; -use solana_program::{pubkey::Pubkey, rent::Rent}; -use solana_program_runtime::{ - execution_budget::SVMTransactionExecutionBudget, - loaded_programs::ProgramRuntimeEnvironments, - solana_sbpf::program::BuiltinProgram, -}; -use solana_sdk_ids::{ - ed25519_program, native_loader, secp256k1_program, secp256r1_program, -}; -use solana_svm::transaction_processor::TransactionProcessingEnvironment; -use tracing::error; - -/// Transaction processing environment plus the exact active Agave feature set. -pub struct SvmEnv { - pub environment: TransactionProcessingEnvironment, - pub feature_set: FeatureSet, -} - -/// Initialize an SVM environment for transaction processing and retain the active feature set. -pub fn build_svm_env( - accountsdb: &AccountsDb, - blockhash: BlockHash, - fee_per_signature: u64, -) -> SvmEnv { - let mut feature_set = FeatureSet::default(); - - // Activate features relevant to ER operations: - // - Rent exemption for all regular accounts (disable collection). - // - Curve25519 syscalls. - // - Poseidon syscall. - // - Fees for failed transaction loading (DoS mitigation). - // - sBPF v3 deployment/execution for cloned Devnet programs. - for id in [ - disable_rent_fees_collection::ID, - curve25519_syscall_enabled::ID, - curve25519_restrict_msm_length::ID, - enable_poseidon_syscall::ID, - enable_sbpf_v3_deployment_and_execution::ID, - enable_transaction_loading_failure_fees::ID, - get_sysvar_syscall_enabled::ID, - ed25519_program_enabled::ID, - secp256k1_program_enabled::ID, - enable_secp256r1_precompile::ID, - ] { - feature_set.activate(&id, 0); - } - - // Persist active features to AccountsDb if they don't already exist. - // This ensures programs checking for these features find them. - for (id, &slot) in feature_set.active() { - ensure_feature_account(accountsdb, id, Some(slot)); - } - - ensure_precompile_account(accountsdb, &ed25519_program::ID); - ensure_precompile_account(accountsdb, &secp256k1_program::ID); - ensure_precompile_account(accountsdb, &secp256r1_program::ID); - ensure_builtin_accounts(accountsdb); - - let budget = SVMTransactionExecutionBudget::new_with_defaults(false); - let runtime_features = feature_set.runtime_features(); - let runtime_v1 = create_program_runtime_environment_v1( - &runtime_features, - &budget, - false, - false, - ) - .map(Into::into) - .unwrap_or_else(|_| { - Arc::new(BuiltinProgram::new_loader(Default::default())) - }); - let runtime_v2 = - create_program_runtime_environment_v2(&budget, false).into(); - let runtime_environments = ProgramRuntimeEnvironments { - program_runtime_v1: runtime_v1, - program_runtime_v2: runtime_v2, - }; - - let environment = TransactionProcessingEnvironment { - blockhash, - blockhash_lamports_per_signature: fee_per_signature, - feature_set: runtime_features, - epoch_total_stake: 0, - program_runtime_environments_for_execution: runtime_environments - .clone(), - program_runtime_environments_for_deployment: runtime_environments, - rent: Rent::default(), - }; - - SvmEnv { - environment, - feature_set, - } -} - -#[cfg(test)] -mod tests { - use solana_program_runtime::{ - invoke_context::InvokeContext, - solana_sbpf::{ebpf, elf::Executable, program::SBPFVersion}, - }; - - use super::*; - - #[test] - fn loads_stripped_sbpf_v3_elf() { - let bytes = minimal_stripped_sbpf_v3_elf(); - let runtime = BuiltinProgram::new_loader(Default::default()); - - let executable = - Executable::::load(&bytes, Arc::new(runtime)) - .unwrap(); - assert_eq!(executable.get_sbpf_version(), SBPFVersion::V3); - } - - fn minimal_stripped_sbpf_v3_elf() -> Vec { - const ELF_HEADER_LEN: usize = 64; - const PROGRAM_HEADER_LEN: usize = 56; - const ET_DYN: u16 = 3; - const EM_BPF: u16 = 247; - const EV_CURRENT: u32 = 1; - const PT_LOAD: u32 = 1; - const PF_X: u32 = 1; - const PF_R: u32 = 4; - - let rodata_offset = ELF_HEADER_LEN + PROGRAM_HEADER_LEN * 3; - let rodata_len = ebpf::INSN_SIZE as u64; - let text_offset = rodata_offset + rodata_len as usize; - let text_len = ebpf::INSN_SIZE as u64; - let mut bytes = Vec::with_capacity(text_offset + text_len as usize); - - bytes.extend_from_slice(&[ - 0x7f, b'E', b'L', b'F', 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - push_u16(&mut bytes, ET_DYN); - push_u16(&mut bytes, EM_BPF); - push_u32(&mut bytes, EV_CURRENT); - push_u64(&mut bytes, ebpf::MM_BYTECODE_START); - push_u64(&mut bytes, ELF_HEADER_LEN as u64); - push_u64(&mut bytes, 0); - push_u32(&mut bytes, 3); - push_u16(&mut bytes, ELF_HEADER_LEN as u16); - push_u16(&mut bytes, PROGRAM_HEADER_LEN as u16); - push_u16(&mut bytes, 3); - push_u16(&mut bytes, 0); - push_u16(&mut bytes, 0); - push_u16(&mut bytes, 0); - assert_eq!(bytes.len(), ELF_HEADER_LEN); - - push_program_header( - &mut bytes, - PT_LOAD, - PF_R, - rodata_offset as u64, - ebpf::MM_RODATA_START, - rodata_len, - ); - push_program_header( - &mut bytes, - PT_LOAD, - PF_X, - text_offset as u64, - ebpf::MM_BYTECODE_START, - text_len, - ); - push_program_header(&mut bytes, 0, 0, 0, 0, 0); - assert_eq!(bytes.len(), rodata_offset); - - bytes.extend_from_slice(b"fake-ro\0"); - assert_eq!(bytes.len(), text_offset); - bytes.extend_from_slice(&[ebpf::EXIT, 0, 0, 0, 0, 0, 0, 0]); - bytes - } - - fn push_program_header( - bytes: &mut Vec, - p_type: u32, - p_flags: u32, - p_offset: u64, - p_vaddr: u64, - p_filesz: u64, - ) { - push_u32(bytes, p_type); - push_u32(bytes, p_flags); - push_u64(bytes, p_offset); - push_u64(bytes, p_vaddr); - push_u64(bytes, p_vaddr); - push_u64(bytes, p_filesz); - push_u64(bytes, p_filesz); - push_u64(bytes, ebpf::INSN_SIZE as u64); - } - - fn push_u16(bytes: &mut Vec, value: u16) { - bytes.extend_from_slice(&value.to_le_bytes()); - } - - fn push_u32(bytes: &mut Vec, value: u32) { - bytes.extend_from_slice(&value.to_le_bytes()); - } - - fn push_u64(bytes: &mut Vec, value: u64) { - bytes.extend_from_slice(&value.to_le_bytes()); - } -} - -/// Helper to create and insert a feature account if it is missing. -fn ensure_feature_account( - accountsdb: &AccountsDb, - id: &Pubkey, - activated_at: Option, -) { - if accountsdb.get_account(id).is_some() { - return; - } - - let feature = Feature { activated_at }; - let account = solana_feature_gate_interface::create_account(&feature, 1); - let _ = accountsdb.insert_account(id, &account); -} - -fn ensure_precompile_account(accountsdb: &AccountsDb, id: &Pubkey) { - if accountsdb.get_account(id).is_some() { - return; - } - - let mut account = AccountSharedData::new(1, 0, &native_loader::ID); - account.set_executable(true); - if let Err(e) = accountsdb.insert_account(id, &account) { - error!("Failed to insert precompile account {}: {:?}", id, e); - } -} - -fn ensure_builtin_accounts(accountsdb: &AccountsDb) { - for builtin in builtins::BUILTINS { - if accountsdb.get_account(&builtin.program_id).is_some() { - continue; - } - - let mut account = AccountSharedData::new(1, 0, &native_loader::ID); - account.set_executable(true); - if let Err(err) = - accountsdb.insert_account(&builtin.program_id, &account) - { - error!( - "Failed to insert builtin account {}: {:?}", - builtin.program_id, err - ); - } - } -} - -mod builtins; -mod executor; -pub mod loader; -pub mod scheduler; diff --git a/magicblock-processor/src/loader.rs b/magicblock-processor/src/loader.rs deleted file mode 100644 index 3a096cf2e..000000000 --- a/magicblock-processor/src/loader.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::{error::Error, path::PathBuf}; - -use magicblock_accounts_db::AccountsDb; -use solana_account::{AccountSharedData, WritableAccount}; -use solana_loader_v3_interface::state::UpgradeableLoaderState; -use solana_program::rent::Rent; -use solana_pubkey::Pubkey; -use solana_sdk_ids::bpf_loader_upgradeable; -use tracing::debug; - -const UPGRADEABLE_LOADER_ID: Pubkey = bpf_loader_upgradeable::ID; - -/// Loads BPF upgradeable programs from file paths directly into the `AccountsDb`. -pub fn load_upgradeable_programs( - accountsdb: &AccountsDb, - progs: &[(Pubkey, PathBuf)], -) -> Result<(), Box> { - debug!(programs = ?progs, "Loading programs"); - for (id, path) in progs { - let elf = std::fs::read(path).map_err(|err| { - format!("Failed to read program file for {id} at {path:?}: {err}") - })?; - add_program(accountsdb, id, &elf)?; - } - Ok(()) -} - -/// Creates and stores the accounts for a BPF upgradeable program. -fn add_program( - accountsdb: &AccountsDb, - id: &Pubkey, - elf: &[u8], -) -> Result<(), Box> { - let rent = Rent::default(); - let min_balance = |len| rent.minimum_balance(len).max(1); - - // 1. Create and store the ProgramData account (header + ELF) - let (programdata_address, _) = - Pubkey::find_program_address(&[id.as_ref()], &UPGRADEABLE_LOADER_ID); - - let mut data = bincode::serialize(&UpgradeableLoaderState::ProgramData { - slot: 0, - upgrade_authority_address: Some(Pubkey::default()), - })?; - data.resize(UpgradeableLoaderState::size_of_programdata_metadata(), 0); - data.extend_from_slice(elf); - - let mut account = AccountSharedData::new( - min_balance(data.len()), - 0, - &UPGRADEABLE_LOADER_ID, - ); - account.set_data(data); - let _ = accountsdb.insert_account(&programdata_address, &account); - - // 2. Create and store the executable Program account - let data = bincode::serialize(&UpgradeableLoaderState::Program { - programdata_address, - })?; - - let mut account = AccountSharedData::new( - min_balance(data.len()), - 0, - &UPGRADEABLE_LOADER_ID, - ); - account.set_data(data); - account.set_executable(true); - let _ = accountsdb.insert_account(id, &account); - - Ok(()) -} diff --git a/magicblock-processor/src/scheduler/coordinator.rs b/magicblock-processor/src/scheduler/coordinator.rs deleted file mode 100644 index 5958db721..000000000 --- a/magicblock-processor/src/scheduler/coordinator.rs +++ /dev/null @@ -1,388 +0,0 @@ -//! Transaction scheduling coordination across multiple executors. -//! -//! # Architecture -//! -//! - **Lock acquisition**: All-or-nothing attempt on all transaction accounts -//! - **Lock contention**: Failed transactions queued behind blocking executor -//! - **FIFO ordering**: Transactions processed in ID order within each blocked queue -//! -//! # Coordination Modes -//! -//! The scheduler operates in one of two modes: -//! - **Primary**: For validators accepting client transactions. Allows concurrent -//! execution of independent transactions, with a limit on blocked transactions. -//! - **Replica**: For validators replaying transactions from a primary. Enforces -//! strict ordering by allowing only one pending blocked transaction at a time. -//! -//! # Key Types -//! -//! - `ExecutorId`: Unique identifier for each executor worker (0..N) -//! - `TransactionId`: Monotonic ID for FIFO queue ordering -//! - `BinaryHeap`: Min-heap ordered by transaction ID - -use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc}; - -use magicblock_core::{ - coordination_mode, - link::transactions::{ProcessableTransaction, TransactionProcessingMode}, -}; -use magicblock_metrics::metrics::MAX_LOCK_CONTENTION_QUEUE_SIZE; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; -use tracing::{error, warn}; - -use super::locks::{ - next_transaction_id, ExecutorId, LocksCache, TransactionId, -}; -use crate::scheduler::locks::RcLock; - -/// Maximum blocked transactions per executor before rejecting new ones (Primary mode). -const BLOCKED_TXN_MULTIPLIER: usize = 2; - -/// Coordinates transaction scheduling across multiple executor workers. -/// -/// # Scheduling Flow -/// -/// 1. Acquire idle executor from pool -/// 2. Try to lock all transaction accounts (all-or-nothing) -/// 3. On success: send to executor; on failure: queue and return executor to pool -/// -/// # Unlock Flow -/// -/// When executor finishes, unlock accounts and retry blocked transactions in FIFO order. -pub(super) struct ExecutionCoordinator { - /// Blocked transactions per executor, ordered by transaction ID (FIFO) - blocked_transactions: Vec>, - /// Pool of idle executors available for work - ready_executors: Vec, - /// Account locks currently held by each running executor (for unlocking) - acquired_locks: Vec>, - /// Global account lock registry - locks: LocksCache, - /// Current coordination mode (Primary or Replica) - mode: CoordinationMode, - /// Semaphore for counting the number of busy executors - /// (i.e. ones executing transaction at any given moment) - execution_permits: Arc, -} - -/// Coordination mode determining how transactions are scheduled. -/// -/// Valid Transitions: -/// StartingUp → Primary [Standalone validators, after replay] -/// StartingUp → Replica [Replica validators, after replay] -/// Primary → Replica [Failover: primary to replica] -/// Replica → Primary [Failover: replica takeover becomes primary] -pub(super) enum CoordinationMode { - /// Ledger replay phase. No validator signer required, no side effects, - /// strict ordering (same constraints as Replica). - StartingUp(ReplicaMode), - /// Primary mode: accepts client transactions, allows concurrent execution. - Primary(PrimaryMode), - /// Replica mode: replays transactions, enforces strict ordering. - Replica(ReplicaMode), -} - -/// State for Primary mode scheduling. -pub(super) struct PrimaryMode { - /// Current number of blocked transactions. - blocked_txn_count: usize, - /// Maximum allowed blocked transactions before rejecting new ones. - max_blocked_txn: usize, -} - -/// State for Replica mode scheduling. -/// -/// In Replica mode, only one transaction can be pending (blocked) at a time. -/// This ensures strict ordering during replay: when a transaction is blocked, -/// new transactions wait in the channel until it completes. -#[derive(Default)] -pub(super) struct ReplicaMode { - /// ID of the currently pending blocked transaction, if any. - pending: Option, -} - -impl CoordinationMode { - /// Returns true if the scheduler is ready to accept new transactions. - fn is_ready(&self) -> bool { - match self { - Self::StartingUp(m) => m.pending.is_none(), - // Primary: ready if we haven't hit the blocked transaction limit - Self::Primary(m) => m.blocked_txn_count < m.max_blocked_txn, - // Replica: ready only if no transaction is pending (strict ordering) - Self::Replica(m) => m.pending.is_none(), - } - } -} - -impl ExecutionCoordinator { - /// Creates a new coordinator starting in StartingUp mode. - /// - /// Starts in StartingUp mode to allow ledger replay before switching to Primary - /// or Replica. - pub(super) fn new(count: usize, execution_permits: Arc) -> Self { - Self { - blocked_transactions: (0..count) - .map(|_| BinaryHeap::new()) - .collect(), - acquired_locks: (0..count).map(|_| Vec::new()).collect(), - ready_executors: (0..count as u32).collect(), - locks: LocksCache::default(), - mode: CoordinationMode::StartingUp(ReplicaMode::default()), - execution_permits, - } - } - - #[inline] - pub(super) fn is_ready(&self) -> bool { - !self.ready_executors.is_empty() && self.mode.is_ready() - } - - #[inline] - pub(super) fn get_ready_executor(&mut self) -> Option { - self.ready_executors.pop() - } - - #[inline] - pub(super) fn release_executor(&mut self, executor: ExecutorId) { - self.ready_executors.push(executor) - } - - pub(super) fn try_schedule( - &mut self, - executor: ExecutorId, - txn: TransactionWithId, - ) -> Result { - match self.try_acquire_locks(executor, &txn.txn) { - Ok(()) => Ok(txn.txn), - Err(blocker) => { - self.release_executor(executor); - self.queue_transaction(blocker, txn); - Err(blocker) - } - } - } - - pub(super) fn try_acquire_locks( - &mut self, - executor: ExecutorId, - txn: &ProcessableTransaction, - ) -> Result<(), ExecutorId> { - let message = txn.transaction.message(); - let accounts = message.account_keys(); - - for (i, &acc) in accounts.iter().enumerate() { - let lock = self.locks.entry(acc).or_default().clone(); - let result = if message.is_writable(i) { - lock.borrow_mut().write(executor) - } else { - lock.borrow_mut().read(executor) - }; - - if let Err(blocker) = result { - // All-or-nothing: release any locks we acquired and fail - self.unlock_accounts(executor); - return Err(blocker); - } - self.acquired_locks[executor as usize].push(lock); - } - Ok(()) - } - - pub(super) fn unlock_accounts(&mut self, executor: ExecutorId) { - for lock in self.acquired_locks[executor as usize].drain(..) { - lock.borrow_mut().unlock(executor); - } - } - - fn queue_transaction( - &mut self, - blocker: ExecutorId, - txn: TransactionWithId, - ) { - match &mut self.mode { - CoordinationMode::StartingUp(r) | CoordinationMode::Replica(r) => { - // In StartingUp/Replica mode, track the pending transaction ID. - // The debug_assert ensures we don't queue when one is already pending - // (enforced by is_ready() returning false when pending.is_some()). - debug_assert!(r.pending.is_none()); - if r.pending.replace(txn.id).is_some() { - error!("Invariant violation: replaced pending transaction"); - } - } - CoordinationMode::Primary(p) => { - p.blocked_txn_count += 1; - } - } - let heap = &mut self.blocked_transactions[blocker as usize]; - heap.push(txn); - MAX_LOCK_CONTENTION_QUEUE_SIZE - .set(MAX_LOCK_CONTENTION_QUEUE_SIZE.get().max(heap.len() as i64)); - } - - pub(super) fn next_blocked_transaction( - &mut self, - executor: ExecutorId, - ) -> Option { - let txn = self.blocked_transactions[executor as usize].pop(); - match &mut self.mode { - CoordinationMode::StartingUp(r) | CoordinationMode::Replica(r) => { - // Clear pending if this was the pending transaction. - if r.pending == txn.as_ref().map(|txn| txn.id) { - r.pending.take(); - } - } - CoordinationMode::Primary(p) => { - p.blocked_txn_count = - p.blocked_txn_count.saturating_sub(txn.is_some() as usize); - } - } - txn - } - - /// Switches to Primary mode from StartingUp or Replica. - /// - /// Called after ledger replay completes on Primary validators, or during - /// failover when a replica takes over as primary. - /// No-op if already in Primary mode. - pub(super) fn switch_to_primary_mode(&mut self) { - if let CoordinationMode::Primary(_) = self.mode { - warn!("Tried to switch to primary mode more than once"); - return; - } - let mode = PrimaryMode { - blocked_txn_count: 0, - max_blocked_txn: self.blocked_transactions.len() - * BLOCKED_TXN_MULTIPLIER, - }; - self.mode = CoordinationMode::Primary(mode); - } - - /// Switches to Replica mode from StartingUp or Primary. - /// - /// Called after ledger replay completes on Replica validators, or during - /// failover when a primary needs to step down to replica. - /// No-op if already in Replica mode. - pub(super) fn switch_to_replica_mode(&mut self) { - match &self.mode { - CoordinationMode::Replica(_) => { - warn!("Tried to switch to replica mode more than once"); - } - CoordinationMode::Primary(_) => { - self.mode = - CoordinationMode::Replica(ReplicaMode { pending: None }); - } - CoordinationMode::StartingUp(r) => { - self.mode = CoordinationMode::Replica(ReplicaMode { - pending: r.pending, - }); - } - } - } - - /// Transitions to Primary mode (scheduler + global state). - /// - /// Called when ledger replay completes on Standalone validators, or during - /// failover when a replica takes over as primary. - /// Coordinates both the local scheduler and global validator state. - pub(super) fn switch_to_primary_mode_globally(&mut self) { - self.switch_to_primary_mode(); - coordination_mode::switch_to_primary_mode(); - } - - /// Transitions to Replica mode (scheduler + global state). - /// - /// Called when ledger replay completes on Replica validators, or during - /// failover when a primary needs to step down to replica. - /// Coordinates both the local scheduler and global validator state. - pub(super) fn switch_to_replica_mode_globally(&mut self) { - self.switch_to_replica_mode(); - coordination_mode::switch_to_replica_mode(); - } - - /// Checks if a transaction mode is compatible with the current coordination mode. - /// - /// - StartingUp mode: rejects Execution transactions (same as Replica) - /// - Primary mode: rejects Replay transactions (only client Execution allowed) - /// - Replica mode: rejects Execution transactions (only Replay allowed) - /// - Simulations are allowed in all modes - pub(super) fn is_transaction_allowed( - &self, - mode: &TransactionProcessingMode, - ) -> bool { - use CoordinationMode::*; - use TransactionProcessingMode::*; - let mode_mismatch = matches!( - (&self.mode, mode), - (Primary(_), Replay(_)) - | (StartingUp(_), Execution(_)) - | (Replica(_), Execution(_)) - ); - !mode_mismatch - } - - /// Check whether the node is acting as an event source for replication - pub(super) fn is_primary(&self) -> bool { - matches!(self.mode, CoordinationMode::Primary(_)) - } - - /// Returns true when no transactions are actively executing. - /// - /// Idle means: all executors are either ready or blocked, so no state - /// transitions are in progress. This is the safe moment for external - /// operations like checksums to access AccountsDb. - pub(super) fn is_idle(&self) -> bool { - self.ready_executors.len() == self.blocked_transactions.len() - } - - /// Waits until all executors are idle, then returns a permit that keeps them paused. - /// - /// This acquires all execution permits, blocking until every executor finishes - /// its current transaction. While holding the returned permit, no executors can - /// start new transactions. - /// - /// Use this for operations that require exclusive access to `AccountsDb`, - /// such as taking snapshots or computing checksums. - pub(super) async fn wait_for_idle(&self) -> OwnedSemaphorePermit { - self.execution_permits - .clone() - .acquire_many_owned(self.blocked_transactions.len() as u32) - .await - .expect("semaphore can never be closed") - } -} - -/// Transaction wrapped with a monotonic ID for FIFO queue ordering. -/// -/// The ID ensures blocked transactions are processed in arrival order: -/// lower IDs (earlier transactions) are dequeued before higher IDs. -pub(super) struct TransactionWithId { - pub(super) id: TransactionId, - pub(super) txn: ProcessableTransaction, -} - -impl TransactionWithId { - pub(super) fn new(txn: ProcessableTransaction) -> Self { - Self { - id: next_transaction_id(), - txn, - } - } -} - -// BinaryHeap is max-heap by default, so we reverse comparison for min-heap behavior -impl Ord for TransactionWithId { - fn cmp(&self, other: &Self) -> Ordering { - other.id.cmp(&self.id) // smaller ID = higher priority - } -} -impl PartialOrd for TransactionWithId { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} -impl PartialEq for TransactionWithId { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } -} -impl Eq for TransactionWithId {} diff --git a/magicblock-processor/src/scheduler/locks.rs b/magicblock-processor/src/scheduler/locks.rs deleted file mode 100644 index 63841f79f..000000000 --- a/magicblock-processor/src/scheduler/locks.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Fast, in-memory account locking for the scheduler. -//! -//! # Bitmask Layout -//! -//! Each `AccountLock` uses a single `u64`: -//! - **MSB (bit 63)**: Write lock flag (exclusive access) -//! - **Bits 0-62**: Read lock flags per executor (concurrent access) -//! -//! # Concurrency Rules -//! -//! - **Write lock**: Requires zero state (no readers or writers) -//! - **Read lock**: Requires MSB clear (no active writer) -//! - **Unlock**: Clears both write bit and executor's read bit -//! -//! # Limits -//! -//! Maximum 63 executors (u64::BITS - 1, reserving MSB for write flag) - -use std::{ - cell::RefCell, - rc::Rc, - sync::atomic::{AtomicU64, Ordering}, -}; - -use rustc_hash::FxHashMap; -use solana_pubkey::Pubkey; - -pub(crate) type ExecutorId = u32; -pub(super) type TransactionId = u64; -pub(super) type RcLock = Rc>; -pub(super) type LocksCache = FxHashMap; - -pub(super) const MAX_SVM_EXECUTORS: u32 = u64::BITS - 1; -const WRITE_BIT: u64 = 1 << MAX_SVM_EXECUTORS; - -pub(super) fn next_transaction_id() -> TransactionId { - // Relaxed ordering is sufficient: IDs only need to be unique per transaction - static COUNTER: AtomicU64 = AtomicU64::new(0); - COUNTER.fetch_add(1, Ordering::Relaxed) -} - -/// Read/write lock on a single account using bitmask state. -/// -/// See [module-level documentation](self) for bitmask layout details. -#[derive(Default)] -pub(super) struct AccountLock { - /// Bitmask lock state (see module docs for layout) - state: u64, -} - -impl AccountLock { - #[inline] - pub(super) fn write( - &mut self, - executor: ExecutorId, - ) -> Result<(), ExecutorId> { - if self.state != 0 { - // trailing_zeros() returns index of first set bit (the blocking executor) - return Err(self.state.trailing_zeros()); - } - // Set write bit AND executor bit to track ownership - self.state = WRITE_BIT | (1 << executor); - Ok(()) - } - - #[inline] - pub(super) fn read( - &mut self, - executor: ExecutorId, - ) -> Result<(), ExecutorId> { - if self.state & WRITE_BIT != 0 { - // trailing_zeros() returns the writer's executor ID - return Err(self.state.trailing_zeros()); - } - self.state |= 1 << executor; - Ok(()) - } - - #[inline] - pub(super) fn unlock(&mut self, executor: ExecutorId) { - // Clear both write bit (if set) and executor's read bit - self.state &= !(WRITE_BIT | (1 << executor)); - } -} diff --git a/magicblock-processor/src/scheduler/mod.rs b/magicblock-processor/src/scheduler/mod.rs deleted file mode 100644 index 8dc5ab2c5..000000000 --- a/magicblock-processor/src/scheduler/mod.rs +++ /dev/null @@ -1,687 +0,0 @@ -//! Central transaction scheduler and event loop. -//! -//! # Architecture -//! -//! - Receives transactions from a global queue -//! - Spawns N `TransactionExecutor` workers (one per OS thread) -//! - Dispatches transactions using account locking to prevent conflicts -//! - Multiplexes between: new transactions, executor readiness, and slot transitions -//! -//! # Streaming Blockhash Algorithm -//! -//! Block hashes are computed incrementally as transactions are processed, rather than -//! waiting until the end of a slot. This enables: -//! -//! - **Early detection of hash divergence** between primary and replica nodes -//! - **Reduced latency** for block finalization -//! -//! The algorithm works as follows: -//! 1. At slot boundary, the hasher is reset and seeded with the previous blockhash -//! 2. Each transaction signature is hashed into the stream as it's scheduled -//! 3. At slot completion, the accumulated hash becomes the new blockhash -//! -//! # Primary vs Replica Responsibilities -//! -//! ## Primary Mode -//! -//! - Acts as the clock source, driving slot transitions via `slot_ticker` -//! - Computes blockhash and broadcasts it via replication channel -//! - Writes completed blocks to the ledger -//! -//! ## Replica Mode -//! -//! - Waits for block production notifications from replication service -//! - Verifies computed blockhash matches the received blockhash -//! - Logs divergence errors (does not halt execution) -//! -//! # Slot Transition Lifecycle -//! -//! 1. **Finalize current block** - compute blockhash, persist to ledger -//! 2. **Broadcast block** - send to replication channel (primary only) -//! 3. **Reset hasher** - seed with previous blockhash for next slot -//! 4. **Advance slot** - increment slot number, reset transaction index -//! 5. **Update program cache** - prune stale programs, re-root to new slot -//! 6. **Update sysvars** - Clock and SlotHashes accounts - -use core::matches; -use std::{ - sync::{Arc, RwLock}, - thread::JoinHandle, - time::{SystemTime, UNIX_EPOCH}, -}; - -use blake3::Hasher; -use coordinator::{ExecutionCoordinator, TransactionWithId}; -use locks::{ExecutorId, MAX_SVM_EXECUTORS}; -use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; -use magicblock_core::{ - link::{ - replication::{self, Message, SuperBlock}, - transactions::{ - ProcessableTransaction, SchedulerCommand, SchedulerCommandResult, - SchedulerMode, TransactionProcessingMode, TransactionToProcessRx, - }, - }, - Slot, -}; -use magicblock_ledger::{LatestBlock, LatestBlockInner, Ledger}; -use magicblock_metrics::metrics; -use solana_account::{from_account, to_account}; -use solana_program::{clock::Clock, hash::Hash, slot_hashes::SlotHashes}; -use solana_program_runtime::loaded_programs::ProgramCache; -use solana_sdk_ids::sysvar::{clock, slot_hashes}; -use state::TransactionSchedulerState; -use tokio::{ - runtime::Builder, - sync::{ - mpsc::{channel, Receiver, Sender}, - oneshot, OwnedSemaphorePermit, Semaphore, - }, - time::{interval, Interval}, -}; -use tokio_util::sync::CancellationToken; -use tracing::{error, info, instrument, warn}; - -use crate::executor::{ - ExecutorCommand, IndexedTransaction, SimpleForkGraph, TransactionExecutor, -}; - -// Capacity of 1 ensures executor processes one transaction at a time -const EXECUTOR_QUEUE_CAPACITY: usize = 1; - -/// Central transaction scheduler managing executor workers and transaction dispatch. -/// -/// Runs in a dedicated thread with a single-threaded Tokio runtime. -pub struct TransactionScheduler { - /// Manages executor pool, account locking, and coordination mode - coordinator: ExecutionCoordinator, - /// Ordered command queue from global processor - transactions_rx: TransactionToProcessRx, - /// Executor readiness notifications (workers signal when idle) - ready_rx: Receiver, - /// Sender channels to each executor worker - executors: Vec>, - /// Shared BPF program cache - program_cache: Arc>>, - /// Accounts database (for sysvar updates on slot transition) - accountsdb: Arc, - /// Global transactions ledger - ledger: Arc, - /// Global shutdown signal. - shutdown: CancellationToken, - /// Receives mode transition commands (Primary or Replica) at runtime. - mode_rx: Receiver, - /// A sink for the events (transactions, blocks etc) that need to be replicated - replication_tx: Sender, - /// Semaphore for coordinating exclusive DB access with external callers. - /// Scheduler acquires permit when scheduling, releases when idle. - pause_permit: Arc, - /// Streaming blockhash state - hasher: Hasher, - /// Time interval between consecutive slots - slot_ticker: Interval, - /// Number of slots to elapse, before we perform snapshot/checksum operations - superblock_size: u64, - /// Current Slot that scheduler is operating on (clock value) - slot: Slot, - /// Current transaction index included into the block under assembly - index: u32, -} - -impl TransactionScheduler { - /// Creates a new scheduler and spawns its executor worker pool. - /// - /// Prepares shared program cache and sysvars, then spawns N executors - /// (one per OS thread, up to MAX_SVM_EXECUTORS). - pub fn new(executors: u32, state: TransactionSchedulerState) -> Self { - let count = executors.clamp(1, MAX_SVM_EXECUTORS) as usize; - let mut executors = Vec::with_capacity(count); - - // Channel for workers to signal when ready for new transactions - let (ready_tx, ready_rx) = channel(count); - let program_cache = state.prepare_programs_cache(); - state.prepare_sysvars(); - - let execution_permits = Arc::new(Semaphore::new(count)); - for id in 0..count { - let (transactions_tx, transactions_rx) = - channel(EXECUTOR_QUEUE_CAPACITY); - let executor = TransactionExecutor::new( - id as u32, - &state, - transactions_rx, - ready_tx.clone(), - execution_permits.clone(), - program_cache.clone(), - ); - executor.populate_builtins(); - executor.spawn(); - executors.push(transactions_tx); - } - - let mut hasher = Hasher::new(); - let slot_ticker = interval(state.block_time); - let latest_block = state.ledger.latest_block().clone(); - hasher.update( - Self::initial_blockhash(&state.accountsdb, &latest_block).as_ref(), - ); - let slot = state.accountsdb.slot() + 1; - - Self { - coordinator: ExecutionCoordinator::new(count, execution_permits), - transactions_rx: state.txn_to_process_rx, - ready_rx, - executors, - ledger: state.ledger, - program_cache, - accountsdb: state.accountsdb, - shutdown: state.shutdown, - mode_rx: state.mode_rx, - replication_tx: state.replication_tx, - pause_permit: state.pause_permit, - superblock_size: state.superblock_size, - hasher, - slot_ticker, - slot, - index: 0, - } - } - - fn initial_blockhash( - accountsdb: &AccountsDb, - latest_block: &LatestBlock, - ) -> Hash { - let ledger_hash = latest_block.load().blockhash; - if ledger_hash != Hash::default() { - return ledger_hash; - } - - let snapshot_hash = accountsdb - .get_account(&slot_hashes::ID) - .and_then(|account| from_account::(&account)) - .and_then(|hashes| { - hashes.slot_hashes().first().map(|(_, hash)| *hash) - }); - if let Some(hash) = snapshot_hash { - return hash; - } - - warn!("initial blockhash was not found, starting with an empty one"); - - Hash::default() - } - - /// Spawns the scheduler's event loop in a dedicated OS thread. - pub fn spawn(self) -> JoinHandle<()> { - std::thread::spawn(move || { - // Single-threaded runtime avoids scheduler contention with other async tasks - let runtime = Builder::new_current_thread() - .thread_name("transaction-scheduler") - .enable_all() - .build() - .expect("Failed to build single-threaded Tokio runtime"); - runtime.block_on(tokio::task::unconstrained(self.run())); - }) - } - - /// Main event loop: processes executor readiness, new transactions, and slot transitions. - /// - /// Uses `biased` select to prioritize in this order: - /// 1. Slot transitions - /// 2. Executor readiness - /// 3. Mode switches (before transactions to avoid race condition) - /// 4. New transactions - #[instrument(skip(self))] - async fn run(mut self) { - // Holds the scheduling permit while transactions are being processed. - // Released when idle so external callers can acquire exclusive access. - let mut scheduling_permit = None; - let mut pending_command = None; - loop { - if let Some(command) = pending_command.take() { - match command { - SchedulerCommand::Transaction(txn) => { - if self.coordinator.is_ready() { - self.handle_new_transaction( - txn, - &mut scheduling_permit, - ) - .await; - continue; - } - pending_command = - Some(SchedulerCommand::Transaction(txn)); - } - SchedulerCommand::Block { block, ack } => { - if self.coordinator.is_idle() { - self.handle_replayed_block(block, ack).await; - continue; - } - pending_command = - Some(SchedulerCommand::Block { block, ack }); - } - SchedulerCommand::Drain { ack } => { - if self.coordinator.is_idle() { - let _ = ack.send(Ok(())); - continue; - } - pending_command = Some(SchedulerCommand::Drain { ack }); - } - } - } - tokio::select! { - biased; - _ = self.slot_ticker.tick() => { - if self.coordinator.is_primary() { - let slot = self.transition_to_new_slot(None).await; - self.handle_superblock(slot).await; - } - } - Some(executor) = self.ready_rx.recv() => { - self.handle_ready_executor(executor).await; - // Release permit when idle: no active work, safe for external access - if self.coordinator.is_idle() { - scheduling_permit.take(); - } - } - Some(mode) = self.mode_rx.recv() => { - match mode { - SchedulerMode::Primary => { - self.coordinator.switch_to_primary_mode_globally(); - } - SchedulerMode::Replica => { - self.coordinator.switch_to_replica_mode_globally(); - } - } - } - Some(command) = self.transactions_rx.recv(), if pending_command.is_none() && self.coordinator.is_ready() => { - pending_command = Some(command); - } - _ = self.shutdown.cancelled() => break, - else => break, - } - } - if self.coordinator.is_primary() { - let slot = self.transition_to_new_slot(None).await; - self.handle_superblock(slot).await; - } - // Shutdown: drop executor channels to signal workers to stop, - // then drain remaining ready notifications - drop(self.executors); - while self.ready_rx.recv().await.is_some() {} - info!("Scheduler terminated"); - } - - /// Sends a replication message, logging any errors. - async fn send_replication(&self, msg: Message) { - if self.replication_tx.is_closed() { - return; - } - let kind = msg.kind(); - if let Err(error) = self.replication_tx.send(msg).await { - error!( - %error, - kind, - slot = self.slot, - index = self.index, - "replication send failed" - ); - } - } - - async fn handle_superblock(&mut self, slot: Slot) { - if !slot.is_multiple_of(self.superblock_size) { - return; - } - - // Wait until the scheduler has no assigned or executing work left, - // then freeze executor starts while snapshotting. - let _guard = self.pause_executors_for_snapshot().await; - // SAFETY: - // we have made sure that no state transitions are in progress via _guard - let Ok(checksum) = (unsafe { self.accountsdb.take_snapshot(slot) }) - else { - error!("failed to create accountsdb snapshot"); - return; - }; - if self.coordinator.is_primary() { - let msg = Message::SuperBlock(SuperBlock { slot, checksum }); - self.send_replication(msg).await; - } - } - - async fn pause_executors_for_snapshot(&mut self) -> OwnedSemaphorePermit { - while !self.coordinator.is_idle() { - if let Some(executor) = self.ready_rx.recv().await { - self.handle_ready_executor(executor).await; - } - } - - self.coordinator.wait_for_idle().await - } - - async fn handle_ready_executor(&mut self, executor: ExecutorId) { - self.coordinator.unlock_accounts(executor); - self.reschedule_blocked_transactions(executor).await; - } - - async fn handle_replayed_block( - &mut self, - block: replication::Block, - ack: oneshot::Sender, - ) { - let result = self.apply_replayed_block(block).await; - let _ = ack.send(result); - } - - async fn apply_replayed_block( - &mut self, - block: replication::Block, - ) -> SchedulerCommandResult { - if self.coordinator.is_primary() { - return Err( - "replicated block received while in primary mode".into() - ); - } - - let permit = self - .pause_permit - .clone() - .acquire_owned() - .await - .map_err(|_| "scheduler semaphore closed".to_string())?; - - let block = - LatestBlockInner::new(block.slot, block.hash, block.timestamp); - self.verify_block_as_replica(&block); - self.ledger - .write_block(block.clone()) - .map_err(|error| error.to_string())?; - self.accountsdb.set_slot(block.slot); - self.update_sysvars(&block); - let slot = block.slot; - let result = self.notify_executors_of_block(block).await; - drop(permit); - // apply_replayed_block advances self.slot before the queued - // latest_block notification can be received. The block_produced path - // then skips transition_to_new_slot for this already-applied slot, so - // handle_superblock must run here for replayed superblock snapshots. - if result.is_ok() { - self.handle_superblock(slot).await; - } - result - } - - async fn notify_executors_of_block( - &mut self, - block: LatestBlockInner, - ) -> SchedulerCommandResult { - let mut acks = Vec::with_capacity(self.executors.len()); - for (executor, tx) in self.executors.iter().enumerate() { - let (ack, rx) = oneshot::channel(); - tx.send(ExecutorCommand::Block { - block: block.clone(), - ack, - }) - .await - .map_err(|error| { - format!("executor {executor} block command failed: {error}") - })?; - acks.push((executor, rx)); - } - - for (executor, ack) in acks { - ack.await.map_err(|_| { - format!("executor {executor} dropped block acknowledgement") - })?; - } - Ok(()) - } - - async fn handle_new_transaction( - &mut self, - txn: ProcessableTransaction, - scheduling_permit: &mut Option, - ) { - if !self.coordinator.is_transaction_allowed(&txn.mode) { - warn!("Dropping transaction due to mode incompatibility"); - return; - } - // Acquire permit if not already held. This blocks if an external caller - // (e.g., checksum) currently holds it, ensuring mutual exclusion. - if scheduling_permit.is_none() { - let permit = self - .pause_permit - .clone() - .acquire_owned() - .await - .expect("scheduler semaphore can never be closed"); - scheduling_permit.replace(permit); - } - // SAFETY: - // the caller ensured that executor was ready before invoking this - // method so the get_ready_executor should always return Some here - let executor = self.coordinator.get_ready_executor().expect( - "unreachable: is_ready() guard ensures an executor is available", - ); - self.schedule_transaction(executor, TransactionWithId::new(txn)) - .await; - } - - async fn reschedule_blocked_transactions(&mut self, blocker: ExecutorId) { - let mut executor = Some(blocker); - while let Some(exec) = executor.take() { - // Try to get next transaction blocked by this executor - let Some(txn) = self.coordinator.next_blocked_transaction(blocker) - else { - // Queue empty: release executor back to pool - self.coordinator.release_executor(exec); - break; - }; - - let blocked = self.schedule_transaction(exec, txn).await; - - // If blocked by the same executor we're draining, stop to avoid infinite loop - if blocked.is_some_and(|b| b == blocker) { - break; - } - // Try to get another executor for the next blocked transaction - executor = self.coordinator.get_ready_executor(); - } - } - - async fn schedule_transaction( - &mut self, - executor: ExecutorId, - txn: TransactionWithId, - ) -> Option { - let txn = match self.coordinator.try_schedule(executor, txn) { - Ok(txn) => txn, - Err(blocker) => return Some(blocker), - }; - let mut is_execution = false; - let (slot, index) = match txn.mode { - TransactionProcessingMode::Replay(ctx) => (ctx.slot, ctx.index), - TransactionProcessingMode::Simulation(_) => (self.slot, 0), - TransactionProcessingMode::Execution(_) => { - is_execution = true; - let index = self.index; - // we only advance the index if we are executing (primary mode) - self.index += 1; - (self.slot, index) - } - }; - if !matches!(txn.mode, TransactionProcessingMode::Simulation(_)) { - self.hasher.update(txn.transaction.signature().as_ref()); - } - - let msg = txn - .encoded - .as_ref() - .cloned() - .filter(|_| is_execution && self.coordinator.is_primary()) - .map(|payload| { - Message::Transaction(replication::Transaction { - index, - slot, - payload, - }) - }); - let txn = IndexedTransaction { slot, index, txn }; - - let sent = self.executors[executor as usize] - .try_send(ExecutorCommand::Transaction(txn)) - .inspect_err( - |error| error!(executor, %error, "Executor channel send failed"), - ) - .is_ok(); - if let Some(msg) = msg.filter(|_| sent) { - self.send_replication(msg).await; - } - None - } - - /// Transitions to the next slot, finalizing the current block. - /// - /// In primary mode, this drives the slot transition clock and broadcasts - /// the new block. In replica mode, this responds to block notifications. - async fn transition_to_new_slot( - &mut self, - block: Option, - ) -> u64 { - let block = self.prepare_block(block).await; - self.finalize_block(block.clone()).await; - self.update_sysvars(&block); - metrics::set_slot(block.slot); - block.slot - } - - /// Prepares the block for the current slot. - /// - /// In primary mode: computes blockhash, creates block from local state. - /// In replica mode: uses the block from the replication stream, verifying hash. - async fn prepare_block( - &mut self, - block: Option, - ) -> LatestBlockInner { - if let Some(block) = block { - self.verify_block_as_replica(&block); - block - } else { - self.prepare_block_as_primary().await - } - } - - /// Prepares block as primary: computes blockhash and broadcasts to replicas. - async fn prepare_block_as_primary(&mut self) -> LatestBlockInner { - let blockhash = (*self.hasher.finalize().as_bytes()).into(); - // NOTE: - // As we have a single node network, we have no - // option but to use the time from host machine - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - // NOTE: since we can tick very frequently, a lot - // of blocks might have identical timestamps - .as_secs() as i64; - let block = LatestBlockInner::new(self.slot, blockhash, timestamp); - let msg = Message::Block(replication::Block { - slot: block.slot, - hash: block.blockhash, - timestamp: block.clock.unix_timestamp, - }); - self.send_replication(msg).await; - block - } - - /// Checks that the blockhash received from replication stream matches the local - fn verify_block_as_replica(&self, block: &LatestBlockInner) { - if block.blockhash.as_ref() != self.hasher.finalize().as_bytes() { - // TODO(bmuddha): - // this should never happen, and it's unclear how - // to recover from it, right now the log is used - // for debugging purposes only - // NOTE: - // This error will always be logged once - // when replica starts up with an empty ledger - error!( - slot = block.slot, - "replication blockhash has diverged from local" - ) - } - } - - /// Finalizes the block: persists to ledger and updates accountsdb slot. - async fn finalize_block(&self, block: LatestBlockInner) { - let slot = block.slot; - let mut block_written = true; - if self.coordinator.is_primary() { - block_written = self.ledger.write_block(block).inspect_err( - |error| error!(%error, %slot, "failed to write block to the ledger"), - ).is_ok(); - } - if block_written { - self.accountsdb.set_slot(slot); - } - } - - /// Updates sysvars and program cache for the new slot. - /// - /// This must be called after block finalization to prepare state for the next slot. - fn update_sysvars(&mut self, block: &LatestBlockInner) { - // Reset hasher and seed with previous blockhash for next slot - self.hasher.reset(); - self.hasher.update(block.blockhash.as_ref()); - - // Advance slot and reset transaction index - self.slot = block.clock.slot; - self.index = 0; - - self.update_program_cache(block.slot); - self.update_clock_sysvar(&block.clock); - self.update_slot_hashes_sysvar(block.slot, &block.blockhash); - } - - /// Updates the program cache for the new slot. - fn update_program_cache(&mut self, slot: Slot) { - let mut cache = self.program_cache.write().unwrap(); - // Prune stale programs and re-root to new slot - cache.prune(slot, None); - // Release lock before syscall lookup (prevents deadlock if sysvar is accessed) - drop(cache); - } - - /// Updates the Clock sysvar account. - fn update_clock_sysvar(&self, clock: &Clock) { - if let Some(mut account) = self.accountsdb.get_account(&clock::ID) { - let _ = account.serialize_data(clock); - let _ = self.accountsdb.insert_account(&clock::ID, &account); - } - } - - /// Updates the SlotHashes sysvar account with the new slot/blockhash pair. - fn update_slot_hashes_sysvar(&self, slot: Slot, blockhash: &Hash) { - if let Some(mut acc) = self.accountsdb.get_account(&slot_hashes::ID) { - let Some(mut hashes) = from_account::(&acc) else { - warn!("failed to read slot hashes from account"); - return; - }; - hashes.add(slot, *blockhash); - if to_account(&hashes, &mut acc).is_none() { - warn!("failed to write slot hashes to account"); - } - let _ = self.accountsdb.insert_account(&slot_hashes::ID, &acc); - } - } -} - -// SAFETY: -// Rc used in the the scheduler is only used in a single -// thread, the scheduler is meant to stay single threaded -unsafe impl Send for TransactionScheduler {} - -pub mod coordinator; -pub mod locks; -pub mod state; -#[cfg(test)] -mod tests; diff --git a/magicblock-processor/src/scheduler/state.rs b/magicblock-processor/src/scheduler/state.rs deleted file mode 100644 index f9c202b03..000000000 --- a/magicblock-processor/src/scheduler/state.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! Shared state initialization for the transaction scheduler. -//! -//! Prepares global resources: program cache, sysvars, and communication channels. -//! `TransactionSchedulerState` is constructed externally and passed to -//! `TransactionScheduler::new()` for initialization. - -use std::{ - sync::{Arc, OnceLock, RwLock}, - time::Duration, -}; - -use magicblock_accounts_db::AccountsDb; -use magicblock_core::link::{ - accounts::AccountUpdateTx, - replication::Message, - transactions::{ - ScheduledTasksTx, SchedulerMode, TransactionStatusTx, - TransactionToProcessRx, - }, -}; -use magicblock_ledger::Ledger; -use serde::Serialize; -use solana_account::AccountSharedData; -use solana_feature_set::FeatureSet; -use solana_program::{ - clock::DEFAULT_SLOTS_PER_EPOCH, - epoch_schedule::EpochSchedule, - slot_hashes::{SlotHashes, MAX_ENTRIES}, - sysvar, -}; -use solana_program_runtime::loaded_programs::ProgramCache; -use solana_pubkey::Pubkey; -use solana_svm::transaction_processor::TransactionProcessingEnvironment; -use tokio::sync::{ - mpsc::{Receiver, Sender}, - Semaphore, -}; -use tokio_util::sync::CancellationToken; - -use crate::executor::SimpleForkGraph; - -/// Container for global state and communication channels. -/// -/// Holds all shared resources needed to bootstrap the scheduler. -/// Created externally and consumed by `TransactionScheduler::new()`. -pub struct TransactionSchedulerState { - // === Global State Handles === - pub accountsdb: Arc, - pub ledger: Arc, - pub environment: TransactionProcessingEnvironment, - pub feature_set: FeatureSet, - - // === Communication Channels === - pub txn_to_process_rx: TransactionToProcessRx, - pub account_update_tx: AccountUpdateTx, - pub transaction_status_tx: TransactionStatusTx, - pub tasks_tx: ScheduledTasksTx, - pub replication_tx: Sender, - /// Semaphore for pausing scheduling during exclusive DB access. - pub pause_permit: Arc, - - pub shutdown: CancellationToken, - pub block_time: Duration, - pub superblock_size: u64, - - /// Receives mode transition commands (Primary or Replica) at runtime. - pub mode_rx: Receiver, -} - -impl TransactionSchedulerState { - /// Initializes the shared BPF program cache. - pub(crate) fn prepare_programs_cache( - &self, - ) -> Arc>> { - static FORK_GRAPH: OnceLock>> = - OnceLock::new(); - - // Singleton fork graph: validator doesn't support forks - let forkgraph = Arc::downgrade( - FORK_GRAPH.get_or_init(|| Arc::new(RwLock::new(SimpleForkGraph))), - ); - - let mut cache = ProgramCache::new(self.accountsdb.slot()); - cache.set_fork_graph(forkgraph); - - Arc::new(RwLock::new(cache)) - } - - /// Ensures essential sysvar accounts exist in `AccountsDb`. - pub(crate) fn prepare_sysvars(&self) { - let block = self.ledger.latest_block().load(); - - // Mutable sysvars (updated on each slot transition) - self.ensure_sysvar(&sysvar::clock::ID, &block.clock); - let slot_hashes = - SlotHashes::new(&[(block.slot, block.blockhash); MAX_ENTRIES]); - self.ensure_sysvar(&sysvar::slot_hashes::ID, &slot_hashes); - - // Immutable/Static sysvars (initialized once) - let epoch_schedule = EpochSchedule::new(DEFAULT_SLOTS_PER_EPOCH); - self.ensure_sysvar(&sysvar::epoch_schedule::ID, &epoch_schedule); - - self.ensure_sysvar(&sysvar::rent::ID, &self.environment.rent); - } - - /// Helper to serialize and insert a sysvar if it doesn't exist. - fn ensure_sysvar(&self, id: &Pubkey, data: &T) { - if self.accountsdb.contains_account(id) { - return; - } - if let Ok(account) = AccountSharedData::new_data(1, data, &sysvar::ID) { - let _ = self.accountsdb.insert_account(id, &account); - } - } -} diff --git a/magicblock-processor/src/scheduler/tests.rs b/magicblock-processor/src/scheduler/tests.rs deleted file mode 100644 index aec709992..000000000 --- a/magicblock-processor/src/scheduler/tests.rs +++ /dev/null @@ -1,491 +0,0 @@ -//! Unit tests for the scheduler's locking and queueing logic. -//! -//! Tests cover: -//! - Lock semantics (write/write, write/read, read/write contention) -//! - FIFO ordering within blocked queues -//! - Executor pool management -//! - Coordination modes (Primary/Replica) -//! - Edge cases (empty transactions, duplicate accounts) - -use std::sync::Arc; - -use magicblock_core::link::transactions::{ - ProcessableTransaction, ReplayPosition, SanitizeableTransaction, - TransactionProcessingMode, -}; -use solana_keypair::Keypair; -use solana_program::{ - hash::Hash, - instruction::{AccountMeta, Instruction}, -}; -use solana_pubkey::Pubkey; -use solana_signer::Signer; -use solana_transaction::Transaction; -use tokio::sync::Semaphore; - -use super::coordinator::{ExecutionCoordinator, TransactionWithId}; - -/// Creates a coordinator for testing with the required execution permits semaphore. -fn new_coordinator(count: usize) -> ExecutionCoordinator { - let execution_permits = Arc::new(Semaphore::new(count)); - ExecutionCoordinator::new(count, execution_permits) -} - -/// Creates a mock transaction with the specified accounts and processing mode. -fn mock_txn_with_mode( - accounts: &[(Pubkey, bool)], - mode: TransactionProcessingMode, -) -> TransactionWithId { - let payer = Keypair::new(); - let instructions: Vec = accounts - .iter() - .map(|(pubkey, is_writable)| { - let meta = if *is_writable { - AccountMeta::new(*pubkey, false) - } else { - AccountMeta::new_readonly(*pubkey, false) - }; - Instruction::new_with_bincode(Pubkey::new_unique(), &(), vec![meta]) - }) - .collect(); - - let transaction = Transaction::new_signed_with_payer( - &instructions, - Some(&payer.pubkey()), - &[payer], - Hash::new_unique(), - ); - - TransactionWithId::new(ProcessableTransaction { - transaction: transaction.sanitize(false).unwrap(), - mode, - encoded: None, - }) -} - -/// Creates a mock execution transaction with the specified accounts. -fn mock_txn(accounts: &[(Pubkey, bool)]) -> TransactionWithId { - mock_txn_with_mode(accounts, TransactionProcessingMode::Execution(None)) -} - -/// Creates a mock replay transaction with the specified accounts. -fn mock_replay_txn( - accounts: &[(Pubkey, bool)], - position: ReplayPosition, -) -> TransactionWithId { - mock_txn_with_mode(accounts, TransactionProcessingMode::Replay(position)) -} - -// ============================================================================= -// Lock Semantics -// ============================================================================= - -#[test] -fn write_blocks_write() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - - assert!(c.try_schedule(e0, mock_txn(&[(acc, true)])).is_ok()); - assert_eq!(c.try_schedule(e1, mock_txn(&[(acc, true)])).err(), Some(e0)); -} - -#[test] -fn write_blocks_read() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - - assert!(c.try_schedule(e0, mock_txn(&[(acc, true)])).is_ok()); - assert_eq!( - c.try_schedule(e1, mock_txn(&[(acc, false)])).err(), - Some(e0) - ); -} - -#[test] -fn read_blocks_write() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - - assert!(c.try_schedule(e0, mock_txn(&[(acc, false)])).is_ok()); - assert_eq!(c.try_schedule(e1, mock_txn(&[(acc, true)])).err(), Some(e0)); -} - -#[test] -fn multiple_readers_allowed() { - let mut c = new_coordinator(3); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - let e2 = c.get_ready_executor().unwrap(); - - assert!(c.try_schedule(e0, mock_txn(&[(acc, false)])).is_ok()); - assert!(c.try_schedule(e1, mock_txn(&[(acc, false)])).is_ok()); - assert!(c.try_schedule(e2, mock_txn(&[(acc, false)])).is_ok()); -} - -// ============================================================================= -// Partial Lock Rollback -// ============================================================================= - -#[test] -fn partial_locks_released_on_failure() { - let mut c = new_coordinator(2); - let (a, b) = (Pubkey::new_unique(), Pubkey::new_unique()); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - - // e0 locks B only - assert!(c.try_schedule(e0, mock_txn(&[(b, true)])).is_ok()); - - // e1 tries [A, B] - locks A, fails on B, should release A - assert_eq!( - c.try_schedule(e1, mock_txn(&[(a, true), (b, true)])).err(), - Some(e0) - ); - - // If A wasn't released, this would fail - let e2 = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e2, mock_txn(&[(a, true)])).is_ok()); -} - -// ============================================================================= -// Queue Ordering (FIFO) -// ============================================================================= - -#[test] -fn blocked_transactions_dequeued_in_fifo_order() { - let mut c = new_coordinator(4); - // Switch to primary mode for tests that need multiple blocked transactions - c.switch_to_primary_mode(); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e0, mock_txn(&[(acc, true)])).is_ok()); - - // Queue 3 transactions - they get IDs in order - let t1 = mock_txn(&[(acc, true)]); - let t2 = mock_txn(&[(acc, true)]); - let t3 = mock_txn(&[(acc, true)]); - let id1 = t1.id; - let id2 = t2.id; - let id3 = t3.id; - assert!(id1 < id2 && id2 < id3); - - // Queue in reverse order to test heap ordering - let e = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e, t3).is_err()); - let e = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e, t1).is_err()); - let e = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e, t2).is_err()); - - // Should dequeue in ID order (FIFO), not insertion order - let d1 = c.next_blocked_transaction(e0).unwrap(); - let d2 = c.next_blocked_transaction(e0).unwrap(); - let d3 = c.next_blocked_transaction(e0).unwrap(); - - assert_eq!(d1.id, id1); - assert_eq!(d2.id, id2); - assert_eq!(d3.id, id3); -} - -// ============================================================================= -// Executor Pool Management -// ============================================================================= - -#[test] -fn blocked_transaction_releases_executor() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - assert!(c.get_ready_executor().is_none()); // Pool exhausted - - assert!(c.try_schedule(e0, mock_txn(&[(acc, true)])).is_ok()); - assert!(c.try_schedule(e1, mock_txn(&[(acc, true)])).is_err()); - - // e1 should be back in pool after being blocked - assert!(c.get_ready_executor().is_some()); -} - -// ============================================================================= -// Unlock and Reschedule -// ============================================================================= - -#[test] -fn unlock_allows_blocked_transaction_to_proceed() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - - assert!(c.try_schedule(e0, mock_txn(&[(acc, true)])).is_ok()); - assert!(c.try_schedule(e1, mock_txn(&[(acc, true)])).is_err()); - - // Unlock e0's locks - c.unlock_accounts(e0); - - // Now blocked txn should be able to acquire locks - let blocked = c.next_blocked_transaction(e0).unwrap(); - let e = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e, blocked).is_ok()); -} - -#[test] -fn transaction_requeued_to_different_executor_keeps_id() { - let mut c = new_coordinator(3); - let (a, b) = (Pubkey::new_unique(), Pubkey::new_unique()); - - let e0 = c.get_ready_executor().unwrap(); - let e1 = c.get_ready_executor().unwrap(); - let e2 = c.get_ready_executor().unwrap(); - - // e0 holds A, e1 holds B - assert!(c.try_schedule(e0, mock_txn(&[(a, true)])).is_ok()); - assert!(c.try_schedule(e1, mock_txn(&[(b, true)])).is_ok()); - - // t1 needs [A, B] - blocked by e0 on A - let t1 = mock_txn(&[(a, true), (b, true)]); - let original_id = t1.id; - assert_eq!(c.try_schedule(e2, t1).err(), Some(e0)); - - // Unlock e0, t1 retries but now blocked by e1 on B - c.unlock_accounts(e0); - let t1 = c.next_blocked_transaction(e0).unwrap(); - assert_eq!(t1.id, original_id); // Same ID - - let e = c.get_ready_executor().unwrap(); - assert_eq!(c.try_schedule(e, t1).err(), Some(e1)); - - // Verify it's in e1's queue with same ID - let t1 = c.next_blocked_transaction(e1).unwrap(); - assert_eq!(t1.id, original_id); -} - -// ============================================================================= -// Edge Cases -// ============================================================================= - -#[test] -fn empty_transaction_always_succeeds() { - let mut c = new_coordinator(1); - let e = c.get_ready_executor().unwrap(); - assert!(c.try_schedule(e, mock_txn(&[])).is_ok()); -} - -#[test] -fn transaction_with_duplicate_accounts() { - // Real transactions shouldn't have duplicates, but verify we don't panic - let mut c = new_coordinator(1); - let acc = Pubkey::new_unique(); - let e = c.get_ready_executor().unwrap(); - - // Same account twice as writable - second lock attempt is no-op (already held) - assert!(c - .try_schedule(e, mock_txn(&[(acc, true), (acc, true)])) - .is_ok()); -} - -// ============================================================================= -// Coordination Modes -// ============================================================================= - -#[test] -fn replica_mode_blocks_new_transactions_when_pending() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - let position = ReplayPosition { - slot: 0, - index: 0, - persist: false, - }; - - // First transaction holds the lock on acc - let e0 = c.get_ready_executor().unwrap(); - assert!(c - .try_schedule(e0, mock_replay_txn(&[(acc, true)], position)) - .is_ok()); - - // Second transaction is BLOCKED because it conflicts with e0's lock - // This sets pending in Replica mode - let e1 = c.get_ready_executor().unwrap(); - assert!(c - .try_schedule(e1, mock_replay_txn(&[(acc, true)], position)) - .is_err()); - - // is_ready should return false because pending is set - assert!(!c.is_ready()); -} - -#[test] -fn replica_mode_unblocks_when_pending_completes() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - let position = ReplayPosition { - slot: 0, - index: 0, - persist: false, - }; - - // First transaction holds the lock - let e0 = c.get_ready_executor().unwrap(); - assert!(c - .try_schedule(e0, mock_replay_txn(&[(acc, true)], position)) - .is_ok()); - - // Second transaction is blocked and sets pending - let e1 = c.get_ready_executor().unwrap(); - assert!(c - .try_schedule(e1, mock_replay_txn(&[(acc, true)], position)) - .is_err()); - assert!(!c.is_ready()); // pending is set - - // Complete e0's transaction (unlock accounts) - c.unlock_accounts(e0); - - // Get the blocked transaction from the queue - let blocked = c.next_blocked_transaction(e0); - assert!(blocked.is_some()); - - // Now is_ready should be true again (pending cleared) - assert!(c.is_ready()); -} - -#[test] -fn starting_up_mode_rejects_execution_transactions() { - let c = new_coordinator(1); - - // StartingUp mode should reject Execution transactions - assert!( - !c.is_transaction_allowed(&TransactionProcessingMode::Execution(None)) - ); - // But allow Replay - assert!(c.is_transaction_allowed(&TransactionProcessingMode::Replay( - ReplayPosition { - slot: 0, - index: 0, - persist: false, - } - ))); -} - -#[test] -fn starting_up_mode_allows_replay_transactions() { - let c = new_coordinator(1); - - // StartingUp mode should allow Replay transactions - assert!(c.is_transaction_allowed(&TransactionProcessingMode::Replay( - ReplayPosition { - slot: 0, - index: 0, - persist: false, - } - ))); -} - -#[test] -fn starting_up_to_primary_mode_switch() { - let mut c = new_coordinator(1); - - // Start in StartingUp, should reject Execution - assert!( - !c.is_transaction_allowed(&TransactionProcessingMode::Execution(None)) - ); - - // Switch to Primary - c.switch_to_primary_mode(); - - // Now should allow Execution - assert!( - c.is_transaction_allowed(&TransactionProcessingMode::Execution(None)) - ); - // But reject Replay - assert!( - !c.is_transaction_allowed(&TransactionProcessingMode::Replay( - ReplayPosition { - slot: 0, - index: 0, - persist: false, - } - )) - ); -} - -#[test] -fn starting_up_to_replica_mode_switch() { - let mut c = new_coordinator(1); - - // Start in StartingUp, should allow Replay - assert!(c.is_transaction_allowed(&TransactionProcessingMode::Replay( - ReplayPosition { - slot: 0, - index: 0, - persist: false, - } - ))); - - // Switch to Replica - c.switch_to_replica_mode(); - - // Should still allow Replay - assert!(c.is_transaction_allowed(&TransactionProcessingMode::Replay( - ReplayPosition { - slot: 0, - index: 0, - persist: false, - } - ))); - // But reject Execution - assert!( - !c.is_transaction_allowed(&TransactionProcessingMode::Execution(None)) - ); -} - -#[test] -fn starting_up_blocks_new_transactions_when_pending() { - let mut c = new_coordinator(2); - let acc = Pubkey::new_unique(); - let position = ReplayPosition { - slot: 0, - index: 0, - persist: false, - }; - - // First transaction holds the lock - let e0 = c.get_ready_executor().unwrap(); - assert!(c - .try_schedule(e0, mock_replay_txn(&[(acc, true)], position)) - .is_ok()); - - // Second transaction is blocked and sets pending - let e1 = c.get_ready_executor().unwrap(); - assert!(c - .try_schedule(e1, mock_replay_txn(&[(acc, true)], position)) - .is_err()); - - // In StartingUp mode, is_ready should be false (pending is set) - assert!(!c.is_ready()); - - // Complete e0's transaction (unlock accounts) - c.unlock_accounts(e0); - - // Get the blocked transaction from the queue - let blocked = c.next_blocked_transaction(e0); - assert!(blocked.is_some()); - - // Now is_ready should be true again (pending cleared) - assert!(c.is_ready()); -} diff --git a/magicblock-processor/tests/ephemeral_accounts.rs b/magicblock-processor/tests/ephemeral_accounts.rs deleted file mode 100644 index b072ca8f5..000000000 --- a/magicblock-processor/tests/ephemeral_accounts.rs +++ /dev/null @@ -1,1115 +0,0 @@ -use guinea::GuineaInstruction; -use magicblock_accounts_db::traits::AccountsBank; -use magicblock_magic_program_api::{ - instruction::MagicBlockInstruction, EPHEMERAL_RENT_PER_BYTE, - EPHEMERAL_VAULT_PUBKEY, ID as MAGIC_PROGRAM_ID, -}; -use solana_account::{AccountSharedData, ReadableAccount, WritableAccount}; -use solana_keypair::Keypair; -use solana_program::instruction::{AccountMeta, Instruction}; -use solana_pubkey::Pubkey; -use test_kit::{ExecutionTestEnv, Signer}; - -/// Calculates rent for an ephemeral account (same logic as magic program) -fn rent_for(data_len: u32) -> u64 { - (u64::from(data_len) + AccountSharedData::ACCOUNT_STATIC_SIZE as u64) - * EPHEMERAL_RENT_PER_BYTE -} - -/// Test context with common setup -struct TestContext { - env: ExecutionTestEnv, - sponsor: Pubkey, - ephemeral: Keypair, -} - -/// Sets up a test with vault, sponsor, and ephemeral account -fn setup_test() -> TestContext { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - TestContext { - env, - sponsor, - ephemeral, - } -} - -/// Executes an instruction and returns the result -async fn execute_instruction( - env: &ExecutionTestEnv, - ix: Instruction, -) -> Result<(), solana_transaction_error::TransactionError> { - let txn = env.build_transaction(&[ix]); - env.execute_transaction(txn).await -} - -/// Executes an instruction with additional signers -async fn execute_instruction_with_signers( - env: &ExecutionTestEnv, - ix: Instruction, - signers: &[&Keypair], -) -> Result<(), solana_transaction_error::TransactionError> { - let txn = env.build_transaction_with_signers(&[ix], signers); - env.execute_transaction(txn).await -} - -/// Helper to initialize the ephemeral vault account in the accounts database -fn init_vault(env: &ExecutionTestEnv) { - // Create vault with enough balance to be rent-exempt (covers the account overhead) - // Using a fixed amount that's sufficient for rent-exempt status - const VAULT_RENT_EXEMPT_BALANCE: u64 = 1_000_000; - env.fund_account_with_owner( - EPHEMERAL_VAULT_PUBKEY, - VAULT_RENT_EXEMPT_BALANCE, - MAGIC_PROGRAM_ID, - ); - let mut vault = env.get_account(EPHEMERAL_VAULT_PUBKEY); - vault.set_ephemeral(true); - vault.commit(); -} - -/// Helper to initialize the sponsor account as delegated -fn init_sponsor(env: &ExecutionTestEnv, sponsor: Pubkey) { - // Ensure sponsor is marked as delegated so it can be modified in gasless mode - let mut sponsor_acc = env.get_account(sponsor); - if !sponsor_acc.delegated() { - sponsor_acc.set_delegated(true); - sponsor_acc.commit(); - } -} - -/// Helper to create an instruction that calls guinea to create an ephemeral account -fn create_ephemeral_account_ix( - magic_program: Pubkey, - sponsor: Pubkey, - ephemeral: Pubkey, - vault: Pubkey, - data_len: u32, -) -> Instruction { - Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::CreateEphemeralAccount { data_len }, - vec![ - AccountMeta::new_readonly(magic_program, false), - AccountMeta::new(sponsor, true), - AccountMeta::new(ephemeral, true), - AccountMeta::new(vault, false), - ], - ) -} - -/// Helper to create an instruction that calls guinea to resize an ephemeral account -fn resize_ephemeral_account_ix( - magic_program: Pubkey, - sponsor: Pubkey, - ephemeral: Pubkey, - vault: Pubkey, - new_data_len: u32, -) -> Instruction { - Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::ResizeEphemeralAccount { new_data_len }, - vec![ - AccountMeta::new_readonly(magic_program, false), - AccountMeta::new(sponsor, true), - AccountMeta::new(ephemeral, false), - AccountMeta::new(vault, false), - ], - ) -} - -/// Helper to create an instruction that calls guinea to close an ephemeral account -fn close_ephemeral_account_ix( - magic_program: Pubkey, - sponsor: Pubkey, - ephemeral: Pubkey, - vault: Pubkey, -) -> Instruction { - Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::CloseEphemeralAccount, - vec![ - AccountMeta::new_readonly(magic_program, false), - AccountMeta::new(sponsor, true), - AccountMeta::new(ephemeral, false), - AccountMeta::new(vault, false), - ], - ) -} - -#[tokio::test] -async fn test_create_ephemeral_account_via_cpi() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Use the payer (which is a signer) as the sponsor - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); // Must start with 0 lamports - - let data_len = 1000; - let expected_rent = rent_for(data_len); - - // Check balances BEFORE operation - let sponsor_before = env.get_account(sponsor); - let vault_before = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_before = env.get_account(ephemeral.pubkey()); - let total_before = sponsor_before.lamports() - + vault_before.lamports() - + ephemeral_before.lamports(); - - println!("=== BEFORE CREATE ==="); - println!("Sponsor: {}", sponsor_before.lamports()); - println!("Vault: {}", vault_before.lamports()); - println!("Ephemeral: {}", ephemeral_before.lamports()); - println!("Total: {}", total_before); - println!("Expected rent: {}", expected_rent); - - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - data_len, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - - let result = env.execute_transaction(txn).await; - if let Err(e) = &result { - println!("Error executing transaction: {:?}", e); - } - assert!(result.is_ok()); - - // Check balances AFTER operation - let sponsor_after = env.get_account(sponsor); - let vault_after = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_after = env.get_account(ephemeral.pubkey()); - let total_after = sponsor_after.lamports() - + vault_after.lamports() - + ephemeral_after.lamports(); - - println!("=== AFTER CREATE ==="); - println!("Sponsor: {}", sponsor_after.lamports()); - println!("Vault: {}", vault_after.lamports()); - println!("Ephemeral: {}", ephemeral_after.lamports()); - println!("Total: {}", total_after); - - // Verify the ephemeral account was created correctly - assert!( - ephemeral_after.ephemeral(), - "Account should be marked as ephemeral" - ); - assert!( - ephemeral_after.delegated(), - "Account should be marked as delegated" - ); - assert_eq!( - *ephemeral_after.owner(), - guinea::ID, - "Owner should be guinea program" - ); - assert_eq!( - ephemeral_after.data().len(), - data_len as usize, - "Data length should match" - ); - assert_eq!( - ephemeral_after.lamports(), - 0, - "Ephemeral account should have 0 lamports" - ); - - // CONSERVATION CHECK: Total lamports should not change - assert_eq!( - total_after, total_before, - "Total lamports should be conserved" - ); - - // VAULT CHECK: Vault should receive exactly the rent amount - assert_eq!( - vault_after.lamports(), - vault_before.lamports() + expected_rent, - "Vault should receive exactly {} lamports", - expected_rent - ); - - // SPONSOR CHECK: Sponsor should be charged exactly the rent amount - assert_eq!( - sponsor_after.lamports(), - sponsor_before.lamports() - expected_rent, - "Sponsor should be charged exactly {} lamports", - expected_rent - ); -} - -#[tokio::test] -async fn test_resize_ephemeral_account_via_cpi() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Use the payer (which is a signer) as the sponsor - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - let initial_data_len = 1000; - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - initial_data_len, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Check balances BEFORE resize - let sponsor_before_resize = env.get_account(sponsor); - let vault_before_resize = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_before_resize = env.get_account(ephemeral.pubkey()); - let total_before_resize = sponsor_before_resize.lamports() - + vault_before_resize.lamports() - + ephemeral_before_resize.lamports(); - - println!("=== BEFORE RESIZE (GROW) ==="); - println!("Sponsor: {}", sponsor_before_resize.lamports()); - println!("Vault: {}", vault_before_resize.lamports()); - println!("Ephemeral: {}", ephemeral_before_resize.lamports()); - println!("Total: {}", total_before_resize); - - // Resize the account - let new_data_len = 2000; - let initial_rent = rent_for(initial_data_len); - let new_rent = rent_for(new_data_len); - let rent_difference = new_rent - initial_rent; - - println!("Expected rent difference: {}", rent_difference); - - let ix = resize_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - new_data_len, - ); - let txn = env.build_transaction(&[ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Check balances AFTER resize - let sponsor_after_resize = env.get_account(sponsor); - let vault_after_resize = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_after_resize = env.get_account(ephemeral.pubkey()); - let total_after_resize = sponsor_after_resize.lamports() - + vault_after_resize.lamports() - + ephemeral_after_resize.lamports(); - - println!("=== AFTER RESIZE (GROW) ==="); - println!("Sponsor: {}", sponsor_after_resize.lamports()); - println!("Vault: {}", vault_after_resize.lamports()); - println!("Ephemeral: {}", ephemeral_after_resize.lamports()); - println!("Total: {}", total_after_resize); - - // Verify the account was resized - assert_eq!( - ephemeral_after_resize.data().len(), - new_data_len as usize, - "Data length should be updated" - ); - assert!( - ephemeral_after_resize.ephemeral(), - "Account should still be ephemeral" - ); - - // CONSERVATION CHECK: Total lamports should not change - assert_eq!( - total_after_resize, total_before_resize, - "Total lamports should be conserved" - ); - - // VAULT CHECK: Vault should receive exactly the rent difference - assert_eq!( - vault_after_resize.lamports(), - vault_before_resize.lamports() + rent_difference, - "Vault should receive exactly {} lamports", - rent_difference - ); - - // SPONSOR CHECK: Sponsor should be charged exactly the rent difference - assert_eq!( - sponsor_after_resize.lamports(), - sponsor_before_resize.lamports() - rent_difference, - "Sponsor should be charged exactly {} lamports", - rent_difference - ); -} - -#[tokio::test] -async fn test_close_ephemeral_account_via_cpi() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Use the payer (which is a signer) as the sponsor - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - let data_len = 1000; - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - data_len, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let expected_rent = rent_for(data_len); - - // Check balances BEFORE close - let sponsor_before_close = env.get_account(sponsor); - let vault_before_close = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_before_close = env.get_account(ephemeral.pubkey()); - let total_before_close = sponsor_before_close.lamports() - + vault_before_close.lamports() - + ephemeral_before_close.lamports(); - - println!("=== BEFORE CLOSE ==="); - println!("Sponsor: {}", sponsor_before_close.lamports()); - println!("Vault: {}", vault_before_close.lamports()); - println!("Ephemeral: {}", ephemeral_before_close.lamports()); - println!("Total: {}", total_before_close); - println!("Expected refund: {}", expected_rent); - - // Close the ephemeral account - let ix = close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - ); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - if let Err(e) = &result { - println!("Error closing account: {:?}", e); - } - assert!(result.is_ok()); - - // Check balances AFTER close - let sponsor_after_close = env.get_account(sponsor); - let vault_after_close = env.get_account(EPHEMERAL_VAULT_PUBKEY); - - // Closed ephemeral accounts are removed from the DB - assert!( - env.try_get_account(ephemeral.pubkey()).is_none(), - "Closed ephemeral account should be removed from DB" - ); - - let total_after_close = - sponsor_after_close.lamports() + vault_after_close.lamports(); - - println!("=== AFTER CLOSE ==="); - println!("Sponsor: {}", sponsor_after_close.lamports()); - println!("Vault: {}", vault_after_close.lamports()); - println!("Total: {}", total_after_close); - - // CONSERVATION CHECK: Total lamports should not change - // (ephemeral had 0 lamports, so sponsor + vault should equal prior total) - assert_eq!( - total_after_close, total_before_close, - "Total lamports should be conserved" - ); - - // VAULT CHECK: Vault should pay out exactly the expected rent - assert_eq!( - vault_after_close.lamports(), - vault_before_close.lamports() - expected_rent, - "Vault should pay out exactly {} lamports", - expected_rent - ); - - // SPONSOR CHECK: Sponsor should receive exactly the expected refund - assert_eq!( - sponsor_after_close.lamports(), - sponsor_before_close.lamports() + expected_rent, - "Sponsor should receive exactly {} lamports refund", - expected_rent - ); -} - -#[tokio::test] -async fn test_resize_smaller_via_cpi() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Use the payer (which is a signer) as the sponsor - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - let initial_data_len = 2000; - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - initial_data_len, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Check balances BEFORE resize (shrink) - let sponsor_before_resize = env.get_account(sponsor); - let vault_before_resize = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_before_resize = env.get_account(ephemeral.pubkey()); - let total_before_resize = sponsor_before_resize.lamports() - + vault_before_resize.lamports() - + ephemeral_before_resize.lamports(); - - println!("=== BEFORE RESIZE (SHRINK) ==="); - println!("Sponsor: {}", sponsor_before_resize.lamports()); - println!("Vault: {}", vault_before_resize.lamports()); - println!("Ephemeral: {}", ephemeral_before_resize.lamports()); - println!("Total: {}", total_before_resize); - - // Resize to smaller - let new_data_len = 1000; - let initial_rent = rent_for(initial_data_len); - let new_rent = rent_for(new_data_len); - let rent_refund = initial_rent - new_rent; - - println!("Expected refund: {}", rent_refund); - - let ix = resize_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - new_data_len, - ); - let txn = env.build_transaction(&[ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Check balances AFTER resize - let sponsor_after_resize = env.get_account(sponsor); - let vault_after_resize = env.get_account(EPHEMERAL_VAULT_PUBKEY); - let ephemeral_after_resize = env.get_account(ephemeral.pubkey()); - let total_after_resize = sponsor_after_resize.lamports() - + vault_after_resize.lamports() - + ephemeral_after_resize.lamports(); - - println!("=== AFTER RESIZE (SHRINK) ==="); - println!("Sponsor: {}", sponsor_after_resize.lamports()); - println!("Vault: {}", vault_after_resize.lamports()); - println!("Ephemeral: {}", ephemeral_after_resize.lamports()); - println!("Total: {}", total_after_resize); - - // Verify the account was resized - assert_eq!( - ephemeral_after_resize.data().len(), - new_data_len as usize, - "Data length should be updated" - ); - - // CONSERVATION CHECK: Total lamports should not change - assert_eq!( - total_after_resize, total_before_resize, - "Total lamports should be conserved" - ); - - // VAULT CHECK: Vault should pay out exactly the refund - assert_eq!( - vault_after_resize.lamports(), - vault_before_resize.lamports() - rent_refund, - "Vault should pay out exactly {} lamports", - rent_refund - ); - - // SPONSOR CHECK: Sponsor should receive exactly the refund - assert_eq!( - sponsor_after_resize.lamports(), - sponsor_before_resize.lamports() + rent_refund, - "Sponsor should receive exactly {} lamports refund", - rent_refund - ); -} - -// ============================================================================ -// Failure & Validation Tests -// ============================================================================ - -/// Helper to create a direct magic-program instruction (not via CPI) -fn direct_create_instruction( - sponsor: Pubkey, - ephemeral: Pubkey, - vault: Pubkey, - data_len: u32, -) -> Instruction { - Instruction::new_with_bincode( - magicblock_magic_program_api::ID, - &MagicBlockInstruction::CreateEphemeralAccount { data_len }, - vec![ - AccountMeta::new(sponsor, true), - AccountMeta::new(ephemeral, false), - AccountMeta::new(vault, false), - ], - ) -} - -#[tokio::test] -async fn test_direct_call_rejected() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - // Call magic-program directly (NOT via CPI) - should fail - let ix = direct_create_instruction( - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert!(result.is_err(), "Direct call should be rejected"); -} - -#[tokio::test] -async fn test_create_with_non_zero_lamports_fails() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(100); // Non-zero lamports! - - let result = execute_instruction_with_signers( - &env, - create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ), - &[&ephemeral], - ) - .await; - - assert!(result.is_err(), "Should fail with non-zero lamports"); -} - -#[tokio::test] -async fn test_create_already_ephemeral_fails() { - let ctx = setup_test(); - - // Simulate an existing ephemeral account (owned by a program, not system) - let mut acc = ctx.env.get_account(ctx.ephemeral.pubkey()); - acc.set_owner(guinea::ID); - acc.set_ephemeral(true); - acc.commit(); - - let result = execute_instruction_with_signers( - &ctx.env, - create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - ctx.sponsor, - ctx.ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ), - &[&ctx.ephemeral], - ) - .await; - - assert!(result.is_err(), "Should fail - already ephemeral"); -} - -#[tokio::test] -async fn test_create_with_wrong_vault_fails() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0).pubkey(); - - // Use wrong vault (not owned by magic-program) - let wrong_vault = env.create_account(1000).pubkey(); - - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::CreateEphemeralAccount { data_len: 1000 }, - vec![ - AccountMeta::new_readonly(magicblock_magic_program_api::ID, false), - AccountMeta::new(sponsor, true), - AccountMeta::new(ephemeral, false), - AccountMeta::new(wrong_vault, false), - ], - ); - - let result = execute_instruction(&env, ix).await; - assert!(result.is_err(), "Should fail with wrong vault"); -} - -#[tokio::test] -async fn test_resize_non_ephemeral_fails() { - let ctx = setup_test(); - let regular = ctx.env.create_account(0).pubkey(); - - // Try to resize a non-ephemeral account - let result = execute_instruction( - &ctx.env, - resize_ephemeral_account_ix( - magicblock_magic_program_api::ID, - ctx.sponsor, - regular, - EPHEMERAL_VAULT_PUBKEY, - 1000, - ), - ) - .await; - - assert!(result.is_err(), "Should fail - not ephemeral"); -} - -#[tokio::test] -async fn test_close_non_ephemeral_fails() { - let ctx = setup_test(); - let regular = ctx.env.create_account(0).pubkey(); - - // Try to close a non-ephemeral account - let result = execute_instruction( - &ctx.env, - close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - ctx.sponsor, - regular, - EPHEMERAL_VAULT_PUBKEY, - ), - ) - .await; - - assert!(result.is_err(), "Should fail - not ephemeral"); -} - -#[tokio::test] -async fn test_resize_to_zero_size() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - // Create with initial size - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Resize to zero - let ix = resize_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 0, - ); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert!(result.is_ok(), "Should allow resize to zero"); - - let ephemeral_after = env.get_account(ephemeral.pubkey()); - assert_eq!(ephemeral_after.data().len(), 0); -} - -#[tokio::test] -async fn test_close_already_closed() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - // Create and close - let create_ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ); - let txn = env.build_transaction_with_signers(&[create_ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let close_ix = close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - ); - let txn = env.build_transaction(&[close_ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Try to close again - should fail because account is now owned by system-program - let close_ix2 = close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - ); - let txn = env.build_transaction(&[close_ix2]); - - let result = env.execute_transaction(txn).await; - assert!( - result.is_err(), - "Re-close should fail (account owned by system-program)" - ); -} - -#[tokio::test] -async fn test_close_already_closed_double_spend() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - // Create and close - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Track balances before first close - let sponsor_before = - env.accountsdb.get_account(&sponsor).unwrap().lamports(); - let vault_before = env - .accountsdb - .get_account(&EPHEMERAL_VAULT_PUBKEY) - .unwrap() - .lamports(); - - let close_ix = close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - ); - let txn = env.build_transaction(&[close_ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let sponsor_after_close = - env.accountsdb.get_account(&sponsor).unwrap().lamports(); - let vault_after_close = env - .accountsdb - .get_account(&EPHEMERAL_VAULT_PUBKEY) - .unwrap() - .lamports(); - - let first_refund = sponsor_after_close - sponsor_before; - let first_vault_debit = vault_before - vault_after_close; - - println!( - "First close - Sponsor refund: {}, Vault debit: {}", - first_refund, first_vault_debit - ); - - let close_ix2 = close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - ); - let txn = env.build_transaction(&[close_ix2]); - let result = env.execute_transaction(txn).await; - - assert!( - result.is_err(), - "Re-close should fail (account not owned by magic-program)" - ); - - // Verify no additional refund was given - let sponsor_after_close2 = - env.accountsdb.get_account(&sponsor).unwrap().lamports(); - let vault_after_close2 = env - .accountsdb - .get_account(&EPHEMERAL_VAULT_PUBKEY) - .unwrap() - .lamports(); - - assert_eq!( - sponsor_after_close2, sponsor_after_close, - "Sponsor should not get second refund" - ); - assert_eq!( - vault_after_close2, vault_after_close, - "Vault should not be debited twice" - ); -} - -#[tokio::test] -async fn test_insufficient_balance_fails() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Use the payer but give it very low balance - let sponsor = env.get_payer().pubkey; - - // Set sponsor balance to very low amount - let mut sponsor_acc = env.get_account(sponsor); - sponsor_acc.set_lamports(100); // Only 100 lamports - sponsor_acc.commit(); - - let ephemeral = env.create_account(0); - - let data_len = 1000; - let required_rent = rent_for(data_len); - - assert!(required_rent > 100, "Rent should exceed sponsor balance"); - - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - data_len, - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - - let result = env.execute_transaction(txn).await; - assert!(result.is_err(), "Should fail - insufficient balance"); -} - -// Tests creating an ephemeral account with a PDA sponsor. -// The guinea program uses `invoke_signed` with proper seeds to sign for the PDA. -#[tokio::test] -async fn test_create_with_pda_sponsor() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // 1. Derive the global sponsor PDA (same seed as in guinea program) - let (global_sponsor_pda, _bump) = - Pubkey::find_program_address(&[b"global_sponsor"], &guinea::ID); - - // 2. Insert into accountsdb with 1 SOL and 32 bytes data, owned by guinea - let mut account = AccountSharedData::new(1_000_000_000, 32, &guinea::ID); - account.set_delegated(true); - let _ = env.accountsdb.insert_account(&global_sponsor_pda, &account); - - let ephemeral = env.create_account(0); - - // 3. Try creating ephemeral account with PDA sponsor using the regular instruction - // guinea will detect the PDA and patch the instruction to use invoke_signed - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::CreateEphemeralAccount { data_len: 1000 }, - vec![ - AccountMeta::new_readonly(magicblock_magic_program_api::ID, false), - AccountMeta::new(global_sponsor_pda, false), // PDA (not a signer in transaction) - AccountMeta::new(ephemeral.pubkey(), true), // Ephemeral must sign - AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), - ], - ); - let txn = env.build_transaction_with_signers(&[ix], &[&ephemeral]); - - let result = env.execute_transaction(txn).await; - - // Should succeed - guinea patches the instruction and uses invoke_signed - match result { - Ok(_) => { - // Verify ephemeral account was created successfully - let ephemeral_acc = env.get_account(ephemeral.pubkey()); - assert!(ephemeral_acc.ephemeral(), "Account should be ephemeral"); - assert_eq!( - ephemeral_acc.data().len(), - 1000, - "Account should have 1000 bytes" - ); - } - Err(e) => { - panic!("PDA sponsor test failed with: {:?}", e); - } - } -} - -#[tokio::test] -async fn test_pda_wrong_owner_fails() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Create a PDA owned by system program (not guinea) - let (pda, _bump) = Pubkey::find_program_address( - &[b"wrong"], - &solana_sdk_ids::system_program::id(), - ); - let mut account = - AccountSharedData::new(0, 0, &solana_sdk_ids::system_program::id()); - account.set_delegated(true); - let _ = env.accountsdb.insert_account(&pda, &account); - - let ephemeral = env.create_account(0); - - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::CreateEphemeralAccount { data_len: 1000 }, - vec![ - AccountMeta::new_readonly(magicblock_magic_program_api::ID, false), - AccountMeta::new(pda, false), // NOT a signer, not owned by guinea - AccountMeta::new(ephemeral.pubkey(), false), - AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), - ], - ); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert!(result.is_err(), "Should fail - PDA not owned by caller"); -} - -#[tokio::test] -async fn test_non_signer_oncurve_sponsor_fails() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Create oncurve account that is NOT a signer - let non_signer = env.create_account(1000); - let ephemeral = env.create_account(0); - - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::CreateEphemeralAccount { data_len: 1000 }, - vec![ - AccountMeta::new_readonly(magicblock_magic_program_api::ID, false), - AccountMeta::new(non_signer.pubkey(), false), // NOT a signer - AccountMeta::new(ephemeral.pubkey(), false), - AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), - ], - ); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert!(result.is_err(), "Should fail - non-signer oncurve account"); -} - -#[tokio::test] -async fn test_full_lifecycle() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - let sponsor = env.get_payer().pubkey; - init_sponsor(&env, sponsor); - let ephemeral = env.create_account(0); - - // Create → Resize (grow) → Resize (shrink) → Close - let create_ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 1000, - ); - let txn = env.build_transaction_with_signers(&[create_ix], &[&ephemeral]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let grow_ix = resize_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 2000, - ); - let txn = env.build_transaction(&[grow_ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let shrink_ix = resize_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - 500, - ); - let txn = env.build_transaction(&[shrink_ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let close_ix = close_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - ephemeral.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - ); - let txn = env.build_transaction(&[close_ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - // Closed ephemeral accounts are removed from the DB - assert!( - env.try_get_account(ephemeral.pubkey()).is_none(), - "Closed ephemeral account should be removed from DB" - ); -} - -#[tokio::test] -async fn test_multiple_accounts_same_sponsor() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - init_vault(&env); - - // Use payer[0] as sponsor - need to be explicit about which payer - let sponsor = env.payers[0].pubkey(); - init_sponsor(&env, sponsor); - - let eph1 = env.create_account(0); - let eph2 = env.create_account(0); - let eph3 = env.create_account(0); - - let total_rent = rent_for(1000) + rent_for(2000) + rent_for(500); - - // Get initial balance directly from AccountsDB to avoid caching issues - let sponsor_balance_before = - env.accountsdb.get_account(&sponsor).unwrap().lamports(); - - // Create 3 accounts with different sizes - for (eph, len) in [(&eph1, 1000), (&eph2, 2000), (&eph3, 500)] { - let ix = create_ephemeral_account_ix( - magicblock_magic_program_api::ID, - sponsor, - eph.pubkey(), - EPHEMERAL_VAULT_PUBKEY, - len, - ); - let txn = env.build_transaction_with_signers(&[ix], &[eph]); - assert!(env.execute_transaction(txn).await.is_ok()); - } - - // Get final balance directly from AccountsDB - let sponsor_balance_after = - env.accountsdb.get_account(&sponsor).unwrap().lamports(); - - assert_eq!( - sponsor_balance_before - sponsor_balance_after, - total_rent, - "Sponsor should be charged total rent for all accounts" - ); -} diff --git a/magicblock-processor/tests/execution.rs b/magicblock-processor/tests/execution.rs deleted file mode 100644 index dc28bf64f..000000000 --- a/magicblock-processor/tests/execution.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{collections::HashSet, time::Duration}; - -use guinea::GuineaInstruction; -use solana_account::ReadableAccount; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use test_kit::{ExecutionTestEnv, Signer}; - -const ACCOUNTS_COUNT: usize = 8; -const TIMEOUT: Duration = Duration::from_millis(200); - -/// Helper to execute a standard "Guinea" transaction on `ACCOUNTS_COUNT` new accounts. -async fn execute_guinea( - env: &ExecutionTestEnv, - ix: GuineaInstruction, - is_writable: bool, -) -> (Signature, Vec) { - let accounts: Vec<_> = (0..ACCOUNTS_COUNT) - .map(|_| { - env.create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID) - }) - .collect(); - - let metas = accounts - .iter() - .map(|a| { - if is_writable { - AccountMeta::new(a.pubkey(), false) - } else { - AccountMeta::new_readonly(a.pubkey(), false) - } - }) - .collect(); - - env.advance_slot(); - let ix = Instruction::new_with_bincode(guinea::ID, &ix, metas); - let txn = env.build_transaction(&[ix]); - let sig = txn.signatures[0]; - - assert!( - env.execute_transaction(txn).await.is_ok(), - "Transaction execution failed" - ); - env.advance_slot(); - - let pubkeys = accounts.iter().map(|a| a.pubkey()).collect(); - (sig, pubkeys) -} - -#[tokio::test] -async fn test_transaction_with_return_data() { - let env = ExecutionTestEnv::new(); - let (sig, _) = - execute_guinea(&env, GuineaInstruction::ComputeBalances, false).await; - - let meta = env.get_transaction(sig).expect("Transaction not in ledger"); - let ret_data = meta.return_data.expect("Return data missing"); - - let expected = (ACCOUNTS_COUNT as u64 * LAMPORTS_PER_SOL).to_le_bytes(); - assert_eq!(ret_data.data, expected, "Incorrect return data balance"); -} - -#[tokio::test] -async fn test_transaction_status_update() { - let env = ExecutionTestEnv::new(); - let (sig, _) = - execute_guinea(&env, GuineaInstruction::PrintSizes, false).await; - - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .expect("Status update missing"); - - assert_eq!(status.txn.signatures()[0], sig); - let logs = status.meta.log_messages.as_ref().expect("Logs missing"); - assert!(logs.len() > ACCOUNTS_COUNT, "Insufficient logs produced"); -} - -#[tokio::test] -async fn test_transaction_modifies_accounts() { - let env = ExecutionTestEnv::new(); - let (_, accounts) = - execute_guinea(&env, GuineaInstruction::WriteByteToData(42), true) - .await; - - // 1. Verify DB state modifications - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .expect("Status update missing"); - - // Skip fee payer, check the guinea accounts - let account_keys: Vec<_> = status - .txn - .message() - .account_keys() - .iter() - .copied() - .collect(); - for pubkey in account_keys.iter().skip(1).take(ACCOUNTS_COUNT) { - let account = env.get_account(*pubkey); - assert_eq!(account.data()[0], 42, "Account data mismatch"); - } - - // 2. Verify update notifications - let mut updated_accounts = HashSet::new(); - while let Ok(update) = env.dispatch.account_update.try_recv() { - updated_accounts.insert(update.account.pubkey); - } - - let expected_accounts: HashSet<_> = HashSet::from_iter(accounts); - assert!( - updated_accounts.is_superset(&expected_accounts), - "Missing account update notifications" - ); -} diff --git a/magicblock-processor/tests/fees.rs b/magicblock-processor/tests/fees.rs deleted file mode 100644 index 6718a7f00..000000000 --- a/magicblock-processor/tests/fees.rs +++ /dev/null @@ -1,275 +0,0 @@ -use std::time::Duration; - -use guinea::GuineaInstruction; -use solana_account::{ReadableAccount, WritableAccount}; -use solana_compute_budget_interface::ComputeBudgetInstruction; -use solana_keypair::Keypair; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, - rent::Rent, -}; -use solana_transaction_error::TransactionError; -use test_kit::{ExecutionTestEnv, Signer}; - -const BASE_FEE: u64 = ExecutionTestEnv::BASE_FEE; -const TIMEOUT: Duration = Duration::from_millis(100); - -/// Helper to setup a guinea instruction with a new account. -fn setup_guinea_ix( - env: &ExecutionTestEnv, - ix_data: GuineaInstruction, -) -> Instruction { - let balance = Rent::default().minimum_balance(128); - let account = env - .create_account_with_config(balance, 128, guinea::ID) - .pubkey(); - - Instruction::new_with_bincode( - guinea::ID, - &ix_data, - vec![AccountMeta::new(account, false)], - ) -} - -#[tokio::test] -async fn test_insufficient_fee() { - let env = ExecutionTestEnv::new(); - let mut payer = env.get_payer(); - payer.set_lamports(BASE_FEE - 1); - payer.commit(); - - let ix = setup_guinea_ix(&env, GuineaInstruction::PrintSizes); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert!(matches!( - result, - Err(TransactionError::InsufficientFundsForFee) - )); -} - -#[tokio::test] -async fn test_separate_fee_payer() { - let env = ExecutionTestEnv::new(); - let sender = - env.create_account_with_config(LAMPORTS_PER_SOL, 0, guinea::ID); - let recipient = env.create_account(LAMPORTS_PER_SOL); - let initial_payer_bal = env.get_payer().lamports(); - const AMOUNT: u64 = 1_000_000; - - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::Transfer(AMOUNT), - vec![ - AccountMeta::new(sender.pubkey(), false), - AccountMeta::new(recipient.pubkey(), false), - ], - ); - let txn = env.build_transaction(&[ix]); - - env.execute_transaction(txn).await.expect("Transfer failed"); - - assert_eq!( - env.get_account(sender.pubkey()).lamports(), - LAMPORTS_PER_SOL - AMOUNT - ); - assert_eq!( - env.get_account(recipient.pubkey()).lamports(), - LAMPORTS_PER_SOL + AMOUNT - ); - assert_eq!(env.get_payer().lamports(), initial_payer_bal - BASE_FEE); -} - -#[tokio::test] -async fn test_compute_unit_price_does_not_change_fee() { - let env = ExecutionTestEnv::new(); - let initial_payer_bal = env.get_payer().lamports(); - - let compute_limit_ix = - ComputeBudgetInstruction::set_compute_unit_limit(1_400_000); - let compute_price_ix = ComputeBudgetInstruction::set_compute_unit_price(1); - let guinea_ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - vec![], - ); - let txn = - env.build_transaction(&[compute_limit_ix, compute_price_ix, guinea_ix]); - - env.execute_transaction(txn) - .await - .expect("Transaction with CU price failed"); - - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .unwrap(); - assert!(status.meta.status.is_ok()); - assert_eq!(status.meta.fee, BASE_FEE); - assert_eq!(env.get_payer().lamports(), initial_payer_bal - BASE_FEE); -} - -#[tokio::test] -async fn test_non_delegated_payer_rejection() { - let env = ExecutionTestEnv::new(); - let mut payer = env.get_payer(); - payer.set_delegated(false); - let initial_bal = payer.lamports(); - payer.commit(); - - let ix = setup_guinea_ix(&env, GuineaInstruction::PrintSizes); - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert!( - matches!(result, Err(TransactionError::InvalidAccountForFee)), - "Non-delegated payer should be rejected" - ); - assert_eq!( - env.get_payer().lamports(), - initial_bal, - "Rejected tx should not be charged" - ); -} - -#[tokio::test] -async fn test_fee_charged_for_failed_transaction() { - let env = ExecutionTestEnv::new(); - env.wait_for_scheduler_ready().await; - let initial_bal = env.get_payer().lamports(); - - // Create invalid instruction (writing to empty data) - let ix = setup_guinea_ix(&env, GuineaInstruction::WriteByteToData(42)); - // Hack: manually set data len to 0 to force failure - let mut acc = env.get_account(ix.accounts[0].pubkey); - acc.set_data(vec![]); - env.accountsdb - .insert_account(&ix.accounts[0].pubkey, &acc) - .unwrap(); - - let txn = env.build_transaction(&[ix]); - env.transaction_scheduler.schedule(txn).await.unwrap(); - - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .unwrap(); - assert!(status.meta.status.is_err(), "Transaction should fail"); - assert_eq!( - env.get_payer().lamports(), - initial_bal - BASE_FEE, - "Fee should be charged on failure" - ); -} - -#[tokio::test] -async fn test_transaction_gasless_mode() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let mut payer = env.get_payer(); - payer.set_lamports(1); - payer.set_delegated(false); - let initial_bal = payer.lamports(); - payer.commit(); - - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - vec![], - ); - let txn = env.build_transaction(&[ix]); - let sig = txn.signatures[0]; - - env.execute_transaction(txn) - .await - .expect("Gasless tx failed"); - - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .unwrap(); - assert_eq!(status.txn.signatures()[0], sig); - assert!(status.meta.status.is_ok()); - assert_eq!( - env.get_payer().lamports(), - initial_bal, - "Balance changed in gasless mode" - ); -} - -#[tokio::test] -async fn test_transaction_gasless_mode_with_cu_price() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let mut payer = env.get_payer(); - payer.set_lamports(1); - payer.set_delegated(false); - let initial_bal = payer.lamports(); - payer.commit(); - - let compute_limit_ix = - ComputeBudgetInstruction::set_compute_unit_limit(1_400_000); - let compute_price_ix = ComputeBudgetInstruction::set_compute_unit_price(1); - let guinea_ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - vec![], - ); - let txn = - env.build_transaction(&[compute_limit_ix, compute_price_ix, guinea_ix]); - let sig = txn.signatures[0]; - - env.execute_transaction(txn) - .await - .expect("Gasless tx with CU price failed"); - - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .unwrap(); - assert_eq!(status.txn.signatures()[0], sig); - assert!(status.meta.status.is_ok()); - assert_eq!(status.meta.fee, 0); - assert_eq!( - env.get_payer().lamports(), - initial_bal, - "Balance changed in gasless mode" - ); -} - -#[tokio::test] -async fn test_transaction_gasless_mode_with_non_existent_account() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let mut payer = env.get_payer(); - payer.set_lamports(1); - payer.set_delegated(false); - let initial_bal = payer.lamports(); - payer.commit(); - - // Use a random non-existent account - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - vec![AccountMeta::new_readonly(Keypair::new().pubkey(), false)], - ); - let txn = env.build_transaction(&[ix]); - - env.execute_transaction(txn) - .await - .expect("Gasless tx with missing acc failed"); - - let status = env - .dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .unwrap(); - assert!(status.meta.status.is_ok()); - assert_eq!( - env.get_payer().lamports(), - initial_bal, - "Balance changed in gasless mode" - ); -} diff --git a/magicblock-processor/tests/post_delegation_actions.rs b/magicblock-processor/tests/post_delegation_actions.rs deleted file mode 100644 index a41b25ea4..000000000 --- a/magicblock-processor/tests/post_delegation_actions.rs +++ /dev/null @@ -1,301 +0,0 @@ -use guinea::GuineaInstruction; -use magicblock_magic_program_api::{ - args::ScheduleTaskArgs, instruction::AccountCloneFields, - MAGIC_CONTEXT_PUBKEY, MAGIC_CONTEXT_SIZE, -}; -use magicblock_program::{ - instruction_utils::InstructionUtils, - validator::{generate_validator_authority_if_needed, validator_authority}, - MagicContext, -}; -use solana_account::{AccountSharedData, ReadableAccount}; -use solana_instruction::{AccountMeta, Instruction}; -use solana_pubkey::Pubkey; -use solana_sdk_ids::system_program; -use solana_signer::Signer; -use test_kit::ExecutionTestEnv; - -/// Inserts the magic context account so that scheduling instructions which -/// write scheduled actions into it can execute against the runtime. -fn insert_magic_context(env: &ExecutionTestEnv) { - let mut magic_context = AccountSharedData::new( - u64::MAX, - MAGIC_CONTEXT_SIZE, - &magicblock_magic_program_api::ID, - ); - magic_context.set_delegated(true); - env.accountsdb - .insert_account(&MAGIC_CONTEXT_PUBKEY, &magic_context) - .unwrap(); -} - -#[tokio::test] -async fn executor_runs_post_delegation_actions_after_clone() { - generate_validator_authority_if_needed(); - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let validator = validator_authority(); - env.fund_account(validator.pubkey(), 10_000_000); - - let target = Pubkey::new_unique(); - env.accountsdb - .insert_account( - &target, - &AccountSharedData::new(100, 0, &system_program::id()), - ) - .unwrap(); - - let counter = Pubkey::new_unique(); - let mut counter_account = AccountSharedData::new(1_000, 1, &guinea::ID); - counter_account.set_delegated(true); - counter_account.set_data_from_slice(&[0]); - env.accountsdb - .insert_account(&counter, &counter_account) - .unwrap(); - - let action_payer = Pubkey::new_unique(); - env.fund_account(action_payer, 1_000_000); - - let schedule_task_action = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::ScheduleTask(ScheduleTaskArgs { - task_id: 1, - execution_interval_millis: 1, - iterations: 1, - instructions: vec![InstructionUtils::noop_instruction(0)], - }), - vec![ - AccountMeta::new_readonly(magicblock_magic_program_api::ID, false), - AccountMeta::new(action_payer, true), - AccountMeta::new(counter, false), - ], - ); - let increment_action = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::Increment, - vec![AccountMeta::new(counter, false)], - ); - let actions = vec![schedule_task_action, increment_action]; - - let clone_ix = InstructionUtils::clone_account_instruction( - target, - vec![9], - AccountCloneFields { - lamports: 1_000_000, - owner: system_program::id(), - delegated: true, - remote_slot: 1, - ..Default::default() - }, - actions.clone(), - ); - let executor_ix = - InstructionUtils::post_delegation_action_executor_instruction( - target, actions, - ); - - let txn = env.build_transaction_with_signers( - &[clone_ix, executor_ix], - &[&validator], - ); - env.execute_transaction(txn).await.unwrap(); - - let target_account = env.get_account(target); - assert!(target_account.delegated()); - assert_eq!(target_account.data(), &[9]); - - let counter_account = env.get_account(counter); - assert_eq!(counter_account.data(), &[1]); -} - -#[tokio::test] -async fn schedule_undelegation_marks_cloned_account_as_undelegated() { - generate_validator_authority_if_needed(); - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let validator = validator_authority(); - env.fund_account(validator.pubkey(), 10_000_000); - insert_magic_context(&env); - - // The account being rescued starts as a plain remote account. - let target = Pubkey::new_unique(); - env.accountsdb - .insert_account( - &target, - &AccountSharedData::new(100, 0, &system_program::id()), - ) - .unwrap(); - - // Clone it as a delegated account, then schedule its undelegation in the - // same transaction (mirrors the cloner's small-account rescue path). - let clone_ix = InstructionUtils::clone_account_instruction( - target, - vec![7], - AccountCloneFields { - lamports: 1_000_000, - owner: system_program::id(), - delegated: true, - remote_slot: 1, - ..Default::default() - }, - Vec::new(), - ); - let schedule_ix = - InstructionUtils::schedule_cloned_account_undelegation_instruction( - target, - ); - - let txn = env.build_transaction_with_signers( - &[clone_ix, schedule_ix], - &[&validator], - ); - env.execute_transaction(txn).await.unwrap(); - - // The schedule instruction mutates the cloned account's owner/state, so it - // must be writable in the instruction. After execution the account is - // marked undelegating and no longer delegated. - let target_account = env.get_account(target); - assert!(target_account.undelegating()); - assert!(!target_account.delegated()); -} - -#[tokio::test] -async fn schedule_undelegation_commits_original_owner() { - generate_validator_authority_if_needed(); - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let validator = validator_authority(); - env.fund_account(validator.pubkey(), 10_000_000); - insert_magic_context(&env); - - let target = Pubkey::new_unique(); - env.accountsdb - .insert_account( - &target, - &AccountSharedData::new(100, 0, &system_program::id()), - ) - .unwrap(); - - // Clone a delegated account owned by a real program, then schedule its - // undelegation in the same transaction. - let clone_ix = InstructionUtils::clone_account_instruction( - target, - vec![7], - AccountCloneFields { - lamports: 1_000_000, - owner: guinea::ID, - delegated: true, - remote_slot: 1, - ..Default::default() - }, - Vec::new(), - ); - let schedule_ix = - InstructionUtils::schedule_cloned_account_undelegation_instruction( - target, - ); - - let txn = env.build_transaction_with_signers( - &[clone_ix, schedule_ix], - &[&validator], - ); - env.execute_transaction(txn).await.unwrap(); - - // The scheduled intent must commit the account with its original owner. - // Scheduling marks the live account as undelegating (owner flipped to the - // delegation program); on mmap-backed accounts that mutation is visible - // through shallow snapshots, so a wrong ordering in the processor bakes - // the delegation program in as the owner. The committor then derives the - // dlp program-config PDA from it and the base-layer commit is rejected - // with InvalidAuthority. - let context_data = env.get_account(MAGIC_CONTEXT_PUBKEY); - let context: MagicContext = - bincode::deserialize(context_data.data()).unwrap(); - let intent = context - .scheduled_base_intents - .first() - .expect("undelegation intent must be scheduled"); - let committed = intent - .intent_bundle - .commit_and_undelegate - .as_ref() - .expect("intent must be a commit-and-undelegate") - .get_committed_accounts() - .first() - .expect("intent must commit the cloned account"); - assert_eq!(committed.pubkey, target); - assert_eq!(committed.account.owner, guinea::ID); - - // The live account is still locked for undelegation as before. - let target_account = env.get_account(target); - assert!(target_account.undelegating()); - assert!(!target_account.delegated()); -} - -#[tokio::test] -async fn chunked_rescue_undelegation_clears_pending_clone() { - generate_validator_authority_if_needed(); - let env = ExecutionTestEnv::new_with_config(0, 1, false); - let validator = validator_authority(); - env.fund_account(validator.pubkey(), 10_000_000); - insert_magic_context(&env); - - let target = Pubkey::new_unique(); - env.accountsdb - .insert_account( - &target, - &AccountSharedData::new(100, 0, &system_program::id()), - ) - .unwrap(); - - let fields = AccountCloneFields { - lamports: 1_000_000, - owner: system_program::id(), - delegated: true, - remote_slot: 1, - ..Default::default() - }; - - // 1. Initialize a chunked (large-account) clone. This registers the pubkey - // in the process-global PENDING_CLONES set. - let init_ix = InstructionUtils::clone_account_init_instruction( - target, - 1, - vec![1], - fields, - ); - let init_tx = env.build_transaction_with_signers(&[init_ix], &[&validator]); - env.execute_transaction(init_tx).await.unwrap(); - - // 2. Final chunk requesting undelegation, paired with the schedule - // instruction. The final `CloneAccountContinue(needs_undelegation=true)` - // intentionally leaves the clone pending so the sibling schedule - // instruction can validate the previous clone; the schedule instruction - // must then clear the pending entry. - let continue_ix = InstructionUtils::clone_account_continue_instruction( - target, - 1, - Vec::new(), - true, - Vec::new(), - true, - ); - let schedule_ix = - InstructionUtils::schedule_cloned_account_undelegation_instruction( - target, - ); - let rescue_tx = env.build_transaction_with_signers( - &[continue_ix, schedule_ix], - &[&validator], - ); - - // Before the rescue, the chunked clone is still pending (the final continue - // intentionally leaves it so the schedule instruction can validate it). - assert!(magicblock_program::is_pending_clone(&target)); - - env.execute_transaction(rescue_tx).await.unwrap(); - assert!(env.get_account(target).undelegating()); - assert!(!env.get_account(target).delegated()); - - // The schedule instruction must clear the process-global pending-clone - // entry; otherwise a later clone init fails with CloneAlreadyPending and - // cleanup could close already-completed state. - assert!(!magicblock_program::is_pending_clone(&target)); -} diff --git a/magicblock-processor/tests/replay.rs b/magicblock-processor/tests/replay.rs deleted file mode 100644 index 23b52f23e..000000000 --- a/magicblock-processor/tests/replay.rs +++ /dev/null @@ -1,121 +0,0 @@ -use std::time::Duration; - -use guinea::GuineaInstruction; -use magicblock_accounts_db::traits::AccountsBank; -use magicblock_core::link::transactions::SanitizeableTransaction; -use solana_account::ReadableAccount; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_pubkey::Pubkey; -use solana_signer::Signer; -use solana_transaction::sanitized::SanitizedTransaction; -use solana_transaction_status::TransactionStatusMeta; -use test_kit::ExecutionTestEnv; - -const ACCOUNTS_COUNT: usize = 8; -const TIMEOUT: Duration = Duration::from_millis(100); - -/// Sets up a replay scenario in Replica mode: -/// - Transaction is written directly to Ledger (no execution) -/// - AccountsDb remains in pre-transaction state -fn setup_replay_scenario_replica( - env: &ExecutionTestEnv, - ix: GuineaInstruction, -) -> (SanitizedTransaction, Vec) { - // 1. Create Accounts - let accounts: Vec<_> = (0..ACCOUNTS_COUNT) - .map(|_| { - env.create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID) - }) - .collect(); - - let metas = accounts - .iter() - .map(|a| AccountMeta::new(a.pubkey(), false)) - .collect(); - let pubkeys: Vec<_> = accounts.iter().map(|a| a.pubkey()).collect(); - - // 2. Build Transaction - let ix = Instruction::new_with_bincode(guinea::ID, &ix, metas); - let txn = env.build_transaction(&[ix]); - let sanitized = txn.sanitize(false).unwrap(); - let sig = *sanitized.signature(); - - // 3. Write transaction to ledger directly (without executing) - // This simulates a transaction that was recorded by the primary - let meta = TransactionStatusMeta { - fee: 5000, - pre_balances: pubkeys.iter().map(|_| LAMPORTS_PER_SOL).collect(), - post_balances: pubkeys.iter().map(|_| LAMPORTS_PER_SOL).collect(), - status: Ok(()), - ..Default::default() - }; - let versioned = sanitized.to_versioned_transaction(); - let encoded = bincode::serialize(&versioned).unwrap(); - let locks = sanitized.get_account_locks_unchecked(); - env.ledger - .write_transaction( - sig, - env.ledger.latest_block().load().slot, - 0, // index - locks.writable, - locks.readonly, - &encoded, - meta, - ) - .expect("Failed to write transaction to ledger"); - - // 4. Verify accounts are still in pre-transaction state - for pubkey in &pubkeys { - let account = env.accountsdb.get_account(pubkey).unwrap(); - assert_eq!( - account.data()[0], - 0, - "Account should be in pre-tx state before replay" - ); - } - - (sanitized, pubkeys) -} - -#[tokio::test] -pub async fn test_replay_state_transition() { - // Run in Replica mode (scheduler starts in Replica, no mode switch) - let env = ExecutionTestEnv::new_replica_mode(1, false); - env.yield_to_scheduler().await; - - let (txn, pubkeys) = setup_replay_scenario_replica( - &env, - GuineaInstruction::WriteByteToData(42), - ); - - // 1. Verify Pre-Replay State - for pubkey in &pubkeys { - let account = env.accountsdb.get_account(pubkey).unwrap(); - assert_eq!(account.data()[0], 0, "Account should be in pre-tx state"); - } - - // 2. Perform Replay (persist=false: no status notifications) - assert!(env.replay_transaction(false, txn).await.is_ok()); - - // 3. Verify No Side Effects (Notifications) - assert!( - env.dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .is_err(), - "Replay should NOT broadcast status updates" - ); - assert!( - env.dispatch.account_update.try_recv().is_err(), - "Replay should NOT broadcast account updates" - ); - - // 4. Verify Post-Replay State (Applied) - for pubkey in &pubkeys { - let account = env.accountsdb.get_account(pubkey).unwrap(); - assert_eq!(account.data()[0], 42, "Replay should update account state"); - } -} diff --git a/magicblock-processor/tests/replica_ordering.rs b/magicblock-processor/tests/replica_ordering.rs deleted file mode 100644 index de72f249d..000000000 --- a/magicblock-processor/tests/replica_ordering.rs +++ /dev/null @@ -1,466 +0,0 @@ -//! Integration tests for Replica mode transaction ordering enforcement. -//! -//! These tests verify that in Replica mode, when transactions conflict on accounts, -//! they are executed in strict submission order (FIFO), even under concurrent pressure. - -use std::{ - collections::HashMap, - time::{Duration, Instant}, -}; - -use guinea::GuineaInstruction; -use magicblock_core::link::{ - replication::{Block, Message}, - transactions::ReplayPosition, -}; -use solana_account::ReadableAccount; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_transaction::Transaction; -use test_kit::{ExecutionTestEnv, Signer}; -use tokio::time::{sleep, timeout}; - -const STRESS_TIMEOUT: Duration = Duration::from_secs(30); -const DEFAULT_BALANCE: u64 = LAMPORTS_PER_SOL * 10; - -// --- Helpers --- - -fn setup_replica_env(executors: u32) -> ExecutionTestEnv { - ExecutionTestEnv::new_replica_mode(executors, true) -} - -fn setup_replica_env_with_superblock_size( - executors: u32, - superblock_size: u64, -) -> ExecutionTestEnv { - ExecutionTestEnv::new_replica_mode_with_superblock_size( - executors, - true, - superblock_size, - ) -} - -fn create_accounts(env: &mut ExecutionTestEnv, count: usize) -> Vec { - (0..count) - .map(|_| { - env.create_account_with_config(DEFAULT_BALANCE, 128, guinea::ID) - .pubkey() - }) - .collect() -} - -fn tx_write( - env: &mut ExecutionTestEnv, - account: Pubkey, - val: u8, -) -> Transaction { - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::WriteByteToData(val), - vec![AccountMeta::new(account, false)], - ); - env.build_transaction(&[ix]) -} - -fn tx_transfer( - env: &mut ExecutionTestEnv, - from: Pubkey, - to: Pubkey, -) -> Transaction { - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::Transfer(1000), - vec![AccountMeta::new(from, false), AccountMeta::new(to, false)], - ); - env.build_transaction(&[ix]) -} - -/// Collects execution statuses until all expected signatures are accounted for or timeout. -async fn collect_statuses( - env: &mut ExecutionTestEnv, - count: usize, - limit: Duration, - context: &str, -) -> Vec { - let start = Instant::now(); - let mut results = Vec::with_capacity(count); - - while results.len() < count { - if start.elapsed() > limit { - panic!( - "[{context}] Timeout waiting for transactions. Got {}/{}.", - results.len(), - count - ); - } - if let Ok(Ok(status)) = timeout( - Duration::from_millis(100), - env.dispatch.transaction_status.recv_async(), - ) - .await - { - assert!( - status.meta.status.is_ok(), - "[{context}] Transaction {} failed: {:?}", - status.txn.signatures()[0], - status.meta.status - ); - results.push(status.txn.signatures()[0]); - } - } - results -} - -/// Verifies that transactions were processed in the exact order they were submitted. -async fn verify_ordered( - env: &mut ExecutionTestEnv, - sigs: &[Signature], - limit: Duration, - context: &str, -) { - let received = collect_statuses(env, sigs.len(), limit, context).await; - assert_eq!( - received, - sigs.to_vec(), - "[{context}] Execution order mismatch.\nExpected: {sigs:?}\nGot: {received:?}" - ); -} - -/// Submits all replay transactions sequentially, then starts the scheduler. -/// Returns signatures in submission order. -async fn submit_all_and_start( - env: &mut ExecutionTestEnv, - txs: Vec, -) -> Vec { - let sigs: Vec = txs.iter().map(|tx| tx.signatures[0]).collect(); - - // Submit all transactions sequentially to preserve order - for (i, tx) in txs.into_iter().enumerate() { - let position = ReplayPosition { - slot: env.accountsdb.slot(), - index: i as u32, - persist: true, - }; - env.transaction_scheduler - .replay(position, tx) - .await - .expect("Failed to submit transaction"); - } - - // Now start the scheduler - env.run_scheduler(); - env.advance_slot(); - - sigs -} - -// --- Tests --- - -/// Stress test: 100 conflicting writes to a single account. -/// In Replica mode, these MUST execute in strict FIFO order. -#[tokio::test] -async fn test_replica_stress_single_account_writes() { - let ctx = "Replica Stress Single Account"; - let mut env = setup_replica_env(8); - let count = 100; - let acc = create_accounts(&mut env, 1)[0]; - - let mut txs = Vec::with_capacity(count); - for i in 0..count { - env.advance_slot(); - txs.push(tx_write(&mut env, acc, (i % 256) as u8)); - } - - let sigs = submit_all_and_start(&mut env, txs).await; - verify_ordered(&mut env, &sigs, STRESS_TIMEOUT, ctx).await; - - // Final state must match last write (count - 1) - let final_val = env.get_account(acc).data()[0]; - assert_eq!( - final_val, - ((count - 1) % 256) as u8, - "[{ctx}] Final data value mismatch" - ); -} - -#[tokio::test] -async fn test_replay_block_waits_for_queued_transactions() { - let mut env = setup_replica_env(4); - let slot = env.advance_slot(); - env.run_scheduler(); - env.yield_to_scheduler().await; - - let acc = create_accounts(&mut env, 1)[0]; - let tx = tx_write(&mut env, acc, 77); - let sig = tx.signatures[0]; - let prev_hash = env.ledger.latest_block().load().blockhash; - - env.transaction_scheduler - .replay( - ReplayPosition { - slot, - index: 0, - persist: true, - }, - tx, - ) - .await - .expect("failed to queue replay transaction"); - - let mut hasher = blake3::Hasher::new(); - hasher.update(prev_hash.as_ref()); - hasher.update(sig.as_ref()); - let hash = (*hasher.finalize().as_bytes()).into(); - - env.transaction_scheduler - .replay_block(Block { - slot, - hash, - timestamp: slot as i64, - }) - .await - .expect("failed to apply replayed block"); - - assert_eq!(env.get_account(acc).data()[0], 77); - assert_eq!(env.ledger.latest_block().load().slot, slot); - assert_eq!(env.accountsdb.slot(), slot); -} - -#[tokio::test] -async fn test_replica_replayed_superblock_takes_snapshot_without_publishing() { - let mut env = setup_replica_env_with_superblock_size(1, 2); - env.run_scheduler(); - env.yield_to_scheduler().await; - - let slot = 2; - let snapshot = env - .accountsdb - .database_directory() - .join(format!("snapshot-{slot:0>12}.tar.gz")); - let prev_hash = env.ledger.latest_block().load().blockhash; - let mut hasher = blake3::Hasher::new(); - hasher.update(prev_hash.as_ref()); - let hash = (*hasher.finalize().as_bytes()).into(); - - env.transaction_scheduler - .replay_block(Block { - slot, - hash, - timestamp: slot as i64, - }) - .await - .expect("failed to apply replayed superblock boundary"); - - timeout(Duration::from_secs(5), async { - while !snapshot.exists() { - sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("timed out waiting for replica accountsdb snapshot"); - - let replication = env - .dispatch - .replication_messages - .as_mut() - .expect("missing replication receiver"); - while let Ok(message) = replication.try_recv() { - assert!( - !matches!(message, Message::SuperBlock(_)), - "replica must not publish SuperBlock messages" - ); - } -} - -/// Stress test: Multiple accounts with cross-conflicts. -/// Transactions form a chain where each conflicts with the next. -#[tokio::test] -async fn test_replica_stress_conflict_chain() { - let ctx = "Replica Stress Conflict Chain"; - let mut env = setup_replica_env(8); - let count = 100; - let accs = create_accounts(&mut env, count + 1); - - // Build a chain: T0 writes A0+A1, T1 writes A1+A2, etc. - // Each transaction conflicts with its neighbors - let mut txs = Vec::with_capacity(count); - for i in 0..count { - env.advance_slot(); - txs.push(tx_transfer(&mut env, accs[i], accs[i + 1])); - } - - let sigs = submit_all_and_start(&mut env, txs).await; - verify_ordered(&mut env, &sigs, STRESS_TIMEOUT, ctx).await; -} - -/// Stress test: Hotspot account with many writers. -/// 50 transactions all write to the same "hot" account plus a unique account each. -#[tokio::test] -async fn test_replica_stress_hotspot() { - let ctx = "Replica Stress Hotspot"; - let mut env = setup_replica_env(8); - let count = 50; - let accs = create_accounts(&mut env, count + 1); - let hotspot = accs[count]; // The last account is the hotspot - - // Each transaction writes to hotspot + its own account - let mut txs = Vec::with_capacity(count); - for (i, acc) in accs.iter().take(count).enumerate() { - env.advance_slot(); - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::WriteByteToData(i as u8), - vec![ - AccountMeta::new(*acc, false), - AccountMeta::new(hotspot, false), - ], - ); - txs.push(env.build_transaction(&[ix])); - } - - let sigs = submit_all_and_start(&mut env, txs).await; - - // All transactions conflict on hotspot, so must be ordered - verify_ordered(&mut env, &sigs, STRESS_TIMEOUT, ctx).await; -} - -/// Stress test: Mixed workload with independent and conflicting transactions. -/// Verifies that independent transactions can proceed while maintaining order -/// for conflicting ones. -#[tokio::test] -async fn test_replica_stress_mixed_workload() { - let ctx = "Replica Stress Mixed Workload"; - let mut env = setup_replica_env(8); - let count = 100; - - // Create 4 groups of accounts - let group_a = create_accounts(&mut env, 1); - let group_b = create_accounts(&mut env, 1); - let independent = create_accounts(&mut env, 50); - - let mut txs = Vec::with_capacity(count); - for i in 0..count { - env.advance_slot(); - match i % 4 { - 0 => txs.push(tx_write(&mut env, group_a[0], i as u8)), - 1 => txs.push(tx_write(&mut env, group_b[0], i as u8)), - _ => { - // Independent writes to unique accounts - let idx = (i / 4) % independent.len(); - txs.push(tx_write(&mut env, independent[idx], i as u8)); - } - } - } - - let sigs = submit_all_and_start(&mut env, txs).await; - let received = - collect_statuses(&mut env, sigs.len(), STRESS_TIMEOUT, ctx).await; - - // Extract positions of group_a and group_b transactions - let group_a_sigs: Vec<_> = sigs - .iter() - .enumerate() - .filter(|(i, _)| *i % 4 == 0) - .map(|(_, s)| *s) - .collect(); - let group_b_sigs: Vec<_> = sigs - .iter() - .enumerate() - .filter(|(i, _)| *i % 4 == 1) - .map(|(_, s)| *s) - .collect(); - - let group_a_positions: Vec<_> = group_a_sigs - .iter() - .map(|s| received.iter().position(|r| r == s).unwrap()) - .collect(); - let group_b_positions: Vec<_> = group_b_sigs - .iter() - .map(|s| received.iter().position(|r| r == s).unwrap()) - .collect(); - - // Group A transactions must be in order - let mut sorted_a = group_a_positions.clone(); - sorted_a.sort(); - assert_eq!(group_a_positions, sorted_a, "[{ctx}] Group A not in order"); - - // Group B transactions must be in order - let mut sorted_b = group_b_positions.clone(); - sorted_b.sort(); - assert_eq!(group_b_positions, sorted_b, "[{ctx}] Group B not in order"); -} - -/// Stress test: Many small transfers between pairs of accounts. -/// Each pair is independent, but transfers within a pair must be ordered. -#[tokio::test] -async fn test_replica_stress_transfer_pairs() { - let ctx = "Replica Stress Transfer Pairs"; - let mut env = setup_replica_env(8); - let pairs = 10; - let transfers_per_pair = 10; - let accs = create_accounts(&mut env, pairs * 2); - - let initial: HashMap<_, _> = accs - .iter() - .map(|k| (*k, env.get_account(*k).lamports())) - .collect(); - - let mut txs = Vec::with_capacity(pairs * transfers_per_pair); - // Submit transfers for each pair in round-robin to interleave them - for _ in 0..transfers_per_pair { - for p in 0..pairs { - env.advance_slot(); - let from = accs[p * 2]; - let to = accs[p * 2 + 1]; - txs.push(tx_transfer(&mut env, from, to)); - } - } - - let sigs = submit_all_and_start(&mut env, txs).await; - let received = - collect_statuses(&mut env, sigs.len(), STRESS_TIMEOUT, ctx).await; - - // For each pair, verify transfers executed in order - for p in 0..pairs { - // Get signatures for this pair's transfers (in submission order) - let pair_sigs: Vec<_> = (0..transfers_per_pair) - .map(|t| sigs[t * pairs + p]) - .collect(); - - // Get their execution positions - let positions: Vec<_> = pair_sigs - .iter() - .map(|s| received.iter().position(|r| r == s).unwrap()) - .collect(); - - // Must be in order - let mut sorted = positions.clone(); - sorted.sort(); - assert_eq!( - positions, sorted, - "[{ctx}] Pair {p} transfers not in order" - ); - } - - // Verify final balances - let transfer_amount = 1000u64; - for p in 0..pairs { - let from = accs[p * 2]; - let to = accs[p * 2 + 1]; - let expected_change = transfer_amount * transfers_per_pair as u64; - assert_eq!( - env.get_account(from).lamports(), - initial[&from] - expected_change, - "[{ctx}] Pair {p} sender balance wrong" - ); - assert_eq!( - env.get_account(to).lamports(), - initial[&to] + expected_change, - "[{ctx}] Pair {p} receiver balance wrong" - ); - } -} diff --git a/magicblock-processor/tests/scheduling.rs b/magicblock-processor/tests/scheduling.rs deleted file mode 100644 index 70e1896e3..000000000 --- a/magicblock-processor/tests/scheduling.rs +++ /dev/null @@ -1,529 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - time::{Duration, Instant}, -}; - -use guinea::GuineaInstruction; -use solana_account::ReadableAccount; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_transaction::Transaction; -use test_kit::{ExecutionTestEnv, Signer}; -use tokio::time::timeout; - -const TIMEOUT: Duration = Duration::from_secs(5); -const STRESS_TIMEOUT: Duration = Duration::from_secs(10); -const DEFAULT_BALANCE: u64 = LAMPORTS_PER_SOL * 10; -const TRANSFER_AMOUNT: u64 = 1000; - -// --- Helpers --- - -fn setup_env(executors: u32) -> ExecutionTestEnv { - ExecutionTestEnv::new_with_config( - ExecutionTestEnv::BASE_FEE, - executors, - true, - ) -} - -fn create_accounts(env: &mut ExecutionTestEnv, count: usize) -> Vec { - (0..count) - .map(|_| { - env.create_account_with_config(DEFAULT_BALANCE, 128, guinea::ID) - .pubkey() - }) - .collect() -} - -fn tx_transfer( - env: &mut ExecutionTestEnv, - from: Pubkey, - to: Pubkey, -) -> Transaction { - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::Transfer(TRANSFER_AMOUNT), - vec![AccountMeta::new(from, false), AccountMeta::new(to, false)], - ); - env.build_transaction(&[ix]) -} - -fn tx_write( - env: &mut ExecutionTestEnv, - account: Pubkey, - val: u8, -) -> Transaction { - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::WriteByteToData(val), - vec![AccountMeta::new(account, false)], - ); - env.build_transaction(&[ix]) -} - -fn tx_read(env: &mut ExecutionTestEnv, accounts: &[Pubkey]) -> Transaction { - let metas = accounts - .iter() - .map(|k| AccountMeta::new_readonly(*k, false)) - .collect(); - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - metas, - ); - env.build_transaction(&[ix]) -} - -/// Schedules transactions, starts the scheduler, and returns signatures in scheduling order. -async fn schedule( - env: &mut ExecutionTestEnv, - txs: Vec, -) -> Vec { - let mut sigs = Vec::with_capacity(txs.len()); - for tx in txs { - sigs.push(tx.signatures[0]); - env.schedule_transaction(tx).await; - } - env.run_scheduler(); - env.advance_slot(); - sigs -} - -/// Collects execution statuses until all expected signatures are accounted for or timeout. -async fn collect_statuses( - env: &mut ExecutionTestEnv, - count: usize, - limit: Duration, - context: &str, -) -> Vec { - let start = Instant::now(); - let mut results = Vec::with_capacity(count); - - while results.len() < count { - if start.elapsed() > limit { - panic!( - "[{context}] Timeout waiting for transactions. Got {}/{count}.", - results.len() - ); - } - // Short poll interval - if let Ok(Ok(status)) = timeout( - Duration::from_millis(100), - env.dispatch.transaction_status.recv_async(), - ) - .await - { - assert!( - status.meta.status.is_ok(), - "[{context}] Transaction {} failed: {:?}", - status.txn.signatures()[0], - status.meta.status - ); - results.push(status.txn.signatures()[0]); - } - } - results -} - -/// Verifies that all signatures were processed successfully (order irrelevant). -async fn verify_unordered( - env: &mut ExecutionTestEnv, - sigs: &[Signature], - limit: Duration, - context: &str, -) { - let received = collect_statuses(env, sigs.len(), limit, context).await; - let expected_set: HashSet<_> = sigs.iter().cloned().collect(); - let received_set: HashSet<_> = received.into_iter().collect(); - - if expected_set != received_set { - let missing: Vec<_> = expected_set.difference(&received_set).collect(); - let extra: Vec<_> = received_set.difference(&expected_set).collect(); - panic!("[{context}] Signature mismatch.\nMissing: {missing:?}\nExtra: {extra:?}"); - } -} - -/// Verifies that transactions were processed in the exact order they were scheduled. -async fn verify_ordered( - env: &mut ExecutionTestEnv, - sigs: &[Signature], - limit: Duration, - context: &str, -) { - let received = collect_statuses(env, sigs.len(), limit, context).await; - assert_eq!( - received, - sigs.to_vec(), - "[{context}] Execution order mismatch.\nExpected: {sigs:?}\nGot: {received:?}" - ); -} - -// --- Scenarios --- - -async fn scenario_parallel_transfers(executors: u32) { - let ctx = format!("Parallel Transfers ({executors} exec)"); - let mut env = setup_env(executors); - let pairs = 20; - let accounts = create_accounts(&mut env, pairs * 2); - - let initial: HashMap<_, _> = accounts - .iter() - .map(|k| (*k, env.get_account(*k).lamports())) - .collect(); - - let mut txs = Vec::with_capacity(pairs); - for i in 0..pairs { - txs.push(tx_transfer(&mut env, accounts[i * 2], accounts[i * 2 + 1])); - } - - let sigs = schedule(&mut env, txs).await; - verify_unordered(&mut env, &sigs, TIMEOUT, &ctx).await; - - for i in 0..pairs { - let from = accounts[i * 2]; - let to = accounts[i * 2 + 1]; - assert_eq!( - env.get_account(from).lamports(), - initial[&from] - TRANSFER_AMOUNT, - "[{ctx}] Sender balance incorrect" - ); - assert_eq!( - env.get_account(to).lamports(), - initial[&to] + TRANSFER_AMOUNT, - "[{ctx}] Recipient balance incorrect" - ); - } -} - -async fn scenario_conflicting_transfers(executors: u32) { - let ctx = format!("Conflicting Transfers ({executors} exec)"); - let mut env = setup_env(executors); - let senders_count = 10; - let mut accounts = create_accounts(&mut env, senders_count + 1); - let recipient = accounts.pop().unwrap(); - let senders = accounts; - - let initial_recip = env.get_account(recipient).lamports(); - let initial_senders: HashMap<_, _> = senders - .iter() - .map(|k| (*k, env.get_account(*k).lamports())) - .collect(); - - let mut txs = Vec::with_capacity(senders.len()); - for sender in &senders { - txs.push(tx_transfer(&mut env, *sender, recipient)); - } - - let sigs = schedule(&mut env, txs).await; - verify_unordered(&mut env, &sigs, TIMEOUT, &ctx).await; - - assert_eq!( - env.get_account(recipient).lamports(), - initial_recip + (TRANSFER_AMOUNT * senders_count as u64), - "[{ctx}] Recipient final balance incorrect" - ); - for s in senders { - assert_eq!( - env.get_account(s).lamports(), - initial_senders[&s] - TRANSFER_AMOUNT, - "[{ctx}] Sender {s} final balance incorrect" - ); - } -} - -async fn scenario_readonly_parallel(executors: u32) { - let ctx = format!("Readonly Parallel ({executors} exec)"); - let mut env = setup_env(executors); - let count = 50; - let accounts = create_accounts(&mut env, 10); - - let mut txs = Vec::with_capacity(count); - for i in 0..count { - txs.push(tx_read( - &mut env, - &[accounts[i % 10], accounts[(i + 1) % 10]], - )); - } - - let sigs = schedule(&mut env, txs).await; - verify_unordered(&mut env, &sigs, TIMEOUT, &ctx).await; -} - -async fn scenario_mixed_workload(executors: u32) { - let ctx = format!("Mixed Workload ({executors} exec)"); - let mut env = setup_env(executors); - let sets = 10; - let accs = create_accounts(&mut env, 6); // A..F - let initial: HashMap<_, _> = accs - .iter() - .map(|k| (*k, env.get_account(*k).lamports())) - .collect(); - - let mut txs = Vec::new(); - for _ in 0..sets { - txs.push(tx_transfer(&mut env, accs[0], accs[1])); // A->B - txs.push(tx_transfer(&mut env, accs[2], accs[3])); // C->D - txs.push(tx_transfer(&mut env, accs[1], accs[0])); // B->A (conflict T1) - txs.push(tx_transfer(&mut env, accs[4], accs[5])); // E->F - } - - let sigs = schedule(&mut env, txs).await; - verify_unordered(&mut env, &sigs, TIMEOUT, &ctx).await; - - let n = sets as u64; - // A & B net zero (transfer back and forth) - assert_eq!( - env.get_account(accs[0]).lamports(), - initial[&accs[0]], - "[{ctx}] Account A balance mismatch" - ); - assert_eq!( - env.get_account(accs[1]).lamports(), - initial[&accs[1]], - "[{ctx}] Account B balance mismatch" - ); - - // C, E lose; D, F gain - assert_eq!( - env.get_account(accs[2]).lamports(), - initial[&accs[2]] - (TRANSFER_AMOUNT * n), - "[{ctx}] Account C balance mismatch" - ); -} - -async fn scenario_conflicting_writes(executors: u32) { - let ctx = format!("Conflicting Writes ({executors} exec)"); - let mut env = setup_env(executors); - let count = 20; - let acc = create_accounts(&mut env, 1)[0]; - - let mut txs = Vec::with_capacity(count); - for i in 1..=count { - txs.push(tx_write(&mut env, acc, i as u8)); - } - - let sigs = schedule(&mut env, txs).await; - verify_unordered(&mut env, &sigs, TIMEOUT, &ctx).await; - - // We can't guarantee *which* write was last, but it must be one of them. - let data = env.get_account(acc).data()[0]; - assert!( - data > 0 && data <= count as u8, - "[{ctx}] Final data value {data} out of expected range (1..={count})" - ); -} - -async fn scenario_serial_conflicting_writes(executors: u32) { - let ctx = format!("Serial Conflicting Writes ({executors} exec)"); - let mut env = setup_env(executors); - let count = 20; - let acc = create_accounts(&mut env, 1)[0]; - - let mut txs = Vec::with_capacity(count); - for i in 0..count { - txs.push(tx_write(&mut env, acc, i as u8)); - } - - // Verify transactions executed in exact scheduling order due to write lock - let sigs = schedule(&mut env, txs).await; - verify_ordered(&mut env, &sigs, TIMEOUT, &ctx).await; - - // Final state must match last write - let final_val = env.get_account(acc).data()[0]; - assert_eq!( - final_val, - (count - 1) as u8, - "[{ctx}] Final data value mismatch" - ); -} - -async fn scenario_serial_transfer_chain(executors: u32) { - let ctx = format!("Serial Transfer Chain ({executors} exec)"); - let mut env = setup_env(executors); - let count = 20; - let accs = create_accounts(&mut env, count + 1); - let initial: HashMap<_, _> = accs - .iter() - .map(|k| (*k, env.get_account(*k).lamports())) - .collect(); - - let mut txs = Vec::with_capacity(count); - for i in 0..count { - txs.push(tx_transfer(&mut env, accs[i], accs[i + 1])); - } - - let sigs = schedule(&mut env, txs).await; - // Note: With current scheduler, we don't guarantee strict FIFO ordering, - // but all transactions should complete and balances should be correct. - verify_unordered(&mut env, &sigs, TIMEOUT, &ctx).await; - - // A loses, Last gains, Middle net zero - assert_eq!( - env.get_account(accs[0]).lamports(), - initial[&accs[0]] - TRANSFER_AMOUNT, - "[{ctx}] First account balance mismatch" - ); - assert_eq!( - env.get_account(accs[count]).lamports(), - initial[&accs[count]] + TRANSFER_AMOUNT, - "[{ctx}] Last account balance mismatch" - ); - for i in 1..count { - assert_eq!( - env.get_account(accs[i]).lamports(), - initial[&accs[i]], - "[{ctx}] Intermediate account {i} balance changed" - ); - } -} - -async fn scenario_stress_test(executors: u32) { - let ctx = "Large Queue Stress Test"; - let mut env = setup_env(executors); - let count = 1000; - let num_accs = 100; - - let t_accs = create_accounts(&mut env, num_accs); - let g_accs = create_accounts(&mut env, num_accs); - - let mut txs = Vec::with_capacity(count); - for i in 0..count { - let idx = i % num_accs; - let tx = match i % 4 { - 0 => { - tx_transfer(&mut env, t_accs[idx], t_accs[(idx + 1) % num_accs]) - } - 1 => tx_write(&mut env, g_accs[0], (i % 255) as u8), - 2 => tx_write(&mut env, g_accs[idx], (i % 255) as u8), - _ => { - tx_read(&mut env, &[g_accs[idx], g_accs[(idx + 1) % num_accs]]) - } - }; - txs.push(tx); - } - - let sigs = schedule(&mut env, txs).await; - verify_unordered(&mut env, &sigs, STRESS_TIMEOUT, ctx).await; -} - -// --- Tests --- - -#[tokio::test] -async fn test_parallel_transfers() { - for executors in [1, 2, 4, 8] { - scenario_parallel_transfers(executors).await; - } -} - -#[tokio::test] -async fn test_conflicting_transfers() { - for executors in [1, 2, 4, 8] { - scenario_conflicting_transfers(executors).await; - } -} - -#[tokio::test] -async fn test_readonly_parallel() { - for executors in [1, 2, 4, 8] { - scenario_readonly_parallel(executors).await; - } -} - -#[tokio::test] -async fn test_mixed_workload() { - for executors in [1, 2, 4, 8] { - scenario_mixed_workload(executors).await; - } -} - -#[tokio::test] -async fn test_conflicting_writes() { - for executors in [1, 2, 4, 8] { - scenario_conflicting_writes(executors).await; - } -} - -#[tokio::test] -async fn test_serial_conflicting_writes() { - for executors in [1, 2, 4, 8] { - scenario_serial_conflicting_writes(executors).await; - } -} - -#[tokio::test] -async fn test_serial_transfer_chain() { - for executors in [1, 2, 4, 8] { - scenario_serial_transfer_chain(executors).await; - } -} - -#[tokio::test] -async fn test_large_queue_mixed_8_executors() { - scenario_stress_test(8).await; -} - -/// Tests that wait_for_idle() blocks scheduling while held, and releases when idle. -/// -/// Flow: -/// 1. Acquire permit before scheduler starts -/// 2. Queue transaction -/// 3. Start scheduler - it blocks waiting for permit -/// 4. Verify transaction doesn't execute for 500ms -/// 5. Release permit - transaction executes -/// 6. Reacquire permit - succeeds quickly (scheduler is idle) -#[tokio::test] -async fn test_wait_for_idle_coordination() { - let mut env = ExecutionTestEnv::new_with_config( - ExecutionTestEnv::BASE_FEE, - 1, - true, // defer_startup - ); - - // 1. Acquire permit before scheduler starts (owned to avoid borrow issues) - let permit = env.transaction_scheduler.wait_for_idle().await; - - // 2. Queue a transaction - let accounts = create_accounts(&mut env, 2); - let tx = tx_transfer(&mut env, accounts[0], accounts[1]); - let sig = tx.signatures[0]; - env.schedule_transaction(tx).await; - - // 3. Start scheduler - env.run_scheduler(); - env.advance_slot(); - - // 4. Transaction should NOT execute while we hold the permit - tokio::time::sleep(Duration::from_millis(500)).await; - assert!( - env.get_transaction(sig).is_none(), - "transaction should not execute while permit is held" - ); - - // 5. Release permit - scheduler should now process - drop(permit); - - // Wait for transaction to complete - let status = timeout(TIMEOUT, async { - loop { - match env.dispatch.transaction_status.recv_async().await { - Ok(s) if s.txn.signatures()[0] == sig => break s, - _ => continue, - } - } - }) - .await - .expect("transaction should complete after permit released"); - assert!(status.meta.status.is_ok(), "transaction should succeed"); - - // 6. Reacquire permit - should succeed quickly (scheduler is idle) - let _permit = timeout( - Duration::from_millis(100), - env.transaction_scheduler.wait_for_idle(), - ) - .await - .expect("should acquire permit quickly when scheduler is idle"); -} diff --git a/magicblock-processor/tests/security.rs b/magicblock-processor/tests/security.rs deleted file mode 100644 index a3ce0a391..000000000 --- a/magicblock-processor/tests/security.rs +++ /dev/null @@ -1,87 +0,0 @@ -use guinea::GuineaInstruction; -use solana_account::{ReadableAccount, WritableAccount}; -use solana_keypair::Keypair; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_pubkey::Pubkey; -use solana_signer::Signer; -use solana_transaction_error::TransactionError; -use test_kit::ExecutionTestEnv; - -fn transfer_ix(from: Pubkey, to: Pubkey, amount: u64) -> Instruction { - Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::Transfer(amount), - vec![AccountMeta::new(from, true), AccountMeta::new(to, false)], - ) -} - -fn setup_confined(env: &ExecutionTestEnv) -> Keypair { - let acc = env.create_account_with_config(LAMPORTS_PER_SOL, 42, guinea::ID); - let mut data = env.get_account(acc.pubkey()); - data.set_confined(true); - env.accountsdb.insert_account(&acc.pubkey(), &data).unwrap(); - acc -} - -#[tokio::test] -async fn test_gasless_undelegated_feepayer_modification_fails() { - let env = ExecutionTestEnv::new_with_config(0, 1, false); - - // 1. Configure Payer: Owned by Guinea (to allow transfer), Undelegated - { - let mut payer = env.get_payer(); - payer.set_owner(guinea::ID); - payer.set_delegated(false); - payer.commit(); - } - - // 2. Execute Transfer (Payer -> Recipient) - let recipient = env.create_account(LAMPORTS_PER_SOL); - let ix = transfer_ix(env.get_payer().pubkey, recipient.pubkey(), 1000); - let txn = env.build_transaction(&[ix]); - - // 3. Assert Failure: Undelegated fee payer cannot be modified in gasless mode - let result = env.execute_transaction(txn).await; - assert_eq!(result.unwrap_err(), TransactionError::InvalidAccountForFee); -} - -#[tokio::test] -async fn test_confined_account_lamport_modification_fails() { - let env = ExecutionTestEnv::new(); - let confined = setup_confined(&env); - let recipient = env.create_account(100); - - // Attempt to move funds FROM confined account - let mut ix = transfer_ix(confined.pubkey(), recipient.pubkey(), 100); - ix.accounts.first_mut().unwrap().is_signer = false; - - let txn = env.build_transaction(&[ix]); - - let result = env.execute_transaction(txn).await; - assert_eq!( - result.unwrap_err(), - TransactionError::UnbalancedTransaction, - "Confined account balance cannot change" - ); -} - -#[tokio::test] -async fn test_confined_account_data_modification_succeeds() { - let env = ExecutionTestEnv::new(); - let confined = setup_confined(&env); - - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::WriteByteToData(99), - vec![AccountMeta::new(confined.pubkey(), false)], - ); - - let txn = env.build_transaction(&[ix]); - assert!(env.execute_transaction(txn).await.is_ok()); - - let acc = env.get_account(confined.pubkey()); - assert_eq!(acc.data()[0], 99, "Data modification should be allowed"); -} diff --git a/magicblock-processor/tests/simulation.rs b/magicblock-processor/tests/simulation.rs deleted file mode 100644 index 7d1e8a37c..000000000 --- a/magicblock-processor/tests/simulation.rs +++ /dev/null @@ -1,198 +0,0 @@ -use std::time::Duration; - -use guinea::GuineaInstruction; -use magicblock_accounts_db::traits::AccountsBank; -use magicblock_core::link::transactions::TransactionSimulationResult; -use solana_account::ReadableAccount; -use solana_compute_budget_interface::ComputeBudgetInstruction; -use solana_instruction::error::InstructionError; -use solana_program::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_transaction_error::TransactionError; -use test_kit::{ExecutionTestEnv, Signer}; - -const ACCOUNTS_COUNT: usize = 8; -const TIMEOUT: Duration = Duration::from_millis(100); - -/// Helper to simulate a standard "Guinea" transaction. -async fn simulate_guinea( - env: &ExecutionTestEnv, - ix: GuineaInstruction, - is_writable: bool, -) -> (TransactionSimulationResult, Signature, Vec) { - let accounts: Vec<_> = (0..ACCOUNTS_COUNT) - .map(|_| { - env.create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID) - }) - .collect(); - - let metas = accounts - .iter() - .map(|a| { - if is_writable { - AccountMeta::new(a.pubkey(), false) - } else { - AccountMeta::new_readonly(a.pubkey(), false) - } - }) - .collect(); - - env.advance_slot(); - let ix = Instruction::new_with_bincode(guinea::ID, &ix, metas); - let txn = env.build_transaction(&[ix]); - let sig = txn.signatures[0]; - - let result = env.simulate_transaction(txn).await; - let pubkeys = accounts.iter().map(|a| a.pubkey()).collect(); - - (result, sig, pubkeys) -} - -#[tokio::test] -async fn test_absent_simulation_side_effects() { - let env = ExecutionTestEnv::new(); - let (_, sig, pubkeys) = simulate_guinea( - &env, - GuineaInstruction::WriteByteToData(42), - true, // writable - ) - .await; - - // 1. Verify No Notifications - assert!( - env.dispatch - .transaction_status - .recv_timeout(TIMEOUT) - .is_err(), - "Simulation triggered status update" - ); - assert!( - env.dispatch.account_update.try_recv().is_err(), - "Simulation triggered account update" - ); - - // 2. Verify No Persistence - assert!( - env.get_transaction(sig).is_none(), - "Simulated transaction written to ledger" - ); - - // 3. Verify No State Change - for pubkey in &pubkeys { - let account = env.accountsdb.get_account(pubkey).unwrap(); - assert_ne!(account.data()[0], 42, "Simulation modified DB state"); - } -} - -#[tokio::test] -async fn test_simulation_logs() { - let env = ExecutionTestEnv::new(); - let (result, _, _) = simulate_guinea( - &env, - GuineaInstruction::PrintSizes, - false, // readonly - ) - .await; - - assert!(result.result.is_ok(), "Simulation failed"); - - let logs = result.logs.expect("Logs missing"); - assert!(logs.len() > ACCOUNTS_COUNT, "Insufficient logs"); - assert!( - result.inner_instructions.is_some(), - "Inner instructions missing" - ); -} - -#[tokio::test] -async fn test_simulation_return_data() { - let env = ExecutionTestEnv::new(); - let (result, _, _) = simulate_guinea( - &env, - GuineaInstruction::ComputeBalances, - false, // readonly - ) - .await; - - assert!(result.result.is_ok(), "Simulation failed"); - - let ret_data = result.return_data.expect("Return data missing"); - let expected = (ACCOUNTS_COUNT as u64 * LAMPORTS_PER_SOL).to_le_bytes(); - assert_eq!(ret_data.data, expected, "Incorrect return data"); -} - -#[tokio::test] -async fn test_simulation_honors_heap_frame_request() { - let env = ExecutionTestEnv::new(); - let account = env - .create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID) - .pubkey(); - let guinea_ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - vec![AccountMeta::new_readonly(account, false)], - ); - - env.advance_slot(); - let default_heap_txn = - env.build_transaction(std::slice::from_ref(&guinea_ix)); - let default_heap_result = env.simulate_transaction(default_heap_txn).await; - assert!( - default_heap_result.result.is_ok(), - "default heap simulation failed: {:?}", - default_heap_result.result - ); - - let heap_frame_ix = ComputeBudgetInstruction::request_heap_frame(64 * 1024); - env.advance_slot(); - let requested_heap_txn = env.build_transaction(&[heap_frame_ix, guinea_ix]); - let requested_heap_result = - env.simulate_transaction(requested_heap_txn).await; - assert!( - requested_heap_result.result.is_ok(), - "requested heap simulation failed: {:?}", - requested_heap_result.result - ); - - assert!( - requested_heap_result.units_consumed - > default_heap_result.units_consumed, - "requesting a larger heap should consume more units: default={}, requested={}", - default_heap_result.units_consumed, - requested_heap_result.units_consumed - ); -} - -#[tokio::test] -async fn test_simulation_honors_compute_unit_limit() { - let env = ExecutionTestEnv::new(); - let account = env - .create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID) - .pubkey(); - let compute_budget_ix = ComputeBudgetInstruction::set_compute_unit_limit(1); - let guinea_ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::PrintSizes, - vec![AccountMeta::new_readonly(account, false)], - ); - - env.advance_slot(); - let txn = env.build_transaction(&[compute_budget_ix, guinea_ix]); - let result = env.simulate_transaction(txn).await; - - assert!( - matches!( - result.result, - Err(TransactionError::InstructionError( - 0, - InstructionError::ComputationalBudgetExceeded - )) - ), - "unexpected result: {:?}", - result.result - ); -} diff --git a/test-kit/Cargo.toml b/test-kit/Cargo.toml deleted file mode 100644 index f71799b01..000000000 --- a/test-kit/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "test-kit" -version.workspace = true -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[dependencies] -guinea = { workspace = true } -magicblock-accounts-db = { workspace = true } -magicblock-core = { workspace = true } -magicblock-ledger = { workspace = true } -magicblock-processor = { workspace = true } - -solana-account = { workspace = true } -solana-instruction = { workspace = true } -solana-keypair = { workspace = true } -solana-program = { workspace = true } -solana-rpc-client = { workspace = true } -solana-signature = { workspace = true } -solana-signer = { workspace = true } -solana-transaction = { workspace = true } -solana-transaction-error = { workspace = true } -solana-transaction-status-client-types = { workspace = true } - -tempfile = { workspace = true } -tokio = { workspace = true } -tokio-util = { workspace = true } -tracing = { workspace = true } -tracing-log = { workspace = true } -tracing-subscriber = { workspace = true } diff --git a/test-kit/src/lib.rs b/test-kit/src/lib.rs deleted file mode 100644 index 4f118c2e5..000000000 --- a/test-kit/src/lib.rs +++ /dev/null @@ -1,539 +0,0 @@ -use std::{ - ops::{Deref, DerefMut}, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, - thread::{self, JoinHandle}, - time::Duration, -}; - -pub use guinea; -use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; -use magicblock_core::{ - link::{ - link, - transactions::{ - ReplayPosition, SanitizeableTransaction, SchedulerMode, - TransactionSchedulerHandle, TransactionSimulationResult, - }, - DispatchEndpoints, - }, - Slot, -}; -use magicblock_ledger::{LatestBlockInner, Ledger}; -use magicblock_processor::{ - build_svm_env, - loader::load_upgradeable_programs, - scheduler::{state::TransactionSchedulerState, TransactionScheduler}, -}; -use solana_account::AccountSharedData; -pub use solana_instruction::*; -use solana_keypair::Keypair; -use solana_program::{ - hash::Hasher, native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, -}; -use solana_signature::Signature; -pub use solana_signer::Signer; -use solana_transaction::Transaction; -use solana_transaction_error::TransactionResult; -use solana_transaction_status_client_types::TransactionStatusMeta; -use tempfile::TempDir; -use tokio::sync::mpsc::Sender; -use tokio_util::sync::CancellationToken; -use tracing::{error, instrument}; - -pub const BLOCK_TIME: Duration = Duration::from_millis(50); -pub const SUPERBLOCK_SIZE: u64 = 72000; -/// A simulated validator backend for integration tests. -/// -/// This struct encapsulates all the core components of a validator, including -/// the `AccountsDb`, a `Ledger`, and a running `TransactionScheduler` with its -/// worker pool. It provides a high-level API for tests to manipulate the blockchain -/// state and process transactions. -pub struct ExecutionTestEnv { - /// Atomic counter to index the payers array - payer_index: AtomicUsize, - /// The default keypair used for paying transaction fees and signing. - pub payers: Vec, - /// A handle to the accounts database, storing all account states. - pub accountsdb: Arc, - /// A handle to the ledger, storing all blocks and transactions. - pub ledger: Arc, - /// The entry point for submitting transactions to the processing pipeline. - pub transaction_scheduler: TransactionSchedulerHandle, - /// The temporary directory holding the `AccountsDb` and `Ledger` files for this test run. - pub dir: TempDir, - /// The "client-side" channel endpoints for listening to validator events. - pub dispatch: DispatchEndpoints, - /// Transaction execution scheduler/backend for deferred launch - pub scheduler: Option, - /// Join handle for the spawned scheduler thread. - scheduler_thread: Option>, - /// Shutdown token for the scheduler runtime. - shutdown: CancellationToken, - /// Sender for transitioning from StartingUp to Primary/Replica mode - mode_tx: Sender, -} - -impl Default for ExecutionTestEnv { - fn default() -> Self { - Self::new() - } -} - -impl ExecutionTestEnv { - pub const BASE_FEE: u64 = 1000; - - /// Creates a new, fully initialized execution test environment. - /// - /// This function sets up a complete validator stack: - /// 1. Creates temporary on-disk storage for the accounts database and ledger. - /// 2. Initializes all the communication channels between the API layer and the core. - /// 3. Spawns a `TransactionScheduler` with one worker thread. - /// 4. Pre-loads a test program (`guinea`) for use in tests. - /// 5. Funds a default `payer` keypair with 1 SOL. - pub fn new() -> Self { - Self::new_with_config(Self::BASE_FEE, 1, false) - } - - /// Creates a new, fully initialized validator test environment with given base fee - /// - /// This function sets up a complete validator stack: - /// 1. Creates temporary on-disk storage for the accounts database and ledger. - /// 2. Initializes all the communication channels between the API layer and the core. - /// 3. Spawns a `TransactionScheduler` with the configured number of worker threads. - /// 4. Pre-loads a test program (`guinea`) for use in tests. - /// 5. Funds a default `payer` keypair with 1 SOL. - pub fn new_with_config( - fee: u64, - executors: u32, - defer_startup: bool, - ) -> Self { - Self::new_internal(fee, executors, defer_startup, true) - } - - /// Creates a test environment in Replica mode for replay ordering tests. - /// - /// The scheduler starts in Replica mode and only accepts replay transactions. - /// Use `switch_to_primary_mode()` to transition to Primary mode. - pub fn new_replica_mode(executors: u32, defer_startup: bool) -> Self { - Self::new_internal(Self::BASE_FEE, executors, defer_startup, false) - } - - /// Creates a replica-mode environment with a custom superblock interval. - pub fn new_replica_mode_with_superblock_size( - executors: u32, - defer_startup: bool, - superblock_size: u64, - ) -> Self { - Self::new_internal_with_superblock_size( - Self::BASE_FEE, - executors, - defer_startup, - false, - superblock_size, - ) - } - - /// Internal constructor with all parameters. - fn new_internal( - fee: u64, - executors: u32, - defer_startup: bool, - primary_mode: bool, - ) -> Self { - Self::new_internal_with_superblock_size( - fee, - executors, - defer_startup, - primary_mode, - SUPERBLOCK_SIZE, - ) - } - - fn new_internal_with_superblock_size( - fee: u64, - executors: u32, - defer_startup: bool, - primary_mode: bool, - superblock_size: u64, - ) -> Self { - init_logger!(); - let dir = - tempfile::tempdir().expect("creating temp dir for validator state"); - let accountsdb = Arc::new( - AccountsDb::open(dir.path()).expect("opening test accountsdb"), - ); - let ledger = - Arc::new(Ledger::open(dir.path()).expect("opening test ledger")); - - let (dispatch, validator_channels) = link(); - let blockhash = ledger.latest_block().load().blockhash; - let svm_env = build_svm_env(&accountsdb, blockhash, fee); - let payers = (0..executors).map(|_| Keypair::new()).collect(); - - let (mode_tx, mode_rx) = tokio::sync::mpsc::channel(1); - - let shutdown = CancellationToken::new(); - let mut this = Self { - payer_index: AtomicUsize::new(0), - payers, - accountsdb: accountsdb.clone(), - ledger: ledger.clone(), - transaction_scheduler: dispatch.transaction_scheduler.clone(), - dir, - dispatch, - scheduler: None, - scheduler_thread: None, - shutdown: shutdown.clone(), - mode_tx, - }; - this.advance_slot(); // Move to slot 1 to ensure a non-genesis state. - - // Load test program - load_upgradeable_programs( - &accountsdb, - &[(guinea::ID, "../programs/elfs/guinea.so".into())], - ) - .expect("failed to load test programs into test env"); - - let scheduler_state = TransactionSchedulerState { - accountsdb, - ledger, - account_update_tx: validator_channels.account_update, - transaction_status_tx: validator_channels.transaction_status, - txn_to_process_rx: validator_channels.transaction_to_process, - tasks_tx: validator_channels.tasks_service, - replication_tx: validator_channels.replication_messages, - environment: svm_env.environment, - feature_set: svm_env.feature_set, - shutdown, - mode_rx, - pause_permit: validator_channels.pause_permit, - block_time: BLOCK_TIME, - superblock_size, - }; - - // Pre-send the target mode so the scheduler picks it up once running. - if primary_mode { - this.mode_tx - .try_send(SchedulerMode::Primary) - .expect("failed to pre-send target mode to mode_tx"); - } - - // Start/Defer the transaction processing backend. - let scheduler = TransactionScheduler::new(executors, scheduler_state); - if defer_startup { - this.scheduler.replace(scheduler); - } else { - this.scheduler_thread = Some(scheduler.spawn()); - } - - for payer in this.payers.iter() { - this.fund_account(payer.pubkey(), LAMPORTS_PER_SOL); - } - this - } - - pub fn run_scheduler(&mut self) { - if let Some(scheduler) = self.scheduler.take() { - self.scheduler_thread = Some(scheduler.spawn()); - } - } - - /// Switches the scheduler from StartingUp to Primary mode. - /// - /// After this call, the scheduler will accept execution transactions - /// in addition to replay transactions. - pub fn switch_to_primary_mode(&self) { - self.mode_tx - .try_send(SchedulerMode::Primary) - .expect("failed to send target mode to mode_tx"); - } - - /// Waits for the scheduler to be ready and in primary mode. - /// - /// This is achieved by waiting for the slot to advance, which indicates - /// the scheduler has processed the mode switch and is running. - pub async fn wait_for_scheduler_ready(&self) { - let initial_slot = self.ledger.latest_block().load().slot; - let start = std::time::Instant::now(); - loop { - let current_slot = self.ledger.latest_block().load().slot; - if current_slot > initial_slot { - break; - } - if start.elapsed() > std::time::Duration::from_secs(5) { - panic!( - "Timed out waiting for scheduler to be ready: slot {current_slot}" - ); - } - tokio::time::sleep(std::time::Duration::from_millis(1)).await; - } - } - - /// Waits for the scheduler to start and be ready to process transactions. - /// - /// This is a simpler version that just yields to allow the scheduler time to start. - /// Use this for tests that don't need to wait for slot advancement (e.g., replay tests). - pub async fn yield_to_scheduler(&self) { - // Give the scheduler time to start and process any pending mode switches - for _ in 0..10 { - tokio::task::yield_now().await; - } - // Small additional delay to ensure the scheduler is in its select! loop - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - - /// Waits for the scheduler to advance to the next slot. - pub fn wait_for_next_slot(&self) { - let initial_slot = self.ledger.latest_block().load().slot; - let start = std::time::Instant::now(); - while self.ledger.latest_block().load().slot <= initial_slot { - if start.elapsed() > std::time::Duration::from_secs(5) { - panic!( - "Timed out waiting for slot to advance: slot {}", - initial_slot - ); - } - std::thread::sleep(std::time::Duration::from_millis(1)); - } - } - - /// Advances the slot and writes a new block to the ledger. - pub fn advance_slot(&self) -> Slot { - let block = self.ledger.latest_block(); - let b = block.load(); - let slot = b.slot + 1; - let hash = { - let mut hasher = Hasher::default(); - hasher.hash(b.blockhash.as_ref()); - hasher.hash(&b.slot.to_le_bytes()); - hasher.result() - }; - let time = slot as i64; - self.ledger - .write_block(LatestBlockInner::new(slot, hash, time)) - .expect("failed to write new block to the ledger"); - self.accountsdb.set_slot(slot); - - // Yield to allow other tasks (like the executor) to process the slot change. - thread::yield_now(); - slot - } - - /// Creates a new account with the specified properties. - /// Note: This helper automatically marks the account as `delegated`. - pub fn create_account_with_config( - &self, - lamports: u64, - space: usize, - owner: Pubkey, - ) -> Keypair { - let keypair = Keypair::new(); - let mut account = AccountSharedData::new(lamports, space, &owner); - account.set_delegated(true); - let _ = self.accountsdb.insert_account(&keypair.pubkey(), &account); - keypair - } - - /// Creates a new, empty system account with the given lamports. - pub fn create_account(&self, lamports: u64) -> Keypair { - self.create_account_with_config(lamports, 0, Default::default()) - } - - /// Funds an existing account with the given lamports. - /// If the account does not exist, it will be created as a system account. - pub fn fund_account(&self, pubkey: Pubkey, lamports: u64) { - self.fund_account_with_owner(pubkey, lamports, Default::default()); - } - - /// Funds an account with a specific owner. - /// Note: This helper automatically marks the account as `delegated`. - pub fn fund_account_with_owner( - &self, - pubkey: Pubkey, - lamports: u64, - owner: Pubkey, - ) { - let mut account = AccountSharedData::new(lamports, 0, &owner); - account.set_delegated(true); - let _ = self.accountsdb.insert_account(&pubkey, &account); - } - - /// Retrieves a transaction's metadata from the ledger by its signature. - pub fn get_transaction( - &self, - sig: Signature, - ) -> Option { - self.ledger - .get_transaction_status(sig, u64::MAX) - .expect("failed to get transaction meta from ledger") - .map(|(_, m)| m) - } - - /// Builds a transaction with the given instructions, signed by the default payer. - pub fn build_transaction(&self, ixs: &[Instruction]) -> Transaction { - let payer = { - let index = self.payer_index.fetch_add(1, Ordering::Relaxed); - &self.payers[index % self.payers.len()] - }; - Transaction::new_signed_with_payer( - ixs, - Some(&payer.pubkey()), - &[payer], - self.ledger.latest_blockhash(), - ) - } - - /// Builds a transaction with additional signers beyond the payer. - pub fn build_transaction_with_signers( - &self, - ixs: &[Instruction], - additional_signers: &[&Keypair], - ) -> Transaction { - let payer = { - let index = self.payer_index.fetch_add(1, Ordering::Relaxed); - &self.payers[index % self.payers.len()] - }; - let mut signers: Vec<&Keypair> = vec![payer]; - signers.extend(additional_signers); - Transaction::new_signed_with_payer( - ixs, - Some(&payer.pubkey()), - &signers, - self.ledger.latest_blockhash(), - ) - } - - /// Submits a transaction for execution and waits for its result. - #[instrument(skip(self, txn))] - pub async fn execute_transaction( - &self, - txn: impl SanitizeableTransaction, - ) -> TransactionResult<()> { - self.transaction_scheduler.execute(txn).await.inspect_err( - |err| error!(error = ?err, "Transaction execution failed"), - ) - } - - /// Submits a transaction for scheduling and returns - #[instrument(skip(self, txn))] - pub async fn schedule_transaction( - &self, - txn: impl SanitizeableTransaction, - ) { - self.transaction_scheduler.schedule(txn).await.unwrap(); - } - - /// Submits a transaction for simulation and waits for the detailed result. - #[instrument(skip(self, txn))] - pub async fn simulate_transaction( - &self, - txn: impl SanitizeableTransaction, - ) -> TransactionSimulationResult { - let result = self - .transaction_scheduler - .simulate(txn) - .await - .expect("transaction executor has shutdown during test"); - if let Err(ref err) = result.result { - error!(error = ?err, "Transaction simulation failed"); - } - result - } - - /// Submits a transaction for replay and waits for its result. - /// - /// # Arguments - /// * `persist` - If true, record the transaction to the ledger and broadcast status - /// * `txn` - The transaction to replay - #[instrument(skip(self, txn))] - pub async fn replay_transaction( - &self, - persist: bool, - txn: impl SanitizeableTransaction, - ) -> TransactionResult<()> { - let position = ReplayPosition { - slot: 0, - index: 0, - persist, - }; - self.transaction_scheduler - .replay(position, txn) - .await - .inspect_err( - |err| error!(error = ?err, "Transaction replay failed"), - ) - } - - pub fn get_account(&self, pubkey: Pubkey) -> CommitableAccount<'_> { - let account = self - .accountsdb - .get_account(&pubkey) - .expect("only existing accounts should be requested during tests"); - CommitableAccount { - pubkey, - account, - db: &self.accountsdb, - } - } - - pub fn try_get_account( - &self, - pubkey: Pubkey, - ) -> Option> { - self.accountsdb - .get_account(&pubkey) - .map(|account| CommitableAccount { - pubkey, - account, - db: &self.accountsdb, - }) - } - - pub fn get_payer(&self) -> CommitableAccount<'_> { - let payer = { - let index = self.payer_index.load(Ordering::Relaxed); - &self.payers[index % self.payers.len()] - }; - self.get_account(payer.pubkey()) - } -} - -impl Drop for ExecutionTestEnv { - fn drop(&mut self) { - self.shutdown.cancel(); - if let Some(handle) = self.scheduler_thread.take() { - let _ = handle.join(); - } - } -} - -pub struct CommitableAccount<'db> { - pub pubkey: Pubkey, - pub account: AccountSharedData, - pub db: &'db AccountsDb, -} - -impl CommitableAccount<'_> { - pub fn commit(self) { - let _ = self.db.insert_account(&self.pubkey, &self.account); - } -} - -impl Deref for CommitableAccount<'_> { - type Target = AccountSharedData; - fn deref(&self) -> &Self::Target { - &self.account - } -} - -impl DerefMut for CommitableAccount<'_> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.account - } -} - -pub mod macros; diff --git a/test-kit/src/macros.rs b/test-kit/src/macros.rs deleted file mode 100644 index 850c428d3..000000000 --- a/test-kit/src/macros.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! ----------------- -//! helper test macros (copy paste from the old test-tools crate) -//! TODO(bmuddha): refactor as part of the tests redesign -//! ----------------- - -use std::{env, path::Path}; - -use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use tracing::instrument; -use tracing_log::LogTracer; -use tracing_subscriber::{fmt, EnvFilter}; - -/// Initialize logging for tests using tracing. -/// -/// This sets up both LogTracer (to capture log crate logs) and the -/// tracing subscriber. It respects RUST_LOG for filtering and optionally -/// appends test file name based on RUST_LOG_STYLE. -pub fn init_logger_for_tests() { - // Capture log records from dependencies using the `log` crate - let _ = LogTracer::init(); - - // Build the RUST_LOG filter - let mut rust_log = env::var("RUST_LOG") - .ok() - .unwrap_or_else(|| "info".to_string()); - - // Support RUST_LOG_STYLE to detect test file and append to RUST_LOG - if let Ok(style_env) = env::var("RUST_LOG_STYLE") { - if style_env.to_lowercase() == "test" { - if let Ok(test_file) = env::var("TEST_FILE_PATH") { - let p = Path::new(&test_file); - if let Some(file_stem) = p.file_stem() { - if let Some(file_name) = file_stem.to_str() { - rust_log.push_str(&format!(",{}=debug", file_name)); - } - } - } - } - } - - let env_filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new(rust_log)); - - let _ = fmt::Subscriber::builder() - .with_env_filter(env_filter) - .with_test_writer() - .try_init(); -} - -/// Initialize logging for a test file using the test file path. -/// -/// This is the legacy entry point that maintains backward compatibility. -/// It extracts the test file name and initializes the logger. -pub fn init_logger_for_test_path(full_path_to_test_file: &str) { - // In order to include logs from the test themselves we need to add the - // name of the test file (minus the extension) to the RUST_LOG filter - let mut rust_log = env::var("RUST_LOG") - .ok() - .unwrap_or_else(|| "info".to_string()); - - if rust_log.ends_with(',') || rust_log == "info" { - let p = Path::new(full_path_to_test_file); - if let Some(file_stem) = p.file_stem() { - if let Some(file_name) = file_stem.to_str() { - let test_level = env::var("RUST_LOG_LEVEL") - .ok() - .unwrap_or_else(|| "info".to_string()); - rust_log.push_str(&format!("{}={}", file_name, test_level)); - } - } - } - - // Capture log records from dependencies using the `log` crate - let _ = LogTracer::init(); - - let env_filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new(rust_log)); - - let _ = fmt::Subscriber::builder() - .with_env_filter(env_filter) - .with_test_writer() - .try_init(); -} - -#[macro_export] -macro_rules! init_logger { - () => { - $crate::macros::init_logger_for_test_path(::std::file!()); - }; -} - -#[instrument] -pub async fn is_devnet_up() -> bool { - RpcClient::new("https://api.devnet.solana.com".to_string()) - .get_version() - .await - .is_ok() -} - -#[macro_export] -macro_rules! skip_if_devnet_down { - () => { - if !$crate::macros::is_devnet_up().await { - ::tracing::warn!("Devnet is down, skipping test"); - return; - } - }; -} diff --git a/tools/genx/Cargo.toml b/tools/genx/Cargo.toml deleted file mode 100644 index 6adb508f3..000000000 --- a/tools/genx/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "genx" -version = "0.0.0" -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[dependencies] -base64 = { workspace = true } -clap = { version = "4.5.23", features = ["derive"] } -magicblock-accounts-db = { workspace = true, features = [ "dev-tools" ] } -json = { workspace = true } -solana-rpc-client = { workspace = true } -solana-commitment-config = { workspace = true } -solana-pubkey = { workspace = true } -tempfile = { workspace = true } diff --git a/tools/genx/README.md b/tools/genx/README.md deleted file mode 100644 index cd941f5c9..000000000 --- a/tools/genx/README.md +++ /dev/null @@ -1,18 +0,0 @@ -## genx - -This is a tool to be used to generate configs/scripts, etc. - -### test-validator - -Used to generate a test-validator script that makes it pre-load accounts from chain as follows: - -1. Find all accounts stored in the ledger (provided via the first arg) -2. Add them to the script as `--maybe-clone` to have them loaded into the validator on startup -if they exist on chain -3. Save the script in a tmp folder and print its path to the terminal - -```sh -cargo run --release --bin genx test-validator \ - --rpc-port 7799 --url 'https://rpc.magicblock.app/mainnet' \ - test-integration/ledgers/ledgers -``` diff --git a/tools/genx/src/main.rs b/tools/genx/src/main.rs deleted file mode 100644 index f9136506d..000000000 --- a/tools/genx/src/main.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::path::PathBuf; - -use clap::{Parser, Subcommand}; -use test_validator::TestValidatorConfig; -mod test_validator; - -#[derive(Debug, Parser)] -#[command(name = "genx")] -#[command(about = "genx CLI tool")] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Debug, Subcommand)] -enum Commands { - /// Generates a script to run a test validator - #[command(name = "test-validator")] - #[command( - about = "Generates a script to run a test validator", - long_about = "Example: genx test-validator --rpc-port 7799 --url devnet path/to/accountsdb" - )] - TestValidator { - accountsdb_path: Option, - - #[arg(long)] - rpc_port: u16, - - #[arg(long)] - url: String, - }, -} - -fn main() { - let cli = Cli::parse(); - match cli.command { - Commands::TestValidator { - accountsdb_path, - rpc_port, - url, - } => { - let config = TestValidatorConfig { rpc_port, url }; - test_validator::gen_test_validator_start_script( - accountsdb_path.as_ref(), - config, - ) - } - } -} diff --git a/tools/genx/src/test_validator.rs b/tools/genx/src/test_validator.rs deleted file mode 100644 index 2a9799665..000000000 --- a/tools/genx/src/test_validator.rs +++ /dev/null @@ -1,158 +0,0 @@ -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; -use std::{ - fs, - path::{Path, PathBuf}, -}; - -use json::{json, Value}; -use magicblock_accounts_db::AccountsDb; -use solana_commitment_config::CommitmentConfig; -use solana_pubkey::Pubkey; -use solana_rpc_client::rpc_client::RpcClient; -use tempfile::tempdir; - -pub struct TestValidatorConfig { - pub rpc_port: u16, - pub url: String, -} - -pub(crate) fn gen_test_validator_start_script( - ledger_path: Option<&PathBuf>, - config: TestValidatorConfig, -) { - let temp_dir = tempdir().expect("Failed to create temporary directory"); - let temp_dir_path = temp_dir.keep(); - let accounts_dir = temp_dir_path.join("accounts"); - fs::create_dir(&accounts_dir).expect("Failed to create accounts directory"); - - let file_path = temp_dir_path.join("run-validator.sh"); - let accounts: Vec = if let Some(ledger_path) = ledger_path { - let adb = - AccountsDb::open(ledger_path).expect("failed to open accountsdb"); - eprintln!( - "Generating test validator script with accounts from accountsdb: {:?}", - ledger_path - ); - adb.iter_all().map(|(pk, _)| pk).collect() - } else { - eprintln!("Generating test validator script without accounts"); - vec![] - }; - - let mut args = vec![ - "--log".to_string(), - "--rpc-port".to_string(), - config.rpc_port.to_string(), - "-r".to_string(), - "--limit-ledger-size".to_string(), - "10000".to_string(), - ]; - - download_accounts_into_from_url_into_dir( - &accounts, - config.url.clone(), - &accounts_dir, - ); - - args.push("--account-dir".into()); - args.push(accounts_dir.to_string_lossy().to_string()); - - args.push("--url".into()); - args.push(config.url); - - let script = format!( - " -#!/usr/bin/env bash -set -e -solana-test-validator \\\n {}", - args.join(" \\\n ") - ); - // chmod u+x - fs::write(&file_path, script) - .expect("Failed to write test validator script"); - // Set permissions - #[cfg(unix)] - { - fs::set_permissions(&file_path, fs::Permissions::from_mode(0o755)) - .expect("Failed to set permissions on Unix"); - } - - #[cfg(windows)] - { - use std::process::Command; - - let output = Command::new("icacls") - .arg(&file_path) - .arg("/grant") - .arg("Everyone:(RX)") - .arg("/grant") - .arg("Users:(RX)") - .arg("/grant") - .arg("Administrators:(F)") - .output() - .expect("Failed to set file permissions on Windows"); - - if !output.status.success() { - eprintln!("Error: {:?}", String::from_utf8_lossy(&output.stderr)); - } else { - println!("Permissions set successfully on Windows!"); - } - } - - eprintln!( - "Run this script to start the test validator: \n\n{}", - file_path.display() - ); -} - -fn download_accounts_into_from_url_into_dir( - pubkeys: &[Pubkey], - url: String, - dir: &Path, -) { - // Derived from error from helius RPC: Failed to download accounts: Error { request: Some(GetMultipleAccounts), kind: RpcError(RpcResponseError { code: -32602, message: "Too many inputs provided; max 100", data: Empty }) } - const MAX_ACCOUNTS: usize = 100; - let rpc_client = - RpcClient::new_with_commitment(url, CommitmentConfig::confirmed()); - let total_len = pubkeys.len(); - for (idx, pubkeys) in pubkeys.chunks(MAX_ACCOUNTS).enumerate() { - let start = idx * MAX_ACCOUNTS; - let end = start + pubkeys.len(); - eprintln!("Downloading {}..{}/{} accounts", start, end, total_len); - match rpc_client.get_multiple_accounts(pubkeys) { - Ok(accs) => accs - .into_iter() - .zip(pubkeys) - .filter_map(|(acc, pubkey)| acc.map(|x| (x, pubkey))) - .for_each(|(acc, pubkey)| { - let path = dir.join(format!("{pubkey}.json")); - let pk = pubkey.to_string(); - let lamports = acc.lamports; - let data = [ - #[allow(deprecated)] // this is just a dev tool - base64::encode(&acc.data), - "base64".to_string(), - ]; - let owner = acc.owner.to_string(); - let executable = acc.executable; - let rent_epoch = acc.rent_epoch; - let space = acc.data.len(); - let json: Value = json! {{ - "pubkey": pk, - "account": { - "lamports": lamports, - "data": data, - "owner": owner, - "executable": executable, - "space": space, - "rentEpoch": rent_epoch - }, - }}; - fs::write(&path, format!("{:#}", json)) - .expect("Failed to write account"); - }), - Err(err) => eprintln!("Failed to download accounts: {:?}", err), - } - } -} diff --git a/tools/keypair-base58/Cargo.toml b/tools/keypair-base58/Cargo.toml deleted file mode 100644 index 6366b3dc9..000000000 --- a/tools/keypair-base58/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "keypair-base58" -authors = ["MagicBlock Maintainers "] -repository = "https://github.com/magicblock-labs/x-validator" -homepage = "https://www.magicblock.gg" -license = "Apache-2.0" -edition = "2021" - -[dependencies] -bs58 = "0.5" -serde_json = "1.0" diff --git a/tools/keypair-base58/README.md b/tools/keypair-base58/README.md deleted file mode 100644 index a133dc9ef..000000000 --- a/tools/keypair-base58/README.md +++ /dev/null @@ -1,10 +0,0 @@ -## keypair-base58 - -Prints the keypair in base58 format for a keypair file. - -Example usage to set the `VALIDATOR_KEYPAIR` environment variable before starting up the -validator that is replaying a ledger for a validator with that keypair: - -```sh -export VALIDATOR_KEYPAIR=`cargo run --bin keypair-base58 -- ledger/validator-keypair.json` -``` diff --git a/tools/keypair-base58/src/main.rs b/tools/keypair-base58/src/main.rs deleted file mode 100644 index 0f29fbd23..000000000 --- a/tools/keypair-base58/src/main.rs +++ /dev/null @@ -1,26 +0,0 @@ -use std::{env, fs::File, io::Read}; - -fn main() { - // Get command line argument - let args: Vec = env::args().collect(); - if args.len() != 2 { - eprintln!("Usage: {} ", args[0]); - std::process::exit(1); - } - - // Read the keypair file - let mut file = File::open(&args[1]).expect("Failed to open keypair file"); - let mut contents = String::new(); - file.read_to_string(&mut contents) - .expect("Failed to read keypair file"); - - // Parse the JSON array - let keypair: Vec = - serde_json::from_str(&contents).expect("Failed to parse keypair JSON"); - - // Convert to base58 - let base58_string = bs58::encode(&keypair).into_string(); - - // Print the result - println!("{}", base58_string); -} diff --git a/tools/ledger-stats/Cargo.toml b/tools/ledger-stats/Cargo.toml deleted file mode 100644 index b39779372..000000000 --- a/tools/ledger-stats/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "ledger-stats" -version = "0.0.0" -authors.workspace = true -repository.workspace = true -homepage.workspace = true -license.workspace = true -edition.workspace = true - -[lib] -path = "src/lib.rs" - -[dependencies] -clap = { workspace = true, features = ["derive"] } -magicblock-accounts-db = { workspace = true } -magicblock-core = { workspace = true } -magicblock-ledger = { workspace = true } -num-format = { workspace = true } -pretty-hex = "0.4.1" -solana-account = { workspace = true } -solana-clock = { workspace = true } -solana-message = { workspace = true } -solana-pubkey = { workspace = true } -solana-signature = { workspace = true } -solana-transaction-status = { workspace = true, features = ["agave-unstable-api"] } diff --git a/tools/ledger-stats/README.md b/tools/ledger-stats/README.md deleted file mode 100644 index b3ddaf9e9..000000000 --- a/tools/ledger-stats/README.md +++ /dev/null @@ -1,191 +0,0 @@ -## Ledger Stats Tool - -Provides diagnostics for ledger data stored in a local directory. - -In order to run it directly from the repo use `cargo run --bin`, i.e.: - -```sh -cargo run --bin ledger-stats -- log ./tools/ledger-stats/ledger/ --success -``` - -The examples use a globally installed version. - -Implemented with subcommands, each of which have a help section, i.e.: - -```sh -❯ ledger-stats log --help -ledger-stats-log 0.0.0 -Transaction logs - -USAGE: - ledger-stats log [FLAGS] [OPTIONS] - -FLAGS: - -h, --help Prints help information - -s, --success Show successful transactions, default: false - -V, --version Prints version information - -OPTIONS: - -e, --end End slot - -s, --start Start slot - -ARGS: - -``` - -The idea is that we keep adding functionality as we need it in order to allow understanding -existing ledgers in order to diagnose user issues quickly. - -We may also add a JSON output format should we ever want to build a UI around it. - -### Summary - -```sh -❯ ledger-stats --help -ledger-stats 0.0.0 - -USAGE: - ledger-stats - -FLAGS: - -h, --help Prints help information - -V, --version Prints version information - -SUBCOMMANDS: - count Counts of items in ledger columns - help Prints this message or the help of the given subcommand(s) - log Transaction logs - sig Transaction details for signature -``` - -### count - -Shown above - -### log - -```sh -❯ ledger-stats log ./tools/ledger-stats/ledger/ --success - -Transaction: 2c1sRDHvvCCF58SVnrq3UnGSDdobHHbgccEgHUbuyzhU5ktgQ3pEXRHyR7JT5M7CUWStMfmRYEVSfLEJwa77Rn3X (4141) - - Program Magic11111111111111111111111111111111111111 invoke [1] - • MutateAccounts: modifying '9twuZbSbuCjErSgMYxPimMKhS5AfhaKJjkdVfW3Ymyhe'. - • MutateAccounts: setting lamports to 1712160 - • MutateAccounts: setting owner to zbtv2cgU1VzSBKNXZ96TcWSRVp1c8HxqCmRp8zPX1uh - • MutateAccounts: setting executable to false - • MutateAccounts: resolved data from id 1 - • MutateAccounts: setting data to len 118 - • MutateAccounts: setting rent_epoch to 18446744073709551615 - • Program Magic11111111111111111111111111111111111111 success - -Transaction: 4cdgG62ut9Hav2WkKjwGgnMTFWGdw7g8gmkdU99CUGfvaR6MyXZZoh7G4CcWWEHmSek1BAfiHjHyKiE2a7U9mMcE (4141) -[ ... ] -``` - -### sig - -
-Transaction details for signature - -```sh -❯ ledger-stats sig ./tools/ledger-stats/ledger/ 3rEEV7SVSXuPYKrW2WgvkGj74mPzzrzbK4Y4dyRZGJYxQDLYiPMQWrQg4nNpwMG16Qc5Ye49jneWSehWJSyEMxH2 - -++++ Transaction Status ++++ - -Field Value -===================== ===================== -Status Ok -Fee 0 -Pre-balances 9,223,368,998,541,374,767 | 0 | 1 -Post-balances 9,223,367,998,541,374,767 | 1,000,000,000,000 | 1 -Inner Instructions 0 -Pre-token Balances None -Post-token Balances None -Rewards None -Loaded Addresses writable: 0, readonly: 0 -Return Data None -Compute Units Consumed 150 - - -++++ Transaction Logs ++++ - - Program Magic11111111111111111111111111111111111111 invoke [1] - • MutateAccounts: modifying 'BPgkXhjdLUMstLjFvrbpG2TWXjSqqukgJ5GEiRnQNhAp'. - • MutateAccounts: setting lamports to 1000000000000 - • MutateAccounts: setting owner to 11111111111111111111111111111111 - • MutateAccounts: setting executable to false - • MutateAccounts: resolved data from id 15 - • MutateAccounts: setting data to len 0 - • MutateAccounts: setting rent_epoch to 0 - • Program Magic11111111111111111111111111111111111111 success - -++++ Transaction ++++ - -num_required_signatures 1 -num_readonly_signed_accounts 0 -num_readonly_unsigned_accounts 1 -block_time 1733731769 - -++++ Account Keys ++++ - - • zbitnhqG6MLu3E6XBJGEd7WarnKDeqzriB14hr74Fjb - • BPgkXhjdLUMstLjFvrbpG2TWXjSqqukgJ5GEiRnQNhAp - • Magic11111111111111111111111111111111111111 - -++++ Instructions ++++ - -#1 Program ID: Magic11111111111111111111111111111111111111 - - Accounts: - • zbitnhqG6MLu3E6XBJGEd7WarnKDeqzriB14hr74Fjb - • BPgkXhjdLUMstLjFvrbpG2TWXjSqqukgJ5GEiRnQNhAp - - Instruction Data Length: 106 (0x6a) bytes - 0000: 00 00 00 00 01 00 00 00 00 00 00 00 9a 64 97 d5 - 0010: a3 ff 66 a8 0f 03 9d cc c2 06 f5 98 a1 e6 68 aa - 0020: 4a d5 6d d5 8b 68 ed 12 fc 65 4e 31 01 00 10 a5 - 0030: d4 e8 00 00 00 01 00 00 00 00 00 00 00 00 00 00 - 0040: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 - 0050: 00 00 00 00 00 00 01 00 01 0f 00 00 00 00 00 00 - 0060: 00 01 00 00 00 00 00 00 00 00 -``` -
- -### accounts - -The accounts subcommand provides details about the accounts in the ledger. It supports the -following options: - -- -c, --count: Print the count of accounts instead of the account details. -- -r, --rent-epoch: Show the rent epoch for each account. -- -f, --filter ...: Filter accounts based on specified criteria. Multiple criteria can be provided as a comma-separated list. Possible values include: - - on or on-curve: Include on-curve accounts. - - off or off-curve: Include off-curve accounts (PDAs). - - executable: Include executable accounts. - - non-executable: Include non-executable accounts. -- -o, --owner : Filter accounts by the specified owner. -- -s, --sort : Sort accounts by the specified column. The default is to sort by the account's public key (Pubkey). -The argument specifies the path to the ledger directory. - -Example usage: - -```sh -❯ ledger-stats accounts ledger -s d -f=off -``` - -This will sort by data size and only include off-curve accounts. - -### account - -The account subcommand provides detailed information about a specific account in the ledger, -including its data. It requires two arguments: - -- ledger-path: The path to the ledger directory. -- pubkey: The public key of the account to retrieve details for. - -Example usage: - -```sh -❯ ledger-stats account ledger 8JSRCegc3J5RqMp8izAZAs23PrmCg6e9TpraVB668xxn -``` diff --git a/tools/ledger-stats/src/account.rs b/tools/ledger-stats/src/account.rs deleted file mode 100644 index 39762c3cc..000000000 --- a/tools/ledger-stats/src/account.rs +++ /dev/null @@ -1,53 +0,0 @@ -use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; -use num_format::{Locale, ToFormattedString}; -use pretty_hex::*; -use solana_account::ReadableAccount; -use solana_pubkey::Pubkey; - -use crate::utils::print_two_col_table; - -pub fn print_account(db: &AccountsDb, pubkey: &Pubkey) { - let account = db.get_account(pubkey).expect("Account not found"); - let oncurve = pubkey.is_on_curve(); - - println!("{} at slot: {}", pubkey, db.slot()); - let rows = vec![ - ("Pubkey".to_string(), pubkey.to_string()), - ("Owner".to_string(), account.owner().to_string()), - ( - "Lamports".to_string(), - account.lamports().to_formatted_string(&Locale::en), - ), - ("Executable".to_string(), account.executable().to_string()), - ( - "Data (Bytes)".to_string(), - account.data().len().to_formatted_string(&Locale::en), - ), - ( - "Curve".to_string(), - if oncurve { "On" } else { "Off" }.to_string(), - ), - ( - "RentEpoch".to_string(), - account.rent_epoch().to_formatted_string(&Locale::en), - ), - ]; - - let data = if !account.data().is_empty() { - let hex = format!( - "{:?}", - account.data().hex_conf(HexConfig { - width: 16, - group: 4, - ascii: true, - ..Default::default() - }) - ); - hex - } else { - "".to_string() - }; - print_two_col_table(None, ["Column", "Value"], &rows); - println!(); - println!("{}", data); -} diff --git a/tools/ledger-stats/src/accounts.rs b/tools/ledger-stats/src/accounts.rs deleted file mode 100644 index be25c87b7..000000000 --- a/tools/ledger-stats/src/accounts.rs +++ /dev/null @@ -1,213 +0,0 @@ -use clap::ValueEnum; -use magicblock_accounts_db::AccountsDb; -use num_format::{Locale, ToFormattedString}; -use solana_account::ReadableAccount; -use solana_clock::Epoch; -use solana_pubkey::Pubkey; - -// ----------------- -// SortAccounts -// ----------------- -#[derive(Clone, Copy, Debug, Default, ValueEnum)] -pub enum SortAccounts { - #[default] - #[value(alias = "p")] - Pubkey, - #[value(alias = "o")] - Owner, - #[value(alias = "l")] - Lamports, - #[value(alias = "e")] - Executable, - #[value(alias = "d")] - DataLen, - #[value(alias = "r")] - RentEpoch, -} - -// ----------------- -// FilterAccounts -// ----------------- -#[derive(Clone, Debug, PartialEq, Eq, ValueEnum)] -pub enum FilterAccounts { - Executable, - NonExecutable, - #[value(alias = "on")] - OnCurve, - #[value(alias = "off")] - OffCurve, -} - -impl FilterAccounts { - pub(crate) fn sanitize(filters: &[Self]) { - if filters.contains(&Self::OnCurve) && filters.contains(&Self::OffCurve) - { - panic!("Cannot filter by both curve and pda"); - } - if filters.contains(&Self::Executable) - && filters.contains(&Self::NonExecutable) - { - panic!("Cannot filter by both executable and non-executable"); - } - } -} - -// ----------------- -// AccountInfo -// ----------------- -struct AccountInfo { - /// Pubkey of the account - pub pubkey: Pubkey, - /// lamports in the account - pub lamports: u64, - /// the epoch at which this account will next owe rent - pub rent_epoch: Epoch, - /// the program that owns this account. If executable, the program that loads this account. - pub owner: Pubkey, - /// this account's data contains a loaded program (and is now read-only) - pub executable: bool, - /// the data in this account - pub data: Vec, -} - -pub fn print_accounts( - adb: &AccountsDb, - sort: SortAccounts, - owner: Option, - filters: &[FilterAccounts], - print_rent_epoch: bool, - count: bool, -) { - let mut accounts = { - let iter = adb.iter_all(); - let all = iter.map(|(pubkey, acc)| AccountInfo { - pubkey, - lamports: acc.lamports(), - rent_epoch: acc.rent_epoch(), - owner: *acc.owner(), - executable: acc.executable(), - data: acc.data().to_vec(), - }); - all.into_iter() - .filter(|acc| { - if !owner.is_none_or(|owner| acc.owner.eq(&owner)) { - return false; - } - if filters.contains(&FilterAccounts::Executable) - && !acc.executable - { - return false; - } - if filters.contains(&FilterAccounts::NonExecutable) - && acc.executable - { - return false; - } - if filters.contains(&FilterAccounts::OnCurve) - && !acc.pubkey.is_on_curve() - { - return false; - } - if filters.contains(&FilterAccounts::OffCurve) - && acc.pubkey.is_on_curve() - { - return false; - } - - true - }) - .collect::>() - }; - accounts.sort_by(|a, b| { - use SortAccounts::*; - match sort { - Pubkey => a.pubkey.cmp(&b.pubkey), - Owner => a.owner.cmp(&b.owner), - Lamports => a.lamports.cmp(&b.lamports), - Executable => a.executable.cmp(&b.executable), - DataLen => a.data.len().cmp(&b.data.len()), - RentEpoch => a.rent_epoch.cmp(&b.rent_epoch), - } - }); - - let slot = adb.slot(); - if count { - if let Some(owner) = owner { - println!( - "Total accounts at slot {} owned by '{}': {}", - slot, - owner, - accounts.len() - ); - } else { - println!("Total accounts at slot {}: {}", slot, accounts.len()); - } - return; - } - - let mut headers = vec![ - "Pubkey", - "Owner", - "Lamports", - "Executable", - "Data(Bytes)", - "Curve", - ]; - if print_rent_epoch { - headers.push("Rent Epoch"); - } - let rows = accounts - .into_iter() - .map(|acc| { - let mut row = vec![ - acc.pubkey.to_string(), - acc.owner.to_string(), - acc.lamports.to_formatted_string(&Locale::en), - acc.executable.to_string(), - acc.data.len().to_string(), - if acc.pubkey.is_on_curve() { - "On".to_string() - } else { - "Off".to_string() - }, - ]; - if print_rent_epoch { - row.push(acc.rent_epoch.to_formatted_string(&Locale::en)); - } - row - }) - .collect::>(); - println!("Accounts at slot {}", slot); - print_rows(&headers, &rows); -} - -fn print_rows(headers: &[&str], rows: &[Vec]) { - let mut widths = headers - .iter() - .map(|header| header.len()) - .collect::>(); - for row in rows { - for (idx, cell) in row.iter().enumerate() { - widths[idx] = widths[idx].max(cell.len()); - } - } - - print_row(headers.iter().copied(), &widths); - print_row(widths.iter().map(|width| "-".repeat(*width)), &widths); - for row in rows { - print_row(row.iter().map(String::as_str), &widths); - } -} - -fn print_row<'a>( - cells: impl IntoIterator + 'a>, - widths: &[usize], -) { - for (idx, (cell, width)) in cells.into_iter().zip(widths).enumerate() { - if idx > 0 { - print!(" "); - } - print!("{: match ledger.get_max_blockhash() { - Ok((slot, hash)) => match ledger.count_blockhashes() { - Ok(count) => { - println!( - "Last blockhash at slot {}: {} of {} total blockhashes", - slot.to_formatted_string(&Locale::en), - hash, - count.to_formatted_string(&Locale::en), - ); - } - Err(err) => { - eprintln!("Failed to count blockhashes: {:?}", err); - } - }, - Err(err) => { - eprintln!("Blockhash not found {:?}", err); - } - }, - }; -} diff --git a/tools/ledger-stats/src/counts.rs b/tools/ledger-stats/src/counts.rs deleted file mode 100644 index a100374ac..000000000 --- a/tools/ledger-stats/src/counts.rs +++ /dev/null @@ -1,67 +0,0 @@ -use magicblock_ledger::Ledger; -use num_format::{Locale, ToFormattedString}; - -use crate::utils::print_two_col_table; - -pub(crate) fn print_counts(ledger: &Ledger) { - let block_times_count = ledger - .count_block_times() - .expect("Failed to count block times") - .to_formatted_string(&Locale::en); - let blockhash_count = ledger - .count_blockhashes() - .expect("Failed to count blockhash") - .to_formatted_string(&Locale::en); - let transaction_status_count = ledger - .count_transaction_status() - .expect("Failed to count transaction status") - .to_formatted_string(&Locale::en); - let successfull_transaction_status_count = ledger - .count_transaction_successful_status() - .expect("Failed to count successful transaction status") - .to_formatted_string(&Locale::en); - let failed_transaction_status_count = ledger - .count_transaction_failed_status() - .expect("Failed to count failed transaction status") - .to_formatted_string(&Locale::en); - let address_signatures_count = ledger - .count_address_signatures() - .expect("Failed to count address signatures") - .to_formatted_string(&Locale::en); - let slot_signatures_count = ledger - .count_slot_signatures() - .expect("Failed to count slot signatures") - .to_formatted_string(&Locale::en); - let transaction_count = ledger - .count_transactions() - .expect("Failed to count transaction") - .to_formatted_string(&Locale::en); - let transaction_memos_count = ledger - .count_transaction_memos() - .expect("Failed to count transaction memos") - .to_formatted_string(&Locale::en); - let perf_samples_count = ledger - .count_perf_samples() - .expect("Failed to count perf samples") - .to_formatted_string(&Locale::en); - - let rows = vec![ - ("Blockhashes".to_string(), blockhash_count), - ("BlockTimes".to_string(), block_times_count), - ("TransactionStatus".to_string(), transaction_status_count), - ("Transactions".to_string(), transaction_count), - ( - "Successful Transactions".to_string(), - successfull_transaction_status_count, - ), - ( - "Failed Transactions".to_string(), - failed_transaction_status_count, - ), - ("SlotSignatures".to_string(), slot_signatures_count), - ("AddressSignatures".to_string(), address_signatures_count), - ("TransactionMemos".to_string(), transaction_memos_count), - ("PerfSamples".to_string(), perf_samples_count), - ]; - print_two_col_table(None, ["Column", "Count"], &rows); -} diff --git a/tools/ledger-stats/src/lib.rs b/tools/ledger-stats/src/lib.rs deleted file mode 100644 index f2aa5413c..000000000 --- a/tools/ledger-stats/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod utils; - -pub use utils::open_ledger; diff --git a/tools/ledger-stats/src/main.rs b/tools/ledger-stats/src/main.rs deleted file mode 100644 index 2b546c0b1..000000000 --- a/tools/ledger-stats/src/main.rs +++ /dev/null @@ -1,184 +0,0 @@ -use std::{collections::HashSet, path::PathBuf, str::FromStr}; - -use clap::{Parser, Subcommand}; -use magicblock_accounts_db::AccountsDb; -use solana_pubkey::Pubkey; - -use crate::utils::open_ledger; - -mod account; -mod accounts; -mod blockhash; -mod counts; -mod transaction_details; -mod transaction_logs; -mod utils; - -#[derive(Debug, Subcommand)] -enum Command { - #[command(name = "count", about = "Counts of items in ledger columns")] - Count { ledger_path: PathBuf }, - #[command(name = "log", about = "Transaction logs")] - Log { - ledger_path: PathBuf, - #[arg( - long, - short = 'u', - help = "Show successful transactions, default: false" - )] - success: bool, - #[arg(long, short, help = "Start slot")] - start: Option, - #[arg(long, short, help = "End slot")] - end: Option, - - #[arg( - long, - short, - value_delimiter = ',', - help = "Accounts in transaction" - )] - accounts: Vec, - }, - #[command(name = "sig", about = "Transaction details for signature")] - Sig { - ledger_path: PathBuf, - #[arg(help = "Signature")] - sig: String, - #[arg(long, short, help = "Show instruction ascii data")] - ascii: bool, - }, - #[command(name = "accounts", about = "Account details")] - Accounts { - ledger_path: PathBuf, - #[arg( - long, - short, - value_enum, - help = "Column by which to sort accounts", - default_value_t = accounts::SortAccounts::Pubkey - )] - sort: accounts::SortAccounts, - #[arg(long, short, help = "Filter by account owner")] - owner: Option, - #[arg(long, short, help = "Show rent epoch")] - rent_epoch: bool, - #[arg( - long, - short, - help = "Filter accounts by specified criteria (comma-separated). PDAs are off-curve", - value_enum, - value_delimiter = ',' - )] - filter: Vec, - #[arg(long, short, help = "Print count instead of account details")] - count: bool, - }, - #[command( - name = "account", - about = "Specific Account Details including Data" - )] - Account { - ledger_path: PathBuf, - #[arg(help = "Pubkey of the account")] - pubkey: String, - }, - Blockhash { - ledger_path: PathBuf, - #[arg( - long, - short, - help = "Prints the highest slot and blockhash for which a blockhash was recorded", - value_enum - )] - query: blockhash::BlockhashQuery, - }, -} - -#[derive(Parser)] -struct Cli { - #[command(subcommand)] - command: Command, -} - -fn main() { - let args = Cli::parse(); - - use Command::*; - match args.command { - Count { ledger_path } => { - counts::print_counts(&open_ledger(&ledger_path)) - } - Log { - ledger_path, - success, - start, - end, - accounts, - } => { - let accounts = (!accounts.is_empty()).then(|| { - accounts - .iter() - .map(|account| { - Pubkey::from_str(account) - .expect("Invalid account pubkey") - }) - .collect::>() - }); - transaction_logs::print_transaction_logs( - &open_ledger(&ledger_path), - start, - end, - accounts, - success, - ); - } - Sig { - ledger_path, - sig, - ascii, - } => { - let ledger = open_ledger(&ledger_path); - transaction_details::print_transaction_details( - &ledger, &sig, ascii, - ); - } - Accounts { - ledger_path, - rent_epoch, - sort, - owner, - filter, - count, - } => { - let owner = owner.map(|owner| { - Pubkey::from_str(&owner).expect("Invalid owner filter pubkey") - }); - accounts::FilterAccounts::sanitize(&filter); - accounts::print_accounts( - &AccountsDb::open(&ledger_path) - .expect("adb couldn't be opened"), - sort, - owner, - &filter, - rent_epoch, - count, - ); - } - Account { - ledger_path, - pubkey, - } => { - let adb = - AccountsDb::open(&ledger_path).expect("adb couldn't be opened"); - let pubkey = Pubkey::from_str(&pubkey).expect("Invalid pubkey"); - account::print_account(&adb, &pubkey); - } - Blockhash { ledger_path, query } => { - blockhash::print_blockhash_details( - &open_ledger(&ledger_path), - query, - ); - } - } -} diff --git a/tools/ledger-stats/src/transaction_details.rs b/tools/ledger-stats/src/transaction_details.rs deleted file mode 100644 index ef845bbac..000000000 --- a/tools/ledger-stats/src/transaction_details.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::str::FromStr; - -use magicblock_ledger::Ledger; -use num_format::{Locale, ToFormattedString}; -use pretty_hex::*; -use solana_message::VersionedMessage; -use solana_signature::Signature; -use solana_transaction_status::ConfirmedTransactionWithStatusMeta; - -use crate::utils::{print_two_col_table, render_logs}; - -pub(crate) fn print_transaction_details( - ledger: &Ledger, - sig: &str, - ix_data_ascii: bool, -) { - let sig = Signature::from_str(sig).expect("Invalid signature"); - let (_slot, status_meta) = match ledger - .get_transaction_status(sig, u64::MAX) - .expect("Failed to get transaction status") - { - Some(val) => val, - None => { - eprintln!("Transaction status not found"); - return; - } - }; - - let status = match &status_meta.status { - Ok(_) => "Ok".to_string(), - Err(err) => format!("{:?}", err), - }; - - let pre_balances = status_meta - .pre_balances - .iter() - .map(|b| b.to_formatted_string(&Locale::en)) - .collect::>() - .join(" | "); - - let post_balances = status_meta - .post_balances - .iter() - .map(|b| b.to_formatted_string(&Locale::en)) - .collect::>() - .join(" | "); - - let inner_instructions = status_meta - .inner_instructions - .as_ref() - .map_or(0, |i| i.len()); - - let pre_token_balances = - status_meta.pre_token_balances.as_ref().map_or_else( - || "None".to_string(), - |b| { - if b.is_empty() { - "None".to_string() - } else { - { - b.iter() - .map(|b| b.ui_token_amount.amount.to_string()) - .collect::>() - .join(" | ") - } - } - }, - ); - - let post_token_balances = - status_meta.post_token_balances.as_ref().map_or_else( - || "None".to_string(), - |b| { - if b.is_empty() { - "None".to_string() - } else { - { - b.iter() - .map(|b| b.ui_token_amount.amount.to_string()) - .collect::>() - .join(" | ") - } - } - }, - ); - - let rewards = status_meta.rewards.as_ref().map_or_else( - || "None".to_string(), - |r| { - if r.is_empty() { - "None".to_string() - } else { - { - r.iter() - .map(|r| r.lamports.to_formatted_string(&Locale::en)) - .collect::>() - .join(" | ") - } - } - }, - ); - - let return_data = status_meta - .return_data - .as_ref() - .map_or("None".to_string(), |d| { - d.data.len().to_formatted_string(&Locale::en) - }); - - let compute_units_consumed = - status_meta.compute_units_consumed.map_or(0, |c| c as usize); - - let rows = vec![ - ("Status".to_string(), status), - ("Fee".to_string(), status_meta.fee.to_string()), - ("Pre-balances".to_string(), pre_balances), - ("Post-balances".to_string(), post_balances), - ( - "Inner Instructions".to_string(), - inner_instructions.to_string(), - ), - ("Pre-token Balances".to_string(), pre_token_balances), - ("Post-token Balances".to_string(), post_token_balances), - ("Rewards".to_string(), rewards), - ( - "Loaded Addresses".to_string(), - format!( - "writable: {}, readonly: {}", - status_meta.loaded_addresses.writable.len(), - status_meta.loaded_addresses.readonly.len() - ), - ), - ("Return Data".to_string(), return_data), - ( - "Compute Units Consumed".to_string(), - compute_units_consumed.to_string(), - ), - ]; - print_two_col_table(Some("Transaction Status"), ["Field", "Value"], &rows); - - match status_meta.log_messages { - None => {} - Some(logs) => { - println!( - "\n++++ Transaction Logs ++++\n{}", - render_logs(&logs, " ") - ); - } - } - - let tx = ledger - .get_complete_transaction(sig, u64::MAX) - .expect("Failed to get transaction"); - - if let Some(ConfirmedTransactionWithStatusMeta { - tx_with_meta, - block_time, - .. - }) = tx - { - if let VersionedMessage::V0(message) = - tx_with_meta.get_transaction().message - { - let rows = vec![ - ( - "num_required_signatures".to_string(), - message.header.num_required_signatures.to_string(), - ), - ( - "num_readonly_signed_accounts".to_string(), - message.header.num_readonly_signed_accounts.to_string(), - ), - ( - "num_readonly_unsigned_accounts".to_string(), - message.header.num_readonly_unsigned_accounts.to_string(), - ), - ( - "block_time".to_string(), - block_time.unwrap_or_default().to_string(), - ), - ]; - print_two_col_table(Some("Transaction"), ["Field", "Value"], &rows); - - println!("++++ Account Keys ++++\n"); - for account_key in &message.account_keys { - println!(" • {}", account_key); - } - - println!("\n++++ Instructions ++++\n"); - for (idx, instruction) in message.instructions.iter().enumerate() { - let program_id = - message.account_keys[instruction.program_id_index as usize]; - println!("#{} Program ID: {}", idx + 1, program_id); - - println!("\n Accounts:"); - for account_index in &instruction.accounts { - let account_key = - message.account_keys[*account_index as usize]; - println!(" • {}", account_key); - } - - print!("\n Instruction Data "); - let hex = format!( - "{:?}", - instruction.data.hex_conf(HexConfig { - width: 16, - group: 4, - ascii: ix_data_ascii, - ..Default::default() - }) - ); - let hex_indented = hex.lines().collect::>().join("\n "); - println!("{}", hex_indented); - } - } - } -} diff --git a/tools/ledger-stats/src/transaction_logs.rs b/tools/ledger-stats/src/transaction_logs.rs deleted file mode 100644 index fb5a0c9c6..000000000 --- a/tools/ledger-stats/src/transaction_logs.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::collections::HashSet; - -use magicblock_ledger::Ledger; -use solana_pubkey::Pubkey; - -use crate::utils::render_logs; - -pub(crate) fn print_transaction_logs( - ledger: &Ledger, - start_slot: Option, - end_slot: Option, - accounts: Option>, - success: bool, -) { - let start_slot = start_slot.unwrap_or(0); - let end_slot = end_slot.unwrap_or(u64::MAX); - let sorted = { - let mut vec = ledger - .iter_transaction_statuses(None, success) - .filter_map(|res| match res { - Ok((slot, sig, status)) - if start_slot <= slot && slot <= end_slot => - { - if let Some(accounts) = &accounts { - // NOTE: I tried to use - // - status.loaded_writable_addresses - // - status.loaded_readonly_addresses - // but those are always empty - let tx = ledger - .get_complete_transaction(sig, u64::MAX) - .expect("Failed to get transaction"); - - let matching_keys = tx - .map(|x| { - x.get_transaction() - .message - .static_account_keys() - .to_vec() - }) - .unwrap_or_default() - .iter() - .filter(|pubkey| accounts.contains(pubkey)) - .cloned() - .collect::>(); - - if !matching_keys.is_empty() { - Some((slot, sig, status, Some(matching_keys))) - } else { - None - } - } else { - Some((slot, sig, status, None)) - } - } - Ok(_) => None, - Err(_) => None, - }) - .collect::>(); - vec.sort_by_key(|(slot, _, _, _)| *slot); - vec - }; - for (slot, sig, status, acc) in sorted { - println!("\n ------------------------------------"); - if let Some(x) = acc { - println!( - "\n## Matched Accounts: {}", - x.iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ") - ); - } - - println!("\n### Transaction: {} ({})", sig, slot); - println!("{}", render_logs(&status.log_messages, " ")); - } -} diff --git a/tools/ledger-stats/src/utils.rs b/tools/ledger-stats/src/utils.rs deleted file mode 100644 index 26b3b1b5d..000000000 --- a/tools/ledger-stats/src/utils.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::path::Path; - -use magicblock_ledger::Ledger; - -#[allow(dead_code)] // shared source file is compiled by both the lib and bin targets -pub(crate) fn print_two_col_table( - title: Option<&str>, - header: [&str; 2], - rows: &[(String, String)], -) { - if let Some(title) = title { - println!("\n++++ {title} ++++\n"); - } - - let left_w = rows - .iter() - .map(|(left, _)| left.len()) - .chain(std::iter::once(header[0].len())) - .max() - .unwrap_or(header[0].len()); - let right_w = rows - .iter() - .map(|(_, right)| right.len()) - .chain(std::iter::once(header[1].len())) - .max() - .unwrap_or(header[1].len()); - - println!( - "{:right_w$}", - header[0], - header[1], - left_w = left_w, - right_w = right_w - ); - println!( - "{:right_w$}", - "=".repeat(left_w), - "=".repeat(right_w), - left_w = left_w, - right_w = right_w - ); - - for (left, right) in rows { - println!( - "{:right_w$}", - left, - right, - left_w = left_w, - right_w = right_w - ); - } -} - -#[allow(dead_code)] // this is actually used from `print_transaction_logs` ./transaction_logs.rs -pub(crate) fn render_logs(logs: &[String], indent: &str) -> String { - logs.iter() - .map(|line| { - let prefix = - if line.contains("Program") && line.contains("invoke [") { - format!("\n{indent}") - } else { - format!("{indent}{indent}• ") - }; - format!("{prefix}{line}") - }) - .collect::>() - .join("\n") -} - -pub fn open_ledger(ledger_path: &Path) -> Ledger { - Ledger::open(ledger_path).expect("Failed to open ledger") -}