From c0cdf5f17128926e36d721b5b69a6b35d5311eb4 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 27 May 2026 16:15:20 +0700 Subject: [PATCH 01/97] refactor: remove unused code --- magicblock-account-cloner/Cargo.toml | 10 --- .../src/account_cloner.rs | 68 ------------------- magicblock-account-cloner/src/lib.rs | 2 - magicblock-account-cloner/src/util.rs | 18 ----- 4 files changed, 98 deletions(-) delete mode 100644 magicblock-account-cloner/src/account_cloner.rs diff --git a/magicblock-account-cloner/Cargo.toml b/magicblock-account-cloner/Cargo.toml index 84e820f99..2c85fdf94 100644 --- a/magicblock-account-cloner/Cargo.toml +++ b/magicblock-account-cloner/Cargo.toml @@ -12,20 +12,15 @@ async-trait = { workspace = true } bincode = { workspace = true } lru = { workspace = true } tracing = { workspace = true } -magicblock-accounts-db = { workspace = true } magicblock-chainlink = { workspace = true } -magicblock-committor-service = { workspace = true } -magicblock-config = { workspace = true } magicblock-core = { workspace = true } magicblock-ledger = { workspace = true } magicblock-magic-program-api = { workspace = true } magicblock-program = { workspace = true } -magicblock-rpc-client = { workspace = true } rand = { workspace = true } solana-account = { workspace = true } solana-hash = { workspace = true } solana-instruction = { workspace = true } -solana-loader-v3-interface = { workspace = true } solana-loader-v4-interface = { workspace = true } solana-pubkey = { workspace = true } solana-sdk-ids = { workspace = true } @@ -35,8 +30,3 @@ solana-sysvar = { workspace = true } solana-transaction = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } - -[dev-dependencies] -magicblock-committor-service = { workspace = true, features = [ - "dev-context-only-utils", -] } diff --git a/magicblock-account-cloner/src/account_cloner.rs b/magicblock-account-cloner/src/account_cloner.rs deleted file mode 100644 index 67437653a..000000000 --- a/magicblock-account-cloner/src/account_cloner.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::sync::Arc; - -use magicblock_committor_service::{ - error::{CommittorServiceError, CommittorServiceResult}, - BaseIntentCommittor, -}; -use thiserror::Error; -use tokio::sync::oneshot; - -pub type AccountClonerResult = Result; - -#[derive(Debug, Clone, Error)] -pub enum AccountClonerError { - #[error(transparent)] - RecvError(#[from] tokio::sync::oneshot::error::RecvError), - - #[error("JoinError ({0})")] - JoinError(String), - - #[error("CommittorServiceError {0}")] - CommittorServiceError(String), -} - -pub async fn map_committor_request_result( - res: oneshot::Receiver>, - intent_committor: Arc, -) -> AccountClonerResult { - match res.await.map_err(|err| { - // Send request error - AccountClonerError::CommittorServiceError(format!( - "error sending request {err:?}" - )) - })? { - Ok(val) => Ok(val), - Err(err) => { - // Commit error - match err { - CommittorServiceError::TableManiaError(table_mania_err) => { - let Some(sig) = table_mania_err.signature() else { - return Err(AccountClonerError::CommittorServiceError( - format!("{:?}", table_mania_err), - )); - }; - let (logs, cus) = crate::util::get_tx_diagnostics( - &sig, - &intent_committor, - ) - .await; - - let cus_str = cus - .map(|cus| format!("{:?}", cus)) - .unwrap_or("N/A".to_string()); - let logs_str = logs - .map(|logs| format!("{:#?}", logs)) - .unwrap_or("N/A".to_string()); - Err(AccountClonerError::CommittorServiceError(format!( - "{:?}\nCUs: {cus_str}\nLogs: {logs_str}", - table_mania_err - ))) - } - _ => Err(AccountClonerError::CommittorServiceError(format!( - "{:?}", - err - ))), - } - } - } -} diff --git a/magicblock-account-cloner/src/lib.rs b/magicblock-account-cloner/src/lib.rs index e39570e08..bb40db757 100644 --- a/magicblock-account-cloner/src/lib.rs +++ b/magicblock-account-cloner/src/lib.rs @@ -72,10 +72,8 @@ use tracing::*; /// Max data that fits in a single transaction (~63KB) pub const MAX_INLINE_DATA_SIZE: usize = 63 * 1024; -mod account_cloner; mod util; -pub use account_cloner::*; pub use util::derive_buffer_pubkey; /// Keep only a bounded history of sent action tx signatures to avoid diff --git a/magicblock-account-cloner/src/util.rs b/magicblock-account-cloner/src/util.rs index 992508faa..fcfe876f6 100644 --- a/magicblock-account-cloner/src/util.rs +++ b/magicblock-account-cloner/src/util.rs @@ -1,10 +1,5 @@ -use std::sync::Arc; - -use magicblock_committor_service::BaseIntentCommittor; use magicblock_program::validator::validator_authority_id; -use magicblock_rpc_client::MagicblockRpcClient; use solana_pubkey::Pubkey; -use solana_signature::Signature; /// Seed for deriving buffer account PDA const BUFFER_SEED: &[u8] = b"buffer"; @@ -15,16 +10,3 @@ pub fn derive_buffer_pubkey(program_pubkey: &Pubkey) -> (Pubkey, u8) { let seeds: &[&[u8]] = &[BUFFER_SEED, program_pubkey.as_ref()]; Pubkey::find_program_address(seeds, &validator_authority_id()) } - -pub(crate) async fn get_tx_diagnostics( - sig: &Signature, - committor: &Arc, -) -> (Option>, Option) { - if let Ok(Ok(transaction)) = committor.get_transaction(sig).await { - let cus = MagicblockRpcClient::get_cus_from_transaction(&transaction); - let logs = MagicblockRpcClient::get_logs_from_transaction(&transaction); - (logs, cus) - } else { - (None, None) - } -} From 07d834f32e68c0bf2faacfd811bc935785c04a50 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Thu, 28 May 2026 19:54:16 +0700 Subject: [PATCH 02/97] wip --- Cargo.lock | 8 +- magicblock-accounts/src/errors.rs | 10 +- .../src/scheduled_commits_processor.rs | 724 +++++------ magicblock-api/src/magic_validator.rs | 47 +- magicblock-api/src/tickers.rs | 63 - magicblock-committor-service/Cargo.toml | 6 + .../src/committor_processor.rs | 138 +- magicblock-committor-service/src/error.rs | 70 +- magicblock-committor-service/src/lib.rs | 3 +- magicblock-committor-service/src/service.rs | 1140 ++++++++--------- .../src/service_ext.rs | 506 ++++---- .../src/service_final.rs | 554 ++++++++ .../tests/replay_base_slot.rs | 173 +++ test-integration/Cargo.lock | 8 +- 14 files changed, 2079 insertions(+), 1371 deletions(-) create mode 100644 magicblock-committor-service/src/service_final.rs create mode 100644 magicblock-processor/tests/replay_base_slot.rs diff --git a/Cargo.lock b/Cargo.lock index 28147b08f..b28916536 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3664,20 +3664,15 @@ dependencies = [ "async-trait", "bincode", "lru 0.16.4", - "magicblock-accounts-db", "magicblock-chainlink", - "magicblock-committor-service", - "magicblock-config", "magicblock-core", "magicblock-ledger", "magicblock-magic-program-api", "magicblock-program", - "magicblock-rpc-client", "rand 0.9.4", "solana-account 3.4.0", "solana-hash 3.1.0", "solana-instruction 3.4.0", - "solana-loader-v3-interface 6.1.1", "solana-loader-v4-interface 3.1.0", "solana-pubkey 3.0.0", "solana-sdk-ids 3.1.0", @@ -3948,6 +3943,9 @@ dependencies = [ "futures-util", "lazy_static", "lru 0.16.4", + "magicblock-account-cloner", + "magicblock-accounts-db", + "magicblock-chainlink", "magicblock-committor-program", "magicblock-core", "magicblock-delegation-program-api", diff --git a/magicblock-accounts/src/errors.rs b/magicblock-accounts/src/errors.rs index 32823039b..3a206dffe 100644 --- a/magicblock-accounts/src/errors.rs +++ b/magicblock-accounts/src/errors.rs @@ -1,9 +1,7 @@ use std::collections::HashSet; -use magicblock_account_cloner::AccountClonerError; use magicblock_committor_service::{ - error::CommittorServiceError, service_ext::CommittorServiceExtError, - ChangesetMeta, + error::CommittorServiceError, ChangesetMeta, }; use solana_pubkey::Pubkey; use solana_transaction_error::TransactionError; @@ -23,15 +21,9 @@ pub enum AccountsError { #[error("CommittorSerivceError: {0}")] CommittorSerivceError(#[from] CommittorServiceError), - #[error("CommittorServiceExtError: {0}")] - CommittorServiceExtError(#[from] CommittorServiceExtError), - #[error("TokioOneshotRecvError")] TokioOneshotRecvError(#[from] Box), - #[error("AccountClonerError")] - AccountClonerError(#[from] AccountClonerError), - #[error("InvalidRpcUrl '{0}'")] InvalidRpcUrl(String), diff --git a/magicblock-accounts/src/scheduled_commits_processor.rs b/magicblock-accounts/src/scheduled_commits_processor.rs index 820bf1fab..1703924d5 100644 --- a/magicblock-accounts/src/scheduled_commits_processor.rs +++ b/magicblock-accounts/src/scheduled_commits_processor.rs @@ -1,362 +1,362 @@ -use std::{ - collections::{HashMap, HashSet}, - sync::{Arc, Mutex}, -}; - -use async_trait::async_trait; -use magicblock_account_cloner::ChainlinkCloner; -use magicblock_accounts_db::AccountsDb; -use magicblock_chainlink::{ - remote_account_provider::{ - chain_rpc_client::ChainRpcClientImpl, - chain_updates_client::ChainUpdatesClient, - }, - submux::SubMuxClient, - Chainlink, -}; -use magicblock_committor_service::{ - intent_execution_manager::BroadcastedIntentExecutionResult, - intent_executor::ExecutionOutput, BaseIntentCommittor, CommittorService, -}; -use magicblock_core::link::transactions::{ - with_encoded, TransactionSchedulerHandle, -}; -use magicblock_metrics::metrics; -use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, - register_scheduled_commit_sent, SentCommit, TransactionScheduler, -}; -use solana_hash::Hash; -use solana_pubkey::Pubkey; -use solana_transaction::Transaction; -use tokio::{ - sync::{broadcast, oneshot}, - task, -}; -use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, instrument}; - -use crate::{ - errors::ScheduledCommitsProcessorResult, ScheduledCommitsProcessor, -}; - -const POISONED_MUTEX_MSG: &str = - "Mutex of RemoteScheduledCommitsProcessor.intents_meta_map is poisoned"; - -pub type ChainlinkImpl = Chainlink< - ChainRpcClientImpl, - SubMuxClient, - AccountsDb, - ChainlinkCloner, ->; - -pub struct ScheduledCommitsProcessorImpl { - committor: Arc, - chainlink: Arc, - cancellation_token: CancellationToken, - intents_meta_map: Arc>>, - transaction_scheduler: TransactionScheduler, -} - -impl ScheduledCommitsProcessorImpl { - pub fn new( - committor: Arc, - chainlink: Arc, - internal_transaction_scheduler: TransactionSchedulerHandle, - ) -> Self { - let result_subscriber = committor.subscribe_for_results(); - let intents_meta_map = Arc::new(Mutex::default()); - let cancellation_token = CancellationToken::new(); - tokio::spawn(Self::result_processor( - result_subscriber, - cancellation_token.clone(), - intents_meta_map.clone(), - internal_transaction_scheduler.clone(), - )); - - Self { - committor, - chainlink, - cancellation_token, - intents_meta_map, - transaction_scheduler: TransactionScheduler::default(), - } - } - - async fn process_undelegation_requests(&self, pubkeys: Vec) { - let mut join_set = task::JoinSet::new(); - for pubkey in pubkeys.into_iter() { - let chainlink = self.chainlink.clone(); - join_set.spawn(async move { - (pubkey, chainlink.undelegation_requested(pubkey).await) - }); - } - let sub_errors = join_set - .join_all() - .await - .into_iter() - .filter_map(|(pubkey, inner_result)| { - if let Err(err) = inner_result { - Some(format!( - "Subscribing to account {} failed: {}", - pubkey, err - )) - } else { - None - } - }) - .collect::>(); - if !sub_errors.is_empty() { - // Instead of aborting the entire commit we log an error here, however - // this means that the undelegated accounts stay in a problematic state - // in the validator and are not synced from chain. - // We could implement a retry mechanism inside of chainlink in the future. - error!( - error_count = sub_errors.len(), - "Failed to subscribe to accounts being undelegated" - ); - } - } - - #[instrument(skip( - result_subscriber, - cancellation_token, - intents_meta_map, - internal_transaction_scheduler - ))] - async fn result_processor( - result_subscriber: oneshot::Receiver< - broadcast::Receiver, - >, - cancellation_token: CancellationToken, - intents_meta_map: Arc>>, - internal_transaction_scheduler: TransactionSchedulerHandle, - ) { - const SUBSCRIPTION_ERR_MSG: &str = - "Failed to get subscription of results of BaseIntents execution"; - - let mut result_receiver = - result_subscriber.await.expect(SUBSCRIPTION_ERR_MSG); - loop { - let execution_result = tokio::select! { - biased; - _ = cancellation_token.cancelled() => { - info!("Shutting down"); - return; - } - execution_result = result_receiver.recv() => { - match execution_result { - Ok(result) => result, - Err(broadcast::error::RecvError::Closed) => { - info!("Intent execution service shut down"); - break; - } - Err(broadcast::error::RecvError::Lagged(skipped)) => { - // SAFETY: This shouldn't happen as our tx execution is faster than Intent execution on Base layer - // If this ever happens it requires investigation - error!(skipped_count = skipped, "Lagging behind intent execution"); - continue; - } - } - } - }; - - let intent_id = execution_result.id; - // Remove intent from metas - let intent_meta = if let Some(intent_meta) = intents_meta_map - .lock() - .expect(POISONED_MUTEX_MSG) - .remove(&intent_id) - { - intent_meta - } else { - // Possible if we have duplicate Intents - // First one will remove id from map and second could fail. - // This should not happen and needs investigation! - error!(intent_id, "Failed to find intent metadata"); - continue; - }; - - Self::process_intent_result( - intent_id, - &internal_transaction_scheduler, - execution_result, - intent_meta, - ) - .await; - } - } - - #[instrument( - skip(internal_transaction_scheduler, result, intent_meta), - fields(intent_id) - )] - async fn process_intent_result( - intent_id: u64, - internal_transaction_scheduler: &TransactionSchedulerHandle, - result: BroadcastedIntentExecutionResult, - mut intent_meta: ScheduledBaseIntentMeta, - ) { - let intent_sent_transaction = - std::mem::take(&mut intent_meta.intent_sent_transaction); - let sent_commit = - Self::build_sent_commit(intent_id, intent_meta, result); - register_scheduled_commit_sent(sent_commit); - let Ok(txn) = with_encoded(intent_sent_transaction) else { - // Unreachable case, all intent transactions are smaller than 64KB by construction - error!("Failed to bincode intent transaction"); - return; - }; - match internal_transaction_scheduler.execute(txn).await { - Ok(()) => { - debug!("Sent commit signaled") - } - Err(err) => { - error!(error = ?err, "Failed to signal sent commit"); - } - } - } - - fn build_sent_commit( - intent_id: u64, - intent_meta: ScheduledBaseIntentMeta, - result: BroadcastedIntentExecutionResult, - ) -> SentCommit { - let error_message = - result.as_ref().err().map(|err| format!("{:?}", err)); - let chain_signatures = match result.inner { - Ok(value) => match value { - ExecutionOutput::SingleStage(signature) => vec![signature], - ExecutionOutput::TwoStage { - commit_signature, - finalize_signature, - } => vec![commit_signature, finalize_signature], - }, - Err(err) => { - error!( - "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", - intent_id, intent_meta.slot, intent_meta.blockhash, err - ); - err.signatures() - .map(|(commit, finalize)| { - finalize - .map(|finalize| vec![commit, finalize]) - .unwrap_or(vec![commit]) - }) - .unwrap_or_default() - } - }; - let patched_errors = result - .patched_errors - .iter() - .map(|err| { - info!("Patched intent: {}. error was: {}", intent_id, err); - err.to_string() - }) - .collect(); - - let callbacks_report = result - .callbacks_report - .iter() - .map(|r| match r { - Ok(sig) => { - format!("OK: {sig}") - } - Err(err) => { - error!( - "Callback failed to schedule: {}. error: {}", - intent_id, err - ); - format!("ERR: {err}") - } - }) - .collect(); - - SentCommit { - message_id: intent_id, - slot: intent_meta.slot, - blockhash: intent_meta.blockhash, - payer: intent_meta.payer, - chain_signatures, - included_pubkeys: intent_meta.included_pubkeys, - excluded_pubkeys: vec![], - requested_undelegation: intent_meta.requested_undelegation, - error_message, - patched_errors, - callbacks_scheduling_results: callbacks_report, - } - } -} - -#[async_trait] -impl ScheduledCommitsProcessor for ScheduledCommitsProcessorImpl { - #[instrument(skip(self))] - async fn process(&self) -> ScheduledCommitsProcessorResult<()> { - let intent_bundles = - self.transaction_scheduler.take_scheduled_intent_bundles(); - - if intent_bundles.is_empty() { - return Ok(()); - } - metrics::inc_committor_intents_count_by(intent_bundles.len() as u64); - - // Add metas for intent we schedule - let pubkeys_being_undelegated = { - let mut intent_metas = - self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); - let mut pubkeys_being_undelegated = HashSet::::new(); - - intent_bundles.iter().for_each(|intent| { - intent_metas - .insert(intent.id, ScheduledBaseIntentMeta::new(intent)); - if let Some(undelegate) = intent.get_undelegate_intent_pubkeys() - { - pubkeys_being_undelegated.extend(undelegate); - } - }); - - pubkeys_being_undelegated.into_iter().collect::>() - }; - - self.process_undelegation_requests(pubkeys_being_undelegated) - .await; - self.committor - .schedule_intent_bundles(intent_bundles) - .await??; - Ok(()) - } - - fn scheduled_commits_len(&self) -> usize { - self.transaction_scheduler.scheduled_actions_len() - } - - fn clear_scheduled_commits(&self) { - self.transaction_scheduler.clear_scheduled_actions(); - } - - fn stop(&self) { - self.cancellation_token.cancel(); - } -} - -struct ScheduledBaseIntentMeta { - slot: u64, - blockhash: Hash, - payer: Pubkey, - included_pubkeys: Vec, - intent_sent_transaction: Transaction, - requested_undelegation: bool, -} - -impl ScheduledBaseIntentMeta { - fn new(intent: &ScheduledIntentBundle) -> Self { - Self { - slot: intent.slot, - blockhash: intent.blockhash, - payer: intent.payer, - included_pubkeys: intent.get_all_committed_pubkeys(), - intent_sent_transaction: intent.sent_transaction.clone(), - requested_undelegation: intent.has_undelegate_intent(), - } - } -} +// use std::{ +// collections::{HashMap, HashSet}, +// sync::{Arc, Mutex}, +// }; +// +// use async_trait::async_trait; +// use magicblock_account_cloner::ChainlinkCloner; +// use magicblock_accounts_db::AccountsDb; +// use magicblock_chainlink::{ +// remote_account_provider::{ +// chain_rpc_client::ChainRpcClientImpl, +// chain_updates_client::ChainUpdatesClient, +// }, +// submux::SubMuxClient, +// Chainlink, +// }; +// use magicblock_committor_service::{ +// intent_execution_manager::BroadcastedIntentExecutionResult, +// intent_executor::ExecutionOutput, BaseIntentCommittor, CommittorService, +// }; +// use magicblock_core::link::transactions::{ +// with_encoded, TransactionSchedulerHandle, +// }; +// use magicblock_metrics::metrics; +// use magicblock_program::{ +// magic_scheduled_base_intent::ScheduledIntentBundle, +// register_scheduled_commit_sent, SentCommit, TransactionScheduler, +// }; +// use solana_hash::Hash; +// use solana_pubkey::Pubkey; +// use solana_transaction::Transaction; +// use tokio::{ +// sync::{broadcast, oneshot}, +// task, +// }; +// use tokio_util::sync::CancellationToken; +// use tracing::{debug, error, info, instrument}; +// +// use crate::{ +// errors::ScheduledCommitsProcessorResult, ScheduledCommitsProcessor, +// }; +// +// const POISONED_MUTEX_MSG: &str = +// "Mutex of RemoteScheduledCommitsProcessor.intents_meta_map is poisoned"; +// +// pub type ChainlinkImpl = Chainlink< +// ChainRpcClientImpl, +// SubMuxClient, +// AccountsDb, +// ChainlinkCloner, +// >; +// +// pub struct ScheduledCommitsProcessorImpl { +// committor: Arc, +// chainlink: Arc, +// cancellation_token: CancellationToken, +// intents_meta_map: Arc>>, +// transaction_scheduler: TransactionScheduler, +// } +// +// impl ScheduledCommitsProcessorImpl { +// pub fn new( +// committor: Arc, +// chainlink: Arc, +// internal_transaction_scheduler: TransactionSchedulerHandle, +// ) -> Self { +// let result_subscriber = committor.subscribe_for_results(); +// let intents_meta_map = Arc::new(Mutex::default()); +// let cancellation_token = CancellationToken::new(); +// tokio::spawn(Self::result_processor( +// result_subscriber, +// cancellation_token.clone(), +// intents_meta_map.clone(), +// internal_transaction_scheduler.clone(), +// )); +// +// Self { +// committor, +// chainlink, +// cancellation_token, +// intents_meta_map, +// transaction_scheduler: TransactionScheduler::default(), +// } +// } +// +// async fn process_undelegation_requests(&self, pubkeys: Vec) { +// let mut join_set = task::JoinSet::new(); +// for pubkey in pubkeys.into_iter() { +// let chainlink = self.chainlink.clone(); +// join_set.spawn(async move { +// (pubkey, chainlink.undelegation_requested(pubkey).await) +// }); +// } +// let sub_errors = join_set +// .join_all() +// .await +// .into_iter() +// .filter_map(|(pubkey, inner_result)| { +// if let Err(err) = inner_result { +// Some(format!( +// "Subscribing to account {} failed: {}", +// pubkey, err +// )) +// } else { +// None +// } +// }) +// .collect::>(); +// if !sub_errors.is_empty() { +// // Instead of aborting the entire commit we log an error here, however +// // this means that the undelegated accounts stay in a problematic state +// // in the validator and are not synced from chain. +// // We could implement a retry mechanism inside of chainlink in the future. +// error!( +// error_count = sub_errors.len(), +// "Failed to subscribe to accounts being undelegated" +// ); +// } +// } +// +// #[instrument(skip( +// result_subscriber, +// cancellation_token, +// intents_meta_map, +// internal_transaction_scheduler +// ))] +// async fn result_processor( +// result_subscriber: oneshot::Receiver< +// broadcast::Receiver, +// >, +// cancellation_token: CancellationToken, +// intents_meta_map: Arc>>, +// internal_transaction_scheduler: TransactionSchedulerHandle, +// ) { +// const SUBSCRIPTION_ERR_MSG: &str = +// "Failed to get subscription of results of BaseIntents execution"; +// +// let mut result_receiver = +// result_subscriber.await.expect(SUBSCRIPTION_ERR_MSG); +// loop { +// let execution_result = tokio::select! { +// biased; +// _ = cancellation_token.cancelled() => { +// info!("Shutting down"); +// return; +// } +// execution_result = result_receiver.recv() => { +// match execution_result { +// Ok(result) => result, +// Err(broadcast::error::RecvError::Closed) => { +// info!("Intent execution service shut down"); +// break; +// } +// Err(broadcast::error::RecvError::Lagged(skipped)) => { +// // SAFETY: This shouldn't happen as our tx execution is faster than Intent execution on Base layer +// // If this ever happens it requires investigation +// error!(skipped_count = skipped, "Lagging behind intent execution"); +// continue; +// } +// } +// } +// }; +// +// let intent_id = execution_result.id; +// // Remove intent from metas +// let intent_meta = if let Some(intent_meta) = intents_meta_map +// .lock() +// .expect(POISONED_MUTEX_MSG) +// .remove(&intent_id) +// { +// intent_meta +// } else { +// // Possible if we have duplicate Intents +// // First one will remove id from map and second could fail. +// // This should not happen and needs investigation! +// error!(intent_id, "Failed to find intent metadata"); +// continue; +// }; +// +// Self::process_intent_result( +// intent_id, +// &internal_transaction_scheduler, +// execution_result, +// intent_meta, +// ) +// .await; +// } +// } +// +// #[instrument( +// skip(internal_transaction_scheduler, result, intent_meta), +// fields(intent_id) +// )] +// async fn process_intent_result( +// intent_id: u64, +// internal_transaction_scheduler: &TransactionSchedulerHandle, +// result: BroadcastedIntentExecutionResult, +// mut intent_meta: ScheduledBaseIntentMeta, +// ) { +// let intent_sent_transaction = +// std::mem::take(&mut intent_meta.intent_sent_transaction); +// let sent_commit = +// Self::build_sent_commit(intent_id, intent_meta, result); +// register_scheduled_commit_sent(sent_commit); +// let Ok(txn) = with_encoded(intent_sent_transaction) else { +// // Unreachable case, all intent transactions are smaller than 64KB by construction +// error!("Failed to bincode intent transaction"); +// return; +// }; +// match internal_transaction_scheduler.execute(txn).await { +// Ok(()) => { +// debug!("Sent commit signaled") +// } +// Err(err) => { +// error!(error = ?err, "Failed to signal sent commit"); +// } +// } +// } +// +// fn build_sent_commit( +// intent_id: u64, +// intent_meta: ScheduledBaseIntentMeta, +// result: BroadcastedIntentExecutionResult, +// ) -> SentCommit { +// let error_message = +// result.as_ref().err().map(|err| format!("{:?}", err)); +// let chain_signatures = match result.inner { +// Ok(value) => match value { +// ExecutionOutput::SingleStage(signature) => vec![signature], +// ExecutionOutput::TwoStage { +// commit_signature, +// finalize_signature, +// } => vec![commit_signature, finalize_signature], +// }, +// Err(err) => { +// error!( +// "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", +// intent_id, intent_meta.slot, intent_meta.blockhash, err +// ); +// err.signatures() +// .map(|(commit, finalize)| { +// finalize +// .map(|finalize| vec![commit, finalize]) +// .unwrap_or(vec![commit]) +// }) +// .unwrap_or_default() +// } +// }; +// let patched_errors = result +// .patched_errors +// .iter() +// .map(|err| { +// info!("Patched intent: {}. error was: {}", intent_id, err); +// err.to_string() +// }) +// .collect(); +// +// let callbacks_report = result +// .callbacks_report +// .iter() +// .map(|r| match r { +// Ok(sig) => { +// format!("OK: {sig}") +// } +// Err(err) => { +// error!( +// "Callback failed to schedule: {}. error: {}", +// intent_id, err +// ); +// format!("ERR: {err}") +// } +// }) +// .collect(); +// +// SentCommit { +// message_id: intent_id, +// slot: intent_meta.slot, +// blockhash: intent_meta.blockhash, +// payer: intent_meta.payer, +// chain_signatures, +// included_pubkeys: intent_meta.included_pubkeys, +// excluded_pubkeys: vec![], +// requested_undelegation: intent_meta.requested_undelegation, +// error_message, +// patched_errors, +// callbacks_scheduling_results: callbacks_report, +// } +// } +// } +// +// #[async_trait] +// impl ScheduledCommitsProcessor for ScheduledCommitsProcessorImpl { +// #[instrument(skip(self))] +// async fn process(&self) -> ScheduledCommitsProcessorResult<()> { +// let intent_bundles = +// self.transaction_scheduler.take_scheduled_intent_bundles(); +// +// if intent_bundles.is_empty() { +// return Ok(()); +// } +// metrics::inc_committor_intents_count_by(intent_bundles.len() as u64); +// +// // Add metas for intent we schedule +// let pubkeys_being_undelegated = { +// let mut intent_metas = +// self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); +// let mut pubkeys_being_undelegated = HashSet::::new(); +// +// intent_bundles.iter().for_each(|intent| { +// intent_metas +// .insert(intent.id, ScheduledBaseIntentMeta::new(intent)); +// if let Some(undelegate) = intent.get_undelegate_intent_pubkeys() +// { +// pubkeys_being_undelegated.extend(undelegate); +// } +// }); +// +// pubkeys_being_undelegated.into_iter().collect::>() +// }; +// +// self.process_undelegation_requests(pubkeys_being_undelegated) +// .await; +// self.committor +// .schedule_intent_bundles(intent_bundles) +// .await??; +// Ok(()) +// } +// +// fn scheduled_commits_len(&self) -> usize { +// self.transaction_scheduler.scheduled_actions_len() +// } +// +// fn clear_scheduled_commits(&self) { +// self.transaction_scheduler.clear_scheduled_actions(); +// } +// +// fn stop(&self) { +// self.cancellation_token.cancel(); +// } +// } +// +// struct ScheduledBaseIntentMeta { +// slot: u64, +// blockhash: Hash, +// payer: Pubkey, +// included_pubkeys: Vec, +// intent_sent_transaction: Transaction, +// requested_undelegation: bool, +// } +// +// impl ScheduledBaseIntentMeta { +// fn new(intent: &ScheduledIntentBundle) -> Self { +// Self { +// slot: intent.slot, +// blockhash: intent.blockhash, +// payer: intent.payer, +// included_pubkeys: intent.get_all_committed_pubkeys(), +// intent_sent_transaction: intent.sent_transaction.clone(), +// requested_undelegation: intent.has_undelegate_intent(), +// } +// } +// } diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 67723a5b2..e04a5eef9 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -28,8 +28,10 @@ use magicblock_chainlink::{ Chainlink, }; use magicblock_committor_service::{ - config::ChainConfig, BaseIntentCommittor, CommittorService, - ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, + config::ChainConfig, + service_final::{IntentExecutionService, InternalIntentRpcClient}, + BaseIntentCommittor, CommittorService, ComputeBudgetConfig, + DEFAULT_ACTIONS_TIMEOUT, }; use magicblock_config::{ config::{ @@ -82,6 +84,7 @@ use tokio::{ runtime::Builder, sync::mpsc::{channel, Sender}, }; +use tokio_util::either::Either; use tokio_util::sync::CancellationToken; use tracing::*; @@ -97,7 +100,7 @@ use crate::{ write_validator_keypair_to_ledger, }, magic_sys_adapter::MagicSysAdapter, - tickers::{init_slot_ticker, init_system_metrics_ticker}, + tickers::init_system_metrics_ticker, }; type ChainlinkImpl = Chainlink< @@ -107,6 +110,11 @@ type ChainlinkImpl = Chainlink< ChainlinkCloner, >; +type IntentExecutionServiceImpl = + IntentExecutionService>; + +type IntentExecutionState = Either>; + // ----------------- // MagicValidator // ----------------- @@ -117,10 +125,8 @@ pub struct MagicValidator { accountsdb: Arc, ledger: Arc, ledger_truncator: LedgerTruncator, - slot_ticker: Option>, - committor_service: Option>, + intent_execution_service: IntentExecutionState, replication_service: Option, - scheduled_commits_processor: Option>, chainlink: Arc, rpc_handle: thread::JoinHandle<()>, identity: Pubkey, @@ -299,7 +305,7 @@ impl MagicValidator { let scheduled_commits_processor = committor_service.as_ref().map(|committor_service| { - Arc::new(ScheduledCommitsProcessorImpl::new( + Arc::new(Intent::new( committor_service.clone(), chainlink.clone(), dispatch.transaction_scheduler.clone(), @@ -426,7 +432,6 @@ impl MagicValidator { exit, _metrics: (metrics_service, system_metrics_ticker), // NOTE: set during [Self::start] - slot_ticker: None, committor_service, replication_service, scheduled_commits_processor, @@ -446,11 +451,22 @@ impl MagicValidator { }) } - #[instrument(skip(config, latest_block))] + #[instrument(skip( + config, latest_block, accounts_db, transaction_scheduler, chainlink + ))] async fn init_committor_service( config: &ValidatorParams, + accounts_db: &Arc, + chainlink: Arc, + transaction_scheduler: &TransactionSchedulerHandle, latest_block: &LatestBlock, - ) -> ApiResult>> { + ) -> ApiResult { + let intent_client = InternalIntentRpcClient::new( + accounts_db.clone(), + transaction_scheduler.clone(), + latest_block.clone() + ); + let committor_persist_path = config.storage.join("committor_service.sqlite"); debug!(path = %committor_persist_path.display(), "Initializing committor service"); @@ -460,8 +476,10 @@ impl MagicValidator { config.validator.keypair.insecure_clone(), latest_block.clone(), ); - let committor_service = Some(Arc::new(CommittorService::try_start( + let committor_service = IntentExecutionServiceImpl::try_new( config.validator.keypair.insecure_clone(), + chainlink, + intent_client, committor_persist_path, ChainConfig { rpc_uri: config.rpc_url().to_owned(), @@ -472,7 +490,7 @@ impl MagicValidator { actions_timeout: DEFAULT_ACTIONS_TIMEOUT, }, actions_callback_executor, - )?)); + )?; Ok(committor_service) } @@ -1014,11 +1032,6 @@ impl MagicValidator { let step_start = Instant::now(); let _ = self.rpc_handle.join(); log_timing("shutdown", "rpc_thread_join", step_start); - if let Some(handle) = self.slot_ticker { - let step_start = Instant::now(); - let _ = handle.await; - log_timing("shutdown", "slot_ticker_join", step_start); - } let step_start = Instant::now(); if let Err(err) = self.ledger_truncator.join() { error!(error = ?err, "Ledger truncator did not gracefully exit"); diff --git a/magicblock-api/src/tickers.rs b/magicblock-api/src/tickers.rs index 30e7e8e9a..72116af55 100644 --- a/magicblock-api/src/tickers.rs +++ b/magicblock-api/src/tickers.rs @@ -19,69 +19,6 @@ use solana_account::ReadableAccount; use tokio_util::sync::CancellationToken; use tracing::*; -pub fn init_slot_ticker( - accountsdb: Arc, - committor_processor: &Option>, - latest_block: LatestBlock, - tick_duration: Duration, - transaction_scheduler: TransactionSchedulerHandle, - exit: Arc, -) -> tokio::task::JoinHandle<()> { - let committor_processor = committor_processor.clone(); - - tokio::task::spawn(async move { - while !exit.load(Ordering::Relaxed) { - tokio::time::sleep(tick_duration).await; - - // Handle intents if such feature enabled - let Some(committor_processor) = &committor_processor else { - continue; - }; - - // If accounts were scheduled to be committed, we accept them here - // and processs the commits - let magic_context_acc = accountsdb.get_account(&magic_program::MAGIC_CONTEXT_PUBKEY) - .expect("Validator found to be running without MagicContext account!"); - if MagicContext::has_scheduled_commits(magic_context_acc.data()) { - handle_scheduled_commits( - committor_processor, - &transaction_scheduler, - &latest_block, - ) - .await; - } - } - }) -} - -#[instrument(skip(committor_processor, transaction_scheduler, latest_block))] -async fn handle_scheduled_commits( - committor_processor: &Arc, - transaction_scheduler: &TransactionSchedulerHandle, - latest_block: &LatestBlock, -) { - // 1. Send the transaction to move the scheduled commits from the MagicContext - // to the global ScheduledCommit store - let tx = InstructionUtils::accept_scheduled_commits( - latest_block.load().blockhash, - ); - let Ok(tx) = with_encoded(tx) else { - // Unreachable case, all schedule commit txns are smaller than 64KB by construction - error!("Failed to bincode intent transaction"); - return; - }; - if let Err(err) = transaction_scheduler.execute(tx).await { - error!(error = ?err, "Failed to accept scheduled commits"); - return; - } - - // 2. Process those scheduled commits - // TODO: fix the possible delay here - // https://github.com/magicblock-labs/magicblock-validator/issues/104 - if let Err(err) = committor_processor.process().await { - error!(error = ?err, "Failed to process scheduled commits"); - } -} #[allow(unused_variables)] pub fn init_system_metrics_ticker( tick_duration: Duration, diff --git a/magicblock-committor-service/Cargo.toml b/magicblock-committor-service/Cargo.toml index 11eec877e..2e681a5dd 100644 --- a/magicblock-committor-service/Cargo.toml +++ b/magicblock-committor-service/Cargo.toml @@ -18,6 +18,12 @@ borsh = { workspace = true } futures-util = { workspace = true } tracing = { workspace = true } lru = { workspace = true } +# TDOO: should be removed +magicblock-account-cloner = { workspace = true } +# TODO: should be removed +magicblock-accounts-db = { workspace = true } +# TODO: should be removed, hidden behinf the trait +magicblock-chainlink = { workspace = true } magicblock-committor-program = { workspace = true, features = [ "no-entrypoint", ] } diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 5439037d8..43998939a 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -1,9 +1,10 @@ use std::{ - collections::{HashMap, HashSet}, + collections::{hash_map::Entry, HashMap, HashSet}, path::Path, - sync::Arc, + sync::{Arc, Mutex}, }; +use futures_util::future::join_all; use magicblock_core::traits::ActionsCallbackScheduler; use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use magicblock_rpc_client::MagicblockRpcClient; @@ -12,12 +13,12 @@ use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_signer::Signer; -use tokio::sync::broadcast; -use tracing::{error, instrument}; +use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; +use tracing::{error, info, instrument}; use crate::{ config::ChainConfig, - error::CommittorServiceResult, + error::{CommittorServiceError, CommittorServiceResult}, intent_execution_manager::{ db::DummyDB, BroadcastedIntentExecutionResult, IntentExecutionManager, }, @@ -33,14 +34,19 @@ use crate::{ MessageSignatures, }, }; +// use crate::service_ext::CommittorServiceExtError; + +const POISONED_MUTEX_MSG: &str = + "CommittorProcessor pending messages mutex poisoned!"; +type BundleResultListener = oneshot::Sender; pub(crate) struct CommittorProcessor { - pub(crate) magicblock_rpc_client: MagicblockRpcClient, pub(crate) table_mania: TableMania, pub(crate) authority: Keypair, persister: IntentPersisterImpl, commits_scheduler: IntentExecutionManager, task_info_fetcher: Arc>, + pending_result_listeners: Arc>>, } impl CommittorProcessor { @@ -91,42 +97,23 @@ impl CommittorProcessor { actions_callback_executor, ); + let result_subscription = commits_scheduler.subscribe_for_results(); + let pending_result_listeners = Arc::new(Mutex::new(HashMap::new())); + tokio::spawn(Self::dispatcher( + result_subscription, + pending_result_listeners.clone(), + )); + Ok(Self { authority, - magicblock_rpc_client: magic_block_rpc_client, table_mania, commits_scheduler, persister, task_info_fetcher, + pending_result_listeners, }) } - pub async fn active_lookup_tables(&self) -> Vec { - self.table_mania.active_table_addresses().await - } - - pub async fn released_lookup_tables(&self) -> Vec { - self.table_mania.released_table_addresses().await - } - - pub fn auth_pubkey(&self) -> Pubkey { - self.authority.pubkey() - } - - pub(crate) async fn reserve_pubkeys( - &self, - pubkeys: HashSet, - ) -> CommittorServiceResult<()> { - Ok(self - .table_mania - .reserve_pubkeys(&self.authority, &pubkeys) - .await?) - } - - pub(crate) async fn release_pubkeys(&self, pubkeys: HashSet) { - self.table_mania.release_pubkeys(&pubkeys).await - } - pub fn get_commit_statuses( &self, message_id: u64, @@ -148,7 +135,7 @@ impl CommittorProcessor { } #[instrument(skip(self, intent_bundles))] - pub async fn schedule_intent_bundle( + pub async fn schedule_intent_bundles( &self, intent_bundles: Vec, ) -> CommittorServiceResult<()> { @@ -169,6 +156,45 @@ impl CommittorProcessor { Ok(()) } + pub async fn execute_intent_bundles( + &self, + intent_bundles: Vec, + ) -> CommittorServiceResult> { + // Critical section + let receivers = { + let mut result_listeners = self + .pending_result_listeners + .lock() + .expect(POISONED_MUTEX_MSG); + + intent_bundles + .iter() + .map(|intent| { + let (sender, receiver) = oneshot::channel(); + match result_listeners.entry(intent.id) { + Entry::Vacant(vacant) => { + vacant.insert(sender); + Ok(receiver) + } + Entry::Occupied(_) => { + Err(CommittorServiceError::RepeatingMessageError( + intent.id, + )) + } + } + }) + .collect::, _>>()? + }; + + self.schedule_intent_bundles(intent_bundles).await?; + let results = join_all(receivers.into_iter()) + .await + .into_iter() + .collect::, RecvError>>()?; + + Ok(results) + } + /// Creates a subscription for results of BaseIntent execution pub fn subscribe_for_results( &self, @@ -186,4 +212,48 @@ impl CommittorProcessor { .fetch_current_commit_nonces(pubkeys, min_context_slot) .await } + + /// Dispatch worker + #[instrument(skip(pending_result_listeners, results_subscription))] + async fn dispatcher( + mut results_subscription: broadcast::Receiver< + BroadcastedIntentExecutionResult, + >, + pending_result_listeners: Arc< + Mutex>, + >, + ) { + loop { + let execution_result = match results_subscription.recv().await { + Ok(result) => result, + Err(broadcast::error::RecvError::Closed) => { + info!("Intent execution shutdown"); + break; + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + // SAFETY: not really feasible to happen as this function is way faster than Intent execution + // requires investigation if ever happens! + error!(skipped, "Dispatcher lag detected"); + continue; + } + }; + + let sender = if let Some(sender) = pending_result_listeners + .lock() + .expect(POISONED_MUTEX_MSG) + .remove(&execution_result.id) + { + sender + } else { + continue; + }; + + if let Err(execution_result) = sender.send(execution_result) { + error!( + intent_id = execution_result.id, + "Failed to send execution result" + ); + } + } + } } diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index fa13e49b8..15535500f 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -2,6 +2,7 @@ use solana_pubkey::Pubkey; use solana_signature::Signature; use solana_transaction_error::TransactionError; use thiserror::Error; +use tokio::sync::oneshot::error::RecvError; use crate::{ intent_execution_manager::IntentExecutionManagerError, @@ -12,74 +13,39 @@ pub type CommittorServiceResult = Result; #[derive(Error, Debug)] pub enum CommittorServiceError { - #[error("CommittorError: {0} ({0:?})")] - CommittorError(#[from] magicblock_committor_program::error::CommittorError), - #[error("CommitPersistError: {0} ({0:?})")] CommitPersistError(#[from] crate::persist::error::CommitPersistError), - #[error("MagicBlockRpcClientError: {0} ({0:?})")] - MagicBlockRpcClientError( - #[from] magicblock_rpc_client::MagicBlockRpcClientError, - ), - - #[error("TableManiaError: {0} ({0:?})")] - TableManiaError(#[from] magicblock_table_mania::error::TableManiaError), + // #[error("MagicBlockRpcClientError: {0} ({0:?})")] + // MagicBlockRpcClientError( + // #[from] magicblock_rpc_client::MagicBlockRpcClientError, + // ), + // #[error("TableManiaError: {0} ({0:?})")] + // TableManiaError(#[from] magicblock_table_mania::error::TableManiaError), #[error("IntentExecutionManagerError: {0} ({0:?})")] IntentExecutionManagerError(#[from] IntentExecutionManagerError), - #[error("TaskInfoFetcherError: {0} ({0:?})")] - TaskInfoFetcherError(#[from] TaskInfoFetcherError), - - #[error( - "Failed send and confirm transaction to {0} on chain: {1} ({1:?})" - )] - FailedToSendAndConfirmTransaction( - String, - magicblock_rpc_client::MagicBlockRpcClientError, - ), - - #[error("The transaction to {0} was sent and confirmed, but encountered an error: {1} ({1:?})")] - EncounteredTransactionError(String, TransactionError), - - #[error("Failed to send init changeset account: {0} ({0:?})")] - FailedToSendInitChangesetAccount( - Box, - ), - - #[error("Failed to confirm init changeset account: {0} ({0:?})")] - FailedToConfirmInitChangesetAccount( - Box, - ), - #[error("Init transaction '{0}' was not confirmed")] - InitChangesetAccountNotConfirmed(String), - - #[error("Task {0} failed to compile transaction message: {1} ({1:?})")] - FailedToCompileTransactionMessage(String, solana_message::CompileError), - - #[error("Task {0} failed to create transaction: {1} ({1:?})")] - FailedToCreateTransaction(String, solana_signer::SignerError), + #[error("RecvError: {0}")] + IntentResultRecvError(#[from] RecvError), - #[error("Could not find commit strategy for bundle {0}")] - CouldNotFindCommitStrategyForBundle(u64), + // #[error("TaskInfoFetcherError: {0} ({0:?})")] + // TaskInfoFetcherError(#[from] TaskInfoFetcherError), - #[error("Failed to fetch metadata account for {0}")] - FailedToFetchDelegationMetadata(Pubkey), + // #[error("Task {0} failed to compile transaction message: {1} ({1:?})")] + // FailedToCompileTransactionMessage(String, solana_message::CompileError), - #[error("Failed to deserialize metadata account for {0}, {1:?}")] - FailedToDeserializeDelegationMetadata( - Pubkey, - solana_program::program_error::ProgramError, - ), + // #[error("Task {0} failed to create transaction: {1} ({1:?})")] + // FailedToCreateTransaction(String, solana_signer::SignerError), + #[error("Attempt to schedule already scheduled message id: {0}")] + RepeatingMessageError(u64), } impl CommittorServiceError { pub fn signature(&self) -> Option { use CommittorServiceError::*; match self { - MagicBlockRpcClientError(e) => e.signature(), - FailedToSendAndConfirmTransaction(_, e) => e.signature(), + // MagicBlockRpcClientError(e) => e.signature(), _ => None, } } diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 9836160f4..fbc314558 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -16,6 +16,7 @@ pub mod transaction_preparator; pub mod transactions; pub(crate) mod utils; +pub mod service_final; #[cfg(test)] pub mod test_utils; @@ -24,4 +25,4 @@ pub use config::DEFAULT_ACTIONS_TIMEOUT; pub use magicblock_committor_program::{ ChangedAccount, Changeset, ChangesetMeta, }; -pub use service::{BaseIntentCommittor, CommittorService}; +// pub use service::{BaseIntentCommittor, CommittorService}; diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index bbec9d805..0c702ea29 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,570 +1,570 @@ -use std::{collections::HashMap, path::Path, sync::Arc, time::Instant}; - -use magicblock_core::traits::ActionsCallbackScheduler; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; -use solana_keypair::Keypair; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta; -use tokio::{ - select, - sync::{ - broadcast, - mpsc::{self, error::TrySendError}, - oneshot, - }, -}; -use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; -use tracing::*; - -use crate::{ - committor_processor::CommittorProcessor, - config::ChainConfig, - error::{CommittorServiceError, CommittorServiceResult}, - intent_execution_manager::BroadcastedIntentExecutionResult, - persist::{CommitStatusRow, MessageSignatures}, - pubkeys_provider::{provide_committee_pubkeys, provide_common_pubkeys}, -}; - -#[derive(Debug)] -pub struct LookupTables { - pub active: Vec, - pub released: Vec, -} - -#[derive(Debug)] -pub enum CommittorMessage { - ReservePubkeysForCommittee { - /// When the request was initiated - initiated: Instant, - /// Called once the pubkeys have been reserved and includes that timestamp - /// at which the request was initiated - respond_to: oneshot::Sender>, - /// The committee whose pubkeys to reserve in a lookup table - /// These pubkeys are used to process/finalize the commit - committee: Pubkey, - /// The owner program of the committee - owner: Pubkey, - }, - ReserveCommonPubkeys { - /// Called once the pubkeys have been reserved - respond_to: oneshot::Sender>, - }, - ReleaseCommonPubkeys { - /// Called once the pubkeys have been released - respond_to: oneshot::Sender<()>, - }, - ScheduleIntentBundle { - /// The [`ScheduleIntentBundle`]s to commit - intent_bundles: Vec, - respond_to: oneshot::Sender>, - }, - GetCommitStatuses { - respond_to: - oneshot::Sender>>, - message_id: u64, - }, - GetCommitSignatures { - respond_to: - oneshot::Sender>>, - commit_id: u64, - pubkey: Pubkey, - }, - GetLookupTables { - respond_to: oneshot::Sender, - }, - GetTransaction { - respond_to: oneshot::Sender< - CommittorServiceResult, - >, - signature: Signature, - }, - SubscribeForResults { - respond_to: oneshot::Sender< - broadcast::Receiver, - >, - }, - FetchCurrentCommitNonces { - respond_to: - oneshot::Sender>>, - pubkeys: Vec, - min_context_slot: u64, - }, - FetchCurrentCommitNoncesSync { - respond_to: std::sync::mpsc::Sender< - CommittorServiceResult>, - >, - pubkeys: Vec, - min_context_slot: u64, - }, -} - -// ----------------- -// CommittorActor -// ----------------- -struct CommittorActor { - receiver: mpsc::Receiver, - processor: Arc, -} - -impl CommittorActor { - pub fn try_new( - receiver: mpsc::Receiver, - authority: Keypair, - persist_file: P, - chain_config: ChainConfig, - actions_callback_executor: A, - ) -> CommittorServiceResult - where - P: AsRef, - A: ActionsCallbackScheduler, - { - let processor = Arc::new(CommittorProcessor::try_new( - authority, - persist_file, - chain_config, - actions_callback_executor, - )?); - - Ok(Self { - receiver, - processor, - }) - } - - #[instrument(skip(self))] - async fn handle_msg(&self, msg: CommittorMessage) { - use CommittorMessage::*; - match msg { - ReservePubkeysForCommittee { - initiated, - respond_to, - committee, - owner, - } => { - let processor = self.processor.clone(); - tokio::task::spawn(async move { - let pubkeys = - provide_committee_pubkeys(&committee, Some(&owner)); - // NOTE: we wait here until the reservation is done which causes the - // cloning of a particular account to be blocked. - // This leads to larger delays on the first clone of an account, but also - // ensures that the account could be committed via a lookup table later. - let result = processor - .reserve_pubkeys(pubkeys) - .await - .map(|_| initiated); - if let Err(e) = respond_to.send(result) { - error!(message_type = "ReservePubkeysForCommittee", error = ?e, "Failed to send response"); - } - }); - } - ReserveCommonPubkeys { respond_to } => { - let processor = self.processor.clone(); - tokio::task::spawn(async move { - let pubkeys = - provide_common_pubkeys(&processor.auth_pubkey()); - let reqid = processor.reserve_pubkeys(pubkeys).await; - if let Err(e) = respond_to.send(reqid) { - error!(message_type = "ReserveCommonPubkeys", error = ?e, "Failed to send response"); - } - }); - } - ReleaseCommonPubkeys { respond_to } => { - let processor = self.processor.clone(); - tokio::task::spawn(async move { - let pubkeys = - provide_common_pubkeys(&processor.auth_pubkey()); - processor.release_pubkeys(pubkeys).await; - if let Err(e) = respond_to.send(()) { - error!(message_type = "ReleaseCommonPubkeys", error = ?e, "Failed to send response"); - } - }); - } - ScheduleIntentBundle { - intent_bundles, - respond_to, - } => { - let result = - self.processor.schedule_intent_bundle(intent_bundles).await; - if let Err(e) = respond_to.send(result) { - error!(message_type = "ScheduleBaseIntents", error = ?e, "Failed to send response"); - } - } - GetCommitStatuses { - message_id, - respond_to, - } => { - let commit_statuses = - self.processor.get_commit_statuses(message_id); - if let Err(e) = respond_to.send(commit_statuses) { - error!(message_type = "GetCommitStatuses", error = ?e, "Failed to send response"); - } - } - GetCommitSignatures { - commit_id, - respond_to, - pubkey, - } => { - let sig = - self.processor.get_commit_signature(commit_id, pubkey); - if let Err(e) = respond_to.send(sig) { - error!(message_type = "GetCommitSignatures", error = ?e, "Failed to send response"); - } - } - GetTransaction { - signature, - respond_to, - } => { - let processor = self.processor.clone(); - tokio::task::spawn(async move { - let res = processor - .magicblock_rpc_client - .get_transaction(&signature, None) - .await - .map_err(Into::into); - if let Err(err) = respond_to.send(res) { - error!(message_type = "GetTransaction", error = ?err, "Failed to send response"); - } - }); - } - GetLookupTables { respond_to } => { - let active_tables = self.processor.active_lookup_tables().await; - let released_tables = - self.processor.released_lookup_tables().await; - if let Err(e) = respond_to.send(LookupTables { - active: active_tables, - released: released_tables, - }) { - error!(message_type = "GetLookupTables", error = ?e, "Failed to send response"); - } - } - SubscribeForResults { respond_to } => { - let subscription = self.processor.subscribe_for_results(); - if let Err(err) = respond_to.send(subscription) { - error!(message_type = "SubscribeForResults", error = ?err, "Failed to send response"); - } - } - FetchCurrentCommitNonces { - respond_to, - pubkeys, - min_context_slot, - } => { - let processor = self.processor.clone(); - tokio::spawn(async move { - let result = processor - .fetch_current_commit_nonces(&pubkeys, min_context_slot) - .await; - if let Err(err) = respond_to - .send(result.map_err(CommittorServiceError::from)) - { - error!(message_type = "FetchCurrentCommitNonces", error = ?err, "Failed to send response"); - } - }); - } - FetchCurrentCommitNoncesSync { - respond_to, - pubkeys, - min_context_slot, - } => { - let processor = self.processor.clone(); - tokio::spawn(async move { - let result = processor - .fetch_current_commit_nonces(&pubkeys, min_context_slot) - .await; - if let Err(err) = respond_to - .send(result.map_err(CommittorServiceError::from)) - { - error!(message_type = "FetchCurrentCommitNoncesSync", error = ?err, "Failed to send response"); - } - }); - } - } - } - - #[instrument(skip(self, cancel_token))] - pub async fn run(&mut self, cancel_token: CancellationToken) { - loop { - select! { - msg = self.receiver.recv() => { - if let Some(msg) = msg { - self.handle_msg(msg).await; - } else { - break; - } - } - _ = cancel_token.cancelled() => { - break; - } - } - } - - info!("Actor shutdown"); - } -} - -// ----------------- -// CommittorService -// ----------------- -pub struct CommittorService { - sender: mpsc::Sender, - cancel_token: CancellationToken, -} - -impl CommittorService { - pub fn try_start( - authority: Keypair, - persist_file: P, - chain_config: ChainConfig, - actions_callback_executor: A, - ) -> CommittorServiceResult - where - P: AsRef, - A: ActionsCallbackScheduler, - { - debug!("Starting committor service"); - let (sender, receiver) = mpsc::channel(1_000); - let cancel_token = CancellationToken::new(); - { - let cancel_token = cancel_token.clone(); - let mut actor = CommittorActor::try_new( - receiver, - authority, - persist_file, - chain_config, - actions_callback_executor, - )?; - tokio::spawn(async move { - actor.run(cancel_token).await; - }); - } - Ok(Self { - sender, - cancel_token, - }) - } - - pub fn reserve_common_pubkeys( - &self, - ) -> oneshot::Receiver> { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::ReserveCommonPubkeys { - respond_to: tx, - }); - rx - } - - pub fn release_common_pubkeys(&self) -> oneshot::Receiver<()> { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::ReleaseCommonPubkeys { - respond_to: tx, - }); - rx - } - - pub fn get_commit_signatures( - &self, - commit_id: u64, - pubkey: Pubkey, - ) -> oneshot::Receiver>> - { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::GetCommitSignatures { - respond_to: tx, - commit_id, - pubkey, - }); - rx - } - - pub fn get_lookup_tables(&self) -> oneshot::Receiver { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::GetLookupTables { respond_to: tx }); - rx - } - - pub fn fetch_current_commit_nonces_sync( - &self, - pubkeys: &[Pubkey], - min_context_slot: u64, - ) -> std::sync::mpsc::Receiver>> - { - let (tx, rx) = std::sync::mpsc::channel(); - self.try_send(CommittorMessage::FetchCurrentCommitNoncesSync { - respond_to: tx, - pubkeys: pubkeys.to_vec(), - min_context_slot, - }); - rx - } - - fn try_send(&self, msg: CommittorMessage) { - if let Err(e) = self.sender.try_send(msg) { - match e { - TrySendError::Full(msg) => error!( - "Channel full, failed to send commit message {:?}", - msg - ), - TrySendError::Closed(msg) => error!( - "Channel closed, failed to send commit message {:?}", - msg - ), - } - } - } -} - -impl BaseIntentCommittor for CommittorService { - fn reserve_pubkeys_for_committee( - &self, - committee: Pubkey, - owner: Pubkey, - ) -> oneshot::Receiver> { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::ReservePubkeysForCommittee { - initiated: Instant::now(), - respond_to: tx, - committee, - owner, - }); - rx - } - - fn schedule_intent_bundles( - &self, - intent_bundles: Vec, - ) -> oneshot::Receiver> { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::ScheduleIntentBundle { - intent_bundles, - respond_to: tx, - }); - rx - } - - fn get_commit_statuses( - &self, - message_id: u64, - ) -> oneshot::Receiver>> { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::GetCommitStatuses { - respond_to: tx, - message_id, - }); - rx - } - - fn get_commit_signatures( - &self, - commit_id: u64, - pubkey: Pubkey, - ) -> oneshot::Receiver>> - { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::GetCommitSignatures { - respond_to: tx, - commit_id, - pubkey, - }); - rx - } - - fn subscribe_for_results( - &self, - ) -> oneshot::Receiver> - { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::SubscribeForResults { respond_to: tx }); - rx - } - - fn get_transaction( - &self, - signature: &Signature, - ) -> oneshot::Receiver< - CommittorServiceResult, - > { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::GetTransaction { - respond_to: tx, - signature: *signature, - }); - - rx - } - - fn fetch_current_commit_nonces( - &self, - pubkeys: &[Pubkey], - min_context_slot: u64, - ) -> oneshot::Receiver>> { - let (tx, rx) = oneshot::channel(); - self.try_send(CommittorMessage::FetchCurrentCommitNonces { - respond_to: tx, - pubkeys: pubkeys.to_vec(), - min_context_slot, - }); - - rx - } - - fn stop(&self) { - self.cancel_token.cancel(); - } - - fn stopped(&self) -> WaitForCancellationFutureOwned { - self.cancel_token.clone().cancelled_owned() - } -} - -pub trait BaseIntentCommittor: Send + Sync + 'static { - /// Reserves pubkeys used in most commits in a lookup table - fn reserve_pubkeys_for_committee( - &self, - committee: Pubkey, - owner: Pubkey, - ) -> oneshot::Receiver>; - - /// Commits the changeset and returns - fn schedule_intent_bundles( - &self, - intent_bundles: Vec, - ) -> oneshot::Receiver>; - - /// Subscribes for results of BaseIntent execution - fn subscribe_for_results( - &self, - ) -> oneshot::Receiver>; - - /// Gets statuses of accounts that were committed as part of a request with provided message_id - fn get_commit_statuses( - &self, - message_id: u64, - ) -> oneshot::Receiver>>; - - /// Gets signatures for commit of particular accounts - fn get_commit_signatures( - &self, - commit_id: u64, - pubkey: Pubkey, - ) -> oneshot::Receiver>>; - - fn get_transaction( - &self, - signature: &Signature, - ) -> oneshot::Receiver< - CommittorServiceResult, - >; - - fn fetch_current_commit_nonces( - &self, - pubkeys: &[Pubkey], - min_context_slot: u64, - ) -> oneshot::Receiver>>; - - /// Stops Committor service - fn stop(&self); - - /// Returns future which resolves once committor `stop` got called - fn stopped(&self) -> WaitForCancellationFutureOwned; -} +// use std::{collections::HashMap, path::Path, sync::Arc, time::Instant}; +// +// use magicblock_core::traits::ActionsCallbackScheduler; +// use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +// use solana_keypair::Keypair; +// use solana_pubkey::Pubkey; +// use solana_signature::Signature; +// use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta; +// use tokio::{ +// select, +// sync::{ +// broadcast, +// mpsc::{self, error::TrySendError}, +// oneshot, +// }, +// }; +// use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; +// use tracing::*; +// +// use crate::{ +// committor_processor::CommittorProcessor, +// config::ChainConfig, +// error::{CommittorServiceError, CommittorServiceResult}, +// intent_execution_manager::BroadcastedIntentExecutionResult, +// persist::{CommitStatusRow, MessageSignatures}, +// pubkeys_provider::{provide_committee_pubkeys, provide_common_pubkeys}, +// }; +// +// #[derive(Debug)] +// pub struct LookupTables { +// pub active: Vec, +// pub released: Vec, +// } +// +// #[derive(Debug)] +// pub enum CommittorMessage { +// ReservePubkeysForCommittee { +// /// When the request was initiated +// initiated: Instant, +// /// Called once the pubkeys have been reserved and includes that timestamp +// /// at which the request was initiated +// respond_to: oneshot::Sender>, +// /// The committee whose pubkeys to reserve in a lookup table +// /// These pubkeys are used to process/finalize the commit +// committee: Pubkey, +// /// The owner program of the committee +// owner: Pubkey, +// }, +// ReserveCommonPubkeys { +// /// Called once the pubkeys have been reserved +// respond_to: oneshot::Sender>, +// }, +// ReleaseCommonPubkeys { +// /// Called once the pubkeys have been released +// respond_to: oneshot::Sender<()>, +// }, +// ScheduleIntentBundle { +// /// The [`ScheduleIntentBundle`]s to commit +// intent_bundles: Vec, +// respond_to: oneshot::Sender>, +// }, +// GetCommitStatuses { +// respond_to: +// oneshot::Sender>>, +// message_id: u64, +// }, +// GetCommitSignatures { +// respond_to: +// oneshot::Sender>>, +// commit_id: u64, +// pubkey: Pubkey, +// }, +// GetLookupTables { +// respond_to: oneshot::Sender, +// }, +// GetTransaction { +// respond_to: oneshot::Sender< +// CommittorServiceResult, +// >, +// signature: Signature, +// }, +// SubscribeForResults { +// respond_to: oneshot::Sender< +// broadcast::Receiver, +// >, +// }, +// FetchCurrentCommitNonces { +// respond_to: +// oneshot::Sender>>, +// pubkeys: Vec, +// min_context_slot: u64, +// }, +// FetchCurrentCommitNoncesSync { +// respond_to: std::sync::mpsc::Sender< +// CommittorServiceResult>, +// >, +// pubkeys: Vec, +// min_context_slot: u64, +// }, +// } +// +// // ----------------- +// // CommittorActor +// // ----------------- +// struct CommittorActor { +// receiver: mpsc::Receiver, +// processor: Arc, +// } +// +// impl CommittorActor { +// pub fn try_new( +// receiver: mpsc::Receiver, +// authority: Keypair, +// persist_file: P, +// chain_config: ChainConfig, +// actions_callback_executor: A, +// ) -> CommittorServiceResult +// where +// P: AsRef, +// A: ActionsCallbackScheduler, +// { +// let processor = Arc::new(CommittorProcessor::try_new( +// authority, +// persist_file, +// chain_config, +// actions_callback_executor, +// )?); +// +// Ok(Self { +// receiver, +// processor, +// }) +// } +// +// #[instrument(skip(self))] +// async fn handle_msg(&self, msg: CommittorMessage) { +// use CommittorMessage::*; +// match msg { +// ReservePubkeysForCommittee { +// initiated, +// respond_to, +// committee, +// owner, +// } => { +// let processor = self.processor.clone(); +// tokio::task::spawn(async move { +// let pubkeys = +// provide_committee_pubkeys(&committee, Some(&owner)); +// // NOTE: we wait here until the reservation is done which causes the +// // cloning of a particular account to be blocked. +// // This leads to larger delays on the first clone of an account, but also +// // ensures that the account could be committed via a lookup table later. +// let result = processor +// .reserve_pubkeys(pubkeys) +// .await +// .map(|_| initiated); +// if let Err(e) = respond_to.send(result) { +// error!(message_type = "ReservePubkeysForCommittee", error = ?e, "Failed to send response"); +// } +// }); +// } +// ReserveCommonPubkeys { respond_to } => { +// let processor = self.processor.clone(); +// tokio::task::spawn(async move { +// let pubkeys = +// provide_common_pubkeys(&processor.auth_pubkey()); +// let reqid = processor.reserve_pubkeys(pubkeys).await; +// if let Err(e) = respond_to.send(reqid) { +// error!(message_type = "ReserveCommonPubkeys", error = ?e, "Failed to send response"); +// } +// }); +// } +// ReleaseCommonPubkeys { respond_to } => { +// let processor = self.processor.clone(); +// tokio::task::spawn(async move { +// let pubkeys = +// provide_common_pubkeys(&processor.auth_pubkey()); +// processor.release_pubkeys(pubkeys).await; +// if let Err(e) = respond_to.send(()) { +// error!(message_type = "ReleaseCommonPubkeys", error = ?e, "Failed to send response"); +// } +// }); +// } +// ScheduleIntentBundle { +// intent_bundles, +// respond_to, +// } => { +// let result = +// self.processor.schedule_intent_bundles(intent_bundles).await; +// if let Err(e) = respond_to.send(result) { +// error!(message_type = "ScheduleBaseIntents", error = ?e, "Failed to send response"); +// } +// } +// GetCommitStatuses { +// message_id, +// respond_to, +// } => { +// let commit_statuses = +// self.processor.get_commit_statuses(message_id); +// if let Err(e) = respond_to.send(commit_statuses) { +// error!(message_type = "GetCommitStatuses", error = ?e, "Failed to send response"); +// } +// } +// GetCommitSignatures { +// commit_id, +// respond_to, +// pubkey, +// } => { +// let sig = +// self.processor.get_commit_signature(commit_id, pubkey); +// if let Err(e) = respond_to.send(sig) { +// error!(message_type = "GetCommitSignatures", error = ?e, "Failed to send response"); +// } +// } +// GetTransaction { +// signature, +// respond_to, +// } => { +// let processor = self.processor.clone(); +// tokio::task::spawn(async move { +// let res = processor +// .magicblock_rpc_client +// .get_transaction(&signature, None) +// .await +// .map_err(Into::into); +// if let Err(err) = respond_to.send(res) { +// error!(message_type = "GetTransaction", error = ?err, "Failed to send response"); +// } +// }); +// } +// GetLookupTables { respond_to } => { +// let active_tables = self.processor.active_lookup_tables().await; +// let released_tables = +// self.processor.released_lookup_tables().await; +// if let Err(e) = respond_to.send(LookupTables { +// active: active_tables, +// released: released_tables, +// }) { +// error!(message_type = "GetLookupTables", error = ?e, "Failed to send response"); +// } +// } +// SubscribeForResults { respond_to } => { +// let subscription = self.processor.subscribe_for_results(); +// if let Err(err) = respond_to.send(subscription) { +// error!(message_type = "SubscribeForResults", error = ?err, "Failed to send response"); +// } +// } +// FetchCurrentCommitNonces { +// respond_to, +// pubkeys, +// min_context_slot, +// } => { +// let processor = self.processor.clone(); +// tokio::spawn(async move { +// let result = processor +// .fetch_current_commit_nonces(&pubkeys, min_context_slot) +// .await; +// if let Err(err) = respond_to +// .send(result.map_err(CommittorServiceError::from)) +// { +// error!(message_type = "FetchCurrentCommitNonces", error = ?err, "Failed to send response"); +// } +// }); +// } +// FetchCurrentCommitNoncesSync { +// respond_to, +// pubkeys, +// min_context_slot, +// } => { +// let processor = self.processor.clone(); +// tokio::spawn(async move { +// let result = processor +// .fetch_current_commit_nonces(&pubkeys, min_context_slot) +// .await; +// if let Err(err) = respond_to +// .send(result.map_err(CommittorServiceError::from)) +// { +// error!(message_type = "FetchCurrentCommitNoncesSync", error = ?err, "Failed to send response"); +// } +// }); +// } +// } +// } +// +// #[instrument(skip(self, cancel_token))] +// pub async fn run(&mut self, cancel_token: CancellationToken) { +// loop { +// select! { +// msg = self.receiver.recv() => { +// if let Some(msg) = msg { +// self.handle_msg(msg).await; +// } else { +// break; +// } +// } +// _ = cancel_token.cancelled() => { +// break; +// } +// } +// } +// +// info!("Actor shutdown"); +// } +// } +// +// // ----------------- +// // CommittorService +// // ----------------- +// pub struct CommittorService { +// sender: mpsc::Sender, +// cancel_token: CancellationToken, +// } +// +// impl CommittorService { +// pub fn try_start( +// authority: Keypair, +// persist_file: P, +// chain_config: ChainConfig, +// actions_callback_executor: A, +// ) -> CommittorServiceResult +// where +// P: AsRef, +// A: ActionsCallbackScheduler, +// { +// debug!("Starting committor service"); +// let (sender, receiver) = mpsc::channel(1_000); +// let cancel_token = CancellationToken::new(); +// { +// let cancel_token = cancel_token.clone(); +// let mut actor = CommittorActor::try_new( +// receiver, +// authority, +// persist_file, +// chain_config, +// actions_callback_executor, +// )?; +// tokio::spawn(async move { +// actor.run(cancel_token).await; +// }); +// } +// Ok(Self { +// sender, +// cancel_token, +// }) +// } +// +// pub fn reserve_common_pubkeys( +// &self, +// ) -> oneshot::Receiver> { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::ReserveCommonPubkeys { +// respond_to: tx, +// }); +// rx +// } +// +// pub fn release_common_pubkeys(&self) -> oneshot::Receiver<()> { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::ReleaseCommonPubkeys { +// respond_to: tx, +// }); +// rx +// } +// +// pub fn get_commit_signatures( +// &self, +// commit_id: u64, +// pubkey: Pubkey, +// ) -> oneshot::Receiver>> +// { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::GetCommitSignatures { +// respond_to: tx, +// commit_id, +// pubkey, +// }); +// rx +// } +// +// pub fn get_lookup_tables(&self) -> oneshot::Receiver { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::GetLookupTables { respond_to: tx }); +// rx +// } +// +// pub fn fetch_current_commit_nonces_sync( +// &self, +// pubkeys: &[Pubkey], +// min_context_slot: u64, +// ) -> std::sync::mpsc::Receiver>> +// { +// let (tx, rx) = std::sync::mpsc::channel(); +// self.try_send(CommittorMessage::FetchCurrentCommitNoncesSync { +// respond_to: tx, +// pubkeys: pubkeys.to_vec(), +// min_context_slot, +// }); +// rx +// } +// +// fn try_send(&self, msg: CommittorMessage) { +// if let Err(e) = self.sender.try_send(msg) { +// match e { +// TrySendError::Full(msg) => error!( +// "Channel full, failed to send commit message {:?}", +// msg +// ), +// TrySendError::Closed(msg) => error!( +// "Channel closed, failed to send commit message {:?}", +// msg +// ), +// } +// } +// } +// } +// +// impl BaseIntentCommittor for CommittorService { +// fn reserve_pubkeys_for_committee( +// &self, +// committee: Pubkey, +// owner: Pubkey, +// ) -> oneshot::Receiver> { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::ReservePubkeysForCommittee { +// initiated: Instant::now(), +// respond_to: tx, +// committee, +// owner, +// }); +// rx +// } +// +// fn schedule_intent_bundles( +// &self, +// intent_bundles: Vec, +// ) -> oneshot::Receiver> { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::ScheduleIntentBundle { +// intent_bundles, +// respond_to: tx, +// }); +// rx +// } +// +// fn get_commit_statuses( +// &self, +// message_id: u64, +// ) -> oneshot::Receiver>> { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::GetCommitStatuses { +// respond_to: tx, +// message_id, +// }); +// rx +// } +// +// fn get_commit_signatures( +// &self, +// commit_id: u64, +// pubkey: Pubkey, +// ) -> oneshot::Receiver>> +// { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::GetCommitSignatures { +// respond_to: tx, +// commit_id, +// pubkey, +// }); +// rx +// } +// +// fn subscribe_for_results( +// &self, +// ) -> oneshot::Receiver> +// { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::SubscribeForResults { respond_to: tx }); +// rx +// } +// +// fn get_transaction( +// &self, +// signature: &Signature, +// ) -> oneshot::Receiver< +// CommittorServiceResult, +// > { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::GetTransaction { +// respond_to: tx, +// signature: *signature, +// }); +// +// rx +// } +// +// fn fetch_current_commit_nonces( +// &self, +// pubkeys: &[Pubkey], +// min_context_slot: u64, +// ) -> oneshot::Receiver>> { +// let (tx, rx) = oneshot::channel(); +// self.try_send(CommittorMessage::FetchCurrentCommitNonces { +// respond_to: tx, +// pubkeys: pubkeys.to_vec(), +// min_context_slot, +// }); +// +// rx +// } +// +// fn stop(&self) { +// self.cancel_token.cancel(); +// } +// +// fn stopped(&self) -> WaitForCancellationFutureOwned { +// self.cancel_token.clone().cancelled_owned() +// } +// } +// +// pub trait BaseIntentCommittor: Send + Sync + 'static { +// /// Reserves pubkeys used in most commits in a lookup table +// fn reserve_pubkeys_for_committee( +// &self, +// committee: Pubkey, +// owner: Pubkey, +// ) -> oneshot::Receiver>; +// +// /// Commits the changeset and returns +// fn schedule_intent_bundles( +// &self, +// intent_bundles: Vec, +// ) -> oneshot::Receiver>; +// +// /// Subscribes for results of BaseIntent execution +// fn subscribe_for_results( +// &self, +// ) -> oneshot::Receiver>; +// +// /// Gets statuses of accounts that were committed as part of a request with provided message_id +// fn get_commit_statuses( +// &self, +// message_id: u64, +// ) -> oneshot::Receiver>>; +// +// /// Gets signatures for commit of particular accounts +// fn get_commit_signatures( +// &self, +// commit_id: u64, +// pubkey: Pubkey, +// ) -> oneshot::Receiver>>; +// +// fn get_transaction( +// &self, +// signature: &Signature, +// ) -> oneshot::Receiver< +// CommittorServiceResult, +// >; +// +// fn fetch_current_commit_nonces( +// &self, +// pubkeys: &[Pubkey], +// min_context_slot: u64, +// ) -> oneshot::Receiver>>; +// +// /// Stops Committor service +// fn stop(&self); +// +// /// Returns future which resolves once committor `stop` got called +// fn stopped(&self) -> WaitForCancellationFutureOwned; +// } diff --git a/magicblock-committor-service/src/service_ext.rs b/magicblock-committor-service/src/service_ext.rs index 024d0423e..d3c0187d4 100644 --- a/magicblock-committor-service/src/service_ext.rs +++ b/magicblock-committor-service/src/service_ext.rs @@ -1,253 +1,253 @@ -use std::{ - collections::{hash_map::Entry, HashMap}, - ops::Deref, - sync::{Arc, Mutex}, - time::Instant, -}; - -use async_trait::async_trait; -use futures_util::future::join_all; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta; -use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; -use tokio_util::sync::WaitForCancellationFutureOwned; -use tracing::{error, info, instrument}; - -use crate::{ - error::{CommittorServiceError, CommittorServiceResult}, - intent_execution_manager::BroadcastedIntentExecutionResult, - persist::{CommitStatusRow, MessageSignatures}, - BaseIntentCommittor, -}; - -const POISONED_MUTEX_MSG: &str = - "CommittorServiceExt pending messages mutex poisoned!"; - -#[async_trait] -pub trait BaseIntentCommittorExt: BaseIntentCommittor { - /// Schedules Base Intents and waits for their results - async fn schedule_intent_bundles_waiting( - &self, - intent_bundles: Vec, - ) -> BaseIntentCommitorExtResult>; -} - -type MessageResultListener = oneshot::Sender; -pub struct CommittorServiceExt { - inner: Arc, - pending_messages: Arc>>, -} - -impl CommittorServiceExt { - pub fn new(inner: Arc) -> Self { - let pending_messages = Arc::new(Mutex::new(HashMap::new())); - let results_subscription = inner.subscribe_for_results(); - let committor_stopped = inner.stopped(); - tokio::spawn(Self::dispatcher( - committor_stopped, - results_subscription, - pending_messages.clone(), - )); - - Self { - inner, - pending_messages, - } - } - - #[instrument(skip( - committor_stopped, - pending_message, - results_subscription - ))] - async fn dispatcher( - committor_stopped: WaitForCancellationFutureOwned, - results_subscription: oneshot::Receiver< - broadcast::Receiver, - >, - pending_message: Arc>>, - ) { - let mut results_subscription = results_subscription.await.unwrap(); - - tokio::pin!(committor_stopped); - loop { - let execution_result = tokio::select! { - biased; - _ = &mut committor_stopped => { - info!("Shutting down extension"); - return; - } - execution_result = results_subscription.recv() => { - match execution_result { - Ok(result) => result, - Err(broadcast::error::RecvError::Closed) => { - info!("Intent execution shutdown"); - break; - } - Err(broadcast::error::RecvError::Lagged(skipped)) => { - // SAFETY: not really feasible to happen as this function is way faster than Intent execution - // requires investigation if ever happens! - error!(skipped, "Dispatcher lag detected"); - continue; - } - } - } - }; - - let sender = if let Some(sender) = pending_message - .lock() - .expect(POISONED_MUTEX_MSG) - .remove(&execution_result.id) - { - sender - } else { - continue; - }; - - if let Err(execution_result) = sender.send(execution_result) { - error!( - intent_id = execution_result.id, - "Failed to send execution result" - ); - } - } - } -} - -#[async_trait] -impl BaseIntentCommittorExt - for CommittorServiceExt -{ - async fn schedule_intent_bundles_waiting( - &self, - base_intents: Vec, - ) -> BaseIntentCommitorExtResult> - { - // Critical section - let receivers = { - let mut pending_messages = - self.pending_messages.lock().expect(POISONED_MUTEX_MSG); - - base_intents - .iter() - .map(|intent| { - let (sender, receiver) = oneshot::channel(); - match pending_messages.entry(intent.id) { - Entry::Vacant(vacant) => { - vacant.insert(sender); - Ok(receiver) - } - Entry::Occupied(_) => Err( - CommittorServiceExtError::RepeatingMessageError( - intent.id, - ), - ), - } - }) - .collect::, _>>()? - }; - - self.schedule_intent_bundles(base_intents).await??; - let results = join_all(receivers.into_iter()) - .await - .into_iter() - .collect::, RecvError>>()?; - - Ok(results) - } -} - -impl BaseIntentCommittor for CommittorServiceExt { - fn reserve_pubkeys_for_committee( - &self, - committee: Pubkey, - owner: Pubkey, - ) -> oneshot::Receiver> { - self.inner.reserve_pubkeys_for_committee(committee, owner) - } - - fn schedule_intent_bundles( - &self, - intent_bundles: Vec, - ) -> oneshot::Receiver> { - self.inner.schedule_intent_bundles(intent_bundles) - } - - fn subscribe_for_results( - &self, - ) -> oneshot::Receiver> - { - self.inner.subscribe_for_results() - } - - fn get_commit_statuses( - &self, - message_id: u64, - ) -> oneshot::Receiver>> { - self.inner.get_commit_statuses(message_id) - } - - fn get_commit_signatures( - &self, - commit_id: u64, - pubkey: Pubkey, - ) -> oneshot::Receiver>> - { - self.inner.get_commit_signatures(commit_id, pubkey) - } - - fn get_transaction( - &self, - signature: &Signature, - ) -> oneshot::Receiver< - CommittorServiceResult, - > { - self.inner.get_transaction(signature) - } - - fn fetch_current_commit_nonces( - &self, - pubkeys: &[Pubkey], - min_context_slot: u64, - ) -> oneshot::Receiver>> { - self.inner - .fetch_current_commit_nonces(pubkeys, min_context_slot) - } - - fn stop(&self) { - self.inner.stop(); - } - - fn stopped(&self) -> WaitForCancellationFutureOwned { - self.inner.stopped() - } -} - -impl Deref for CommittorServiceExt { - type Target = Arc; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -#[derive(thiserror::Error, Debug)] -pub enum CommittorServiceExtError { - #[error("Attempt to schedule already scheduled message id: {0}")] - RepeatingMessageError(u64), - #[error("RecvError: {0}")] - RecvError(#[from] RecvError), - #[error("CommittorServiceError: {0:?}")] - CommittorServiceError(Box), -} - -impl From for CommittorServiceExtError { - fn from(e: CommittorServiceError) -> Self { - Self::CommittorServiceError(Box::new(e)) - } -} - -pub type BaseIntentCommitorExtResult = - Result; +// use std::{ +// collections::{hash_map::Entry, HashMap}, +// ops::Deref, +// sync::{Arc, Mutex}, +// time::Instant, +// }; +// +// use async_trait::async_trait; +// use futures_util::future::join_all; +// use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +// use solana_pubkey::Pubkey; +// use solana_signature::Signature; +// use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta; +// use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; +// use tokio_util::sync::WaitForCancellationFutureOwned; +// use tracing::{error, info, instrument}; +// +// use crate::{ +// error::{CommittorServiceError, CommittorServiceResult}, +// intent_execution_manager::BroadcastedIntentExecutionResult, +// persist::{CommitStatusRow, MessageSignatures}, +// BaseIntentCommittor, +// }; +// +// const POISONED_MUTEX_MSG: &str = +// "CommittorServiceExt pending messages mutex poisoned!"; +// +// #[async_trait] +// pub trait BaseIntentCommittorExt: BaseIntentCommittor { +// /// Schedules Base Intents and waits for their results +// async fn schedule_intent_bundles_waiting( +// &self, +// intent_bundles: Vec, +// ) -> BaseIntentCommitorExtResult>; +// } +// +// type MessageResultListener = oneshot::Sender; +// pub struct CommittorServiceExt { +// inner: Arc, +// pending_messages: Arc>>, +// } +// +// impl CommittorServiceExt { +// pub fn new(inner: Arc) -> Self { +// let pending_messages = Arc::new(Mutex::new(HashMap::new())); +// let results_subscription = inner.subscribe_for_results(); +// let committor_stopped = inner.stopped(); +// tokio::spawn(Self::dispatcher( +// committor_stopped, +// results_subscription, +// pending_messages.clone(), +// )); +// +// Self { +// inner, +// pending_messages, +// } +// } +// +// #[instrument(skip( +// committor_stopped, +// pending_message, +// results_subscription +// ))] +// async fn dispatcher( +// committor_stopped: WaitForCancellationFutureOwned, +// results_subscription: oneshot::Receiver< +// broadcast::Receiver, +// >, +// pending_message: Arc>>, +// ) { +// let mut results_subscription = results_subscription.await.unwrap(); +// +// tokio::pin!(committor_stopped); +// loop { +// let execution_result = tokio::select! { +// biased; +// _ = &mut committor_stopped => { +// info!("Shutting down extension"); +// return; +// } +// execution_result = results_subscription.recv() => { +// match execution_result { +// Ok(result) => result, +// Err(broadcast::error::RecvError::Closed) => { +// info!("Intent execution shutdown"); +// break; +// } +// Err(broadcast::error::RecvError::Lagged(skipped)) => { +// // SAFETY: not really feasible to happen as this function is way faster than Intent execution +// // requires investigation if ever happens! +// error!(skipped, "Dispatcher lag detected"); +// continue; +// } +// } +// } +// }; +// +// let sender = if let Some(sender) = pending_message +// .lock() +// .expect(POISONED_MUTEX_MSG) +// .remove(&execution_result.id) +// { +// sender +// } else { +// continue; +// }; +// +// if let Err(execution_result) = sender.send(execution_result) { +// error!( +// intent_id = execution_result.id, +// "Failed to send execution result" +// ); +// } +// } +// } +// } +// +// #[async_trait] +// impl BaseIntentCommittorExt +// for CommittorServiceExt +// { +// async fn schedule_intent_bundles_waiting( +// &self, +// base_intents: Vec, +// ) -> BaseIntentCommitorExtResult> +// { +// // Critical section +// let receivers = { +// let mut pending_messages = +// self.pending_messages.lock().expect(POISONED_MUTEX_MSG); +// +// base_intents +// .iter() +// .map(|intent| { +// let (sender, receiver) = oneshot::channel(); +// match pending_messages.entry(intent.id) { +// Entry::Vacant(vacant) => { +// vacant.insert(sender); +// Ok(receiver) +// } +// Entry::Occupied(_) => Err( +// CommittorServiceExtError::RepeatingMessageError( +// intent.id, +// ), +// ), +// } +// }) +// .collect::, _>>()? +// }; +// +// self.schedule_intent_bundles(base_intents).await??; +// let results = join_all(receivers.into_iter()) +// .await +// .into_iter() +// .collect::, RecvError>>()?; +// +// Ok(results) +// } +// } +// +// impl BaseIntentCommittor for CommittorServiceExt { +// fn reserve_pubkeys_for_committee( +// &self, +// committee: Pubkey, +// owner: Pubkey, +// ) -> oneshot::Receiver> { +// self.inner.reserve_pubkeys_for_committee(committee, owner) +// } +// +// fn schedule_intent_bundles( +// &self, +// intent_bundles: Vec, +// ) -> oneshot::Receiver> { +// self.inner.schedule_intent_bundles(intent_bundles) +// } +// +// fn subscribe_for_results( +// &self, +// ) -> oneshot::Receiver> +// { +// self.inner.subscribe_for_results() +// } +// +// fn get_commit_statuses( +// &self, +// message_id: u64, +// ) -> oneshot::Receiver>> { +// self.inner.get_commit_statuses(message_id) +// } +// +// fn get_commit_signatures( +// &self, +// commit_id: u64, +// pubkey: Pubkey, +// ) -> oneshot::Receiver>> +// { +// self.inner.get_commit_signatures(commit_id, pubkey) +// } +// +// fn get_transaction( +// &self, +// signature: &Signature, +// ) -> oneshot::Receiver< +// CommittorServiceResult, +// > { +// self.inner.get_transaction(signature) +// } +// +// fn fetch_current_commit_nonces( +// &self, +// pubkeys: &[Pubkey], +// min_context_slot: u64, +// ) -> oneshot::Receiver>> { +// self.inner +// .fetch_current_commit_nonces(pubkeys, min_context_slot) +// } +// +// fn stop(&self) { +// self.inner.stop(); +// } +// +// fn stopped(&self) -> WaitForCancellationFutureOwned { +// self.inner.stopped() +// } +// } +// +// impl Deref for CommittorServiceExt { +// type Target = Arc; +// +// fn deref(&self) -> &Self::Target { +// &self.inner +// } +// } +// +// #[derive(thiserror::Error, Debug)] +// pub enum CommittorServiceExtError { +// #[error("Attempt to schedule already scheduled message id: {0}")] +// RepeatingMessageError(u64), +// #[error("RecvError: {0}")] +// RecvError(#[from] RecvError), +// #[error("CommittorServiceError: {0:?}")] +// CommittorServiceError(Box), +// } +// +// impl From for CommittorServiceExtError { +// fn from(e: CommittorServiceError) -> Self { +// Self::CommittorServiceError(Box::new(e)) +// } +// } +// +// pub type BaseIntentCommitorExtResult = +// Result; diff --git a/magicblock-committor-service/src/service_final.rs b/magicblock-committor-service/src/service_final.rs new file mode 100644 index 000000000..5021a59f4 --- /dev/null +++ b/magicblock-committor-service/src/service_final.rs @@ -0,0 +1,554 @@ +use std::{ + collections::{HashMap, HashSet}, + iter::chain, + path::Path, + sync::{Arc, Mutex}, + time::Duration, +}; + +use async_trait::async_trait; +use base64::encode; +use magicblock_account_cloner::ChainlinkCloner; +use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; +use magicblock_chainlink::{ + remote_account_provider::{ + chain_rpc_client::ChainRpcClientImpl, + chain_updates_client::ChainUpdatesClient, + }, + submux::SubMuxClient, + Chainlink, +}; +use magicblock_core::{ + link::transactions::{with_encoded, TransactionSchedulerHandle}, + traits::{ActionsCallbackScheduler, LatestBlockProvider}, +}; +use magicblock_metrics::metrics; +use magicblock_program::{ + instruction_utils::InstructionUtils, + magic_scheduled_base_intent::ScheduledIntentBundle, + register_scheduled_commit_sent, MagicContext, Pubkey, SentCommit, + TransactionScheduler, MAGIC_CONTEXT_PUBKEY, +}; +use solana_account::ReadableAccount; +use solana_hash::Hash; +use solana_keypair::Keypair; +use solana_transaction::Transaction; +use solana_transaction_error::TransactionError; +use tokio::{sync::broadcast, task, task::JoinHandle}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, instrument}; + +use crate::{ + committor_processor::CommittorProcessor, config::ChainConfig, + error::CommittorServiceError, + intent_execution_manager::BroadcastedIntentExecutionResult, + intent_executor::ExecutionOutput, +}; + +// TODO(edwin): adapt +const POISONED_MUTEX_MSG: &str = + "Mutex of RemoteScheduledCommitsProcessor.intents_meta_map is poisoned"; + +pub type ChainlinkImpl = Chainlink< + ChainRpcClientImpl, + SubMuxClient, + AccountsDb, + ChainlinkCloner, +>; + +#[async_trait] +pub trait IntentRpcClient: Send + Sync + 'static { + type Error: std::error::Error + Send; + + /// Executes `Accept` tx and returns accepted intents + async fn accept_scheduled_intents( + &self, + ) -> Result, Self::Error>; + + /// Processes intent results, submitting them on chain(ER) + async fn finalize_intent( + &self, + sent_tx: Transaction, + sent_commit: SentCommit, + ) -> Result<(), Self::Error>; +} + +pub struct InternalIntentRpcClient { + /// Provides access to MagicContext + accounts_db: Arc, + /// Internal endpoint for scheduling ER TXs + transaction_scheduler: TransactionSchedulerHandle, + /// Provides access to ER latest block for TX creation + latest_block_provider: L, +} + +impl InternalIntentRpcClient { + pub fn new( + accounts_db: Arc, + transaction_scheduler: TransactionSchedulerHandle, + latest_block_provider: L, + ) -> Self { + Self { + accounts_db, + transaction_scheduler, + latest_block_provider, + } + } + + /// Sends transaction to move the scheduled commits from the `MagicContext` + /// to the global ScheduledCommit store + async fn execute_accept_tx(&self) -> Result<(), InternalRpcClientError> { + let tx = InstructionUtils::accept_scheduled_commits( + self.latest_block_provider.blockhash(), + ); + let encoded_tx = with_encoded(tx).inspect_err(|err| { + error!("Failed to bincode intent transaction"); + })?; + self.transaction_scheduler + .execute(encoded_tx) + .await + .inspect_err(|err| { + error!(error = ?err, "Failed to accept scheduled commits"); + })?; + + Ok(()) + } +} + +#[async_trait] +impl IntentRpcClient for InternalIntentRpcClient { + type Error = InternalRpcClientError; + + async fn accept_scheduled_intents( + &self, + ) -> Result, Self::Error> { + // If accounts were scheduled to be committed, we accept them here + // and processs the commits + let magic_context_acc = + self.accounts_db.get_account(&MAGIC_CONTEXT_PUBKEY).expect( + "Validator found to be running without MagicContext account!", + ); + if !MagicContext::has_scheduled_commits(magic_context_acc.data()) { + return Ok(vec![]); + } + self.execute_accept_tx().await?; + + // Return intents from global store + Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) + } + + async fn finalize_intent( + &self, + sent_tx: Transaction, + sent_commit: SentCommit, + ) -> Result<(), Self::Error> { + register_scheduled_commit_sent(sent_commit); + let txn = with_encoded(sent_tx).inspect_err(|err| { + // Unreachable case, all intent transactions are smaller than 64KB by construction + error!("Failed to bincode intent transaction"); + })?; + self.transaction_scheduler + .execute(txn) + .await + .inspect(|_| debug!("Sent commit signaled")) + .inspect_err( + |err| error!(error = ?err, "Failed to signal sent commit"), + )?; + + Ok(()) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum InternalRpcClientError { + #[error("asd")] + TransactionError(#[from] TransactionError), +} + +pub struct IntentExecutionServiceConfig> { + pub authority: Keypair, + pub slot_interval: Duration, + pub persist_file: P, + pub base_chain_config: ChainConfig, +} + +pub struct IntentExecutionService { + /// Validator/ER authority keypair + authority: Keypair, + /// Chainlink for notifying of undelegations + chainlink: Arc, + /// ER client specific for Intent needs + intent_rpc_client: Arc, + /// Processor of accepted intents + processor: Arc, + /// Time interval to scrape MagicContext(ER slot interval) + // TODO(edwin): can be removed if LatestBlocK moved into magicblock-core + slot_interval: Duration, + cancellation_token: CancellationToken, + /// Meta for ongoing executing intents + intents_meta_map: Arc>>, +} + +impl IntentExecutionService { + pub fn try_new( + authority: Keypair, + chainlink: Arc, + intent_rpc_client: R, + persist_file: P, + base_chain_config: ChainConfig, + actions_callback_executor: A, + slot_interval: Duration, + cancellation_token: CancellationToken, + ) -> Self + where + P: AsRef, + A: ActionsCallbackScheduler, + { + let intents_meta_map = Arc::new(Mutex::default()); + let processor = CommittorProcessor::try_new( + authority.insecure_clone(), + persist_file, + base_chain_config, + actions_callback_executor, + ) + .unwrap(); // TODO(edwin): remove + + Self { + authority, + chainlink, + intent_rpc_client: Arc::new(intent_rpc_client), + processor: Arc::new(processor), + slot_interval, + cancellation_token, + intents_meta_map, + } + } + + pub async fn start(self) -> JoinHandle<()> { + tokio::task::spawn(self.run()) + } + + async fn run(self) { + // TODO(edwin): move to start and make ExecutionHandle{main, result_worker} + let result_subscriber = self.processor.subscribe_for_results(); + let cancellation_token = self.cancellation_token.clone(); + let handle = tokio::spawn(Self::result_processor( + result_subscriber, + cancellation_token, + self.intents_meta_map.clone(), + self.intent_rpc_client.clone(), + )); + + let mut interval = tokio::time::interval(self.slot_interval); + loop { + tokio::select! { + biased; + _ = self.cancellation_token.cancelled() => { + break; + } + _ = interval.tick() => { + let accept_result = self + .intent_rpc_client + .accept_scheduled_intents() + .await; + let intent_bundles = match accept_result { + Ok(value) => value, + Err(err) => { + error!("Failed to accept intents: {}", err); + continue; + } + }; + + if let Err(err) = self.schedule_intent_execution(intent_bundles).await { + error!("Failed to schedule intent execution: {}", err); + } + } + } + } + + // TODO(edwin): handle cancellation + todo!() + } + + async fn schedule_intent_execution( + &self, + intent_bundles: Vec, + ) -> Result<(), CommittorServiceError> { + if intent_bundles.is_empty() { + return Ok(()); + } + + metrics::inc_committor_intents_count_by(intent_bundles.len() as u64); + + // Add metas for intent we schedule + let pubkeys_being_undelegated = { + let mut intent_metas = + self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); + let mut pubkeys_being_undelegated = HashSet::::new(); + + intent_bundles.iter().for_each(|intent| { + intent_metas + .insert(intent.id, ScheduledBaseIntentMeta::new(intent)); + if let Some(undelegate) = intent.get_undelegate_intent_pubkeys() + { + pubkeys_being_undelegated.extend(undelegate); + } + }); + + pubkeys_being_undelegated.into_iter().collect::>() + }; + + self.process_undelegation_requests(pubkeys_being_undelegated) + .await; + self.processor + .schedule_intent_bundles(intent_bundles) + .await?; + Ok(()) + } + + async fn process_undelegation_requests(&self, pubkeys: Vec) { + let mut join_set = task::JoinSet::new(); + for pubkey in pubkeys.into_iter() { + let chainlink = self.chainlink.clone(); + join_set.spawn(async move { + (pubkey, chainlink.undelegation_requested(pubkey).await) + }); + } + let sub_errors = join_set + .join_all() + .await + .into_iter() + .filter_map(|(pubkey, inner_result)| { + if let Err(err) = inner_result { + Some(format!( + "Subscribing to account {} failed: {}", + pubkey, err + )) + } else { + None + } + }) + .collect::>(); + if !sub_errors.is_empty() { + // Instead of aborting the entire commit we log an error here, however + // this means that the undelegated accounts stay in a problematic state + // in the validator and are not synced from chain. + // We could implement a retry mechanism inside of chainlink in the future. + error!( + error_count = sub_errors.len(), + "Failed to subscribe to accounts being undelegated" + ); + } + } + + #[instrument(skip( + result_subscription, + cancellation_token, + intents_meta_map, + intent_client + ))] + async fn result_processor( + mut result_subscription: broadcast::Receiver< + BroadcastedIntentExecutionResult, + >, + cancellation_token: CancellationToken, + intents_meta_map: Arc>>, + intent_client: Arc, + ) { + const SUBSCRIPTION_ERR_MSG: &str = + "Failed to get subscription of results of BaseIntents execution"; + + loop { + let execution_result = tokio::select! { + biased; + _ = cancellation_token.cancelled() => { + // TODO(edwin): validate shutdown correctness + info!("Shutting down"); + return; + } + execution_result = result_subscription.recv() => { + match execution_result { + Ok(result) => result, + Err(broadcast::error::RecvError::Closed) => { + info!("Intent execution service shut down"); + break; + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + // SAFETY: This shouldn't happen as our tx execution is faster than Intent execution on Base layer + // If this ever happens it requires investigation + error!(skipped_count = skipped, "Lagging behind intent execution"); + continue; + } + } + } + }; + + IntentExecutionService::::process_execution_result( + &intent_client, + execution_result, + &intents_meta_map, + ) + .await; + } + } + + async fn process_execution_result( + intent_client: &Arc, + execution_result: BroadcastedIntentExecutionResult, + intents_meta_map: &Arc>>, + ) -> Result<(), IntentExecutionServiceError> { + // Create IntentMeta + let intent_id = execution_result.id; + // Remove intent from metas + let Some(mut intent_meta) = intents_meta_map + .lock() + .expect(POISONED_MUTEX_MSG) + .remove(&intent_id) + else { + // Possible if we have duplicate Intents + // First one will remove id from map and second could fail. + // This should not happen and needs investigation! + error!(intent_id, "Failed to find intent metadata"); + return Ok(()); + }; + + let sent_transaction = + std::mem::take(&mut intent_meta.intent_sent_transaction); + let sent_commit = IntentExecutionService::::build_sent_commit( + intent_id, + intent_meta, + execution_result, + ); + // TODO(edwin): convert + intent_client + .finalize_intent(sent_transaction, sent_commit) + .await; + + Ok(()) + } + + #[instrument( + skip(internal_transaction_scheduler, result, intent_meta), + fields(intent_id) + )] + async fn process_intent_result( + intent_id: u64, + internal_transaction_scheduler: &TransactionSchedulerHandle, + result: BroadcastedIntentExecutionResult, + mut intent_meta: ScheduledBaseIntentMeta, + ) { + let intent_sent_transaction = + std::mem::take(&mut intent_meta.intent_sent_transaction); + let sent_commit = + Self::build_sent_commit(intent_id, intent_meta, result); + register_scheduled_commit_sent(sent_commit); + let Ok(txn) = with_encoded(intent_sent_transaction) else { + // Unreachable case, all intent transactions are smaller than 64KB by construction + error!("Failed to bincode intent transaction"); + return; + }; + match internal_transaction_scheduler.execute(txn).await { + Ok(()) => { + debug!("Sent commit signaled") + } + Err(err) => { + error!(error = ?err, "Failed to signal sent commit"); + } + } + } + + fn build_sent_commit( + intent_id: u64, + intent_meta: ScheduledBaseIntentMeta, + result: BroadcastedIntentExecutionResult, + ) -> SentCommit { + let error_message = + result.as_ref().err().map(|err| format!("{:?}", err)); + let chain_signatures = match result.inner { + Ok(value) => match value { + ExecutionOutput::SingleStage(signature) => vec![signature], + ExecutionOutput::TwoStage { + commit_signature, + finalize_signature, + } => vec![commit_signature, finalize_signature], + }, + Err(err) => { + error!( + "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", + intent_id, intent_meta.slot, intent_meta.blockhash, err + ); + err.signatures() + .map(|(commit, finalize)| { + finalize + .map(|finalize| vec![commit, finalize]) + .unwrap_or(vec![commit]) + }) + .unwrap_or_default() + } + }; + let patched_errors = result + .patched_errors + .iter() + .map(|err| { + info!("Patched intent: {}. error was: {}", intent_id, err); + err.to_string() + }) + .collect(); + + let callbacks_report = result + .callbacks_report + .iter() + .map(|r| match r { + Ok(sig) => { + format!("OK: {sig}") + } + Err(err) => { + error!( + "Callback failed to schedule: {}. error: {}", + intent_id, err + ); + format!("ERR: {err}") + } + }) + .collect(); + + SentCommit { + message_id: intent_id, + slot: intent_meta.slot, + blockhash: intent_meta.blockhash, + payer: intent_meta.payer, + chain_signatures, + included_pubkeys: intent_meta.included_pubkeys, + excluded_pubkeys: vec![], + requested_undelegation: intent_meta.requested_undelegation, + error_message, + patched_errors, + callbacks_scheduling_results: callbacks_report, + } + } +} + +struct ScheduledBaseIntentMeta { + slot: u64, + blockhash: Hash, + payer: Pubkey, + included_pubkeys: Vec, + intent_sent_transaction: Transaction, + requested_undelegation: bool, +} + +impl ScheduledBaseIntentMeta { + fn new(intent: &ScheduledIntentBundle) -> Self { + Self { + slot: intent.slot, + blockhash: intent.blockhash, + payer: intent.payer, + included_pubkeys: intent.get_all_committed_pubkeys(), + intent_sent_transaction: intent.sent_transaction.clone(), + requested_undelegation: intent.has_undelegate_intent(), + } + } +} + +#[derive(thiserror::Error, Debug)] +pub enum IntentExecutionServiceError {} diff --git a/magicblock-processor/tests/replay_base_slot.rs b/magicblock-processor/tests/replay_base_slot.rs new file mode 100644 index 000000000..37d6bcc7d --- /dev/null +++ b/magicblock-processor/tests/replay_base_slot.rs @@ -0,0 +1,173 @@ +// Regression test for: replay base slot must be accountsdb_slot + 1, not accountsdb_slot. +// +// The bug: process_ledger was called with full_process_starting_slot = accountsdb_slot +// (inclusive). Because accountsdb_slot is the last slot whose effects are already in +// AccountsDb, this caused every successful transaction in that slot to be applied a +// second time. +// +// The fix: start replay at accountsdb_slot + 1 so the last persisted slot is never re-run. +// +// Test strategy: +// 1. Set up an account C with data[0] = 0 in AccountsDb. +// 2. Build a non-idempotent Increment transaction T1 targeting C. +// 3. Write T1 to the ledger at slot S (marked successful) but do NOT execute it yet. +// 4. Apply T1 to AccountsDb via replay_transaction — AccountsDb is now post-T1 (data[0] = 1) +// and accountsdb_slot = S, exactly as after a clean shutdown. +// 5. Inject an empty block header at S+1 directly into the ledger WITHOUT calling +// advance_slot() — ledger_slot is now S+1 while accountsdb_slot stays at S. +// 6. Call process_ledger with the correct starting slot (S+1): T1 must NOT be re-applied, +// so data[0] stays 1. +// 7. A second test demonstrates the bug: process_ledger starting at S (the old code) +// re-applies T1 → data[0] becomes 2. + +use guinea::GuineaInstruction; +use magicblock_accounts_db::traits::AccountsBank; +use magicblock_core::link::transactions::SanitizeableTransaction; +use magicblock_ledger::{ + blockstore_processor::process_ledger, LatestBlockInner, +}; +use solana_account::ReadableAccount; +use solana_keypair::Keypair; +use solana_program::{ + hash::Hash, + instruction::{AccountMeta, Instruction}, + native_token::LAMPORTS_PER_SOL, +}; +use solana_signer::Signer; +use solana_transaction::sanitized::SanitizedTransaction; +use solana_transaction_status::TransactionStatusMeta; +use test_kit::ExecutionTestEnv; + +async fn setup(env: &ExecutionTestEnv) -> Keypair { + env.yield_to_scheduler().await; + let kp = env.create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID); + assert_eq!( + env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], + 0, + "account data must start at 0" + ); + kp +} + +fn build_increment_tx( + env: &ExecutionTestEnv, + kp: &Keypair, +) -> SanitizedTransaction { + let ix = Instruction::new_with_bincode( + guinea::ID, + &GuineaInstruction::Increment, + vec![AccountMeta::new(kp.pubkey(), false)], + ); + env.build_transaction(&[ix]).sanitize(true).unwrap() +} + +fn write_tx_to_ledger( + env: &ExecutionTestEnv, + sanitized: &SanitizedTransaction, + slot: u64, +) { + let sig = *sanitized.signature(); + let meta = TransactionStatusMeta { + fee: 0, + pre_balances: vec![LAMPORTS_PER_SOL], + post_balances: vec![LAMPORTS_PER_SOL], + status: Ok(()), + ..Default::default() + }; + let versioned = sanitized.to_versioned_transaction(); + let encoded = bincode::serialize(&versioned).unwrap(); + let locks = sanitized.get_account_locks_unchecked(); + env.ledger + .write_transaction( + sig, + slot, + 0, + locks.writable, + locks.readonly, + &encoded, + meta, + ) + .expect("failed to write transaction to ledger"); +} + +fn inject_empty_block(env: &ExecutionTestEnv, slot: u64) { + let latest = env.ledger.latest_block().load(); + let time = latest.clock.unix_timestamp; + drop(latest); + env.ledger + .write_block(LatestBlockInner::new(slot, Hash::new_unique(), time + 1)) + .expect("failed to write injected block"); +} + +/// Correct behavior: process_ledger starting at accountsdb_slot + 1 does not re-apply T1. +#[tokio::test] +async fn test_replay_starts_after_accountsdb_slot() { + let env = ExecutionTestEnv::new_replica_mode(1, false); + let kp = setup(&env).await; + + let slot_s = env.ledger.latest_block().load().slot; + + let sanitized = build_increment_tx(&env, &kp); + write_tx_to_ledger(&env, &sanitized, slot_s); + + // Apply T1 so AccountsDb reflects post-T1 state (data[0] = 1). + env.replay_transaction(false, sanitized).await.unwrap(); + assert_eq!( + env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], + 1, + "T1 must be applied to AccountsDb before replay test" + ); + assert_eq!(env.accountsdb.slot(), slot_s); + + // Inject empty block at S+1 without advancing accountsdb_slot. + inject_empty_block(&env, slot_s + 1); + + // Fix: replay starts at S+1, T1 at slot S is not re-applied. + process_ledger( + &env.ledger, + slot_s + 1, + env.transaction_scheduler.clone(), + 0, + ) + .await + .unwrap(); + + assert_eq!( + env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], + 1, + "data[0] must remain 1 — T1 must not be replayed again" + ); +} + +/// Bug demonstration: process_ledger starting at accountsdb_slot re-applies T1. +#[tokio::test] +async fn test_replay_from_accountsdb_slot_double_applies() { + let env = ExecutionTestEnv::new_replica_mode(1, false); + let kp = setup(&env).await; + + let slot_s = env.ledger.latest_block().load().slot; + + let sanitized = build_increment_tx(&env, &kp); + write_tx_to_ledger(&env, &sanitized, slot_s); + + // Apply T1 (data[0] = 1). + env.replay_transaction(false, sanitized).await.unwrap(); + assert_eq!( + env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], + 1, + "T1 must be applied to AccountsDb before replay test" + ); + + inject_empty_block(&env, slot_s + 1); + + // Bug: replay starts at S (inclusive), so T1 is applied a second time. + process_ledger(&env.ledger, slot_s, env.transaction_scheduler.clone(), 0) + .await + .unwrap(); + + assert_eq!( + env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], + 2, + "data[0] must be 2 — T1 was replayed a second time (double-apply bug)" + ); +} diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index baffc161b..ff51b643d 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -4202,20 +4202,15 @@ dependencies = [ "async-trait", "bincode", "lru 0.16.2", - "magicblock-accounts-db", "magicblock-chainlink", - "magicblock-committor-service", - "magicblock-config", "magicblock-core", "magicblock-ledger", "magicblock-magic-program-api 0.11.1", "magicblock-program", - "magicblock-rpc-client", "rand 0.9.2", "solana-account", "solana-hash 3.1.0", "solana-instruction", - "solana-loader-v3-interface", "solana-loader-v4-interface", "solana-pubkey 3.0.0", "solana-sdk-ids", @@ -4474,6 +4469,9 @@ dependencies = [ "borsh", "futures-util", "lru 0.16.2", + "magicblock-account-cloner", + "magicblock-accounts-db", + "magicblock-chainlink", "magicblock-committor-program", "magicblock-core", "magicblock-delegation-program-api 0.3.0 (git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b)", From 79464a91a77f9998f44cd63d4e36ab460c243dae Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 12:52:05 +0700 Subject: [PATCH 03/97] wip: services integrated --- magicblock-api/src/errors.rs | 9 - magicblock-api/src/magic_sys_adapter.rs | 21 +-- magicblock-api/src/magic_validator.rs | 163 +++++++++--------- .../src/committor_processor.rs | 6 +- magicblock-committor-service/src/lib.rs | 2 +- .../src/service_final.rs | 100 +++++++---- 6 files changed, 154 insertions(+), 147 deletions(-) diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 86c2c8834..0c7e9a6ce 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -15,9 +15,6 @@ pub enum ApiError { #[error("Accounts error: {0}")] AccountsError(Box), - #[error("AccountCloner error: {0}")] - AccountClonerError(Box), - #[error("Ledger error: {0}")] LedgerError(Box), @@ -118,12 +115,6 @@ impl From for ApiError { } } -impl From for ApiError { - fn from(e: magicblock_account_cloner::AccountClonerError) -> Self { - Self::AccountClonerError(Box::new(e)) - } -} - impl From for ApiError { fn from(e: magicblock_ledger::errors::LedgerError) -> Self { Self::LedgerError(Box::new(e)) diff --git a/magicblock-api/src/magic_sys_adapter.rs b/magicblock-api/src/magic_sys_adapter.rs index 1e13b44a7..daa67f6ab 100644 --- a/magicblock-api/src/magic_sys_adapter.rs +++ b/magicblock-api/src/magic_sys_adapter.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; -use magicblock_committor_service::CommittorService; +use magicblock_committor_service::committor_processor::CommittorProcessor; use magicblock_core::{intent::CommittedAccount, traits::MagicSys}; use magicblock_metrics::metrics; use solana_instruction::error::InstructionError; @@ -9,7 +9,7 @@ use tracing::error; #[derive(Clone)] pub struct MagicSysAdapter { - committor_service: Option>, + committor_processor: Arc, } impl MagicSysAdapter { @@ -19,13 +19,13 @@ impl MagicSysAdapter { const TIMEOUT_ERR: u32 = 0xE000_0001; /// Returned when the fetch of current commit nonces fails. const FETCH_ERR: u32 = 0xE000_0002; - /// Returned when no committor service is configured. - const NO_COMMITTOR_ERR: u32 = 0xE000_0003; const FETCH_TIMEOUT: Duration = Duration::from_secs(30); - pub fn new(committor_service: Option>) -> Self { - Self { committor_service } + pub fn new(committor_processor: Arc) -> Self { + Self { + committor_processor, + } } } @@ -37,12 +37,6 @@ impl MagicSys for MagicSysAdapter { if commits.is_empty() { return Ok(HashMap::new()); } - let committor_service = - if let Some(committor_service) = &self.committor_service { - Ok(committor_service) - } else { - Err(InstructionError::Custom(Self::NO_COMMITTOR_ERR)) - }?; let min_context_slot = commits .iter() @@ -53,7 +47,8 @@ impl MagicSys for MagicSysAdapter { commits.iter().map(|account| account.pubkey).collect(); let _timer = metrics::start_fetch_commit_nonces_wait_timer(); - let receiver = committor_service + let receiver = self + .committor_processor .fetch_current_commit_nonces_sync(&pubkeys, min_context_slot); receiver .recv_timeout(Self::FETCH_TIMEOUT) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index e04a5eef9..e58a1caf8 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -9,10 +9,7 @@ use std::{ }; use magicblock_account_cloner::ChainlinkCloner; -use magicblock_accounts::{ - scheduled_commits_processor::ScheduledCommitsProcessorImpl, - ScheduledCommitsProcessor, -}; +use magicblock_accounts::ScheduledCommitsProcessor; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_aperture::{ initialize_aperture, @@ -28,10 +25,13 @@ use magicblock_chainlink::{ Chainlink, }; use magicblock_committor_service::{ + committor_processor::CommittorProcessor, config::ChainConfig, - service_final::{IntentExecutionService, InternalIntentRpcClient}, - BaseIntentCommittor, CommittorService, ComputeBudgetConfig, - DEFAULT_ACTIONS_TIMEOUT, + service_final::{ + IntentExecutionService, IntentExecutionServiceConfig, + InternalIntentRpcClient, + }, + ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, }; use magicblock_config::{ config::{ @@ -84,7 +84,6 @@ use tokio::{ runtime::Builder, sync::mpsc::{channel, Sender}, }; -use tokio_util::either::Either; use tokio_util::sync::CancellationToken; use tracing::*; @@ -113,8 +112,6 @@ type ChainlinkImpl = Chainlink< type IntentExecutionServiceImpl = IntentExecutionService>; -type IntentExecutionState = Either>; - // ----------------- // MagicValidator // ----------------- @@ -125,7 +122,7 @@ pub struct MagicValidator { accountsdb: Arc, ledger: Arc, ledger_truncator: LedgerTruncator, - intent_execution_service: IntentExecutionState, + intent_execution_service: IntentExecutionServiceImpl, replication_service: Option, chainlink: Arc, rpc_handle: thread::JoinHandle<()>, @@ -226,15 +223,6 @@ impl MagicValidator { let accountsdb = Arc::new(accountsdb); let (mut dispatch, validator_channels) = link(); - let step_start = Instant::now(); - let committor_service = - Self::init_committor_service(&config, ledger.latest_block()) - .await?; - log_timing("startup", "committor_service_init", step_start); - init_magic_sys(Arc::new(MagicSysAdapter::new( - committor_service.clone(), - ))); - let step_start = Instant::now(); let chainlink = Arc::new( Self::init_chainlink( @@ -247,6 +235,28 @@ impl MagicValidator { ); log_timing("startup", "chainlink_init", step_start); + let step_start = Instant::now(); + let committor_processor = { + let processor = Self::init_committor_processor( + &config, + &ledger.latest_block(), + )?; + Arc::new(processor) + }; + let intent_execution_service = Self::init_intent_execution_service( + &accountsdb, + &chainlink, + &dispatch.transaction_scheduler, + ledger.latest_block(), + &committor_processor, + config.ledger.block_time, + &token, + ); + log_timing("startup", "committor_service_init", step_start); + init_magic_sys(Arc::new(MagicSysAdapter::new( + committor_processor.clone(), + ))); + let replication_service = if let Some((broker, is_fresh_start)) = broker { let messages_rx = dispatch.replication_messages.take().expect( @@ -303,15 +313,6 @@ impl MagicValidator { ); log_timing("startup", "system_metrics_ticker_start", step_start); - let scheduled_commits_processor = - committor_service.as_ref().map(|committor_service| { - Arc::new(Intent::new( - committor_service.clone(), - chainlink.clone(), - dispatch.transaction_scheduler.clone(), - )) - }); - let step_start = Instant::now(); load_upgradeable_programs( &accountsdb, @@ -432,9 +433,8 @@ impl MagicValidator { exit, _metrics: (metrics_service, system_metrics_ticker), // NOTE: set during [Self::start] - committor_service, + intent_execution_service, replication_service, - scheduled_commits_processor, chainlink, token, ledger, @@ -451,48 +451,57 @@ impl MagicValidator { }) } - #[instrument(skip( - config, latest_block, accounts_db, transaction_scheduler, chainlink - ))] - async fn init_committor_service( + pub fn init_committor_processor( config: &ValidatorParams, - accounts_db: &Arc, - chainlink: Arc, - transaction_scheduler: &TransactionSchedulerHandle, latest_block: &LatestBlock, - ) -> ApiResult { - let intent_client = InternalIntentRpcClient::new( - accounts_db.clone(), - transaction_scheduler.clone(), - latest_block.clone() - ); - + ) -> ApiResult { + let authority = config.validator.keypair.insecure_clone(); let committor_persist_path = config.storage.join("committor_service.sqlite"); - debug!(path = %committor_persist_path.display(), "Initializing committor service"); - // TODO(thlorenz): when we support lifecycle modes again, only start it when needed + let base_chain_config = ChainConfig { + rpc_uri: config.rpc_url().to_owned(), + commitment: CommitmentConfig::confirmed(), + compute_budget_config: ComputeBudgetConfig::new( + config.commit.compute_unit_price, + ), + actions_timeout: DEFAULT_ACTIONS_TIMEOUT, + }; + let actions_callback_executor = ActionsCallbackService::new( Arc::new(RpcClient::new(config.aperture.listen.http())), config.validator.keypair.insecure_clone(), latest_block.clone(), ); - let committor_service = IntentExecutionServiceImpl::try_new( - config.validator.keypair.insecure_clone(), - chainlink, - intent_client, + Ok(CommittorProcessor::try_new( + authority, committor_persist_path, - ChainConfig { - rpc_uri: config.rpc_url().to_owned(), - commitment: CommitmentConfig::confirmed(), - compute_budget_config: ComputeBudgetConfig::new( - config.commit.compute_unit_price, - ), - actions_timeout: DEFAULT_ACTIONS_TIMEOUT, - }, + base_chain_config, actions_callback_executor, - )?; + )?) + } + + fn init_intent_execution_service( + accounts_db: &Arc, + chainlink: &Arc, + transaction_scheduler: &TransactionSchedulerHandle, + latest_block: &LatestBlock, + committor_processor: &Arc, + slot_interval: Duration, + cancellation_token: &CancellationToken, + ) -> IntentExecutionServiceImpl { + let intent_client = InternalIntentRpcClient::new( + accounts_db.clone(), + transaction_scheduler.clone(), + latest_block.clone(), + ); - Ok(committor_service) + IntentExecutionServiceImpl::new( + chainlink.clone(), + intent_client, + committor_processor.clone(), + slot_interval, + cancellation_token.clone(), + ) } #[allow(clippy::too_many_arguments)] @@ -926,15 +935,8 @@ impl MagicValidator { } let step_start = Instant::now(); - self.slot_ticker = Some(init_slot_ticker( - self.accountsdb.clone(), - &self.scheduled_commits_processor, - self.ledger.latest_block().clone(), - self.config.ledger.block_time, - self.transaction_scheduler.clone(), - self.exit.clone(), - )); - log_timing("startup", "slot_ticker_start", step_start); + self.intent_execution_service.start(); + log_timing("startup", "intent_execution_service_start", step_start); let step_start = Instant::now(); self.ledger_truncator.start(); @@ -987,22 +989,11 @@ impl MagicValidator { // Ordering is important here // Commitor service shall be stopped last self.token.cancel(); - if let Some(ref scheduled_commits_processor) = - self.scheduled_commits_processor - { - let step_start = Instant::now(); - scheduled_commits_processor.stop(); - log_timing( - "shutdown", - "scheduled_commits_processor_stop", - step_start, - ); - } - if let Some(ref committor_service) = self.committor_service { - let step_start = Instant::now(); - committor_service.stop(); - log_timing("shutdown", "committor_service_stop", step_start); - } + + let step_start = Instant::now(); + // TODO(edwin): handle returned errs, at least log + let _ = self.intent_execution_service.stop().await; + log_timing("shutdown", "intent_execution_service_stop", step_start); let step_start = Instant::now(); self.claim_fees_task.stop().await; diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 43998939a..84b56a32d 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -40,9 +40,9 @@ const POISONED_MUTEX_MSG: &str = "CommittorProcessor pending messages mutex poisoned!"; type BundleResultListener = oneshot::Sender; -pub(crate) struct CommittorProcessor { - pub(crate) table_mania: TableMania, - pub(crate) authority: Keypair, +pub struct CommittorProcessor { + table_mania: TableMania, + authority: Keypair, persister: IntentPersisterImpl, commits_scheduler: IntentExecutionManager, task_info_fetcher: Arc>, diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index fbc314558..b16c76b58 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -1,4 +1,4 @@ -mod committor_processor; +pub mod committor_processor; mod compute_budget; pub mod config; mod consts; diff --git a/magicblock-committor-service/src/service_final.rs b/magicblock-committor-service/src/service_final.rs index 5021a59f4..375ca24af 100644 --- a/magicblock-committor-service/src/service_final.rs +++ b/magicblock-committor-service/src/service_final.rs @@ -1,6 +1,7 @@ use std::{ collections::{HashMap, HashSet}, iter::chain, + mem, path::Path, sync::{Arc, Mutex}, time::Duration, @@ -34,7 +35,11 @@ use solana_hash::Hash; use solana_keypair::Keypair; use solana_transaction::Transaction; use solana_transaction_error::TransactionError; -use tokio::{sync::broadcast, task, task::JoinHandle}; +use tokio::{ + sync::broadcast, + task, + task::{JoinError, JoinHandle}, +}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument}; @@ -161,20 +166,62 @@ impl IntentRpcClient for InternalIntentRpcClient { #[derive(thiserror::Error, Debug)] pub enum InternalRpcClientError { + // TODO(edwin) #[error("asd")] TransactionError(#[from] TransactionError), } -pub struct IntentExecutionServiceConfig> { - pub authority: Keypair, - pub slot_interval: Duration, - pub persist_file: P, - pub base_chain_config: ChainConfig, +pub enum IntentExecutionService { + Created(ServiceInner), + Started(JoinHandle<()>), + Stopped, + Error, } -pub struct IntentExecutionService { - /// Validator/ER authority keypair - authority: Keypair, +impl IntentExecutionService { + pub fn new( + chainlink: Arc, + intent_rpc_client: R, + processor: Arc, + slot_interval: Duration, + cancellation_token: CancellationToken, + ) -> Self { + Self::Created(ServiceInner::new( + chainlink, + intent_rpc_client, + processor, + slot_interval, + cancellation_token, + )) + } + + fn take(&mut self) -> Self { + mem::replace(self, Self::Error) + } + + pub fn start(&mut self) { + let Self::Created(service) = self.take() else { + // TODO(edwin): err InvalidState(String) + return; + }; + + let handle = service.start(); + *self = Self::Started(handle); + } + + pub async fn stop(&mut self) -> Result<(), JoinError> { + let Self::Started(handle) = self.take() else { + // TODO(edwin): err InvalidState(String) + return Ok(()); + }; + + handle.await?; + *self = Self::Stopped; + Ok(()) + } +} + +pub struct ServiceInner { /// Chainlink for notifying of undelegations chainlink: Arc, /// ER client specific for Intent needs @@ -189,42 +236,25 @@ pub struct IntentExecutionService { intents_meta_map: Arc>>, } -impl IntentExecutionService { - pub fn try_new( - authority: Keypair, +impl ServiceInner { + pub fn new( chainlink: Arc, intent_rpc_client: R, - persist_file: P, - base_chain_config: ChainConfig, - actions_callback_executor: A, + processor: Arc, slot_interval: Duration, cancellation_token: CancellationToken, - ) -> Self - where - P: AsRef, - A: ActionsCallbackScheduler, - { - let intents_meta_map = Arc::new(Mutex::default()); - let processor = CommittorProcessor::try_new( - authority.insecure_clone(), - persist_file, - base_chain_config, - actions_callback_executor, - ) - .unwrap(); // TODO(edwin): remove - + ) -> Self { Self { - authority, chainlink, intent_rpc_client: Arc::new(intent_rpc_client), - processor: Arc::new(processor), + processor, slot_interval, cancellation_token, - intents_meta_map, + intents_meta_map: Arc::new(Mutex::default()), } } - pub async fn start(self) -> JoinHandle<()> { + pub fn start(self) -> JoinHandle<()> { tokio::task::spawn(self.run()) } @@ -383,7 +413,7 @@ impl IntentExecutionService { } }; - IntentExecutionService::::process_execution_result( + ServiceInner::::process_execution_result( &intent_client, execution_result, &intents_meta_map, @@ -414,7 +444,7 @@ impl IntentExecutionService { let sent_transaction = std::mem::take(&mut intent_meta.intent_sent_transaction); - let sent_commit = IntentExecutionService::::build_sent_commit( + let sent_commit = ServiceInner::::build_sent_commit( intent_id, intent_meta, execution_result, From 33906010ab7ca35dd898afa7187d918ffbff2f6f Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 12:57:51 +0700 Subject: [PATCH 04/97] wip: remove unused files --- magicblock-api/src/magic_validator.rs | 2 +- magicblock-api/src/tickers.rs | 12 +- .../src/committor_processor.rs | 5 +- magicblock-committor-service/src/error.rs | 25 +- magicblock-committor-service/src/lib.rs | 3 - magicblock-committor-service/src/service.rs | 570 ------------------ .../src/service_ext.rs | 253 -------- .../src/service_final.rs | 15 +- 8 files changed, 13 insertions(+), 872 deletions(-) delete mode 100644 magicblock-committor-service/src/service.rs delete mode 100644 magicblock-committor-service/src/service_ext.rs diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index e58a1caf8..d5c9ef70f 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -28,7 +28,7 @@ use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, service_final::{ - IntentExecutionService, IntentExecutionServiceConfig, + IntentExecutionService, InternalIntentRpcClient, }, ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, diff --git a/magicblock-api/src/tickers.rs b/magicblock-api/src/tickers.rs index 72116af55..0825a58a3 100644 --- a/magicblock-api/src/tickers.rs +++ b/magicblock-api/src/tickers.rs @@ -1,21 +1,17 @@ use std::{ sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{ Ordering}, Arc, }, time::Duration, }; -use magicblock_accounts::ScheduledCommitsProcessor; -use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; +use magicblock_accounts_db::{AccountsDb}; use magicblock_core::link::transactions::{ - with_encoded, TransactionSchedulerHandle, + TransactionSchedulerHandle, }; -use magicblock_ledger::{LatestBlock, Ledger}; -use magicblock_magic_program_api as magic_program; +use magicblock_ledger::{Ledger}; use magicblock_metrics::metrics; -use magicblock_program::{instruction_utils::InstructionUtils, MagicContext}; -use solana_account::ReadableAccount; use tokio_util::sync::CancellationToken; use tracing::*; diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 84b56a32d..599f07c35 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -1,5 +1,5 @@ use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, + collections::{hash_map::Entry, HashMap}, path::Path, sync::{Arc, Mutex}, }; @@ -12,7 +12,6 @@ use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_signer::Signer; use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; use tracing::{error, info, instrument}; @@ -42,7 +41,6 @@ type BundleResultListener = oneshot::Sender; pub struct CommittorProcessor { table_mania: TableMania, - authority: Keypair, persister: IntentPersisterImpl, commits_scheduler: IntentExecutionManager, task_info_fetcher: Arc>, @@ -105,7 +103,6 @@ impl CommittorProcessor { )); Ok(Self { - authority, table_mania, commits_scheduler, persister, diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index 15535500f..87c653d4e 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,13 +1,8 @@ -use solana_pubkey::Pubkey; use solana_signature::Signature; -use solana_transaction_error::TransactionError; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::{ - intent_execution_manager::IntentExecutionManagerError, - intent_executor::task_info_fetcher::TaskInfoFetcherError, -}; +use crate::intent_execution_manager::IntentExecutionManagerError; pub type CommittorServiceResult = Result; @@ -16,36 +11,20 @@ pub enum CommittorServiceError { #[error("CommitPersistError: {0} ({0:?})")] CommitPersistError(#[from] crate::persist::error::CommitPersistError), - // #[error("MagicBlockRpcClientError: {0} ({0:?})")] - // MagicBlockRpcClientError( - // #[from] magicblock_rpc_client::MagicBlockRpcClientError, - // ), - - // #[error("TableManiaError: {0} ({0:?})")] - // TableManiaError(#[from] magicblock_table_mania::error::TableManiaError), #[error("IntentExecutionManagerError: {0} ({0:?})")] IntentExecutionManagerError(#[from] IntentExecutionManagerError), #[error("RecvError: {0}")] IntentResultRecvError(#[from] RecvError), - // #[error("TaskInfoFetcherError: {0} ({0:?})")] - // TaskInfoFetcherError(#[from] TaskInfoFetcherError), - - // #[error("Task {0} failed to compile transaction message: {1} ({1:?})")] - // FailedToCompileTransactionMessage(String, solana_message::CompileError), - - // #[error("Task {0} failed to create transaction: {1} ({1:?})")] - // FailedToCreateTransaction(String, solana_signer::SignerError), #[error("Attempt to schedule already scheduled message id: {0}")] RepeatingMessageError(u64), } +// TODO(edwin): use or remove impl CommittorServiceError { pub fn signature(&self) -> Option { - use CommittorServiceError::*; match self { - // MagicBlockRpcClientError(e) => e.signature(), _ => None, } } diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index b16c76b58..8fdc819a5 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -7,8 +7,6 @@ pub mod intent_execution_manager; pub mod intent_executor; pub mod persist; mod pubkeys_provider; -mod service; -pub mod service_ext; #[cfg(feature = "dev-context-only-utils")] pub mod stubs; pub mod tasks; @@ -25,4 +23,3 @@ pub use config::DEFAULT_ACTIONS_TIMEOUT; pub use magicblock_committor_program::{ ChangedAccount, Changeset, ChangesetMeta, }; -// pub use service::{BaseIntentCommittor, CommittorService}; diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs deleted file mode 100644 index 0c702ea29..000000000 --- a/magicblock-committor-service/src/service.rs +++ /dev/null @@ -1,570 +0,0 @@ -// use std::{collections::HashMap, path::Path, sync::Arc, time::Instant}; -// -// use magicblock_core::traits::ActionsCallbackScheduler; -// use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; -// use solana_keypair::Keypair; -// use solana_pubkey::Pubkey; -// use solana_signature::Signature; -// use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta; -// use tokio::{ -// select, -// sync::{ -// broadcast, -// mpsc::{self, error::TrySendError}, -// oneshot, -// }, -// }; -// use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; -// use tracing::*; -// -// use crate::{ -// committor_processor::CommittorProcessor, -// config::ChainConfig, -// error::{CommittorServiceError, CommittorServiceResult}, -// intent_execution_manager::BroadcastedIntentExecutionResult, -// persist::{CommitStatusRow, MessageSignatures}, -// pubkeys_provider::{provide_committee_pubkeys, provide_common_pubkeys}, -// }; -// -// #[derive(Debug)] -// pub struct LookupTables { -// pub active: Vec, -// pub released: Vec, -// } -// -// #[derive(Debug)] -// pub enum CommittorMessage { -// ReservePubkeysForCommittee { -// /// When the request was initiated -// initiated: Instant, -// /// Called once the pubkeys have been reserved and includes that timestamp -// /// at which the request was initiated -// respond_to: oneshot::Sender>, -// /// The committee whose pubkeys to reserve in a lookup table -// /// These pubkeys are used to process/finalize the commit -// committee: Pubkey, -// /// The owner program of the committee -// owner: Pubkey, -// }, -// ReserveCommonPubkeys { -// /// Called once the pubkeys have been reserved -// respond_to: oneshot::Sender>, -// }, -// ReleaseCommonPubkeys { -// /// Called once the pubkeys have been released -// respond_to: oneshot::Sender<()>, -// }, -// ScheduleIntentBundle { -// /// The [`ScheduleIntentBundle`]s to commit -// intent_bundles: Vec, -// respond_to: oneshot::Sender>, -// }, -// GetCommitStatuses { -// respond_to: -// oneshot::Sender>>, -// message_id: u64, -// }, -// GetCommitSignatures { -// respond_to: -// oneshot::Sender>>, -// commit_id: u64, -// pubkey: Pubkey, -// }, -// GetLookupTables { -// respond_to: oneshot::Sender, -// }, -// GetTransaction { -// respond_to: oneshot::Sender< -// CommittorServiceResult, -// >, -// signature: Signature, -// }, -// SubscribeForResults { -// respond_to: oneshot::Sender< -// broadcast::Receiver, -// >, -// }, -// FetchCurrentCommitNonces { -// respond_to: -// oneshot::Sender>>, -// pubkeys: Vec, -// min_context_slot: u64, -// }, -// FetchCurrentCommitNoncesSync { -// respond_to: std::sync::mpsc::Sender< -// CommittorServiceResult>, -// >, -// pubkeys: Vec, -// min_context_slot: u64, -// }, -// } -// -// // ----------------- -// // CommittorActor -// // ----------------- -// struct CommittorActor { -// receiver: mpsc::Receiver, -// processor: Arc, -// } -// -// impl CommittorActor { -// pub fn try_new( -// receiver: mpsc::Receiver, -// authority: Keypair, -// persist_file: P, -// chain_config: ChainConfig, -// actions_callback_executor: A, -// ) -> CommittorServiceResult -// where -// P: AsRef, -// A: ActionsCallbackScheduler, -// { -// let processor = Arc::new(CommittorProcessor::try_new( -// authority, -// persist_file, -// chain_config, -// actions_callback_executor, -// )?); -// -// Ok(Self { -// receiver, -// processor, -// }) -// } -// -// #[instrument(skip(self))] -// async fn handle_msg(&self, msg: CommittorMessage) { -// use CommittorMessage::*; -// match msg { -// ReservePubkeysForCommittee { -// initiated, -// respond_to, -// committee, -// owner, -// } => { -// let processor = self.processor.clone(); -// tokio::task::spawn(async move { -// let pubkeys = -// provide_committee_pubkeys(&committee, Some(&owner)); -// // NOTE: we wait here until the reservation is done which causes the -// // cloning of a particular account to be blocked. -// // This leads to larger delays on the first clone of an account, but also -// // ensures that the account could be committed via a lookup table later. -// let result = processor -// .reserve_pubkeys(pubkeys) -// .await -// .map(|_| initiated); -// if let Err(e) = respond_to.send(result) { -// error!(message_type = "ReservePubkeysForCommittee", error = ?e, "Failed to send response"); -// } -// }); -// } -// ReserveCommonPubkeys { respond_to } => { -// let processor = self.processor.clone(); -// tokio::task::spawn(async move { -// let pubkeys = -// provide_common_pubkeys(&processor.auth_pubkey()); -// let reqid = processor.reserve_pubkeys(pubkeys).await; -// if let Err(e) = respond_to.send(reqid) { -// error!(message_type = "ReserveCommonPubkeys", error = ?e, "Failed to send response"); -// } -// }); -// } -// ReleaseCommonPubkeys { respond_to } => { -// let processor = self.processor.clone(); -// tokio::task::spawn(async move { -// let pubkeys = -// provide_common_pubkeys(&processor.auth_pubkey()); -// processor.release_pubkeys(pubkeys).await; -// if let Err(e) = respond_to.send(()) { -// error!(message_type = "ReleaseCommonPubkeys", error = ?e, "Failed to send response"); -// } -// }); -// } -// ScheduleIntentBundle { -// intent_bundles, -// respond_to, -// } => { -// let result = -// self.processor.schedule_intent_bundles(intent_bundles).await; -// if let Err(e) = respond_to.send(result) { -// error!(message_type = "ScheduleBaseIntents", error = ?e, "Failed to send response"); -// } -// } -// GetCommitStatuses { -// message_id, -// respond_to, -// } => { -// let commit_statuses = -// self.processor.get_commit_statuses(message_id); -// if let Err(e) = respond_to.send(commit_statuses) { -// error!(message_type = "GetCommitStatuses", error = ?e, "Failed to send response"); -// } -// } -// GetCommitSignatures { -// commit_id, -// respond_to, -// pubkey, -// } => { -// let sig = -// self.processor.get_commit_signature(commit_id, pubkey); -// if let Err(e) = respond_to.send(sig) { -// error!(message_type = "GetCommitSignatures", error = ?e, "Failed to send response"); -// } -// } -// GetTransaction { -// signature, -// respond_to, -// } => { -// let processor = self.processor.clone(); -// tokio::task::spawn(async move { -// let res = processor -// .magicblock_rpc_client -// .get_transaction(&signature, None) -// .await -// .map_err(Into::into); -// if let Err(err) = respond_to.send(res) { -// error!(message_type = "GetTransaction", error = ?err, "Failed to send response"); -// } -// }); -// } -// GetLookupTables { respond_to } => { -// let active_tables = self.processor.active_lookup_tables().await; -// let released_tables = -// self.processor.released_lookup_tables().await; -// if let Err(e) = respond_to.send(LookupTables { -// active: active_tables, -// released: released_tables, -// }) { -// error!(message_type = "GetLookupTables", error = ?e, "Failed to send response"); -// } -// } -// SubscribeForResults { respond_to } => { -// let subscription = self.processor.subscribe_for_results(); -// if let Err(err) = respond_to.send(subscription) { -// error!(message_type = "SubscribeForResults", error = ?err, "Failed to send response"); -// } -// } -// FetchCurrentCommitNonces { -// respond_to, -// pubkeys, -// min_context_slot, -// } => { -// let processor = self.processor.clone(); -// tokio::spawn(async move { -// let result = processor -// .fetch_current_commit_nonces(&pubkeys, min_context_slot) -// .await; -// if let Err(err) = respond_to -// .send(result.map_err(CommittorServiceError::from)) -// { -// error!(message_type = "FetchCurrentCommitNonces", error = ?err, "Failed to send response"); -// } -// }); -// } -// FetchCurrentCommitNoncesSync { -// respond_to, -// pubkeys, -// min_context_slot, -// } => { -// let processor = self.processor.clone(); -// tokio::spawn(async move { -// let result = processor -// .fetch_current_commit_nonces(&pubkeys, min_context_slot) -// .await; -// if let Err(err) = respond_to -// .send(result.map_err(CommittorServiceError::from)) -// { -// error!(message_type = "FetchCurrentCommitNoncesSync", error = ?err, "Failed to send response"); -// } -// }); -// } -// } -// } -// -// #[instrument(skip(self, cancel_token))] -// pub async fn run(&mut self, cancel_token: CancellationToken) { -// loop { -// select! { -// msg = self.receiver.recv() => { -// if let Some(msg) = msg { -// self.handle_msg(msg).await; -// } else { -// break; -// } -// } -// _ = cancel_token.cancelled() => { -// break; -// } -// } -// } -// -// info!("Actor shutdown"); -// } -// } -// -// // ----------------- -// // CommittorService -// // ----------------- -// pub struct CommittorService { -// sender: mpsc::Sender, -// cancel_token: CancellationToken, -// } -// -// impl CommittorService { -// pub fn try_start( -// authority: Keypair, -// persist_file: P, -// chain_config: ChainConfig, -// actions_callback_executor: A, -// ) -> CommittorServiceResult -// where -// P: AsRef, -// A: ActionsCallbackScheduler, -// { -// debug!("Starting committor service"); -// let (sender, receiver) = mpsc::channel(1_000); -// let cancel_token = CancellationToken::new(); -// { -// let cancel_token = cancel_token.clone(); -// let mut actor = CommittorActor::try_new( -// receiver, -// authority, -// persist_file, -// chain_config, -// actions_callback_executor, -// )?; -// tokio::spawn(async move { -// actor.run(cancel_token).await; -// }); -// } -// Ok(Self { -// sender, -// cancel_token, -// }) -// } -// -// pub fn reserve_common_pubkeys( -// &self, -// ) -> oneshot::Receiver> { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::ReserveCommonPubkeys { -// respond_to: tx, -// }); -// rx -// } -// -// pub fn release_common_pubkeys(&self) -> oneshot::Receiver<()> { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::ReleaseCommonPubkeys { -// respond_to: tx, -// }); -// rx -// } -// -// pub fn get_commit_signatures( -// &self, -// commit_id: u64, -// pubkey: Pubkey, -// ) -> oneshot::Receiver>> -// { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::GetCommitSignatures { -// respond_to: tx, -// commit_id, -// pubkey, -// }); -// rx -// } -// -// pub fn get_lookup_tables(&self) -> oneshot::Receiver { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::GetLookupTables { respond_to: tx }); -// rx -// } -// -// pub fn fetch_current_commit_nonces_sync( -// &self, -// pubkeys: &[Pubkey], -// min_context_slot: u64, -// ) -> std::sync::mpsc::Receiver>> -// { -// let (tx, rx) = std::sync::mpsc::channel(); -// self.try_send(CommittorMessage::FetchCurrentCommitNoncesSync { -// respond_to: tx, -// pubkeys: pubkeys.to_vec(), -// min_context_slot, -// }); -// rx -// } -// -// fn try_send(&self, msg: CommittorMessage) { -// if let Err(e) = self.sender.try_send(msg) { -// match e { -// TrySendError::Full(msg) => error!( -// "Channel full, failed to send commit message {:?}", -// msg -// ), -// TrySendError::Closed(msg) => error!( -// "Channel closed, failed to send commit message {:?}", -// msg -// ), -// } -// } -// } -// } -// -// impl BaseIntentCommittor for CommittorService { -// fn reserve_pubkeys_for_committee( -// &self, -// committee: Pubkey, -// owner: Pubkey, -// ) -> oneshot::Receiver> { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::ReservePubkeysForCommittee { -// initiated: Instant::now(), -// respond_to: tx, -// committee, -// owner, -// }); -// rx -// } -// -// fn schedule_intent_bundles( -// &self, -// intent_bundles: Vec, -// ) -> oneshot::Receiver> { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::ScheduleIntentBundle { -// intent_bundles, -// respond_to: tx, -// }); -// rx -// } -// -// fn get_commit_statuses( -// &self, -// message_id: u64, -// ) -> oneshot::Receiver>> { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::GetCommitStatuses { -// respond_to: tx, -// message_id, -// }); -// rx -// } -// -// fn get_commit_signatures( -// &self, -// commit_id: u64, -// pubkey: Pubkey, -// ) -> oneshot::Receiver>> -// { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::GetCommitSignatures { -// respond_to: tx, -// commit_id, -// pubkey, -// }); -// rx -// } -// -// fn subscribe_for_results( -// &self, -// ) -> oneshot::Receiver> -// { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::SubscribeForResults { respond_to: tx }); -// rx -// } -// -// fn get_transaction( -// &self, -// signature: &Signature, -// ) -> oneshot::Receiver< -// CommittorServiceResult, -// > { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::GetTransaction { -// respond_to: tx, -// signature: *signature, -// }); -// -// rx -// } -// -// fn fetch_current_commit_nonces( -// &self, -// pubkeys: &[Pubkey], -// min_context_slot: u64, -// ) -> oneshot::Receiver>> { -// let (tx, rx) = oneshot::channel(); -// self.try_send(CommittorMessage::FetchCurrentCommitNonces { -// respond_to: tx, -// pubkeys: pubkeys.to_vec(), -// min_context_slot, -// }); -// -// rx -// } -// -// fn stop(&self) { -// self.cancel_token.cancel(); -// } -// -// fn stopped(&self) -> WaitForCancellationFutureOwned { -// self.cancel_token.clone().cancelled_owned() -// } -// } -// -// pub trait BaseIntentCommittor: Send + Sync + 'static { -// /// Reserves pubkeys used in most commits in a lookup table -// fn reserve_pubkeys_for_committee( -// &self, -// committee: Pubkey, -// owner: Pubkey, -// ) -> oneshot::Receiver>; -// -// /// Commits the changeset and returns -// fn schedule_intent_bundles( -// &self, -// intent_bundles: Vec, -// ) -> oneshot::Receiver>; -// -// /// Subscribes for results of BaseIntent execution -// fn subscribe_for_results( -// &self, -// ) -> oneshot::Receiver>; -// -// /// Gets statuses of accounts that were committed as part of a request with provided message_id -// fn get_commit_statuses( -// &self, -// message_id: u64, -// ) -> oneshot::Receiver>>; -// -// /// Gets signatures for commit of particular accounts -// fn get_commit_signatures( -// &self, -// commit_id: u64, -// pubkey: Pubkey, -// ) -> oneshot::Receiver>>; -// -// fn get_transaction( -// &self, -// signature: &Signature, -// ) -> oneshot::Receiver< -// CommittorServiceResult, -// >; -// -// fn fetch_current_commit_nonces( -// &self, -// pubkeys: &[Pubkey], -// min_context_slot: u64, -// ) -> oneshot::Receiver>>; -// -// /// Stops Committor service -// fn stop(&self); -// -// /// Returns future which resolves once committor `stop` got called -// fn stopped(&self) -> WaitForCancellationFutureOwned; -// } diff --git a/magicblock-committor-service/src/service_ext.rs b/magicblock-committor-service/src/service_ext.rs deleted file mode 100644 index d3c0187d4..000000000 --- a/magicblock-committor-service/src/service_ext.rs +++ /dev/null @@ -1,253 +0,0 @@ -// use std::{ -// collections::{hash_map::Entry, HashMap}, -// ops::Deref, -// sync::{Arc, Mutex}, -// time::Instant, -// }; -// -// use async_trait::async_trait; -// use futures_util::future::join_all; -// use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; -// use solana_pubkey::Pubkey; -// use solana_signature::Signature; -// use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta; -// use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; -// use tokio_util::sync::WaitForCancellationFutureOwned; -// use tracing::{error, info, instrument}; -// -// use crate::{ -// error::{CommittorServiceError, CommittorServiceResult}, -// intent_execution_manager::BroadcastedIntentExecutionResult, -// persist::{CommitStatusRow, MessageSignatures}, -// BaseIntentCommittor, -// }; -// -// const POISONED_MUTEX_MSG: &str = -// "CommittorServiceExt pending messages mutex poisoned!"; -// -// #[async_trait] -// pub trait BaseIntentCommittorExt: BaseIntentCommittor { -// /// Schedules Base Intents and waits for their results -// async fn schedule_intent_bundles_waiting( -// &self, -// intent_bundles: Vec, -// ) -> BaseIntentCommitorExtResult>; -// } -// -// type MessageResultListener = oneshot::Sender; -// pub struct CommittorServiceExt { -// inner: Arc, -// pending_messages: Arc>>, -// } -// -// impl CommittorServiceExt { -// pub fn new(inner: Arc) -> Self { -// let pending_messages = Arc::new(Mutex::new(HashMap::new())); -// let results_subscription = inner.subscribe_for_results(); -// let committor_stopped = inner.stopped(); -// tokio::spawn(Self::dispatcher( -// committor_stopped, -// results_subscription, -// pending_messages.clone(), -// )); -// -// Self { -// inner, -// pending_messages, -// } -// } -// -// #[instrument(skip( -// committor_stopped, -// pending_message, -// results_subscription -// ))] -// async fn dispatcher( -// committor_stopped: WaitForCancellationFutureOwned, -// results_subscription: oneshot::Receiver< -// broadcast::Receiver, -// >, -// pending_message: Arc>>, -// ) { -// let mut results_subscription = results_subscription.await.unwrap(); -// -// tokio::pin!(committor_stopped); -// loop { -// let execution_result = tokio::select! { -// biased; -// _ = &mut committor_stopped => { -// info!("Shutting down extension"); -// return; -// } -// execution_result = results_subscription.recv() => { -// match execution_result { -// Ok(result) => result, -// Err(broadcast::error::RecvError::Closed) => { -// info!("Intent execution shutdown"); -// break; -// } -// Err(broadcast::error::RecvError::Lagged(skipped)) => { -// // SAFETY: not really feasible to happen as this function is way faster than Intent execution -// // requires investigation if ever happens! -// error!(skipped, "Dispatcher lag detected"); -// continue; -// } -// } -// } -// }; -// -// let sender = if let Some(sender) = pending_message -// .lock() -// .expect(POISONED_MUTEX_MSG) -// .remove(&execution_result.id) -// { -// sender -// } else { -// continue; -// }; -// -// if let Err(execution_result) = sender.send(execution_result) { -// error!( -// intent_id = execution_result.id, -// "Failed to send execution result" -// ); -// } -// } -// } -// } -// -// #[async_trait] -// impl BaseIntentCommittorExt -// for CommittorServiceExt -// { -// async fn schedule_intent_bundles_waiting( -// &self, -// base_intents: Vec, -// ) -> BaseIntentCommitorExtResult> -// { -// // Critical section -// let receivers = { -// let mut pending_messages = -// self.pending_messages.lock().expect(POISONED_MUTEX_MSG); -// -// base_intents -// .iter() -// .map(|intent| { -// let (sender, receiver) = oneshot::channel(); -// match pending_messages.entry(intent.id) { -// Entry::Vacant(vacant) => { -// vacant.insert(sender); -// Ok(receiver) -// } -// Entry::Occupied(_) => Err( -// CommittorServiceExtError::RepeatingMessageError( -// intent.id, -// ), -// ), -// } -// }) -// .collect::, _>>()? -// }; -// -// self.schedule_intent_bundles(base_intents).await??; -// let results = join_all(receivers.into_iter()) -// .await -// .into_iter() -// .collect::, RecvError>>()?; -// -// Ok(results) -// } -// } -// -// impl BaseIntentCommittor for CommittorServiceExt { -// fn reserve_pubkeys_for_committee( -// &self, -// committee: Pubkey, -// owner: Pubkey, -// ) -> oneshot::Receiver> { -// self.inner.reserve_pubkeys_for_committee(committee, owner) -// } -// -// fn schedule_intent_bundles( -// &self, -// intent_bundles: Vec, -// ) -> oneshot::Receiver> { -// self.inner.schedule_intent_bundles(intent_bundles) -// } -// -// fn subscribe_for_results( -// &self, -// ) -> oneshot::Receiver> -// { -// self.inner.subscribe_for_results() -// } -// -// fn get_commit_statuses( -// &self, -// message_id: u64, -// ) -> oneshot::Receiver>> { -// self.inner.get_commit_statuses(message_id) -// } -// -// fn get_commit_signatures( -// &self, -// commit_id: u64, -// pubkey: Pubkey, -// ) -> oneshot::Receiver>> -// { -// self.inner.get_commit_signatures(commit_id, pubkey) -// } -// -// fn get_transaction( -// &self, -// signature: &Signature, -// ) -> oneshot::Receiver< -// CommittorServiceResult, -// > { -// self.inner.get_transaction(signature) -// } -// -// fn fetch_current_commit_nonces( -// &self, -// pubkeys: &[Pubkey], -// min_context_slot: u64, -// ) -> oneshot::Receiver>> { -// self.inner -// .fetch_current_commit_nonces(pubkeys, min_context_slot) -// } -// -// fn stop(&self) { -// self.inner.stop(); -// } -// -// fn stopped(&self) -> WaitForCancellationFutureOwned { -// self.inner.stopped() -// } -// } -// -// impl Deref for CommittorServiceExt { -// type Target = Arc; -// -// fn deref(&self) -> &Self::Target { -// &self.inner -// } -// } -// -// #[derive(thiserror::Error, Debug)] -// pub enum CommittorServiceExtError { -// #[error("Attempt to schedule already scheduled message id: {0}")] -// RepeatingMessageError(u64), -// #[error("RecvError: {0}")] -// RecvError(#[from] RecvError), -// #[error("CommittorServiceError: {0:?}")] -// CommittorServiceError(Box), -// } -// -// impl From for CommittorServiceExtError { -// fn from(e: CommittorServiceError) -> Self { -// Self::CommittorServiceError(Box::new(e)) -// } -// } -// -// pub type BaseIntentCommitorExtResult = -// Result; diff --git a/magicblock-committor-service/src/service_final.rs b/magicblock-committor-service/src/service_final.rs index 375ca24af..b830ce93f 100644 --- a/magicblock-committor-service/src/service_final.rs +++ b/magicblock-committor-service/src/service_final.rs @@ -1,14 +1,11 @@ use std::{ collections::{HashMap, HashSet}, - iter::chain, mem, - path::Path, sync::{Arc, Mutex}, time::Duration, }; use async_trait::async_trait; -use base64::encode; use magicblock_account_cloner::ChainlinkCloner; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_chainlink::{ @@ -21,7 +18,7 @@ use magicblock_chainlink::{ }; use magicblock_core::{ link::transactions::{with_encoded, TransactionSchedulerHandle}, - traits::{ActionsCallbackScheduler, LatestBlockProvider}, + traits::LatestBlockProvider, }; use magicblock_metrics::metrics; use magicblock_program::{ @@ -32,7 +29,6 @@ use magicblock_program::{ }; use solana_account::ReadableAccount; use solana_hash::Hash; -use solana_keypair::Keypair; use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tokio::{ @@ -44,8 +40,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument}; use crate::{ - committor_processor::CommittorProcessor, config::ChainConfig, - error::CommittorServiceError, + committor_processor::CommittorProcessor, error::CommittorServiceError, intent_execution_manager::BroadcastedIntentExecutionResult, intent_executor::ExecutionOutput, }; @@ -107,7 +102,7 @@ impl InternalIntentRpcClient { self.latest_block_provider.blockhash(), ); let encoded_tx = with_encoded(tx).inspect_err(|err| { - error!("Failed to bincode intent transaction"); + error!(error = ?err, "Failed to bincode intent transaction"); })?; self.transaction_scheduler .execute(encoded_tx) @@ -150,7 +145,7 @@ impl IntentRpcClient for InternalIntentRpcClient { register_scheduled_commit_sent(sent_commit); let txn = with_encoded(sent_tx).inspect_err(|err| { // Unreachable case, all intent transactions are smaller than 64KB by construction - error!("Failed to bincode intent transaction"); + error!(error = ?err, "Failed to bincode intent transaction"); })?; self.transaction_scheduler .execute(txn) @@ -443,7 +438,7 @@ impl ServiceInner { }; let sent_transaction = - std::mem::take(&mut intent_meta.intent_sent_transaction); + mem::take(&mut intent_meta.intent_sent_transaction); let sent_commit = ServiceInner::::build_sent_commit( intent_id, intent_meta, From 4931a62e26d3d0798b988e7133ecc23c8de89e47 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 15:13:06 +0700 Subject: [PATCH 05/97] feat: integration + fixes --- magicblock-api/src/errors.rs | 4 + magicblock-api/src/magic_sys_adapter.rs | 43 +++++++- magicblock-api/src/magic_validator.rs | 9 +- magicblock-api/src/tickers.rs | 15 +-- .../src/service_final.rs | 98 ++++++++----------- 5 files changed, 89 insertions(+), 80 deletions(-) diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 0c7e9a6ce..21af35c23 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -1,4 +1,5 @@ use magicblock_accounts_db::error::AccountsDbError; +use magicblock_committor_service::service_final::IntentExecutionServiceError; use solana_pubkey::Pubkey; use thiserror::Error; @@ -38,6 +39,9 @@ pub enum ApiError { Box, ), + #[error("IntentExecutionServiceError: {0}")] + IntentExecutionServiceError(#[from] IntentExecutionServiceError), + #[error("Failed to load programs into bank: {0}")] FailedToLoadProgramsIntoBank(String), diff --git a/magicblock-api/src/magic_sys_adapter.rs b/magicblock-api/src/magic_sys_adapter.rs index daa67f6ab..3347d4dfa 100644 --- a/magicblock-api/src/magic_sys_adapter.rs +++ b/magicblock-api/src/magic_sys_adapter.rs @@ -1,6 +1,9 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; -use magicblock_committor_service::committor_processor::CommittorProcessor; +use magicblock_committor_service::{ + committor_processor::CommittorProcessor, + intent_executor::task_info_fetcher::TaskInfoFetcherResult, +}; use magicblock_core::{intent::CommittedAccount, traits::MagicSys}; use magicblock_metrics::metrics; use solana_instruction::error::InstructionError; @@ -9,6 +12,7 @@ use tracing::error; #[derive(Clone)] pub struct MagicSysAdapter { + handle: tokio::runtime::Handle, committor_processor: Arc, } @@ -22,11 +26,41 @@ impl MagicSysAdapter { const FETCH_TIMEOUT: Duration = Duration::from_secs(30); - pub fn new(committor_processor: Arc) -> Self { + pub fn new( + handle: tokio::runtime::Handle, + committor_processor: Arc, + ) -> Self { Self { + handle, committor_processor, } } + + fn fetch_current_commit_nonces_sync( + &self, + pubkeys: &[Pubkey], + min_context_slot: u64, + ) -> std::sync::mpsc::Receiver>> + { + let (sender, receiver) = std::sync::mpsc::channel(); + let committor_processor = self.committor_processor.clone(); + let pubkeys = pubkeys.to_owned(); + + // This is required to switch from TransactionExecutor runtime + // blocking on it would cause a panic + let _guard = self.handle.enter(); + tokio::spawn(async move { + let result = committor_processor + .fetch_current_commit_nonces(&pubkeys, min_context_slot) + .await; + if let Err(err) = sender.send(result) { + error!(error = ?err, "Failed to send result back"); + } + }); + drop(_guard); + + receiver + } } impl MagicSys for MagicSysAdapter { @@ -47,9 +81,8 @@ impl MagicSys for MagicSysAdapter { commits.iter().map(|account| account.pubkey).collect(); let _timer = metrics::start_fetch_commit_nonces_wait_timer(); - let receiver = self - .committor_processor - .fetch_current_commit_nonces_sync(&pubkeys, min_context_slot); + let receiver = + self.fetch_current_commit_nonces_sync(&pubkeys, min_context_slot); receiver .recv_timeout(Self::FETCH_TIMEOUT) .map_err(|err| match err { diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index d5c9ef70f..2cf2ca1db 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -9,7 +9,6 @@ use std::{ }; use magicblock_account_cloner::ChainlinkCloner; -use magicblock_accounts::ScheduledCommitsProcessor; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_aperture::{ initialize_aperture, @@ -27,10 +26,7 @@ use magicblock_chainlink::{ use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, - service_final::{ - IntentExecutionService, - InternalIntentRpcClient, - }, + service_final::{IntentExecutionService, InternalIntentRpcClient}, ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, }; use magicblock_config::{ @@ -254,6 +250,7 @@ impl MagicValidator { ); log_timing("startup", "committor_service_init", step_start); init_magic_sys(Arc::new(MagicSysAdapter::new( + tokio::runtime::Handle::current(), committor_processor.clone(), ))); @@ -935,7 +932,7 @@ impl MagicValidator { } let step_start = Instant::now(); - self.intent_execution_service.start(); + self.intent_execution_service.start()?; log_timing("startup", "intent_execution_service_start", step_start); let step_start = Instant::now(); diff --git a/magicblock-api/src/tickers.rs b/magicblock-api/src/tickers.rs index 0825a58a3..c011e3f80 100644 --- a/magicblock-api/src/tickers.rs +++ b/magicblock-api/src/tickers.rs @@ -1,16 +1,7 @@ -use std::{ - sync::{ - atomic::{ Ordering}, - Arc, - }, - time::Duration, -}; +use std::{sync::Arc, time::Duration}; -use magicblock_accounts_db::{AccountsDb}; -use magicblock_core::link::transactions::{ - TransactionSchedulerHandle, -}; -use magicblock_ledger::{Ledger}; +use magicblock_accounts_db::AccountsDb; +use magicblock_ledger::Ledger; use magicblock_metrics::metrics; use tokio_util::sync::CancellationToken; use tracing::*; diff --git a/magicblock-committor-service/src/service_final.rs b/magicblock-committor-service/src/service_final.rs index b830ce93f..dbcd0f2ce 100644 --- a/magicblock-committor-service/src/service_final.rs +++ b/magicblock-committor-service/src/service_final.rs @@ -45,9 +45,7 @@ use crate::{ intent_executor::ExecutionOutput, }; -// TODO(edwin): adapt -const POISONED_MUTEX_MSG: &str = - "Mutex of RemoteScheduledCommitsProcessor.intents_meta_map is poisoned"; +const POISONED_MUTEX_MSG: &str = "ServiceInner intents_meta_map mutex poisoned"; pub type ChainlinkImpl = Chainlink< ChainRpcClientImpl, @@ -161,8 +159,7 @@ impl IntentRpcClient for InternalIntentRpcClient { #[derive(thiserror::Error, Debug)] pub enum InternalRpcClientError { - // TODO(edwin) - #[error("asd")] + #[error("TransactionError: {0}")] TransactionError(#[from] TransactionError), } @@ -173,7 +170,11 @@ pub enum IntentExecutionService { Error, } -impl IntentExecutionService { +impl IntentExecutionService +where + R: IntentRpcClient, + R::Error: Into, +{ pub fn new( chainlink: Arc, intent_rpc_client: R, @@ -194,20 +195,23 @@ impl IntentExecutionService { mem::replace(self, Self::Error) } - pub fn start(&mut self) { + pub fn start(&mut self) -> Result<(), IntentExecutionServiceError> { let Self::Created(service) = self.take() else { - // TODO(edwin): err InvalidState(String) - return; + return Err(IntentExecutionServiceError::InvalidState( + "service must be in Created state to start".into(), + )); }; let handle = service.start(); *self = Self::Started(handle); + Ok(()) } - pub async fn stop(&mut self) -> Result<(), JoinError> { + pub async fn stop(&mut self) -> Result<(), IntentExecutionServiceError> { let Self::Started(handle) = self.take() else { - // TODO(edwin): err InvalidState(String) - return Ok(()); + return Err(IntentExecutionServiceError::InvalidState( + "service must be in Started state to stop".into(), + )); }; handle.await?; @@ -231,7 +235,11 @@ pub struct ServiceInner { intents_meta_map: Arc>>, } -impl ServiceInner { +impl ServiceInner +where + R: IntentRpcClient, + R::Error: Into, +{ pub fn new( chainlink: Arc, intent_rpc_client: R, @@ -250,20 +258,19 @@ impl ServiceInner { } pub fn start(self) -> JoinHandle<()> { - tokio::task::spawn(self.run()) - } - - async fn run(self) { - // TODO(edwin): move to start and make ExecutionHandle{main, result_worker} let result_subscriber = self.processor.subscribe_for_results(); let cancellation_token = self.cancellation_token.clone(); - let handle = tokio::spawn(Self::result_processor( + tokio::spawn(Self::result_processor( result_subscriber, cancellation_token, self.intents_meta_map.clone(), self.intent_rpc_client.clone(), )); + tokio::task::spawn(self.accept_worker()) + } + + async fn accept_worker(self) { let mut interval = tokio::time::interval(self.slot_interval); loop { tokio::select! { @@ -380,9 +387,6 @@ impl ServiceInner { intents_meta_map: Arc>>, intent_client: Arc, ) { - const SUBSCRIPTION_ERR_MSG: &str = - "Failed to get subscription of results of BaseIntents execution"; - loop { let execution_result = tokio::select! { biased; @@ -408,12 +412,15 @@ impl ServiceInner { } }; - ServiceInner::::process_execution_result( + if let Err(err) = ServiceInner::::process_execution_result( &intent_client, execution_result, &intents_meta_map, ) - .await; + .await + { + error!(error = ?err, "Failed process intent execution results"); + } } } @@ -444,44 +451,14 @@ impl ServiceInner { intent_meta, execution_result, ); - // TODO(edwin): convert intent_client .finalize_intent(sent_transaction, sent_commit) - .await; + .await + .map_err(Into::into)?; Ok(()) } - #[instrument( - skip(internal_transaction_scheduler, result, intent_meta), - fields(intent_id) - )] - async fn process_intent_result( - intent_id: u64, - internal_transaction_scheduler: &TransactionSchedulerHandle, - result: BroadcastedIntentExecutionResult, - mut intent_meta: ScheduledBaseIntentMeta, - ) { - let intent_sent_transaction = - std::mem::take(&mut intent_meta.intent_sent_transaction); - let sent_commit = - Self::build_sent_commit(intent_id, intent_meta, result); - register_scheduled_commit_sent(sent_commit); - let Ok(txn) = with_encoded(intent_sent_transaction) else { - // Unreachable case, all intent transactions are smaller than 64KB by construction - error!("Failed to bincode intent transaction"); - return; - }; - match internal_transaction_scheduler.execute(txn).await { - Ok(()) => { - debug!("Sent commit signaled") - } - Err(err) => { - error!(error = ?err, "Failed to signal sent commit"); - } - } - } - fn build_sent_commit( intent_id: u64, intent_meta: ScheduledBaseIntentMeta, @@ -576,4 +553,11 @@ impl ScheduledBaseIntentMeta { } #[derive(thiserror::Error, Debug)] -pub enum IntentExecutionServiceError {} +pub enum IntentExecutionServiceError { + #[error("Invalid state: {0}")] + InvalidState(String), + #[error("JoinError: {0}")] + JoinError(#[from] JoinError), + #[error("IntentRpcClientError: {0}")] + IntentRpcClientError(#[from] InternalRpcClientError), +} From 0fbb24a57cb47b3790ddddfd84884605077a5b0c Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 15:17:33 +0700 Subject: [PATCH 06/97] refactor: remove unused file --- Cargo.lock | 10 - magicblock-accounts/Cargo.toml | 16 - magicblock-accounts/src/lib.rs | 1 - .../src/scheduled_commits_processor.rs | 362 ------------------ .../src/service_final.rs | 4 - test-integration/Cargo.lock | 10 - 6 files changed, 403 deletions(-) delete mode 100644 magicblock-accounts/src/scheduled_commits_processor.rs diff --git a/Cargo.lock b/Cargo.lock index b28916536..63b57b609 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3690,21 +3690,11 @@ name = "magicblock-accounts" version = "0.11.1" dependencies = [ "async-trait", - "magicblock-account-cloner", - "magicblock-accounts-db", - "magicblock-chainlink", "magicblock-committor-service", - "magicblock-core", - "magicblock-metrics", - "magicblock-program", - "solana-hash 3.1.0", "solana-pubkey 3.0.0", - "solana-transaction", "solana-transaction-error 3.2.0", "thiserror 2.0.18", "tokio", - "tokio-util", - "tracing", "url", ] diff --git a/magicblock-accounts/Cargo.toml b/magicblock-accounts/Cargo.toml index aaf0879d4..5b48785a5 100644 --- a/magicblock-accounts/Cargo.toml +++ b/magicblock-accounts/Cargo.toml @@ -9,26 +9,10 @@ edition.workspace = true [dependencies] async-trait = { workspace = true } -tracing = { workspace = true } -magicblock-account-cloner = { workspace = true } -magicblock-accounts-db = { workspace = true } -magicblock-chainlink = { workspace = true } magicblock-committor-service = { workspace = true } -magicblock-core = { workspace = true } -magicblock-metrics = { workspace = true } -magicblock-program = { workspace = true } -solana-hash = { workspace = true } solana-pubkey = { workspace = true } solana-transaction-error = { workspace = true } -solana-transaction = { workspace = true } tokio = { workspace = true } -tokio-util = { workspace = true } thiserror = { workspace = true } url = { workspace = true } - -[dev-dependencies] -magicblock-committor-service = { workspace = true, features = [ - "dev-context-only-utils", -] } -tokio-util = { workspace = true } diff --git a/magicblock-accounts/src/lib.rs b/magicblock-accounts/src/lib.rs index d11a013b2..c7eac2118 100644 --- a/magicblock-accounts/src/lib.rs +++ b/magicblock-accounts/src/lib.rs @@ -1,6 +1,5 @@ mod config; pub mod errors; -pub mod scheduled_commits_processor; mod traits; pub use config::*; diff --git a/magicblock-accounts/src/scheduled_commits_processor.rs b/magicblock-accounts/src/scheduled_commits_processor.rs deleted file mode 100644 index 1703924d5..000000000 --- a/magicblock-accounts/src/scheduled_commits_processor.rs +++ /dev/null @@ -1,362 +0,0 @@ -// use std::{ -// collections::{HashMap, HashSet}, -// sync::{Arc, Mutex}, -// }; -// -// use async_trait::async_trait; -// use magicblock_account_cloner::ChainlinkCloner; -// use magicblock_accounts_db::AccountsDb; -// use magicblock_chainlink::{ -// remote_account_provider::{ -// chain_rpc_client::ChainRpcClientImpl, -// chain_updates_client::ChainUpdatesClient, -// }, -// submux::SubMuxClient, -// Chainlink, -// }; -// use magicblock_committor_service::{ -// intent_execution_manager::BroadcastedIntentExecutionResult, -// intent_executor::ExecutionOutput, BaseIntentCommittor, CommittorService, -// }; -// use magicblock_core::link::transactions::{ -// with_encoded, TransactionSchedulerHandle, -// }; -// use magicblock_metrics::metrics; -// use magicblock_program::{ -// magic_scheduled_base_intent::ScheduledIntentBundle, -// register_scheduled_commit_sent, SentCommit, TransactionScheduler, -// }; -// use solana_hash::Hash; -// use solana_pubkey::Pubkey; -// use solana_transaction::Transaction; -// use tokio::{ -// sync::{broadcast, oneshot}, -// task, -// }; -// use tokio_util::sync::CancellationToken; -// use tracing::{debug, error, info, instrument}; -// -// use crate::{ -// errors::ScheduledCommitsProcessorResult, ScheduledCommitsProcessor, -// }; -// -// const POISONED_MUTEX_MSG: &str = -// "Mutex of RemoteScheduledCommitsProcessor.intents_meta_map is poisoned"; -// -// pub type ChainlinkImpl = Chainlink< -// ChainRpcClientImpl, -// SubMuxClient, -// AccountsDb, -// ChainlinkCloner, -// >; -// -// pub struct ScheduledCommitsProcessorImpl { -// committor: Arc, -// chainlink: Arc, -// cancellation_token: CancellationToken, -// intents_meta_map: Arc>>, -// transaction_scheduler: TransactionScheduler, -// } -// -// impl ScheduledCommitsProcessorImpl { -// pub fn new( -// committor: Arc, -// chainlink: Arc, -// internal_transaction_scheduler: TransactionSchedulerHandle, -// ) -> Self { -// let result_subscriber = committor.subscribe_for_results(); -// let intents_meta_map = Arc::new(Mutex::default()); -// let cancellation_token = CancellationToken::new(); -// tokio::spawn(Self::result_processor( -// result_subscriber, -// cancellation_token.clone(), -// intents_meta_map.clone(), -// internal_transaction_scheduler.clone(), -// )); -// -// Self { -// committor, -// chainlink, -// cancellation_token, -// intents_meta_map, -// transaction_scheduler: TransactionScheduler::default(), -// } -// } -// -// async fn process_undelegation_requests(&self, pubkeys: Vec) { -// let mut join_set = task::JoinSet::new(); -// for pubkey in pubkeys.into_iter() { -// let chainlink = self.chainlink.clone(); -// join_set.spawn(async move { -// (pubkey, chainlink.undelegation_requested(pubkey).await) -// }); -// } -// let sub_errors = join_set -// .join_all() -// .await -// .into_iter() -// .filter_map(|(pubkey, inner_result)| { -// if let Err(err) = inner_result { -// Some(format!( -// "Subscribing to account {} failed: {}", -// pubkey, err -// )) -// } else { -// None -// } -// }) -// .collect::>(); -// if !sub_errors.is_empty() { -// // Instead of aborting the entire commit we log an error here, however -// // this means that the undelegated accounts stay in a problematic state -// // in the validator and are not synced from chain. -// // We could implement a retry mechanism inside of chainlink in the future. -// error!( -// error_count = sub_errors.len(), -// "Failed to subscribe to accounts being undelegated" -// ); -// } -// } -// -// #[instrument(skip( -// result_subscriber, -// cancellation_token, -// intents_meta_map, -// internal_transaction_scheduler -// ))] -// async fn result_processor( -// result_subscriber: oneshot::Receiver< -// broadcast::Receiver, -// >, -// cancellation_token: CancellationToken, -// intents_meta_map: Arc>>, -// internal_transaction_scheduler: TransactionSchedulerHandle, -// ) { -// const SUBSCRIPTION_ERR_MSG: &str = -// "Failed to get subscription of results of BaseIntents execution"; -// -// let mut result_receiver = -// result_subscriber.await.expect(SUBSCRIPTION_ERR_MSG); -// loop { -// let execution_result = tokio::select! { -// biased; -// _ = cancellation_token.cancelled() => { -// info!("Shutting down"); -// return; -// } -// execution_result = result_receiver.recv() => { -// match execution_result { -// Ok(result) => result, -// Err(broadcast::error::RecvError::Closed) => { -// info!("Intent execution service shut down"); -// break; -// } -// Err(broadcast::error::RecvError::Lagged(skipped)) => { -// // SAFETY: This shouldn't happen as our tx execution is faster than Intent execution on Base layer -// // If this ever happens it requires investigation -// error!(skipped_count = skipped, "Lagging behind intent execution"); -// continue; -// } -// } -// } -// }; -// -// let intent_id = execution_result.id; -// // Remove intent from metas -// let intent_meta = if let Some(intent_meta) = intents_meta_map -// .lock() -// .expect(POISONED_MUTEX_MSG) -// .remove(&intent_id) -// { -// intent_meta -// } else { -// // Possible if we have duplicate Intents -// // First one will remove id from map and second could fail. -// // This should not happen and needs investigation! -// error!(intent_id, "Failed to find intent metadata"); -// continue; -// }; -// -// Self::process_intent_result( -// intent_id, -// &internal_transaction_scheduler, -// execution_result, -// intent_meta, -// ) -// .await; -// } -// } -// -// #[instrument( -// skip(internal_transaction_scheduler, result, intent_meta), -// fields(intent_id) -// )] -// async fn process_intent_result( -// intent_id: u64, -// internal_transaction_scheduler: &TransactionSchedulerHandle, -// result: BroadcastedIntentExecutionResult, -// mut intent_meta: ScheduledBaseIntentMeta, -// ) { -// let intent_sent_transaction = -// std::mem::take(&mut intent_meta.intent_sent_transaction); -// let sent_commit = -// Self::build_sent_commit(intent_id, intent_meta, result); -// register_scheduled_commit_sent(sent_commit); -// let Ok(txn) = with_encoded(intent_sent_transaction) else { -// // Unreachable case, all intent transactions are smaller than 64KB by construction -// error!("Failed to bincode intent transaction"); -// return; -// }; -// match internal_transaction_scheduler.execute(txn).await { -// Ok(()) => { -// debug!("Sent commit signaled") -// } -// Err(err) => { -// error!(error = ?err, "Failed to signal sent commit"); -// } -// } -// } -// -// fn build_sent_commit( -// intent_id: u64, -// intent_meta: ScheduledBaseIntentMeta, -// result: BroadcastedIntentExecutionResult, -// ) -> SentCommit { -// let error_message = -// result.as_ref().err().map(|err| format!("{:?}", err)); -// let chain_signatures = match result.inner { -// Ok(value) => match value { -// ExecutionOutput::SingleStage(signature) => vec![signature], -// ExecutionOutput::TwoStage { -// commit_signature, -// finalize_signature, -// } => vec![commit_signature, finalize_signature], -// }, -// Err(err) => { -// error!( -// "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", -// intent_id, intent_meta.slot, intent_meta.blockhash, err -// ); -// err.signatures() -// .map(|(commit, finalize)| { -// finalize -// .map(|finalize| vec![commit, finalize]) -// .unwrap_or(vec![commit]) -// }) -// .unwrap_or_default() -// } -// }; -// let patched_errors = result -// .patched_errors -// .iter() -// .map(|err| { -// info!("Patched intent: {}. error was: {}", intent_id, err); -// err.to_string() -// }) -// .collect(); -// -// let callbacks_report = result -// .callbacks_report -// .iter() -// .map(|r| match r { -// Ok(sig) => { -// format!("OK: {sig}") -// } -// Err(err) => { -// error!( -// "Callback failed to schedule: {}. error: {}", -// intent_id, err -// ); -// format!("ERR: {err}") -// } -// }) -// .collect(); -// -// SentCommit { -// message_id: intent_id, -// slot: intent_meta.slot, -// blockhash: intent_meta.blockhash, -// payer: intent_meta.payer, -// chain_signatures, -// included_pubkeys: intent_meta.included_pubkeys, -// excluded_pubkeys: vec![], -// requested_undelegation: intent_meta.requested_undelegation, -// error_message, -// patched_errors, -// callbacks_scheduling_results: callbacks_report, -// } -// } -// } -// -// #[async_trait] -// impl ScheduledCommitsProcessor for ScheduledCommitsProcessorImpl { -// #[instrument(skip(self))] -// async fn process(&self) -> ScheduledCommitsProcessorResult<()> { -// let intent_bundles = -// self.transaction_scheduler.take_scheduled_intent_bundles(); -// -// if intent_bundles.is_empty() { -// return Ok(()); -// } -// metrics::inc_committor_intents_count_by(intent_bundles.len() as u64); -// -// // Add metas for intent we schedule -// let pubkeys_being_undelegated = { -// let mut intent_metas = -// self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); -// let mut pubkeys_being_undelegated = HashSet::::new(); -// -// intent_bundles.iter().for_each(|intent| { -// intent_metas -// .insert(intent.id, ScheduledBaseIntentMeta::new(intent)); -// if let Some(undelegate) = intent.get_undelegate_intent_pubkeys() -// { -// pubkeys_being_undelegated.extend(undelegate); -// } -// }); -// -// pubkeys_being_undelegated.into_iter().collect::>() -// }; -// -// self.process_undelegation_requests(pubkeys_being_undelegated) -// .await; -// self.committor -// .schedule_intent_bundles(intent_bundles) -// .await??; -// Ok(()) -// } -// -// fn scheduled_commits_len(&self) -> usize { -// self.transaction_scheduler.scheduled_actions_len() -// } -// -// fn clear_scheduled_commits(&self) { -// self.transaction_scheduler.clear_scheduled_actions(); -// } -// -// fn stop(&self) { -// self.cancellation_token.cancel(); -// } -// } -// -// struct ScheduledBaseIntentMeta { -// slot: u64, -// blockhash: Hash, -// payer: Pubkey, -// included_pubkeys: Vec, -// intent_sent_transaction: Transaction, -// requested_undelegation: bool, -// } -// -// impl ScheduledBaseIntentMeta { -// fn new(intent: &ScheduledIntentBundle) -> Self { -// Self { -// slot: intent.slot, -// blockhash: intent.blockhash, -// payer: intent.payer, -// included_pubkeys: intent.get_all_committed_pubkeys(), -// intent_sent_transaction: intent.sent_transaction.clone(), -// requested_undelegation: intent.has_undelegate_intent(), -// } -// } -// } diff --git a/magicblock-committor-service/src/service_final.rs b/magicblock-committor-service/src/service_final.rs index dbcd0f2ce..10697c597 100644 --- a/magicblock-committor-service/src/service_final.rs +++ b/magicblock-committor-service/src/service_final.rs @@ -297,9 +297,6 @@ where } } } - - // TODO(edwin): handle cancellation - todo!() } async fn schedule_intent_execution( @@ -391,7 +388,6 @@ where let execution_result = tokio::select! { biased; _ = cancellation_token.cancelled() => { - // TODO(edwin): validate shutdown correctness info!("Shutting down"); return; } diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index ff51b643d..debbf6bf2 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -4228,21 +4228,11 @@ name = "magicblock-accounts" version = "0.11.1" dependencies = [ "async-trait", - "magicblock-account-cloner", - "magicblock-accounts-db", - "magicblock-chainlink", "magicblock-committor-service", - "magicblock-core", - "magicblock-metrics", - "magicblock-program", - "solana-hash 3.1.0", "solana-pubkey 3.0.0", - "solana-transaction", "solana-transaction-error", "thiserror 2.0.18", "tokio", - "tokio-util 0.7.17", - "tracing", "url", ] From 9db0cc33a5d1f804e3db13e953faf3b5759210fb Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 15:23:06 +0700 Subject: [PATCH 07/97] refactor: renaming and file split --- magicblock-api/src/errors.rs | 2 +- magicblock-api/src/magic_validator.rs | 2 +- magicblock-committor-service/src/lib.rs | 2 +- .../src/{service_final.rs => service.rs} | 128 +----------------- .../src/service/intent_client.rs | 127 +++++++++++++++++ 5 files changed, 136 insertions(+), 125 deletions(-) rename magicblock-committor-service/src/{service_final.rs => service.rs} (77%) create mode 100644 magicblock-committor-service/src/service/intent_client.rs diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 21af35c23..4f80a6b72 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -1,5 +1,5 @@ use magicblock_accounts_db::error::AccountsDbError; -use magicblock_committor_service::service_final::IntentExecutionServiceError; +use magicblock_committor_service::service::IntentExecutionServiceError; use solana_pubkey::Pubkey; use thiserror::Error; diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 2cf2ca1db..7f0f5c4a3 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -26,7 +26,7 @@ use magicblock_chainlink::{ use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, - service_final::{IntentExecutionService, InternalIntentRpcClient}, + service::{intent_client::InternalIntentRpcClient, IntentExecutionService}, ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, }; use magicblock_config::{ diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 8fdc819a5..6d5f3e12b 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -14,7 +14,7 @@ pub mod transaction_preparator; pub mod transactions; pub(crate) mod utils; -pub mod service_final; +pub mod service; #[cfg(test)] pub mod test_utils; diff --git a/magicblock-committor-service/src/service_final.rs b/magicblock-committor-service/src/service.rs similarity index 77% rename from magicblock-committor-service/src/service_final.rs rename to magicblock-committor-service/src/service.rs index 10697c597..997c4c355 100644 --- a/magicblock-committor-service/src/service_final.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,3 +1,5 @@ +pub mod intent_client; + use std::{ collections::{HashMap, HashSet}, mem, @@ -5,9 +7,9 @@ use std::{ time::Duration, }; -use async_trait::async_trait; +use intent_client::{IntentRpcClient, InternalRpcClientError}; use magicblock_account_cloner::ChainlinkCloner; -use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; +use magicblock_accounts_db::AccountsDb; use magicblock_chainlink::{ remote_account_provider::{ chain_rpc_client::ChainRpcClientImpl, @@ -16,28 +18,19 @@ use magicblock_chainlink::{ submux::SubMuxClient, Chainlink, }; -use magicblock_core::{ - link::transactions::{with_encoded, TransactionSchedulerHandle}, - traits::LatestBlockProvider, -}; use magicblock_metrics::metrics; use magicblock_program::{ - instruction_utils::InstructionUtils, - magic_scheduled_base_intent::ScheduledIntentBundle, - register_scheduled_commit_sent, MagicContext, Pubkey, SentCommit, - TransactionScheduler, MAGIC_CONTEXT_PUBKEY, + magic_scheduled_base_intent::ScheduledIntentBundle, Pubkey, SentCommit, }; -use solana_account::ReadableAccount; use solana_hash::Hash; use solana_transaction::Transaction; -use solana_transaction_error::TransactionError; use tokio::{ sync::broadcast, task, task::{JoinError, JoinHandle}, }; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, instrument}; +use tracing::{error, info, instrument}; use crate::{ committor_processor::CommittorProcessor, error::CommittorServiceError, @@ -54,115 +47,6 @@ pub type ChainlinkImpl = Chainlink< ChainlinkCloner, >; -#[async_trait] -pub trait IntentRpcClient: Send + Sync + 'static { - type Error: std::error::Error + Send; - - /// Executes `Accept` tx and returns accepted intents - async fn accept_scheduled_intents( - &self, - ) -> Result, Self::Error>; - - /// Processes intent results, submitting them on chain(ER) - async fn finalize_intent( - &self, - sent_tx: Transaction, - sent_commit: SentCommit, - ) -> Result<(), Self::Error>; -} - -pub struct InternalIntentRpcClient { - /// Provides access to MagicContext - accounts_db: Arc, - /// Internal endpoint for scheduling ER TXs - transaction_scheduler: TransactionSchedulerHandle, - /// Provides access to ER latest block for TX creation - latest_block_provider: L, -} - -impl InternalIntentRpcClient { - pub fn new( - accounts_db: Arc, - transaction_scheduler: TransactionSchedulerHandle, - latest_block_provider: L, - ) -> Self { - Self { - accounts_db, - transaction_scheduler, - latest_block_provider, - } - } - - /// Sends transaction to move the scheduled commits from the `MagicContext` - /// to the global ScheduledCommit store - async fn execute_accept_tx(&self) -> Result<(), InternalRpcClientError> { - let tx = InstructionUtils::accept_scheduled_commits( - self.latest_block_provider.blockhash(), - ); - let encoded_tx = with_encoded(tx).inspect_err(|err| { - error!(error = ?err, "Failed to bincode intent transaction"); - })?; - self.transaction_scheduler - .execute(encoded_tx) - .await - .inspect_err(|err| { - error!(error = ?err, "Failed to accept scheduled commits"); - })?; - - Ok(()) - } -} - -#[async_trait] -impl IntentRpcClient for InternalIntentRpcClient { - type Error = InternalRpcClientError; - - async fn accept_scheduled_intents( - &self, - ) -> Result, Self::Error> { - // If accounts were scheduled to be committed, we accept them here - // and processs the commits - let magic_context_acc = - self.accounts_db.get_account(&MAGIC_CONTEXT_PUBKEY).expect( - "Validator found to be running without MagicContext account!", - ); - if !MagicContext::has_scheduled_commits(magic_context_acc.data()) { - return Ok(vec![]); - } - self.execute_accept_tx().await?; - - // Return intents from global store - Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) - } - - async fn finalize_intent( - &self, - sent_tx: Transaction, - sent_commit: SentCommit, - ) -> Result<(), Self::Error> { - register_scheduled_commit_sent(sent_commit); - let txn = with_encoded(sent_tx).inspect_err(|err| { - // Unreachable case, all intent transactions are smaller than 64KB by construction - error!(error = ?err, "Failed to bincode intent transaction"); - })?; - self.transaction_scheduler - .execute(txn) - .await - .inspect(|_| debug!("Sent commit signaled")) - .inspect_err( - |err| error!(error = ?err, "Failed to signal sent commit"), - )?; - - Ok(()) - } -} - -#[derive(thiserror::Error, Debug)] -pub enum InternalRpcClientError { - #[error("TransactionError: {0}")] - TransactionError(#[from] TransactionError), -} - pub enum IntentExecutionService { Created(ServiceInner), Started(JoinHandle<()>), diff --git a/magicblock-committor-service/src/service/intent_client.rs b/magicblock-committor-service/src/service/intent_client.rs new file mode 100644 index 000000000..20670b606 --- /dev/null +++ b/magicblock-committor-service/src/service/intent_client.rs @@ -0,0 +1,127 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; +use magicblock_core::{ + link::transactions::{with_encoded, TransactionSchedulerHandle}, + traits::LatestBlockProvider, +}; +use magicblock_program::{ + instruction_utils::InstructionUtils, + magic_scheduled_base_intent::ScheduledIntentBundle, + register_scheduled_commit_sent, MagicContext, SentCommit, + TransactionScheduler, MAGIC_CONTEXT_PUBKEY, +}; +use solana_account::ReadableAccount; +use solana_transaction::Transaction; +use solana_transaction_error::TransactionError; +use tracing::{debug, error}; + +#[async_trait] +pub trait IntentRpcClient: Send + Sync + 'static { + type Error: std::error::Error + Send; + + /// Executes `Accept` tx and returns accepted intents + async fn accept_scheduled_intents( + &self, + ) -> Result, Self::Error>; + + /// Processes intent results, submitting them on chain(ER) + async fn finalize_intent( + &self, + sent_tx: Transaction, + sent_commit: SentCommit, + ) -> Result<(), Self::Error>; +} + +pub struct InternalIntentRpcClient { + /// Provides access to MagicContext + accounts_db: Arc, + /// Internal endpoint for scheduling ER TXs + transaction_scheduler: TransactionSchedulerHandle, + /// Provides access to ER latest block for TX creation + latest_block_provider: L, +} + +impl InternalIntentRpcClient { + pub fn new( + accounts_db: Arc, + transaction_scheduler: TransactionSchedulerHandle, + latest_block_provider: L, + ) -> Self { + Self { + accounts_db, + transaction_scheduler, + latest_block_provider, + } + } + + /// Sends transaction to move the scheduled commits from the `MagicContext` + /// to the global ScheduledCommit store + async fn execute_accept_tx(&self) -> Result<(), InternalRpcClientError> { + let tx = InstructionUtils::accept_scheduled_commits( + self.latest_block_provider.blockhash(), + ); + let encoded_tx = with_encoded(tx).inspect_err(|err| { + error!(error = ?err, "Failed to bincode intent transaction"); + })?; + self.transaction_scheduler + .execute(encoded_tx) + .await + .inspect_err(|err| { + error!(error = ?err, "Failed to accept scheduled commits"); + })?; + + Ok(()) + } +} + +#[async_trait] +impl IntentRpcClient for InternalIntentRpcClient { + type Error = InternalRpcClientError; + + async fn accept_scheduled_intents( + &self, + ) -> Result, Self::Error> { + // If accounts were scheduled to be committed, we accept them here + // and processs the commits + let magic_context_acc = + self.accounts_db.get_account(&MAGIC_CONTEXT_PUBKEY).expect( + "Validator found to be running without MagicContext account!", + ); + if !MagicContext::has_scheduled_commits(magic_context_acc.data()) { + return Ok(vec![]); + } + self.execute_accept_tx().await?; + + // Return intents from global store + Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) + } + + async fn finalize_intent( + &self, + sent_tx: Transaction, + sent_commit: SentCommit, + ) -> Result<(), Self::Error> { + register_scheduled_commit_sent(sent_commit); + let txn = with_encoded(sent_tx).inspect_err(|err| { + // Unreachable case, all intent transactions are smaller than 64KB by construction + error!(error = ?err, "Failed to bincode intent transaction"); + })?; + self.transaction_scheduler + .execute(txn) + .await + .inspect(|_| debug!("Sent commit signaled")) + .inspect_err( + |err| error!(error = ?err, "Failed to signal sent commit"), + )?; + + Ok(()) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum InternalRpcClientError { + #[error("TransactionError: {0}")] + TransactionError(#[from] TransactionError), +} From 501741f54572ccc52d3fed3caad243ccef3f82c5 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 15:31:51 +0700 Subject: [PATCH 08/97] feat: renamings --- magicblock-committor-service/src/service.rs | 10 +++++----- .../src/service/intent_client.rs | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 997c4c355..f9a86c99b 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use intent_client::{IntentRpcClient, InternalRpcClientError}; +use intent_client::{ERIntentClient, InternalIntentClientError}; use magicblock_account_cloner::ChainlinkCloner; use magicblock_accounts_db::AccountsDb; use magicblock_chainlink::{ @@ -56,7 +56,7 @@ pub enum IntentExecutionService { impl IntentExecutionService where - R: IntentRpcClient, + R: ERIntentClient, R::Error: Into, { pub fn new( @@ -121,7 +121,7 @@ pub struct ServiceInner { impl ServiceInner where - R: IntentRpcClient, + R: ERIntentClient, R::Error: Into, { pub fn new( @@ -332,7 +332,7 @@ where execution_result, ); intent_client - .finalize_intent(sent_transaction, sent_commit) + .notify_commit_sent(sent_transaction, sent_commit) .await .map_err(Into::into)?; @@ -439,5 +439,5 @@ pub enum IntentExecutionServiceError { #[error("JoinError: {0}")] JoinError(#[from] JoinError), #[error("IntentRpcClientError: {0}")] - IntentRpcClientError(#[from] InternalRpcClientError), + IntentRpcClientError(#[from] InternalIntentClientError), } diff --git a/magicblock-committor-service/src/service/intent_client.rs b/magicblock-committor-service/src/service/intent_client.rs index 20670b606..65aaa2ca3 100644 --- a/magicblock-committor-service/src/service/intent_client.rs +++ b/magicblock-committor-service/src/service/intent_client.rs @@ -18,7 +18,7 @@ use solana_transaction_error::TransactionError; use tracing::{debug, error}; #[async_trait] -pub trait IntentRpcClient: Send + Sync + 'static { +pub trait ERIntentClient: Send + Sync + 'static { type Error: std::error::Error + Send; /// Executes `Accept` tx and returns accepted intents @@ -27,7 +27,7 @@ pub trait IntentRpcClient: Send + Sync + 'static { ) -> Result, Self::Error>; /// Processes intent results, submitting them on chain(ER) - async fn finalize_intent( + async fn notify_commit_sent( &self, sent_tx: Transaction, sent_commit: SentCommit, @@ -58,7 +58,7 @@ impl InternalIntentRpcClient { /// Sends transaction to move the scheduled commits from the `MagicContext` /// to the global ScheduledCommit store - async fn execute_accept_tx(&self) -> Result<(), InternalRpcClientError> { + async fn send_accept_tx(&self) -> Result<(), InternalIntentClientError> { let tx = InstructionUtils::accept_scheduled_commits( self.latest_block_provider.blockhash(), ); @@ -77,8 +77,8 @@ impl InternalIntentRpcClient { } #[async_trait] -impl IntentRpcClient for InternalIntentRpcClient { - type Error = InternalRpcClientError; +impl ERIntentClient for InternalIntentRpcClient { + type Error = InternalIntentClientError; async fn accept_scheduled_intents( &self, @@ -92,13 +92,13 @@ impl IntentRpcClient for InternalIntentRpcClient { if !MagicContext::has_scheduled_commits(magic_context_acc.data()) { return Ok(vec![]); } - self.execute_accept_tx().await?; + self.send_accept_tx().await?; // Return intents from global store Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) } - async fn finalize_intent( + async fn notify_commit_sent( &self, sent_tx: Transaction, sent_commit: SentCommit, @@ -121,7 +121,7 @@ impl IntentRpcClient for InternalIntentRpcClient { } #[derive(thiserror::Error, Debug)] -pub enum InternalRpcClientError { +pub enum InternalIntentClientError { #[error("TransactionError: {0}")] TransactionError(#[from] TransactionError), } From 6039cdd6f42bfa6caf62070410c49b9a673f79f2 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 17:39:59 +0700 Subject: [PATCH 09/97] refactor: cleanup --- .../src/stubs/changeset_committor_stub.rs | 299 -------- magicblock-committor-service/src/stubs/mod.rs | 2 - .../src/transactions.rs | 664 ------------------ .../tests/test_ix_commit_local.rs | 167 ++--- 4 files changed, 47 insertions(+), 1085 deletions(-) delete mode 100644 magicblock-committor-service/src/stubs/changeset_committor_stub.rs diff --git a/magicblock-committor-service/src/stubs/changeset_committor_stub.rs b/magicblock-committor-service/src/stubs/changeset_committor_stub.rs deleted file mode 100644 index 0b417db47..000000000 --- a/magicblock-committor-service/src/stubs/changeset_committor_stub.rs +++ /dev/null @@ -1,299 +0,0 @@ -use std::{ - collections::HashMap, - sync::{Arc, Mutex}, - time::{Instant, SystemTime, UNIX_EPOCH}, -}; - -use async_trait::async_trait; -use magicblock_program::magic_scheduled_base_intent::{ - CommitType, ScheduledIntentBundle, UndelegateType, -}; -use solana_account::Account; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_transaction_status_client_types::{ - EncodedConfirmedTransactionWithStatusMeta, EncodedTransaction, - EncodedTransactionWithStatusMeta, -}; -use tokio::sync::{broadcast, oneshot}; -use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; - -use crate::{ - error::CommittorServiceResult, - intent_execution_manager::BroadcastedIntentExecutionResult, - intent_executor::ExecutionOutput, - persist::{CommitStatusRow, IntentPersisterImpl, MessageSignatures}, - service_ext::{BaseIntentCommitorExtResult, BaseIntentCommittorExt}, - BaseIntentCommittor, -}; - -#[derive(Default)] -pub struct ChangesetCommittorStub { - cancellation_token: CancellationToken, - reserved_pubkeys_for_committee: Arc>>, - #[allow(clippy::type_complexity)] - committed_changesets: Arc>>, - committed_accounts: Arc>>, -} - -impl ChangesetCommittorStub { - #[allow(clippy::len_without_is_empty)] - pub fn len(&self) -> usize { - self.committed_changesets.lock().unwrap().len() - } - - pub fn committed(&self, pubkey: &Pubkey) -> Option { - self.committed_accounts.lock().unwrap().get(pubkey).cloned() - } -} - -impl BaseIntentCommittor for ChangesetCommittorStub { - fn reserve_pubkeys_for_committee( - &self, - committee: Pubkey, - owner: Pubkey, - ) -> oneshot::Receiver> { - let initiated = Instant::now(); - let (tx, rx) = oneshot::channel::>(); - self.reserved_pubkeys_for_committee - .lock() - .unwrap() - .insert(committee, owner); - - tx.send(Ok(initiated)).unwrap_or_else(|_| { - tracing::error!( - message_type = "ReservePubkeysForCommittee", - "Failed to send response" - ); - }); - rx - } - - fn schedule_intent_bundles( - &self, - intent_bundles: Vec, - ) -> oneshot::Receiver> { - let (sender, receiver) = oneshot::channel(); - let _ = sender.send(Ok(())); - - { - let mut committed_accounts = - self.committed_accounts.lock().unwrap(); - intent_bundles.iter().for_each(|intent| { - intent - .get_all_committed_accounts() - .iter() - .for_each(|account| { - committed_accounts - .insert(account.pubkey, account.account.clone()); - }) - }) - } - - { - let mut changesets = self.committed_changesets.lock().unwrap(); - intent_bundles.into_iter().for_each(|intent| { - changesets.insert(intent.id, intent); - }); - } - - receiver - } - - fn subscribe_for_results( - &self, - ) -> oneshot::Receiver> - { - let (_, receiver) = oneshot::channel(); - receiver - } - - fn get_commit_statuses( - &self, - message_id: u64, - ) -> oneshot::Receiver>> { - let (tx, rx) = oneshot::channel(); - - let commit = self - .committed_changesets - .lock() - .unwrap() - .remove(&message_id); - let Some(base_intent) = commit else { - tx.send(Ok(vec![])).unwrap_or_else(|_| { - tracing::error!( - message_type = "GetCommitStatuses", - "Failed to send response" - ); - }); - return rx; - }; - - let status_rows = IntentPersisterImpl::create_commit_rows(&base_intent); - tx.send(Ok(status_rows)).unwrap_or_else(|_| { - tracing::error!( - message_type = "GetCommitStatuses", - "Failed to send response" - ); - }); - - rx - } - - fn get_commit_signatures( - &self, - _commit_id: u64, - _pubkey: Pubkey, - ) -> oneshot::Receiver>> - { - let (tx, rx) = oneshot::channel(); - let message_signature = MessageSignatures { - commit_stage_signature: Signature::new_unique(), - finalize_stage_signature: Some(Signature::new_unique()), - created_at: now(), - }; - - tx.send(Ok(Some(message_signature))).unwrap_or_else(|_| { - tracing::error!( - message_type = "GetCommitSignatures", - "Failed to send response" - ); - }); - - rx - } - - fn get_transaction( - &self, - _: &Signature, - ) -> oneshot::Receiver< - CommittorServiceResult, - > { - let (tx, rx) = oneshot::channel(); - if let Err(_err) = - tx.send(Ok(EncodedConfirmedTransactionWithStatusMeta { - slot: 0, - transaction: EncodedTransactionWithStatusMeta { - transaction: EncodedTransaction::LegacyBinary( - "".to_string(), - ), - meta: None, - version: None, - }, - block_time: None, - })) - { - tracing::error!( - message_type = "GetTransaction", - "Failed to send response" - ); - }; - - rx - } - - fn fetch_current_commit_nonces( - &self, - pubkeys: &[Pubkey], - _min_context_slot: u64, - ) -> oneshot::Receiver>> { - let (tx, rx) = oneshot::channel(); - let nonces = pubkeys.iter().map(|p| (*p, 0u64)).collect(); - tx.send(Ok(nonces)).unwrap_or_else(|_| { - tracing::error!( - message_type = "FetchCurrentCommitNonces", - "Failed to send response" - ); - }); - rx - } - - fn stop(&self) { - self.cancellation_token.cancel(); - } - - fn stopped(&self) -> WaitForCancellationFutureOwned { - self.cancellation_token.clone().cancelled_owned() - } -} - -#[async_trait] -impl BaseIntentCommittorExt for ChangesetCommittorStub { - async fn schedule_intent_bundles_waiting( - &self, - intent_bundles: Vec, - ) -> BaseIntentCommitorExtResult> - { - self.schedule_intent_bundles(intent_bundles.clone()) - .await??; - let res = intent_bundles - .into_iter() - .map(|message| { - let callbacks_report = (0..count_bundle_callbacks(&message)) - .map(|_| Ok(Signature::new_unique())) - .collect(); - BroadcastedIntentExecutionResult { - id: message.id, - inner: Ok(ExecutionOutput::TwoStage { - commit_signature: Signature::new_unique(), - finalize_signature: Signature::new_unique(), - }), - patched_errors: Arc::new(vec![]), - callbacks_report, - } - }) - .collect::>(); - - Ok(res) - } -} - -fn count_bundle_callbacks(bundle: &ScheduledIntentBundle) -> usize { - let ib = &bundle.intent_bundle; - - let from_commit = ib - .commit - .as_ref() - .map(|ct| match ct { - CommitType::Standalone(_) => 0, - CommitType::WithBaseActions { base_actions, .. } => { - base_actions.iter().filter(|a| a.callback.is_some()).count() - } - }) - .unwrap_or(0); - - let from_cau = ib - .commit_and_undelegate - .as_ref() - .map(|cau| { - let from_commit_action = match &cau.commit_action { - CommitType::Standalone(_) => 0, - CommitType::WithBaseActions { base_actions, .. } => { - base_actions.iter().filter(|a| a.callback.is_some()).count() - } - }; - let from_undelegate = match &cau.undelegate_action { - UndelegateType::Standalone => 0, - UndelegateType::WithBaseActions(actions) => { - actions.iter().filter(|a| a.callback.is_some()).count() - } - }; - from_commit_action + from_undelegate - }) - .unwrap_or(0); - - let from_standalone = ib - .standalone_actions - .iter() - .filter(|a| a.callback.is_some()) - .count(); - - from_commit + from_cau + from_standalone -} - -fn now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs() -} diff --git a/magicblock-committor-service/src/stubs/mod.rs b/magicblock-committor-service/src/stubs/mod.rs index 9cfb6e45c..e69de29bb 100644 --- a/magicblock-committor-service/src/stubs/mod.rs +++ b/magicblock-committor-service/src/stubs/mod.rs @@ -1,2 +0,0 @@ -mod changeset_committor_stub; -pub use changeset_committor_stub::ChangesetCommittorStub; diff --git a/magicblock-committor-service/src/transactions.rs b/magicblock-committor-service/src/transactions.rs index 84bf3180c..e24f8504c 100644 --- a/magicblock-committor-service/src/transactions.rs +++ b/magicblock-committor-service/src/transactions.rs @@ -1,676 +1,12 @@ use base64::{prelude::BASE64_STANDARD, Engine}; use solana_rpc_client::rpc_client::SerializableTransaction; -use static_assertions::const_assert; /// From agave rpc/src/rpc.rs [MAX_BASE64_SIZE] pub(crate) const MAX_ENCODED_TRANSACTION_SIZE: usize = 1644; -/// How many process and commit buffer instructions fit into a single transaction -#[allow(unused)] // serves as documentation as well -pub const MAX_PROCESS_PER_TX: u8 = 3; - -/// How many process and commit buffer instructions fit into a single transaction -/// when using address lookup tables but not including the buffer account in the -/// lookup table -#[allow(unused)] // serves as documentation as well -pub const MAX_PROCESS_PER_TX_USING_LOOKUP: u8 = 12; - -/// How many close buffer instructions fit into a single transaction -#[allow(unused)] // serves as documentation as well -pub const MAX_CLOSE_PER_TX: u8 = 8; - -/// How many close buffer instructions fit into a single transaction -/// when using address lookup tables but not including the buffer account -/// nor chunk account in the lookup table -#[allow(unused)] // serves as documentation as well -pub const MAX_CLOSE_PER_TX_USING_LOOKUP: u8 = 8; - -/// How many process and commit buffer instructions combined with close buffer instructions -/// fit into a single transaction -#[allow(unused)] // serves as documentation as well -pub const MAX_PROCESS_AND_CLOSE_PER_TX: u8 = 2; - -/// How many process and commit buffer instructions combined with -/// close buffer instructions fit into a single transaction when -/// using lookup tables but not including the buffer account -#[allow(unused)] // serves as documentation as well -pub const MAX_PROCESS_AND_CLOSE_PER_TX_USING_LOOKUP: u8 = 5; - -/// How many finalize instructions fit into a single transaction -#[allow(unused)] // serves as documentation as well -pub const MAX_FINALIZE_PER_TX: u8 = 5; - -/// How many finalize instructions fit into a single transaction -/// when using address lookup tables -#[allow(unused)] // serves as documentation as well -pub const MAX_FINALIZE_PER_TX_USING_LOOKUP: u8 = 48; - -/// How many undelegate instructions fit into a single transaction -/// NOTE: that we assume the rent reimbursement account to be the delegated account -#[allow(unused)] // serves as documentation as well -pub const MAX_UNDELEGATE_PER_TX: u8 = 3; - -/// How many undelegate instructions fit into a single transaction -/// when using address lookup tables -/// NOTE: that we assume the rent reimbursement account to be the delegated account -#[allow(unused)] // serves as documentation as well -pub const MAX_UNDELEGATE_PER_TX_USING_LOOKUP: u8 = 16; - -// Allows us to run undelegate instructions without rechunking them since we know -// that we didn't process more than we also can undelegate -const_assert!(MAX_PROCESS_PER_TX <= MAX_UNDELEGATE_PER_TX,); - -// Allows us to run undelegate instructions using lookup tables without rechunking -// them since we know that we didn't process more than we also can undelegate -const_assert!( - MAX_PROCESS_PER_TX_USING_LOOKUP <= MAX_UNDELEGATE_PER_TX_USING_LOOKUP -); - pub fn serialize_and_encode_base64( transaction: &impl SerializableTransaction, ) -> String { - // SAFETY: runs statically let serialized = bincode::serialize(transaction).unwrap(); BASE64_STANDARD.encode(serialized) } - -#[cfg(test)] -mod test { - use std::collections::HashSet; - - use dlp_api::args::{CommitStateArgs, CommitStateFromBufferArgs}; - use lazy_static::lazy_static; - use magicblock_committor_program::instruction_builder::close_buffer::{ - create_close_ix, CreateCloseIxArgs, - }; - use solana_address_lookup_table_interface::state::LOOKUP_TABLE_MAX_ADDRESSES; - use solana_hash::Hash; - use solana_instruction::Instruction; - use solana_keypair::Keypair; - use solana_message::{ - v0::Message, AddressLookupTableAccount, VersionedMessage, - }; - use solana_pubkey::Pubkey; - use solana_signer::Signer; - use solana_transaction::versioned::VersionedTransaction; - use tracing::info; - - use super::*; - use crate::{ - compute_budget::{Budget, ComputeBudget}, - error::{ - CommittorServiceError::{ - FailedToCompileTransactionMessage, FailedToCreateTransaction, - }, - CommittorServiceResult, - }, - pubkeys_provider::{provide_committee_pubkeys, provide_common_pubkeys}, - test_utils, - }; - - fn get_lookup_tables( - ixs: &[Instruction], - ) -> Vec { - let pubkeys = ixs - .iter() - .flat_map(|ix| ix.accounts.iter().map(|acc| acc.pubkey)) - .collect::>(); - - let lookup_table = AddressLookupTableAccount { - key: Pubkey::default(), - addresses: pubkeys.into_iter().collect(), - }; - vec![lookup_table] - } - - // ----------------- - // Helpers - // ----------------- - #[allow(clippy::enum_variant_names)] - enum TransactionOpts { - NoLookupTable, - UseLookupTable, - } - fn encoded_tx_size( - auth: &Keypair, - ixs: &[Instruction], - opts: &TransactionOpts, - ) -> CommittorServiceResult { - use TransactionOpts::*; - let lookup_tables = match opts { - NoLookupTable => vec![], - UseLookupTable => get_lookup_tables(ixs), - }; - - let versioned_msg = Message::try_compile( - &auth.pubkey(), - ixs, - &lookup_tables, - Hash::default(), - ) - .map_err(|err| { - FailedToCompileTransactionMessage( - "Calculating transaction size".to_string(), - err, - ) - })?; - let versioned_tx = VersionedTransaction::try_new( - VersionedMessage::V0(versioned_msg), - &[&auth], - ) - .map_err(|err| { - FailedToCreateTransaction( - "Calculating transaction size".to_string(), - err, - ) - })?; - - let encoded = serialize_and_encode_base64(&versioned_tx); - Ok(encoded.len()) - } - - // ----------------- - // Process Commitables and Close Buffers - // ----------------- - pub(crate) fn process_commits_ix( - validator_auth: Pubkey, - pubkey: &Pubkey, - delegated_account_owner: &Pubkey, - buffer_pda: &Pubkey, - commit_args: CommitStateFromBufferArgs, - ) -> Instruction { - dlp_api::instruction_builder::commit_state_from_buffer( - validator_auth, - *pubkey, - *delegated_account_owner, - *buffer_pda, - commit_args, - ) - } - - pub(crate) fn close_buffers_ix( - validator_auth: Pubkey, - pubkey: &Pubkey, - commit_id: u64, - ) -> Instruction { - create_close_ix(CreateCloseIxArgs { - authority: validator_auth, - pubkey: *pubkey, - commit_id, - }) - } - - pub(crate) fn process_and_close_ixs( - validator_auth: Pubkey, - pubkey: &Pubkey, - delegated_account_owner: &Pubkey, - buffer_pda: &Pubkey, - commit_id: u64, - commit_args: CommitStateFromBufferArgs, - ) -> Vec { - let process_ix = process_commits_ix( - validator_auth, - pubkey, - delegated_account_owner, - buffer_pda, - commit_args, - ); - let close_ix = close_buffers_ix(validator_auth, pubkey, commit_id); - - vec![process_ix, close_ix] - } - - // ----------------- - // Finalize - // ----------------- - pub(crate) fn finalize_ix( - validator_auth: Pubkey, - pubkey: &Pubkey, - ) -> Instruction { - dlp_api::instruction_builder::finalize(validator_auth, *pubkey) - } - - // These tests statically determine the optimal ix count to fit into a single - // transaction and assert that the const we export in prod match those numbers. - // Thus when an instruction changes and one of those numbers with it a failing - // test alerts us. - // This is less overhead than running those static functions each time at - // startup. - - #[test] - fn test_max_process_per_tx() { - test_utils::init_test_logger(); - assert_eq!(super::MAX_PROCESS_PER_TX, *MAX_PROCESS_PER_TX); - assert_eq!( - super::MAX_PROCESS_PER_TX_USING_LOOKUP, - *MAX_PROCESS_PER_TX_USING_LOOKUP - ); - } - - #[test] - fn test_max_close_per_tx() { - test_utils::init_test_logger(); - assert_eq!(super::MAX_CLOSE_PER_TX, *MAX_CLOSE_PER_TX); - assert_eq!( - super::MAX_CLOSE_PER_TX_USING_LOOKUP, - *MAX_CLOSE_PER_TX_USING_LOOKUP - ); - } - - #[test] - fn test_max_process_and_closes_per_tx() { - test_utils::init_test_logger(); - assert_eq!( - super::MAX_PROCESS_AND_CLOSE_PER_TX, - *MAX_PROCESS_AND_CLOSE_PER_TX - ); - assert_eq!( - super::MAX_PROCESS_AND_CLOSE_PER_TX_USING_LOOKUP, - *MAX_PROCESS_AND_CLOSE_PER_TX_USING_LOOKUP - ); - } - - #[test] - fn test_max_finalize_per_tx() { - test_utils::init_test_logger(); - assert_eq!(super::MAX_FINALIZE_PER_TX, *MAX_FINALIZE_PER_TX); - assert_eq!( - super::MAX_FINALIZE_PER_TX_USING_LOOKUP, - *MAX_FINALIZE_PER_TX_USING_LOOKUP - ); - } - - #[test] - fn test_max_undelegate_per_tx() { - test_utils::init_test_logger(); - assert_eq!(super::MAX_UNDELEGATE_PER_TX, *MAX_UNDELEGATE_PER_TX); - assert_eq!( - super::MAX_UNDELEGATE_PER_TX_USING_LOOKUP, - *MAX_UNDELEGATE_PER_TX_USING_LOOKUP - ); - } - - // ----------------- - // Process Commitables using Args - // ----------------- - #[test] - fn test_log_commit_args_ix_sizes() { - test_utils::init_test_logger(); - // This test is used to investigate the size of the transaction related to - // the amount of committed accounts and their data size. - fn run(auth: &Keypair, ixs: usize) { - let mut tx_lines = vec![]; - use TransactionOpts::*; - for tx_opts in [NoLookupTable, UseLookupTable] { - let mut tx_sizes = vec![]; - for size in [0, 10, 20, 50, 100, 200, 500, 1024] { - let ixs = (0..ixs) - .map(|_| make_ix(auth, size)) - .collect::>(); - - let tx_size = - encoded_tx_size(auth, &ixs, &tx_opts).unwrap(); - tx_sizes.push((size, tx_size)); - } - tx_lines.push(tx_sizes); - } - let sizes = tx_lines - .into_iter() - .map(|line| { - line.into_iter() - .map(|(size, len)| format!("{:4}:{:5}", size, len)) - .collect::>() - .join("|") - }) - .collect::>() - .join("\n"); - info!(instruction_count = ixs, sizes = %sizes, "Transaction size report"); - } - fn make_ix(auth: &Keypair, data_size: usize) -> Instruction { - let data = vec![1; data_size]; - let args = CommitStateArgs { - data, - ..CommitStateArgs::default() - }; - dlp_api::instruction_builder::commit_state( - auth.pubkey(), - Pubkey::new_unique(), - Pubkey::new_unique(), - args, - ) - } - - let auth = &Keypair::new(); - run(auth, 0); - run(auth, 1); - run(auth, 2); - run(auth, 5); - run(auth, 8); - run(auth, 10); - run(auth, 15); - run(auth, 20); - /* - 0 ixs: - 0: 184| 10: 184| 20: 184| 50: 184| 100: 184| 200: 184| 500: 184|1024: 184 - 0: 184| 10: 184| 20: 184| 50: 184| 100: 184| 200: 184| 500: 184|1024: 184 - 1 ixs: - 0: 620| 10: 636| 20: 648| 50: 688| 100: 756| 200: 888| 500: 1288|1024: 1988 - 0: 336| 10: 348| 20: 364| 50: 404| 100: 472| 200: 604| 500: 1004|1024: 1704 - 2 ixs: - 0: 932| 10: 960| 20: 984| 50: 1064| 100: 1200| 200: 1468| 500: 2268|1024: 3664 - 0: 400| 10: 424| 20: 452| 50: 532| 100: 668| 200: 936| 500: 1736|1024: 3132 - 5 ixs: - 0: 1864| 10: 1932| 20: 1996| 50: 2196| 100: 2536| 200: 3204| 500: 5204|1024: 8696 - 0: 588| 10: 652| 20: 720| 50: 920| 100: 1260| 200: 1928| 500: 3928|1024: 7420 - 8 ixs: - 0: 2796| 10: 2904| 20: 3008| 50: 3328| 100: 3872| 200: 4940| 500: 8140|1024:13728 - 0: 776| 10: 880| 20: 988| 50: 1308| 100: 1852| 200: 2920| 500: 6120|1024:11708 - 10 ixs: - 0: 3416| 10: 3552| 20: 3684| 50: 4084| 100: 4764| 200: 6096| 500:10096|1024:17084 - 0: 900| 10: 1032| 20: 1168| 50: 1568| 100: 2248| 200: 3580| 500: 7580|1024:14568 - 15 ixs: - 0: 4972| 10: 5172| 20: 5372| 50: 5972| 100: 6992| 200: 8992| 500:14992|1024:25472 - 0: 1212| 10: 1412| 20: 1612| 50: 2212| 100: 3232| 200: 5232| 500:11232|1024:21712 - 20 ixs: - 0: 6524| 10: 6792| 20: 7056| 50: 7856| 100: 9216| 200:11884| 500:19884|1024:33856 - 0: 1528| 10: 1792| 20: 2060| 50: 2860| 100: 4220| 200: 6888| 500:14888|1024:28860 - - Legend: - - x ixs: - data size/ix: encoded size | ... - data size/ix: encoded size | ... (using lookup tables) - - Given that max transaction size is 1644 bytes, we can see that the max data size is: - - - 1 ixs: slightly larger than 500 bytes - - 2 ixs: slightly larger than 200 bytes - - 5 ixs: slightly larger than 100 bytes - - 8 ixs: slightly larger than 50 bytes - - 10 ixs: slightly larger than 20 bytes - - 15 ixs: slightly larger than 10 bytes - - 20 ixs: no data supported (only lamport changes) - - Also it is clear that using a lookup table makes a huge difference especially if we commit - lots of different accounts. - */ - } - - // ----------------- - // Process Commitables and Close Buffers - // ----------------- - lazy_static! { - pub(crate) static ref MAX_PROCESS_PER_TX: u8 = { - max_chunks_per_transaction("Max process per tx", |auth_pubkey| { - let pubkey = Pubkey::new_unique(); - let delegated_account_owner = Pubkey::new_unique(); - let buffer_pda = Pubkey::new_unique(); - let commit_args = CommitStateFromBufferArgs::default(); - vec![process_commits_ix( - auth_pubkey, - &pubkey, - &delegated_account_owner, - &buffer_pda, - commit_args, - )] - }) - }; - pub(crate) static ref MAX_PROCESS_PER_TX_USING_LOOKUP: u8 = { - max_chunks_per_transaction_using_lookup_table( - "Max process per tx using lookup", - |auth_pubkey, committee, delegated_account_owner| { - let buffer_pda = Pubkey::new_unique(); - let commit_args = CommitStateFromBufferArgs::default(); - vec![process_commits_ix( - auth_pubkey, - &committee, - &delegated_account_owner, - &buffer_pda, - commit_args, - )] - }, - None, - ) - }; - pub(crate) static ref MAX_CLOSE_PER_TX: u8 = { - let commit_id = 0; - max_chunks_per_transaction("Max close per tx", |auth_pubkey| { - let pubkey = Pubkey::new_unique(); - vec![close_buffers_ix(auth_pubkey, &pubkey, commit_id)] - }) - }; - pub(crate) static ref MAX_CLOSE_PER_TX_USING_LOOKUP: u8 = { - let commit_id = 0; - max_chunks_per_transaction_using_lookup_table( - "Max close per tx using lookup", - |auth_pubkey, committee, _| { - vec![close_buffers_ix(auth_pubkey, &committee, commit_id)] - }, - None, - ) - }; - pub(crate) static ref MAX_PROCESS_AND_CLOSE_PER_TX: u8 = { - let commit_id = 0; - max_chunks_per_transaction( - "Max process and close per tx", - |auth_pubkey| { - let pubkey = Pubkey::new_unique(); - let delegated_account_owner = Pubkey::new_unique(); - let buffer_pda = Pubkey::new_unique(); - let commit_args = CommitStateFromBufferArgs::default(); - process_and_close_ixs( - auth_pubkey, - &pubkey, - &delegated_account_owner, - &buffer_pda, - commit_id, - commit_args, - ) - }, - ) - }; - pub(crate) static ref MAX_PROCESS_AND_CLOSE_PER_TX_USING_LOOKUP: u8 = { - let commit_id = 0; - max_chunks_per_transaction_using_lookup_table( - "Max process and close per tx using lookup", - |auth_pubkey, committee, delegated_account_owner| { - let commit_args = CommitStateFromBufferArgs::default(); - let buffer_pda = Pubkey::new_unique(); - process_and_close_ixs( - auth_pubkey, - &committee, - &delegated_account_owner, - &buffer_pda, - commit_id, - commit_args, - ) - }, - None, - ) - }; - } - - // ----------------- - // Finalize - // ----------------- - lazy_static! { - pub(crate) static ref MAX_FINALIZE_PER_TX: u8 = { - max_chunks_per_transaction("Max finalize per tx", |auth_pubkey| { - let pubkey = Pubkey::new_unique(); - vec![finalize_ix(auth_pubkey, &pubkey)] - }) - }; - pub(crate) static ref MAX_FINALIZE_PER_TX_USING_LOOKUP: u8 = { - max_chunks_per_transaction_using_lookup_table( - "Max finalize per tx using lookup", - |auth_pubkey, committee, _| { - vec![finalize_ix(auth_pubkey, &committee)] - }, - Some(40), - ) - }; - } - - // ----------------- - // Undelegate - // ----------------- - lazy_static! { - pub(crate) static ref MAX_UNDELEGATE_PER_TX: u8 = { - max_chunks_per_transaction("Max undelegate per tx", |auth_pubkey| { - let pubkey = Pubkey::new_unique(); - let owner_program = Pubkey::new_unique(); - vec![dlp_api::instruction_builder::undelegate( - auth_pubkey, - pubkey, - owner_program, - auth_pubkey, - )] - }) - }; - pub(crate) static ref MAX_UNDELEGATE_PER_TX_USING_LOOKUP: u8 = { - max_chunks_per_transaction_using_lookup_table( - "Max undelegate per tx using lookup", - |auth_pubkey, committee, owner_program| { - vec![dlp_api::instruction_builder::undelegate( - auth_pubkey, - committee, - owner_program, - auth_pubkey, - )] - }, - None, - ) - }; - } - - // ----------------- - // Max Chunks Per Transaction - // ----------------- - - fn max_chunks_per_transaction Vec>( - label: &str, - create_ixs: F, - ) -> u8 { - info!(test_label = label, "Starting transaction chunking test"); - - let auth = Keypair::new(); - let auth_pubkey = auth.pubkey(); - // NOTE: the size of the budget instructions is always the same, no matter - // which budget we provide - let mut ixs = ComputeBudget::Process(Budget::default()).instructions(1); - let mut chunks = 0_u8; - loop { - ixs.extend(create_ixs(auth_pubkey)); - chunks += 1; - - // SAFETY: runs statically - let versioned_msg = - Message::try_compile(&auth_pubkey, &ixs, &[], Hash::default()) - .unwrap(); - // SAFETY: runs statically - let versioned_tx = VersionedTransaction::try_new( - VersionedMessage::V0(versioned_msg), - &[&auth], - ) - .unwrap(); - let encoded = serialize_and_encode_base64(&versioned_tx); - info!( - chunks = chunks, - size_bytes = encoded.len(), - "Transaction size measured" - ); - if encoded.len() > MAX_ENCODED_TRANSACTION_SIZE { - return chunks - 1; - } - } - } - - fn extend_lookup_table( - lookup_table: &mut AddressLookupTableAccount, - auth_pubkey: Pubkey, - committee: Pubkey, - owner: Option<&Pubkey>, - ) { - let keys = provide_committee_pubkeys(&committee, owner) - .into_iter() - .chain(provide_common_pubkeys(&auth_pubkey)) - .chain(lookup_table.addresses.iter().cloned()) - .collect::>(); - lookup_table.addresses = keys.into_iter().collect(); - assert!( - lookup_table.addresses.len() <= LOOKUP_TABLE_MAX_ADDRESSES, - "Lookup table has too many ({}) addresses", - lookup_table.addresses.len() - ); - } - - fn max_chunks_per_transaction_using_lookup_table< - FI: Fn(Pubkey, Pubkey, Pubkey) -> Vec, - >( - label: &str, - create_ixs: FI, - start_at: Option, - ) -> u8 { - info!(test_label = label, start_at = ?start_at, "Starting lookup table transaction test"); - let auth = Keypair::new(); - let auth_pubkey = auth.pubkey(); - let mut ixs = ComputeBudget::Process(Budget::default()).instructions(1); - let mut chunks = start_at.unwrap_or_default(); - let mut lookup_table = AddressLookupTableAccount { - key: Pubkey::default(), - addresses: vec![], - }; - // If we start at specific chunk size let's prep the ixs and assume - // we are using the same addresses to avoid blowing out the lookup table - if chunks > 0 { - let committee = Pubkey::new_unique(); - let owner_program = Pubkey::new_unique(); - extend_lookup_table( - &mut lookup_table, - auth_pubkey, - committee, - Some(&owner_program), - ); - for _ in 0..chunks { - ixs.extend(create_ixs(auth_pubkey, committee, owner_program)); - } - } - loop { - let committee = Pubkey::new_unique(); - let owner_program = Pubkey::new_unique(); - ixs.extend(create_ixs(auth_pubkey, committee, owner_program)); - - chunks += 1; - extend_lookup_table( - &mut lookup_table, - auth_pubkey, - committee, - Some(&owner_program), - ); - - // SAFETY: runs statically - let versioned_msg = Message::try_compile( - &auth_pubkey, - &ixs, - &[lookup_table.clone()], - Hash::default(), - ) - .unwrap(); - // SAFETY: runs statically - let versioned_tx = VersionedTransaction::try_new( - VersionedMessage::V0(versioned_msg), - &[&auth], - ) - .unwrap(); - let encoded = serialize_and_encode_base64(&versioned_tx); - info!( - chunks = chunks, - size_bytes = encoded.len(), - "Transaction size measured with lookup table" - ); - if encoded.len() > MAX_ENCODED_TRANSACTION_SIZE { - return chunks - 1; - } - } - } -} diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index d18d0575f..221667f6e 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -1,23 +1,18 @@ -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, sync::Arc}; use borsh::to_vec; use magicblock_committor_service::{ + committor_processor::CommittorProcessor, config::ChainConfig, intent_executor::{error::IntentExecutorError, ExecutionOutput}, persist::CommitStrategy, - service_ext::{BaseIntentCommittorExt, CommittorServiceExt}, - BaseIntentCommittor, CommittorService, ComputeBudgetConfig, + ComputeBudgetConfig, }; use magicblock_core::intent::CommittedAccount; use magicblock_program::magic_scheduled_base_intent::{ CommitAndUndelegate, CommitType, MagicBaseIntent, MagicIntentBundle, ScheduledIntentBundle, UndelegateType, }; -use magicblock_rpc_client::MagicblockRpcClient; use program_flexi_counter::state::FlexiCounter; use solana_account::{Account, ReadableAccount}; use solana_commitment_config::CommitmentConfig; @@ -245,14 +240,15 @@ async fn commit_single_account( fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; // Run each test with and without finalizing - let service = CommittorService::try_start( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(); - let service = CommittorServiceExt::new(Arc::new(service)); + let processor = Arc::new( + CommittorProcessor::try_new( + validator_auth.insecure_clone(), + ":memory:", + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + common::MockActionsCallbackExecutor::default(), + ) + .unwrap(), + ); let counter_auth = Keypair::new(); let (pubkey, mut account) = @@ -308,7 +304,7 @@ async fn commit_single_account( // We should always be able to Commit & Finalize 1 account either with Args or Buffers ix_commit_local( - service, + processor, vec![intent], expect_strategies(&[(expected_strategy, 1)]), program_flexi_counter::ID, @@ -327,14 +323,15 @@ async fn commit_book_order_account( fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; // Run each test with and without finalizing - let service = CommittorService::try_start( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(); - let service = CommittorServiceExt::new(Arc::new(service)); + let processor = Arc::new( + CommittorProcessor::try_new( + validator_auth.insecure_clone(), + ":memory:", + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + common::MockActionsCallbackExecutor::default(), + ) + .unwrap(), + ); let payer = Keypair::new(); let (order_book_pk, mut order_book_ac) = @@ -387,7 +384,7 @@ async fn commit_book_order_account( }; ix_commit_local( - service, + processor, vec![intent], expect_strategies(&[(expected_strategy, 1)]), program_schedulecommit::ID, @@ -800,14 +797,15 @@ async fn commit_multiple_accounts( let validator_auth = ensure_validator_authority(); fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; - let service = CommittorService::try_start( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(); - let service = CommittorServiceExt::new(Arc::new(service)); + let processor = Arc::new( + CommittorProcessor::try_new( + validator_auth.insecure_clone(), + ":memory:", + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + common::MockActionsCallbackExecutor::default(), + ) + .unwrap(), + ); // Create bundles of committed accounts let bundles_of_committees = create_bundles(bundle_size, bytess).await; @@ -850,7 +848,7 @@ async fn commit_multiple_accounts( .collect::>(); ix_commit_local( - service, + processor, intents, expected_strategies, program_flexi_counter::ID, @@ -869,14 +867,15 @@ async fn execute_intent_bundle( let validator_auth = ensure_validator_authority(); fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; - let service = CommittorService::try_start( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(); - let service = CommittorServiceExt::new(Arc::new(service)); + let processor = Arc::new( + CommittorProcessor::try_new( + validator_auth.insecure_clone(), + ":memory:", + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + common::MockActionsCallbackExecutor::default(), + ) + .unwrap(), + ); // Create bundles of committed accounts let to_commit = create_and_delegate_accounts(bytess_to_commit); @@ -911,7 +910,7 @@ async fn execute_intent_bundle( intent_bundle, }; ix_commit_local( - service, + processor, vec![intent_bundle], expected_strategies, program_flexi_counter::id(), @@ -947,13 +946,13 @@ async fn execute_intent_bundle( // Test Executor // ----------------- async fn ix_commit_local( - service: CommittorServiceExt, + processor: Arc, intent_bundles: Vec, expected_strategies: ExpectedStrategies, program_id: Pubkey, ) { - let execution_outputs = service - .schedule_intent_bundles_waiting(intent_bundles.clone()) + let execution_outputs = processor + .execute_intent_bundles(intent_bundles.clone()) .await .unwrap() .into_iter() @@ -961,7 +960,6 @@ async fn ix_commit_local( // Assert that all completed assert_eq!(execution_outputs.len(), intent_bundles.len()); - service.release_common_pubkeys().await.unwrap(); let rpc_client = RpcClient::new("http://localhost:7799".to_string()); let mut strategies = ExpectedStrategies::new(); @@ -1061,10 +1059,8 @@ async fn ix_commit_local( }) .collect(); - let statuses = service + let statuses = processor .get_commit_statuses(base_intent.id) - .await - .unwrap() .unwrap(); debug!( "{}", @@ -1122,75 +1118,6 @@ async fn ix_commit_local( "Strategies used do not match expected ones" ); - let expect_empty_lookup_tables = false; - // changeset.accounts.len() == changeset.accounts_to_undelegate.len(); - if expect_empty_lookup_tables { - let lookup_tables = service.get_lookup_tables().await.unwrap(); - assert!(lookup_tables.active.is_empty()); - - if utils::TEST_TABLE_CLOSE { - let mut closing_tables = lookup_tables.released; - - // Tables deactivate after ~2.5 mins (150secs), but most times - // it takes a lot longer so we allow double the time - const MAX_TIME_TO_CLOSE: Duration = Duration::from_secs(300); - info!( - "Waiting for lookup tables close for up to {} secs", - MAX_TIME_TO_CLOSE.as_secs() - ); - - let start = Instant::now(); - let rpc_client = MagicblockRpcClient::from(rpc_client); - loop { - let accs = rpc_client - .get_multiple_accounts_with_commitment( - &closing_tables, - CommitmentConfig::confirmed(), - None, - ) - .await - .unwrap(); - let closed_pubkeys = accs - .into_iter() - .zip(closing_tables.iter()) - .filter_map(|(acc, pubkey)| { - if acc.is_none() { - Some(*pubkey) - } else { - None - } - }) - .collect::>(); - closing_tables.retain(|pubkey| { - if closed_pubkeys.contains(pubkey) { - debug!("Table {} closed", pubkey); - false - } else { - true - } - }); - if closing_tables.is_empty() { - break; - } - debug!( - "Still waiting for {} released table(s) to close", - closing_tables.len() - ); - if Instant::now() - start > MAX_TIME_TO_CLOSE { - panic!( - "Timed out waiting for tables close after {} seconds. Still open: {}", - MAX_TIME_TO_CLOSE.as_secs(), - closing_tables - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ") - ); - } - utils::sleep_millis(10_000).await; - } - } - } } fn validate_account( From 641e4bc11653b0cab3f8b8c3fb2f0d915ffe966e Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 17:42:30 +0700 Subject: [PATCH 10/97] refactor: cleanup --- .../src/committor_processor.rs | 4 +- .../src/compute_budget.rs | 138 ------------------ magicblock-committor-service/src/stubs/mod.rs | 1 + .../tests/test_ix_commit_local.rs | 5 +- 4 files changed, 4 insertions(+), 144 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 599f07c35..acf5cfb91 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -40,7 +40,7 @@ const POISONED_MUTEX_MSG: &str = type BundleResultListener = oneshot::Sender; pub struct CommittorProcessor { - table_mania: TableMania, + _table_mania: TableMania, persister: IntentPersisterImpl, commits_scheduler: IntentExecutionManager, task_info_fetcher: Arc>, @@ -103,7 +103,7 @@ impl CommittorProcessor { )); Ok(Self { - table_mania, + _table_mania: table_mania, commits_scheduler, persister, task_info_fetcher, diff --git a/magicblock-committor-service/src/compute_budget.rs b/magicblock-committor-service/src/compute_budget.rs index 93a40e8e7..4a6b5e79f 100644 --- a/magicblock-committor-service/src/compute_budget.rs +++ b/magicblock-committor-service/src/compute_budget.rs @@ -1,26 +1,6 @@ use solana_compute_budget_interface::ComputeBudgetInstruction; use solana_instruction::Instruction; -// ----------------- -// Budgets -// ----------------- -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Budget { - base_budget: u32, - per_committee: u32, - compute_unit_price: u64, -} - -impl Default for Budget { - fn default() -> Self { - Self { - base_budget: 80_000, - per_committee: 45_000, - compute_unit_price: 1_000_000, - } - } -} - #[derive(Debug, Clone)] pub struct BufferWithReallocBudget { base_budget: u32, @@ -67,20 +47,9 @@ impl BufferWriteChunkBudget { } } -// ----------------- -// ComputeBudgetConfig -// ----------------- #[derive(Debug, Clone)] pub struct ComputeBudgetConfig { pub compute_unit_price: u64, - pub args_process: Budget, - pub finalize: Budget, - pub buffer_close: Budget, - /// The budget used for processing and process + closing a buffer. - /// Since we mix pure process and process + close instructions, we need to - /// assume the worst case and use the process + close budget for all. - pub buffer_process_and_close: Budget, - pub undelegate: Budget, pub buffer_init: BufferWithReallocBudget, pub buffer_realloc: BufferWithReallocBudget, pub buffer_write: BufferWriteChunkBudget, @@ -90,31 +59,6 @@ impl ComputeBudgetConfig { pub fn new(compute_unit_price: u64) -> Self { Self { compute_unit_price, - args_process: Budget { - compute_unit_price, - base_budget: 80_000, - per_committee: 35_000, - }, - buffer_close: Budget { - compute_unit_price, - base_budget: 10_000, - per_committee: 25_000, - }, - buffer_process_and_close: Budget { - compute_unit_price, - base_budget: 40_000, - per_committee: 45_000, - }, - finalize: Budget { - compute_unit_price, - base_budget: 80_000, - per_committee: 25_000, - }, - undelegate: Budget { - compute_unit_price, - base_budget: 70_000, - per_committee: 35_000, - }, buffer_init: BufferWithReallocBudget { base_budget: 12_000, per_realloc_ix: 6_000, @@ -134,88 +78,6 @@ impl ComputeBudgetConfig { } } -impl ComputeBudgetConfig { - pub fn args_process_budget(&self) -> ComputeBudget { - ComputeBudget::Process(self.args_process) - } - pub fn buffer_close_budget(&self) -> ComputeBudget { - ComputeBudget::Close(self.buffer_close) - } - pub fn buffer_process_and_close_budget(&self) -> ComputeBudget { - ComputeBudget::ProcessAndClose(self.buffer_process_and_close) - } - pub fn finalize_budget(&self) -> ComputeBudget { - ComputeBudget::Finalize(self.finalize) - } - pub fn undelegate_budget(&self) -> ComputeBudget { - ComputeBudget::Undelegate(self.undelegate) - } -} - -// ----------------- -// ComputeBudget -// ----------------- -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ComputeBudget { - Process(Budget), - Close(Budget), - ProcessAndClose(Budget), - Finalize(Budget), - Undelegate(Budget), -} - -impl ComputeBudget { - fn base_budget(&self) -> u32 { - use ComputeBudget::*; - match self { - Process(budget) => budget.base_budget, - Close(budget) => budget.base_budget, - ProcessAndClose(budget) => budget.base_budget, - Finalize(budget) => budget.base_budget, - Undelegate(budget) => budget.base_budget, - } - } - - fn per_committee(&self) -> u32 { - use ComputeBudget::*; - match self { - Process(budget) => budget.per_committee, - Close(budget) => budget.per_committee, - ProcessAndClose(budget) => budget.per_committee, - Finalize(budget) => budget.per_committee, - Undelegate(budget) => budget.per_committee, - } - } - - pub fn compute_unit_price(&self) -> u64 { - use ComputeBudget::*; - match self { - Process(budget) => budget.compute_unit_price, - Close(budget) => budget.compute_unit_price, - ProcessAndClose(budget) => budget.compute_unit_price, - Finalize(budget) => budget.compute_unit_price, - Undelegate(budget) => budget.compute_unit_price, - } - } - - fn total_budget(&self, committee_count: u32) -> u32 { - self.per_committee() - .checked_mul(committee_count) - .and_then(|product| product.checked_add(self.base_budget())) - .unwrap_or(u32::MAX) - } - - pub fn instructions(&self, committee_count: usize) -> Vec { - let committee_count = - u32::try_from(committee_count).unwrap_or(u32::MAX); - - instructions( - self.total_budget(committee_count), - self.compute_unit_price(), - ) - } -} - fn instructions( compute_budget: u32, compute_unit_price: u64, diff --git a/magicblock-committor-service/src/stubs/mod.rs b/magicblock-committor-service/src/stubs/mod.rs index e69de29bb..8b1378917 100644 --- a/magicblock-committor-service/src/stubs/mod.rs +++ b/magicblock-committor-service/src/stubs/mod.rs @@ -0,0 +1 @@ + diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 221667f6e..14526a18c 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -1059,9 +1059,7 @@ async fn ix_commit_local( }) .collect(); - let statuses = processor - .get_commit_statuses(base_intent.id) - .unwrap(); + let statuses = processor.get_commit_statuses(base_intent.id).unwrap(); debug!( "{}", statuses @@ -1117,7 +1115,6 @@ async fn ix_commit_local( strategies, expected_strategies, "Strategies used do not match expected ones" ); - } fn validate_account( From c380be777c63040270844662d961e488d8fdfd86 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 17:45:39 +0700 Subject: [PATCH 11/97] refactor: cleanup --- Cargo.lock | 7 -- magicblock-committor-service/Cargo.toml | 8 --- magicblock-committor-service/src/error.rs | 10 --- magicblock-committor-service/src/lib.rs | 3 - .../src/pubkeys_provider.rs | 67 ------------------- magicblock-committor-service/src/stubs/mod.rs | 1 - 6 files changed, 96 deletions(-) delete mode 100644 magicblock-committor-service/src/pubkeys_provider.rs delete mode 100644 magicblock-committor-service/src/stubs/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 63b57b609..68d159661 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3931,7 +3931,6 @@ dependencies = [ "bincode", "borsh", "futures-util", - "lazy_static", "lru 0.16.4", "magicblock-account-cloner", "magicblock-accounts-db", @@ -3945,10 +3944,8 @@ dependencies = [ "magicblock-table-mania", "rand 0.9.4", "rusqlite", - "serde_json", "solana-account 3.4.0", "solana-account-decoder", - "solana-address-lookup-table-interface 3.1.0", "solana-commitment-config", "solana-compute-budget-interface", "solana-hash 3.1.0", @@ -3961,19 +3958,15 @@ dependencies = [ "solana-rpc-client-api", "solana-signature 3.4.0", "solana-signer", - "solana-system-program", "solana-transaction", "solana-transaction-error 3.2.0", "solana-transaction-status-client-types", - "static_assertions", "tempfile", "test-kit", "thiserror 2.0.18", "tokio", "tokio-util", "tracing", - "tracing-log", - "tracing-subscriber", ] [[package]] diff --git a/magicblock-committor-service/Cargo.toml b/magicblock-committor-service/Cargo.toml index 2e681a5dd..e878e7185 100644 --- a/magicblock-committor-service/Cargo.toml +++ b/magicblock-committor-service/Cargo.toml @@ -36,7 +36,6 @@ magicblock-table-mania = { workspace = true } rusqlite = { workspace = true } solana-account = { workspace = true } solana-account-decoder = { workspace = true } -solana-address-lookup-table-interface = { workspace = true } solana-commitment-config = { workspace = true } solana-compute-budget-interface = { workspace = true } solana-hash = { workspace = true } @@ -49,25 +48,18 @@ solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-signature = { workspace = true } solana-signer = { workspace = true } -solana-system-program = { workspace = true } solana-transaction = { workspace = true } solana-transaction-error = { workspace = true } solana-transaction-status-client-types = { workspace = true } -static_assertions = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } [dev-dependencies] solana-signature = { workspace = true, features = ["rand"] } -serde_json = { workspace = true } -lazy_static = { workspace = true } rand = { workspace = true } tempfile = { workspace = true } test-kit = { workspace = true } -tracing-subscriber = { workspace = true } -tracing-log = { workspace = true } [features] default = [] -dev-context-only-utils = [] diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index 87c653d4e..2b4ce584b 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,4 +1,3 @@ -use solana_signature::Signature; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; @@ -20,12 +19,3 @@ pub enum CommittorServiceError { #[error("Attempt to schedule already scheduled message id: {0}")] RepeatingMessageError(u64), } - -// TODO(edwin): use or remove -impl CommittorServiceError { - pub fn signature(&self) -> Option { - match self { - _ => None, - } - } -} diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 6d5f3e12b..48d90930f 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -6,9 +6,6 @@ pub mod error; pub mod intent_execution_manager; pub mod intent_executor; pub mod persist; -mod pubkeys_provider; -#[cfg(feature = "dev-context-only-utils")] -pub mod stubs; pub mod tasks; pub mod transaction_preparator; pub mod transactions; diff --git a/magicblock-committor-service/src/pubkeys_provider.rs b/magicblock-committor-service/src/pubkeys_provider.rs deleted file mode 100644 index 9995c174e..000000000 --- a/magicblock-committor-service/src/pubkeys_provider.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::collections::HashSet; - -use dlp_api::pda; -use solana_pubkey::Pubkey; -use solana_system_program::id as system_program_id; -use tracing::*; - -/// Returns all accounts needed to process/finalize a commit for the account -/// with the provided `delegated_account`. -/// NOTE: that buffer and chunk accounts are different for each commit and -/// thus are not included -pub fn provide_committee_pubkeys( - committee: &Pubkey, - owner_program: Option<&Pubkey>, -) -> HashSet { - let mut set = HashSet::new(); - set.insert(*committee); - set.insert(pda::delegation_record_pda_from_delegated_account(committee)); - set.insert(pda::delegation_metadata_pda_from_delegated_account( - committee, - )); - set.insert(pda::commit_state_pda_from_delegated_account(committee)); - set.insert(pda::commit_record_pda_from_delegated_account(committee)); - set.insert(pda::undelegate_buffer_pda_from_delegated_account(committee)); - - // NOTE: ideally we'd also include the rent_fee_payer here, but that is - // not known to the validator at the time of cloning since it is - // stored inside the delegation metadata account instead of the - // delegation record - - if let Some(owner_program) = owner_program { - set.insert(pda::program_config_from_program_id(owner_program)); - } else { - warn!(committee = %committee, "No owner program provided for committee"); - } - set -} - -/// Returns common accounts needed for process/finalize transactions, -/// namely the program ids used and the fees vaults and the validator itself. -pub fn provide_common_pubkeys(validator: &Pubkey) -> HashSet { - let mut set = HashSet::new(); - - let deleg_program = dlp_api::id(); - let protocol_fees_vault = pda::fees_vault_pda(); - let validator_fees_vault = - pda::validator_fees_vault_pda_from_validator(validator); - let committor_program = magicblock_committor_program::id(); - - trace!( - validator = %validator, - deleg_program = %deleg_program, - protocol_fees_vault = %protocol_fees_vault, - validator_fees_vault = %validator_fees_vault, - committor_program = %committor_program, - "Common pubkeys loaded" - ); - - set.insert(*validator); - set.insert(system_program_id()); - set.insert(deleg_program); - set.insert(protocol_fees_vault); - set.insert(validator_fees_vault); - set.insert(committor_program); - - set -} diff --git a/magicblock-committor-service/src/stubs/mod.rs b/magicblock-committor-service/src/stubs/mod.rs deleted file mode 100644 index 8b1378917..000000000 --- a/magicblock-committor-service/src/stubs/mod.rs +++ /dev/null @@ -1 +0,0 @@ - From 3a3a177e370fefe058fc6afbe2613d7cf3f1cf99 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 17:47:02 +0700 Subject: [PATCH 12/97] fix: lint --- magicblock-api/src/magic_validator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 7f0f5c4a3..eba8f3405 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -235,7 +235,7 @@ impl MagicValidator { let committor_processor = { let processor = Self::init_committor_processor( &config, - &ledger.latest_block(), + ledger.latest_block(), )?; Arc::new(processor) }; From b49c17ccc50cbabacf3876a94cc7a331b2326be0 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 17:57:42 +0700 Subject: [PATCH 13/97] feat: update locks --- test-integration/Cargo.lock | 3 --- test-integration/test-committor-service/Cargo.toml | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 678b5403d..e5b712eaf 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -4452,7 +4452,6 @@ dependencies = [ "rusqlite", "solana-account", "solana-account-decoder", - "solana-address-lookup-table-interface", "solana-commitment-config", "solana-compute-budget-interface", "solana-hash 3.1.0", @@ -4465,11 +4464,9 @@ dependencies = [ "solana-rpc-client-api", "solana-signature", "solana-signer", - "solana-system-program", "solana-transaction", "solana-transaction-error", "solana-transaction-status-client-types", - "static_assertions", "thiserror 2.0.18", "tokio", "tokio-util 0.7.17", diff --git a/test-integration/test-committor-service/Cargo.toml b/test-integration/test-committor-service/Cargo.toml index 53e29921d..6035b5ec8 100644 --- a/test-integration/test-committor-service/Cargo.toml +++ b/test-integration/test-committor-service/Cargo.toml @@ -12,9 +12,7 @@ magicblock-core = { workspace = true } magicblock-committor-program = { workspace = true, features = [ "no-entrypoint", ] } -magicblock-committor-service = { workspace = true, features = [ - "dev-context-only-utils", -] } +magicblock-committor-service = { workspace = true } magicblock-delegation-program-api = { workspace = true } magicblock-program = { workspace = true } magicblock-rpc-client = { workspace = true } From 01ce24f711bd67543d76d0e0499c8e2785423a77 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 29 May 2026 18:02:31 +0700 Subject: [PATCH 14/97] fix: remove outdated file --- .../tests/replay_base_slot.rs | 173 ------------------ 1 file changed, 173 deletions(-) delete mode 100644 magicblock-processor/tests/replay_base_slot.rs diff --git a/magicblock-processor/tests/replay_base_slot.rs b/magicblock-processor/tests/replay_base_slot.rs deleted file mode 100644 index 37d6bcc7d..000000000 --- a/magicblock-processor/tests/replay_base_slot.rs +++ /dev/null @@ -1,173 +0,0 @@ -// Regression test for: replay base slot must be accountsdb_slot + 1, not accountsdb_slot. -// -// The bug: process_ledger was called with full_process_starting_slot = accountsdb_slot -// (inclusive). Because accountsdb_slot is the last slot whose effects are already in -// AccountsDb, this caused every successful transaction in that slot to be applied a -// second time. -// -// The fix: start replay at accountsdb_slot + 1 so the last persisted slot is never re-run. -// -// Test strategy: -// 1. Set up an account C with data[0] = 0 in AccountsDb. -// 2. Build a non-idempotent Increment transaction T1 targeting C. -// 3. Write T1 to the ledger at slot S (marked successful) but do NOT execute it yet. -// 4. Apply T1 to AccountsDb via replay_transaction — AccountsDb is now post-T1 (data[0] = 1) -// and accountsdb_slot = S, exactly as after a clean shutdown. -// 5. Inject an empty block header at S+1 directly into the ledger WITHOUT calling -// advance_slot() — ledger_slot is now S+1 while accountsdb_slot stays at S. -// 6. Call process_ledger with the correct starting slot (S+1): T1 must NOT be re-applied, -// so data[0] stays 1. -// 7. A second test demonstrates the bug: process_ledger starting at S (the old code) -// re-applies T1 → data[0] becomes 2. - -use guinea::GuineaInstruction; -use magicblock_accounts_db::traits::AccountsBank; -use magicblock_core::link::transactions::SanitizeableTransaction; -use magicblock_ledger::{ - blockstore_processor::process_ledger, LatestBlockInner, -}; -use solana_account::ReadableAccount; -use solana_keypair::Keypair; -use solana_program::{ - hash::Hash, - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, -}; -use solana_signer::Signer; -use solana_transaction::sanitized::SanitizedTransaction; -use solana_transaction_status::TransactionStatusMeta; -use test_kit::ExecutionTestEnv; - -async fn setup(env: &ExecutionTestEnv) -> Keypair { - env.yield_to_scheduler().await; - let kp = env.create_account_with_config(LAMPORTS_PER_SOL, 128, guinea::ID); - assert_eq!( - env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], - 0, - "account data must start at 0" - ); - kp -} - -fn build_increment_tx( - env: &ExecutionTestEnv, - kp: &Keypair, -) -> SanitizedTransaction { - let ix = Instruction::new_with_bincode( - guinea::ID, - &GuineaInstruction::Increment, - vec![AccountMeta::new(kp.pubkey(), false)], - ); - env.build_transaction(&[ix]).sanitize(true).unwrap() -} - -fn write_tx_to_ledger( - env: &ExecutionTestEnv, - sanitized: &SanitizedTransaction, - slot: u64, -) { - let sig = *sanitized.signature(); - let meta = TransactionStatusMeta { - fee: 0, - pre_balances: vec![LAMPORTS_PER_SOL], - post_balances: vec![LAMPORTS_PER_SOL], - status: Ok(()), - ..Default::default() - }; - let versioned = sanitized.to_versioned_transaction(); - let encoded = bincode::serialize(&versioned).unwrap(); - let locks = sanitized.get_account_locks_unchecked(); - env.ledger - .write_transaction( - sig, - slot, - 0, - locks.writable, - locks.readonly, - &encoded, - meta, - ) - .expect("failed to write transaction to ledger"); -} - -fn inject_empty_block(env: &ExecutionTestEnv, slot: u64) { - let latest = env.ledger.latest_block().load(); - let time = latest.clock.unix_timestamp; - drop(latest); - env.ledger - .write_block(LatestBlockInner::new(slot, Hash::new_unique(), time + 1)) - .expect("failed to write injected block"); -} - -/// Correct behavior: process_ledger starting at accountsdb_slot + 1 does not re-apply T1. -#[tokio::test] -async fn test_replay_starts_after_accountsdb_slot() { - let env = ExecutionTestEnv::new_replica_mode(1, false); - let kp = setup(&env).await; - - let slot_s = env.ledger.latest_block().load().slot; - - let sanitized = build_increment_tx(&env, &kp); - write_tx_to_ledger(&env, &sanitized, slot_s); - - // Apply T1 so AccountsDb reflects post-T1 state (data[0] = 1). - env.replay_transaction(false, sanitized).await.unwrap(); - assert_eq!( - env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], - 1, - "T1 must be applied to AccountsDb before replay test" - ); - assert_eq!(env.accountsdb.slot(), slot_s); - - // Inject empty block at S+1 without advancing accountsdb_slot. - inject_empty_block(&env, slot_s + 1); - - // Fix: replay starts at S+1, T1 at slot S is not re-applied. - process_ledger( - &env.ledger, - slot_s + 1, - env.transaction_scheduler.clone(), - 0, - ) - .await - .unwrap(); - - assert_eq!( - env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], - 1, - "data[0] must remain 1 — T1 must not be replayed again" - ); -} - -/// Bug demonstration: process_ledger starting at accountsdb_slot re-applies T1. -#[tokio::test] -async fn test_replay_from_accountsdb_slot_double_applies() { - let env = ExecutionTestEnv::new_replica_mode(1, false); - let kp = setup(&env).await; - - let slot_s = env.ledger.latest_block().load().slot; - - let sanitized = build_increment_tx(&env, &kp); - write_tx_to_ledger(&env, &sanitized, slot_s); - - // Apply T1 (data[0] = 1). - env.replay_transaction(false, sanitized).await.unwrap(); - assert_eq!( - env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], - 1, - "T1 must be applied to AccountsDb before replay test" - ); - - inject_empty_block(&env, slot_s + 1); - - // Bug: replay starts at S (inclusive), so T1 is applied a second time. - process_ledger(&env.ledger, slot_s, env.transaction_scheduler.clone(), 0) - .await - .unwrap(); - - assert_eq!( - env.accountsdb.get_account(&kp.pubkey()).unwrap().data()[0], - 2, - "data[0] must be 2 — T1 was replayed a second time (double-apply bug)" - ); -} From d292b3e30305814b26f1428bba70c93abdf69eb1 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 1 Jun 2026 11:22:10 +0700 Subject: [PATCH 15/97] feat: some docs --- magicblock-committor-service/src/service.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index f3dd3a6b2..ec4a6ffe9 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -95,7 +95,7 @@ where pub struct ServiceInner { /// Chainlink for notifying of undelegations chainlink: Arc, - /// ER client specific for Intent needs + /// ER client specific for Intent needs. Could be switched to RpcClient intent_rpc_client: Arc, /// Processor of accepted intents processor: Arc, @@ -129,7 +129,8 @@ where } } - pub fn start(self) -> JoinHandle<()> { + /// Starts 2 workers: one accepting intents, another awaiting and handling results + fn start(self) -> JoinHandle<()> { let result_subscriber = self.processor.subscribe_for_results(); let cancellation_token = self.cancellation_token.clone(); tokio::spawn(Self::result_processor( @@ -296,7 +297,7 @@ where intent_client: &Arc, execution_result: BroadcastedIntentExecutionResult, intents_meta_map: &Arc>>, - ) -> Result<(), IntentExecutionServiceError> { + ) -> Result<(), R::Error> { // Create IntentMeta let intent_id = execution_result.id; // Remove intent from metas @@ -321,8 +322,7 @@ where ); intent_client .notify_commit_sent(sent_transaction, sent_commit) - .await - .map_err(Into::into)?; + .await?; Ok(()) } From 345f1f1b35dd0b8c6f9e126d0dd4659fc5a95755 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 1 Jun 2026 13:19:40 +0700 Subject: [PATCH 16/97] refactor: fetch of pending commits and refresh --- .../src/committor_processor.rs | 309 ++++-------------- magicblock-committor-service/src/error.rs | 5 + .../src/persist/commit_persister.rs | 242 ++++++++++++++ 3 files changed, 312 insertions(+), 244 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index fd14ddff4..c6eec4369 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -1,27 +1,21 @@ use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap}, + collections::{hash_map::Entry, HashMap}, path::Path, sync::{atomic::AtomicU64, Arc, Mutex}, time::{SystemTime, UNIX_EPOCH}, }; use futures_util::future::join_all; -use magicblock_core::{ - intent::CommittedAccount, traits::ActionsCallbackScheduler, -}; -use magicblock_program::magic_scheduled_base_intent::{ - CommitAndUndelegate, CommitType, MagicIntentBundle, ScheduledIntentBundle, - UndelegateType, -}; +use magicblock_core::traits::ActionsCallbackScheduler; +use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; -use solana_account::Account; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_transaction::Transaction; +use solana_signer::Signer; use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; -use tracing::{error, info, instrument, warn}; +use tracing::{error, info, instrument}; use crate::{ config::ChainConfig, @@ -37,18 +31,18 @@ use crate::{ }, }, persist::{ - CommitStatusRow, CommitType as PersistCommitType, IntentPersister, - IntentPersisterImpl, MessageSignatures, + CommitStatusRow, IntentPersister, IntentPersisterImpl, + MessageSignatures, }, }; - -const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; const POISONED_MUTEX_MSG: &str = "CommittorProcessor pending messages mutex poisoned!"; type BundleResultListener = oneshot::Sender; pub struct CommittorProcessor { + authority: Keypair, _table_mania: TableMania, + magic_rpc_client: MagicblockRpcClient, persister: IntentPersisterImpl, commits_scheduler: IntentExecutionManager, task_info_fetcher: Arc>, @@ -116,7 +110,9 @@ impl CommittorProcessor { )); Ok(Self { + authority, _table_mania: table_mania, + magic_rpc_client: magic_block_rpc_client, commits_scheduler, persister, task_info_fetcher, @@ -144,22 +140,29 @@ impl CommittorProcessor { Ok(signatures) } - pub async fn pending_intent_bundles( + pub async fn pending_intent_bundles( &self, - ) -> CommittorServiceResult> { - let rows = self.persister.get_pending_commit_statuses()?; - if rows.is_empty() { - return Ok(Vec::new()); + ) -> CommittorServiceResult> + where + F: Fn(&CommitStatusRow) -> bool, + { + const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; + + // Extract pending bundles satisfying predicate + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let mut bundles = self.persister.pending_intent_bundles(|row| { + now.saturating_sub(row.last_retried_at) <= RECOVERY_MIN_AGE_SECS + })?; + + if bundles.is_empty() { + return Ok(bundles); } - let recovery_base_slot = self.magicblock_rpc_client.get_slot().await?; - let bundles = pending_rows_to_scheduled_intent_bundles( - rows, - self.auth_pubkey(), - recovery_base_slot, - unix_timestamp(), - ); - if !bundles.is_empty() { + // Log info about extracted bundles + { let accounts_count: usize = bundles .iter() .map(|bundle| bundle.get_all_committed_pubkeys().len()) @@ -171,9 +174,43 @@ impl CommittorProcessor { ); } + // Extracted bundles are out of data and missing some of the info + self.refresh_intent_bundles(&mut bundles).await?; + Ok(bundles) } + async fn refresh_intent_bundles( + &self, + intent_bundles: &mut [ScheduledIntentBundle], + ) -> CommittorServiceResult<()> { + let payer = self.authority.pubkey(); + let slot = self.magic_rpc_client.get_slot().await?; + + macro_rules! set_remote_slot { + ($field:expr, $slot:expr) => { + if let Some(ref mut v) = $field { + v.get_committed_accounts_mut() + .iter_mut() + .for_each(|a| a.remote_slot = slot); + } + }; + } + + intent_bundles.iter_mut().for_each(|b| { + b.payer = payer; + set_remote_slot!(b.intent_bundle.commit, slot); + set_remote_slot!(b.intent_bundle.commit_finalize, slot); + set_remote_slot!(b.intent_bundle.commit_and_undelegate, slot); + set_remote_slot!( + b.intent_bundle.commit_finalize_and_undelegate, + slot + ); + }); + + Ok(()) + } + #[instrument(skip(self, intent_bundles))] pub async fn schedule_intent_bundles( &self, @@ -312,219 +349,3 @@ impl CommittorProcessor { } } } - - fn pending_rows_to_scheduled_intent_bundles( - rows: Vec, - payer: Pubkey, - recovery_base_slot: u64, - recovery_time: u64, -) -> Vec { - let mut grouped_rows = BTreeMap::>::new(); - for row in rows { - grouped_rows.entry(row.message_id).or_default().push(row); - } - - grouped_rows - .into_iter() - .filter_map(|(message_id, rows)| { - if rows - .iter() - .any(|row| !pending_row_is_old_enough(row, recovery_time)) - { - warn!( - intent_id = message_id, - "Skipping pending commit intent because it is not old enough to recover" - ); - return None; - } - - let first = rows.first()?; - let slot = first.slot; - let blockhash = first.ephemeral_blockhash; - if rows.iter().any(|r| { - r.slot != slot || r.ephemeral_blockhash != blockhash - }) { - warn!( - intent_id = message_id, - "Skipping pending commit intent: rows disagree on slot or ephemeral_blockhash" - ); - return None; - } - let mut commit_finalize_accounts = Vec::new(); - let mut commit_finalize_and_undelegate_accounts = Vec::new(); - - for row in rows { - let Some((account, undelegate)) = - committed_account_from_pending_row(row, recovery_base_slot) - else { - continue; - }; - if undelegate { - commit_finalize_and_undelegate_accounts.push(account); - } else { - commit_finalize_accounts.push(account); - } - } - - let mut intent_bundle = MagicIntentBundle::default(); - if !commit_finalize_accounts.is_empty() { - intent_bundle.commit_finalize = - Some(CommitType::Standalone(commit_finalize_accounts)); - } - if !commit_finalize_and_undelegate_accounts.is_empty() { - intent_bundle.commit_finalize_and_undelegate = - Some(CommitAndUndelegate { - commit_action: CommitType::Standalone( - commit_finalize_and_undelegate_accounts, - ), - undelegate_action: UndelegateType::Standalone, - }); - } - if intent_bundle.is_empty() { - return None; - } - - Some(ScheduledIntentBundle { - id: message_id, - slot, - blockhash, - sent_transaction: Transaction::default(), - payer, - intent_bundle, - }) - }) - .collect() -} - -fn pending_row_is_old_enough( - row: &CommitStatusRow, - recovery_time: u64, -) -> bool { - recovery_time.saturating_sub(row.last_retried_at) > RECOVERY_MIN_AGE_SECS -} - -fn unix_timestamp() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - -fn committed_account_from_pending_row( - row: CommitStatusRow, - recovery_base_slot: u64, -) -> Option<(CommittedAccount, bool)> { - if row.commit_type == PersistCommitType::DataAccount && row.data.is_none() { - warn!( - intent_id = row.message_id, - pubkey = %row.pubkey, - "Skipping pending data-account commit row without account data" - ); - return None; - } - - Some(( - CommittedAccount { - pubkey: row.pubkey, - account: Account { - lamports: row.lamports, - data: row.data.unwrap_or_default(), - owner: row.delegated_account_owner, - executable: false, - rent_epoch: 0, - }, - remote_slot: recovery_base_slot, - }, - row.undelegate, - )) -} - -#[cfg(test)] -mod tests { - use solana_hash::Hash; - - use super::*; - use crate::persist::CommitStatus; - - fn pending_row( - message_id: u64, - pubkey: Pubkey, - owner: Pubkey, - blockhash: Hash, - undelegate: bool, - data: Option>, - ) -> CommitStatusRow { - let commit_type = - if data.as_ref().map(|data| data.is_empty()) == Some(false) { - PersistCommitType::DataAccount - } else { - PersistCommitType::EmptyAccount - }; - - CommitStatusRow { - message_id, - pubkey, - commit_id: 0, - delegated_account_owner: owner, - slot: 42, - ephemeral_blockhash: blockhash, - undelegate, - lamports: 1_000, - data, - commit_type, - created_at: 1, - commit_status: CommitStatus::Pending, - commit_strategy: Default::default(), - last_retried_at: 1, - retries_count: 0, - } - } - - #[test] - fn pending_rows_reconstruct_commit_finalize_bundle() { - let payer = Pubkey::new_unique(); - let owner = Pubkey::new_unique(); - let blockhash = Hash::new_unique(); - let commit_pubkey = Pubkey::new_unique(); - let undelegate_pubkey = Pubkey::new_unique(); - - let bundles = pending_rows_to_scheduled_intent_bundles( - vec![ - pending_row( - 9, - commit_pubkey, - owner, - blockhash, - false, - Some(vec![1, 2, 3]), - ), - pending_row(9, undelegate_pubkey, owner, blockhash, true, None), - ], - payer, - 7, - RECOVERY_MIN_AGE_SECS + 2, - ); - - assert_eq!(bundles.len(), 1); - let bundle = &bundles[0]; - assert_eq!(bundle.id, 9); - assert_eq!(bundle.slot, 42); - assert_eq!(bundle.blockhash, blockhash); - assert_eq!(bundle.payer, payer); - - let commit_accounts = - bundle.get_commit_finalize_intent_accounts().unwrap(); - assert_eq!(commit_accounts.len(), 1); - assert_eq!(commit_accounts[0].pubkey, commit_pubkey); - assert_eq!(commit_accounts[0].account.data, vec![1, 2, 3]); - assert_eq!(commit_accounts[0].remote_slot, 7); - - let undelegate_accounts = bundle - .get_commit_finalize_and_undelegate_intent_accounts() - .unwrap(); - assert_eq!(undelegate_accounts.len(), 1); - assert_eq!(undelegate_accounts[0].pubkey, undelegate_pubkey); - assert_eq!(undelegate_accounts[0].account.owner, owner); - assert!(bundle.has_undelegate_intent()); - } -} diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index 2b4ce584b..a44977104 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -16,6 +16,11 @@ pub enum CommittorServiceError { #[error("RecvError: {0}")] IntentResultRecvError(#[from] RecvError), + #[error("MagicBlockRpcClientError: {0} ({0:?})")] + MagicBlockRpcClientError( + #[from] magicblock_rpc_client::MagicBlockRpcClientError, + ), + #[error("Attempt to schedule already scheduled message id: {0}")] RepeatingMessageError(u64), } diff --git a/magicblock-committor-service/src/persist/commit_persister.rs b/magicblock-committor-service/src/persist/commit_persister.rs index 940f3425b..e5ae1119d 100644 --- a/magicblock-committor-service/src/persist/commit_persister.rs +++ b/magicblock-committor-service/src/persist/commit_persister.rs @@ -67,6 +67,12 @@ pub trait IntentPersister: Send + Sync + Clone + 'static { fn get_pending_commit_statuses( &self, ) -> CommitPersistResult>; + fn pending_intent_bundles( + &self, + predicate: F, + ) -> CommitPersistResult> + where + F: Fn(&CommitStatusRow) -> bool; fn get_commit_status_by_message( &self, message_id: u64, @@ -262,6 +268,22 @@ impl IntentPersister for IntentPersisterImpl { .get_pending_commit_statuses() } + /// Returns pending bundles satisfying predicate + /// NOTE: this constructs `ScheduleIntentBundle` only from existing information + /// As persister doesn't save `ScheduleIntentBundle::payer` info, Pubkey::default is used + /// `CommittedAccount` information like slot maybe also outdated + /// It is responsibility of calling site to refresh data in intent + fn pending_intent_bundles( + &self, + predicate: F, + ) -> CommitPersistResult> + where + F: Fn(&CommitStatusRow) -> bool, + { + let rows = self.get_pending_commit_statuses()?; + Ok(pending_rows_to_scheduled_intent_bundles(rows, predicate)) + } + fn get_commit_status_by_message( &self, message_id: u64, @@ -417,6 +439,19 @@ impl IntentPersister for Option { } } + fn pending_intent_bundles( + &self, + predicate: F, + ) -> CommitPersistResult> + where + F: Fn(&CommitStatusRow) -> bool, + { + match self { + Some(persister) => persister.pending_intent_bundles(predicate), + None => Ok(Vec::new()), + } + } + fn get_commit_status_by_message( &self, message_id: u64, @@ -467,6 +502,129 @@ impl IntentPersister for Option { } } +fn pending_rows_to_scheduled_intent_bundles( + rows: Vec, + filter: F, +) -> Vec +where + F: Fn(&CommitStatusRow) -> bool, +{ + let mut grouped_rows = BTreeMap::>::new(); + for row in rows { + grouped_rows.entry(row.message_id).or_default().push(row); + } + + grouped_rows + .into_iter() + // Filter row if any item doesn't satisfy predicate + .filter(|(message_id, rows)| { + if rows.iter().any(|row| filter(row)) { + warn!( + intent_id = message_id, + "Skipping pending commit intent because it is not old enough to recover" + ); + false + } else { + true + } + }) + // Filter malformed items + .filter_map(|(message_id, rows)| { + let first = rows.first()?; + let slot = first.slot; + let blockhash = first.ephemeral_blockhash; + if rows.iter().any(|r| r.slot != slot || r.ephemeral_blockhash != blockhash) { + warn!( + intent_id = message_id, + "Skipping pending commit intent: rows disagree on slot or ephemeral_blockhash" + ); + None + } else { + Some(rows) + } + }) + .filter_map(intent_bundle_from_rows) + .collect() +} + +fn intent_bundle_from_rows( + rows: Vec, +) -> Option { + let first = rows.first()?; + let message_id = first.message_id; + let slot = first.slot; + let blockhash = first.ephemeral_blockhash; + + let mut commit_finalize_accounts = Vec::new(); + let mut commit_finalize_and_undelegate_accounts = Vec::new(); + for row in rows { + let Some((account, undelegate)) = committed_account_from_row(row) + else { + continue; + }; + if undelegate { + commit_finalize_and_undelegate_accounts.push(account); + } else { + commit_finalize_accounts.push(account); + } + } + + let mut intent_bundle = MagicIntentBundle::default(); + if !commit_finalize_accounts.is_empty() { + intent_bundle.commit_finalize = + Some(IntentCommitType::Standalone(commit_finalize_accounts)); + } + if !commit_finalize_and_undelegate_accounts.is_empty() { + intent_bundle.commit_finalize_and_undelegate = + Some(CommitAndUndelegate { + commit_action: IntentCommitType::Standalone( + commit_finalize_and_undelegate_accounts, + ), + undelegate_action: UndelegateType::Standalone, + }); + } + if intent_bundle.is_empty() { + return None; + } + + Some(ScheduledIntentBundle { + id: message_id, + slot, + blockhash, + sent_transaction: Transaction::default(), + payer: Pubkey::default(), + intent_bundle, + }) +} + +fn committed_account_from_row( + row: CommitStatusRow, +) -> Option<(CommittedAccount, bool)> { + if row.commit_type == CommitType::DataAccount && row.data.is_none() { + warn!( + intent_id = row.message_id, + pubkey = %row.pubkey, + "Skipping pending data-account commit row without account data" + ); + return None; + } + + Some(( + CommittedAccount { + pubkey: row.pubkey, + account: Account { + lamports: row.lamports, + data: row.data.unwrap_or_default(), + owner: row.delegated_account_owner, + executable: false, + rent_epoch: 0, + }, + remote_slot: 0, + }, + row.undelegate, + )) +} + #[cfg(test)] mod tests { use magicblock_core::intent::CommittedAccount; @@ -481,6 +639,8 @@ mod tests { use tempfile::NamedTempFile; use super::*; + + const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; use crate::{ persist::{types, CommitStatusSignatures}, test_utils, @@ -794,4 +954,86 @@ mod tests { let statuses = persister.get_commit_statuses_by_message(1).unwrap(); assert_eq!(statuses.len(), 0); } + + fn pending_row( + message_id: u64, + pubkey: Pubkey, + owner: Pubkey, + blockhash: Hash, + undelegate: bool, + data: Option>, + ) -> CommitStatusRow { + let commit_type = + if data.as_ref().map(|data| data.is_empty()) == Some(false) { + types::CommitType::DataAccount + } else { + types::CommitType::EmptyAccount + }; + CommitStatusRow { + message_id, + pubkey, + commit_id: 0, + delegated_account_owner: owner, + slot: 42, + ephemeral_blockhash: blockhash, + undelegate, + lamports: 1_000, + data, + commit_type, + created_at: 1, + commit_status: CommitStatus::Pending, + commit_strategy: Default::default(), + last_retried_at: 1, + retries_count: 0, + } + } + + #[test] + fn pending_rows_reconstruct_commit_finalize_bundle() { + let owner = Pubkey::new_unique(); + let blockhash = Hash::new_unique(); + let commit_pubkey = Pubkey::new_unique(); + let undelegate_pubkey = Pubkey::new_unique(); + + let recovery_time = RECOVERY_MIN_AGE_SECS + 2; + // filter returns true = skip; rows with last_retried_at=1 are old enough → don't skip + let bundles = pending_rows_to_scheduled_intent_bundles( + vec![ + pending_row( + 9, + commit_pubkey, + owner, + blockhash, + false, + Some(vec![1, 2, 3]), + ), + pending_row(9, undelegate_pubkey, owner, blockhash, true, None), + ], + |row| { + recovery_time.saturating_sub(row.last_retried_at) + <= RECOVERY_MIN_AGE_SECS + }, + ); + + assert_eq!(bundles.len(), 1); + let bundle = &bundles[0]; + assert_eq!(bundle.id, 9); + assert_eq!(bundle.slot, 42); + assert_eq!(bundle.blockhash, blockhash); + + let commit_accounts = + bundle.get_commit_finalize_intent_accounts().unwrap(); + assert_eq!(commit_accounts.len(), 1); + assert_eq!(commit_accounts[0].pubkey, commit_pubkey); + assert_eq!(commit_accounts[0].account.data, vec![1, 2, 3]); + assert_eq!(commit_accounts[0].remote_slot, 0); + + let undelegate_accounts = bundle + .get_commit_finalize_and_undelegate_intent_accounts() + .unwrap(); + assert_eq!(undelegate_accounts.len(), 1); + assert_eq!(undelegate_accounts[0].pubkey, undelegate_pubkey); + assert_eq!(undelegate_accounts[0].account.owner, owner); + assert!(bundle.has_undelegate_intent()); + } } From 07866271b7c15f6814a909884e36aa4faff617e6 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 1 Jun 2026 13:28:20 +0700 Subject: [PATCH 17/97] refactor: unused generic --- magicblock-committor-service/src/committor_processor.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index c6eec4369..d7a30842d 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -140,12 +140,9 @@ impl CommittorProcessor { Ok(signatures) } - pub async fn pending_intent_bundles( + pub async fn pending_intent_bundles( &self, - ) -> CommittorServiceResult> - where - F: Fn(&CommitStatusRow) -> bool, - { + ) -> CommittorServiceResult> { const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; // Extract pending bundles satisfying predicate From 88216f720d6324aaf75b0cc761aeb77d49c25c65 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 1 Jun 2026 14:16:32 +0700 Subject: [PATCH 18/97] feat: integrate rescheduling of pending commits --- magicblock-api/src/magic_validator.rs | 11 +- .../src/committor_processor.rs | 1 + .../src/persist/commit_persister.rs | 4 +- magicblock-committor-service/src/service.rs | 106 ++++++++++++++++-- .../src/service/intent_client.rs | 3 + 5 files changed, 105 insertions(+), 20 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 8a6491ba8..245f6f8a8 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -935,12 +935,6 @@ impl MagicValidator { log_timing("startup", "reset_accounts_bank", step_start); } - // Recovery of persisted pending commit intents reads the local accounts - // bank for delegation checks, so it must run only after replay + reset. - if let Some(processor) = self.scheduled_commits_processor.as_ref() { - processor.spawn_pending_intents_recovery(); - } - // Notify the scheduler that ledger replay and bank cleanup is complete. if self.is_standalone { self.mode_tx @@ -1030,8 +1024,9 @@ impl MagicValidator { self.token.cancel(); let step_start = Instant::now(); - // TODO(edwin): handle returned errs, at least log - let _ = self.intent_execution_service.stop().await; + if let Err(err) = self.intent_execution_service.stop().await { + error!(error =? err, "Failure during stopping Intent Execution Service") + } log_timing("shutdown", "intent_execution_service_stop", step_start); let step_start = Instant::now(); diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index d7a30842d..e44878675 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -140,6 +140,7 @@ impl CommittorProcessor { Ok(signatures) } + /// Fetches pending bundles from DB pub async fn pending_intent_bundles( &self, ) -> CommittorServiceResult> { diff --git a/magicblock-committor-service/src/persist/commit_persister.rs b/magicblock-committor-service/src/persist/commit_persister.rs index e5ae1119d..e0ca2464d 100644 --- a/magicblock-committor-service/src/persist/commit_persister.rs +++ b/magicblock-committor-service/src/persist/commit_persister.rs @@ -504,7 +504,7 @@ impl IntentPersister for Option { fn pending_rows_to_scheduled_intent_bundles( rows: Vec, - filter: F, + predicate: F, ) -> Vec where F: Fn(&CommitStatusRow) -> bool, @@ -518,7 +518,7 @@ where .into_iter() // Filter row if any item doesn't satisfy predicate .filter(|(message_id, rows)| { - if rows.iter().any(|row| filter(row)) { + if rows.iter().any(&predicate) { warn!( intent_id = message_id, "Skipping pending commit intent because it is not old enough to recover" diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index ec4a6ffe9..595b482fe 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -2,15 +2,17 @@ pub mod intent_client; use std::{ collections::{HashMap, HashSet}, + future::Future, mem, sync::{Arc, Mutex}, time::Duration, }; +use futures_util::future::join_all; use intent_client::{ERIntentClient, InternalIntentClientError}; use magicblock_account_cloner::ChainlinkCloner; use magicblock_chainlink::{ProdChainlink, ProdInnerChainlink}; -use magicblock_metrics::metrics; +use magicblock_metrics::metrics::{self, AccountFetchOrigin}; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, Pubkey, SentCommit, }; @@ -22,10 +24,10 @@ use tokio::{ task::{JoinError, JoinHandle}, }; use tokio_util::sync::CancellationToken; -use tracing::{error, info, instrument}; +use tracing::{error, info, instrument, warn}; use crate::{ - committor_processor::CommittorProcessor, error::CommittorServiceError, + committor_processor::CommittorProcessor, error::CommittorServiceResult, intent_execution_manager::BroadcastedIntentExecutionResult, intent_executor::ExecutionOutput, }; @@ -144,6 +146,10 @@ where } async fn accept_worker(self) { + if let Err(err) = self.reschedule_pending_bundles().await { + error!(error = ?err, "Failed to reschedule pending bundles") + } + let mut interval = tokio::time::interval(self.slot_interval); loop { tokio::select! { @@ -172,22 +178,60 @@ where } } + async fn reschedule_pending_bundles(&self) -> CommittorServiceResult<()> { + // Fetch pending bundles from DB + let mut bundles = + self.processor.pending_intent_bundles().await.inspect_err(|err| { + error!(error = ?err, "Failed to load pending intent bundles for recovery"); + })?; + if bundles.is_empty() { + return Ok(()); + } + + // Retain only recoverable bundles + self.retain_recoverable_intent_bundles(&mut bundles).await; + + // Schedule without initial persisitance as bundle already exists in db + self.process_intent_bundles(bundles, |bundles| { + self.processor.schedule_recovered_intent_bundles(bundles) + }) + .await + } + async fn schedule_intent_execution( &self, intent_bundles: Vec, - ) -> Result<(), CommittorServiceError> { + ) -> CommittorServiceResult<()> { if intent_bundles.is_empty() { return Ok(()); } metrics::inc_committor_intents_count_by(intent_bundles.len() as u64); + self.process_intent_bundles(intent_bundles, |bundles| { + self.processor.schedule_intent_bundles(bundles) + }) + .await + } + + async fn process_intent_bundles( + &self, + intent_bundles: Vec, + schedule: F, + ) -> CommittorServiceResult<()> + where + F: FnOnce(Vec) -> Fut, + Fut: Future>, + { + if intent_bundles.is_empty() { + return Ok(()); + } + // Add metas for intent we schedule let pubkeys_being_undelegated = { let mut intent_metas = self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); let mut pubkeys_being_undelegated = HashSet::::new(); - intent_bundles.iter().for_each(|intent| { intent_metas .insert(intent.id, ScheduledBaseIntentMeta::new(intent)); @@ -196,16 +240,12 @@ where pubkeys_being_undelegated.extend(undelegate); } }); - pubkeys_being_undelegated.into_iter().collect::>() }; self.process_undelegation_requests(pubkeys_being_undelegated) .await; - self.processor - .schedule_intent_bundles(intent_bundles) - .await?; - Ok(()) + schedule(intent_bundles).await } async fn process_undelegation_requests(&self, pubkeys: Vec) { @@ -396,6 +436,52 @@ where callbacks_scheduling_results: callbacks_report, } } + + /// Retains bundles whose accounts are still delegated + async fn retain_recoverable_intent_bundles( + &self, + bundles: &mut Vec, + ) { + let results = join_all( + bundles.iter().map(|b| b.get_all_committed_pubkeys()).map( + |pubkeys| async move { + self.chainlink + .accounts_delegated_on_base_and_er( + &pubkeys, + AccountFetchOrigin::GetAccount, + ) + .await + }, + ), + ) + .await; + + let mut results_iter = results.into_iter(); + bundles.retain(|bundle| { + let Some(result) = results_iter.next() else { + error!("Results and bundles must have equal length"); + return false; + }; + match result { + Ok(delegated) if delegated.iter().all(|d| *d) => true, + Ok(_) => { + warn!( + intent_id = bundle.id, + "Skipping recovered commit intent because not all accounts are delegated on base and ER" + ); + false + } + Err(err) => { + error!( + intent_id = bundle.id, + error = ?err, + "Failed to verify recovered commit intent accounts" + ); + false + } + } + }); + } } struct ScheduledBaseIntentMeta { diff --git a/magicblock-committor-service/src/service/intent_client.rs b/magicblock-committor-service/src/service/intent_client.rs index 65aaa2ca3..e22ee704a 100644 --- a/magicblock-committor-service/src/service/intent_client.rs +++ b/magicblock-committor-service/src/service/intent_client.rs @@ -32,6 +32,9 @@ pub trait ERIntentClient: Send + Sync + 'static { sent_tx: Transaction, sent_commit: SentCommit, ) -> Result<(), Self::Error>; + + // TODO(edwin): probably more proper place to load pending intent + // CommittorProcessor::pending_intent_bundles could be moved here in the future } pub struct InternalIntentRpcClient { From 6b4c4aa4c07393373fef53171b54ac4f72906232 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 1 Jun 2026 14:24:09 +0700 Subject: [PATCH 19/97] fix: cleanup intent map if scheduling of intent failed --- magicblock-committor-service/src/service.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 595b482fe..cfcd4f10f 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -228,6 +228,7 @@ where } // Add metas for intent we schedule + let intent_ids: Vec; let pubkeys_being_undelegated = { let mut intent_metas = self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); @@ -240,12 +241,23 @@ where pubkeys_being_undelegated.extend(undelegate); } }); + intent_ids = intent_bundles.iter().map(|b| b.id).collect(); pubkeys_being_undelegated.into_iter().collect::>() }; self.process_undelegation_requests(pubkeys_being_undelegated) .await; - schedule(intent_bundles).await + + let result = schedule(intent_bundles).await; + // If scheduling failed remove from map + if result.is_err() { + let mut intent_metas = + self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); + intent_ids.iter().for_each(|id| { + intent_metas.remove(id); + }); + } + result } async fn process_undelegation_requests(&self, pubkeys: Vec) { From fff494e883dcb981f23f3f3ed7247df87fbffce3 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 1 Jun 2026 14:31:44 +0700 Subject: [PATCH 20/97] fix: failure path --- magicblock-api/src/magic_validator.rs | 1 - .../src/committor_processor.rs | 49 +++++++++++++------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 245f6f8a8..64bab9835 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -429,7 +429,6 @@ impl MagicValidator { config, exit, _metrics: (metrics_service, system_metrics_ticker), - // NOTE: set during [Self::start] intent_execution_service, replication_service, chainlink, diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index e44878675..2b9ec862e 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -236,32 +236,49 @@ impl CommittorProcessor { intent_bundles: Vec, ) -> CommittorServiceResult> { // Critical section - let receivers = { + let (receivers, inserted_ids) = { let mut result_listeners = self .pending_result_listeners .lock() .expect(POISONED_MUTEX_MSG); - intent_bundles - .iter() - .map(|intent| { - let (sender, receiver) = oneshot::channel(); - match result_listeners.entry(intent.id) { - Entry::Vacant(vacant) => { - vacant.insert(sender); - Ok(receiver) + let mut receivers = Vec::with_capacity(intent_bundles.len()); + let mut inserted_ids = Vec::with_capacity(intent_bundles.len()); + + for intent in &intent_bundles { + let (sender, receiver) = oneshot::channel(); + match result_listeners.entry(intent.id) { + Entry::Vacant(vacant) => { + vacant.insert(sender); + inserted_ids.push(intent.id); + receivers.push(receiver); + } + Entry::Occupied(_) => { + for id in &inserted_ids { + result_listeners.remove(id); } - Entry::Occupied(_) => { - Err(CommittorServiceError::RepeatingMessageError( + return Err( + CommittorServiceError::RepeatingMessageError( intent.id, - )) - } + ), + ); } - }) - .collect::, _>>()? + } + } + (receivers, inserted_ids) }; - self.schedule_intent_bundles(intent_bundles).await?; + if let Err(err) = self.schedule_intent_bundles(intent_bundles).await { + let mut result_listeners = self + .pending_result_listeners + .lock() + .expect(POISONED_MUTEX_MSG); + for id in &inserted_ids { + result_listeners.remove(id); + } + return Err(err); + } + let results = join_all(receivers.into_iter()) .await .into_iter() From 3cb9ec67c47a8fe3ebab8b6cf98096c88805662c Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 13:36:31 +0700 Subject: [PATCH 21/97] wip: some outbox components --- magicblock-committor-service/src/service.rs | 89 ++++++++++++++++--- .../src/service/intent_client.rs | 15 ++++ .../service/outbox_intent_bundles_reader.rs | 74 +++++++++++++++ programs/magicblock/src/lib.rs | 1 + .../magicblock/src/outbox_intent_bundles.rs | 29 ++++++ 5 files changed, 198 insertions(+), 10 deletions(-) create mode 100644 magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs create mode 100644 programs/magicblock/src/outbox_intent_bundles.rs diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index cfcd4f10f..f437f1fbc 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,9 +1,11 @@ pub mod intent_client; +pub mod outbox_intent_bundles_reader; use std::{ collections::{HashMap, HashSet}, future::Future, mem, + num::NonZeroUsize, sync::{Arc, Mutex}, time::Duration, }; @@ -27,9 +29,13 @@ use tokio_util::sync::CancellationToken; use tracing::{error, info, instrument, warn}; use crate::{ - committor_processor::CommittorProcessor, error::CommittorServiceResult, + committor_processor::CommittorProcessor, + error::CommittorServiceResult, intent_execution_manager::BroadcastedIntentExecutionResult, intent_executor::ExecutionOutput, + service::outbox_intent_bundles_reader::{ + InternalOutboxIntentBundlesReaderError, OutboxIntentBundlesReader, + }, }; const POISONED_MUTEX_MSG: &str = "ServiceInner intents_meta_map mutex poisoned"; @@ -47,18 +53,22 @@ pub enum IntentExecutionService { impl IntentExecutionService where R: ERIntentClient, + // ERIntentClient errors should be convertible to Service errors R::Error: Into, + // OutboxReader errors should be convertible to Service errors + ::Error: + Into, { pub fn new( chainlink: Arc, - intent_rpc_client: R, + intent_client: R, processor: Arc, slot_interval: Duration, cancellation_token: CancellationToken, ) -> Self { Self::Created(ServiceInner::new( chainlink, - intent_rpc_client, + intent_client, processor, slot_interval, cancellation_token, @@ -98,7 +108,7 @@ pub struct ServiceInner { /// Chainlink for notifying of undelegations chainlink: Arc, /// ER client specific for Intent needs. Could be switched to RpcClient - intent_rpc_client: Arc, + intent_client: Arc, /// Processor of accepted intents processor: Arc, /// Time interval to scrape MagicContext(ER slot interval) @@ -112,7 +122,11 @@ pub struct ServiceInner { impl ServiceInner where R: ERIntentClient, + // ERIntentClient errors should be convertible to Service errors R::Error: Into, + // OutboxReader errors should be convertible to Service errors + ::Error: + Into, { pub fn new( chainlink: Arc, @@ -123,7 +137,7 @@ where ) -> Self { Self { chainlink, - intent_rpc_client: Arc::new(intent_rpc_client), + intent_client: Arc::new(intent_rpc_client), processor, slot_interval, cancellation_token, @@ -139,16 +153,27 @@ where result_subscriber, cancellation_token, self.intents_meta_map.clone(), - self.intent_rpc_client.clone(), + self.intent_client.clone(), )); tokio::task::spawn(self.accept_worker()) } async fn accept_worker(self) { - if let Err(err) = self.reschedule_pending_bundles().await { - error!(error = ?err, "Failed to reschedule pending bundles") - } + // Reschedule existing outbox intents first + // We need to ensure that accounts in outbox a scheduled before + // we accept new incoming Intents + self.reschedule_intents() + .await + .inspect_err(|err| { + error!(error = ?err, "Failed to reschedule pending bundles") + }) + // TODO(edwin): early shutdown or cleanup errors to avoid this + .expect("Failed to reschedule intents"); + + // if let Err(err) = self.reschedule_pending_bundles().await { + // error!(error = ?err, "Failed to reschedule pending bundles") + // } let mut interval = tokio::time::interval(self.slot_interval); loop { @@ -159,7 +184,7 @@ where } _ = interval.tick() => { let accept_result = self - .intent_rpc_client + .intent_client .accept_scheduled_intents() .await; let intent_bundles = match accept_result { @@ -178,6 +203,48 @@ where } } + async fn reschedule_intents( + &self, + ) -> Result<(), IntentExecutionServiceError> { + /// Number of intents rescheduled at once + const RESCHEDULE_CHUNK_SIZE: NonZeroUsize = + NonZeroUsize::new(1000).unwrap(); + + let mut outbox_bundles_reader = self.intent_client.outbox_reader(); + loop { + // Read by chunks in order not to overload `IntentExecutionEngine` + let intent_bundles_chunk = outbox_bundles_reader + .read(RESCHEDULE_CHUNK_SIZE) + .await + .map_err(Into::into)?; + if intent_bundles_chunk.is_empty() { + return Ok(()); + } + + // TODO(edwin): use status + let read_len = intent_bundles_chunk.len(); + let intent_bundles = intent_bundles_chunk + .into_iter() + .map(|el| el.intent_bundle) + .collect(); + + // Schedule without initial persistence as bundle already exists in db + let result = self + .process_intent_bundles(intent_bundles, |bundles| { + self.processor.schedule_recovered_intent_bundles(bundles) + }) + .await; + if let Err(err) = result { + error!(error = ?err, "Failed to reschedule pending bundles") + } + + // Check if we've rescheduled intents from Outbox + if read_len != RESCHEDULE_CHUNK_SIZE.get() { + return Ok(()); + } + } + } + async fn reschedule_pending_bundles(&self) -> CommittorServiceResult<()> { // Fetch pending bundles from DB let mut bundles = @@ -526,4 +593,6 @@ pub enum IntentExecutionServiceError { JoinError(#[from] JoinError), #[error("IntentRpcClientError: {0}")] IntentRpcClientError(#[from] InternalIntentClientError), + #[error("OutboxReaderError")] + OutboxReaderError(#[from] InternalOutboxIntentBundlesReaderError), } diff --git a/magicblock-committor-service/src/service/intent_client.rs b/magicblock-committor-service/src/service/intent_client.rs index e22ee704a..f89d4d515 100644 --- a/magicblock-committor-service/src/service/intent_client.rs +++ b/magicblock-committor-service/src/service/intent_client.rs @@ -17,8 +17,15 @@ use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tracing::{debug, error}; +use crate::service::outbox_intent_bundles_reader::{ + InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, +}; + #[async_trait] pub trait ERIntentClient: Send + Sync + 'static { + /// Type that is able to read IntentBundles from Outbox + /// Can be via AccountsDB, RpcClient or any other means + type OutboxReader: OutboxIntentBundlesReader; type Error: std::error::Error + Send; /// Executes `Accept` tx and returns accepted intents @@ -33,6 +40,9 @@ pub trait ERIntentClient: Send + Sync + 'static { sent_commit: SentCommit, ) -> Result<(), Self::Error>; + /// Returns reader capable of reading IntentBundles from Outbox + fn outbox_reader(&self) -> Self::OutboxReader; + // TODO(edwin): probably more proper place to load pending intent // CommittorProcessor::pending_intent_bundles could be moved here in the future } @@ -82,6 +92,7 @@ impl InternalIntentRpcClient { #[async_trait] impl ERIntentClient for InternalIntentRpcClient { type Error = InternalIntentClientError; + type OutboxReader = InternalOutboxIntentBundlesReader; async fn accept_scheduled_intents( &self, @@ -121,6 +132,10 @@ impl ERIntentClient for InternalIntentRpcClient { Ok(()) } + + fn outbox_reader(&self) -> Self::OutboxReader { + InternalOutboxIntentBundlesReader::new(self.accounts_db.clone()) + } } #[derive(thiserror::Error, Debug)] diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs new file mode 100644 index 000000000..f7cd5baf6 --- /dev/null +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -0,0 +1,74 @@ +use std::{num::NonZeroUsize, sync::Arc}; + +use async_trait::async_trait; +use magicblock_accounts_db::AccountsDb; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; +use solana_account::ReadableAccount; +use solana_pubkey::Pubkey; +// TODO(edwin): name - OutboxIntentAccount/PendingIntentAccount + +// TODO(edwin): name - PendingIntentBundlesReader? +#[async_trait] +pub trait OutboxIntentBundlesReader: Send + 'static { + type Error: Send; + + /// Reads `n` outbox intents + /// Returns `OutboxIntentBundle` in ascending order by `ScheduledIntentBundle::id` + /// If Vec::len != n, that means that there's no more Outbox intents + async fn read( + &mut self, + n: NonZeroUsize, + ) -> Result, Self::Error>; +} + +pub struct InternalOutboxIntentBundlesReader { + /// Current intent id pos + intent_id_pos: Option, + /// Accounts DB where we read intents from + /// Could be an RpcClient in the future + accounts_db: Arc, +} + +impl InternalOutboxIntentBundlesReader { + const TARGET_PROGRAM_ID: Pubkey = magicblock_program::ID; + const ACCOUNT_DISCRIMINATOR: [u8; 8] = todo!(); + + pub fn new(accounts_db: Arc) -> Self { + Self { + // Uninitialized state + intent_id_pos: None, + accounts_db, + } + } + + pub fn initialize_intent_pos( + &mut self, + ) -> Result<(), InternalOutboxIntentBundlesReader> { + let asd = self + .accounts_db + .get_program_accounts(&Self::TARGET_PROGRAM_ID, |account| { + if !account.data().starts_with(&Self::ACCOUNT_DISCRIMINATOR) { + return false; + } + + true + }) + .unwrap(); + todo!() + } +} + +#[async_trait] +impl OutboxIntentBundlesReader for InternalOutboxIntentBundlesReader { + type Error = InternalOutboxIntentBundlesReaderError; + + async fn read( + &mut self, + n: NonZeroUsize, + ) -> Result, Self::Error> { + todo!() + } +} + +#[derive(thiserror::Error, Debug)] +pub enum InternalOutboxIntentBundlesReaderError {} diff --git a/programs/magicblock/src/lib.rs b/programs/magicblock/src/lib.rs index 684e1c067..f3ae44268 100644 --- a/programs/magicblock/src/lib.rs +++ b/programs/magicblock/src/lib.rs @@ -9,6 +9,7 @@ mod schedule_transactions; pub use magic_context::MagicContext; pub mod magic_scheduled_base_intent; pub mod magicblock_processor; +pub mod outbox_intent_bundles; pub mod test_utils; mod utils; pub mod validator; diff --git a/programs/magicblock/src/outbox_intent_bundles.rs b/programs/magicblock/src/outbox_intent_bundles.rs new file mode 100644 index 000000000..bdf12a48c --- /dev/null +++ b/programs/magicblock/src/outbox_intent_bundles.rs @@ -0,0 +1,29 @@ +use solana_signature::Signature; + +use crate::magic_scheduled_base_intent::ScheduledIntentBundle; + +// TODO(edwin): naming Outbox/Pending +pub struct OutboxIntentBundle { + // TODO(edwin): define visability + pub intent_bundle: ScheduledIntentBundle, + // TODO(edwin): define visability + pub status: OutboxIntentBundleStatus, +} + +pub enum OutboxIntentBundleStatus { + Accepted, + Executing(ExecutionStage), +} + +pub enum ExecutionStage { + SingleStage(Signature), + TwoStage(TwoStageProgress), +} + +pub enum TwoStageProgress { + Committing(Signature), + Finalizing { + commit: Signature, + finalize: Signature, + }, +} From d402c92aa9086270a3eeb3eeead99c3fae747bae Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 16:27:53 +0700 Subject: [PATCH 22/97] fix: compilation --- magicblock-api/src/magic_validator.rs | 8 ++-- .../src/committor_processor.rs | 15 ++++-- .../src/persist/commit_persister.rs | 47 +++++++------------ 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 65ba7daab..7c6e523f8 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -458,10 +458,10 @@ impl MagicValidator { let base_chain_config = ChainConfig { rpc_uri: config.rpc_url().to_owned(), commitment: CommitmentConfig::confirmed(), - websocket_uri: config - .websocket_urls() - .next() - .map(ToOwned::to_owned), + websocket_uri: config + .websocket_urls() + .next() + .map(ToOwned::to_owned), compute_budget_config: ComputeBudgetConfig::new( config.commit.compute_unit_price, ), diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 5f060d5a8..0f942e1e7 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -157,17 +157,22 @@ impl CommittorProcessor { &self, ) -> CommittorServiceResult> { const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; - const RECOVERY_MAX_AGE_SECS: u64 = 14 * 24 * 60 * 60; + const RECOVERY_MAX_AGE_SECS: u64 = 14 * 24 * 60 * 60; // Extract pending bundles satisfying predicate let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); - let mut bundles = self.persister.pending_intent_bundles(|row| { - now.saturating_sub(row.last_retried_at) <= RECOVERY_MIN_AGE_SECS - && now.saturating_sub(row.last_retried_at) >= RECOVERY_MAX_AGE_SECS - })?; + let mut bundles = self.persister.pending_intent_bundles( + now.saturating_sub(RECOVERY_MAX_AGE_SECS), + now.saturating_sub(RECOVERY_MIN_AGE_SECS), + |row| { + now.saturating_sub(row.last_retried_at) <= RECOVERY_MIN_AGE_SECS + && now.saturating_sub(row.last_retried_at) + >= RECOVERY_MAX_AGE_SECS + }, + )?; if bundles.is_empty() { return Ok(bundles); diff --git a/magicblock-committor-service/src/persist/commit_persister.rs b/magicblock-committor-service/src/persist/commit_persister.rs index 531281347..1f7af7dca 100644 --- a/magicblock-committor-service/src/persist/commit_persister.rs +++ b/magicblock-committor-service/src/persist/commit_persister.rs @@ -64,13 +64,10 @@ pub trait IntentPersister: Send + Sync + Clone + 'static { &self, message_id: u64, ) -> CommitPersistResult>; - fn get_pending_commit_statuses( - &self, - min_created_at: u64, - max_last_retried_at: u64, - ) -> CommitPersistResult>; fn pending_intent_bundles( &self, + min_created_at: u64, + max_created_at: u64, predicate: F, ) -> CommitPersistResult> where @@ -261,17 +258,6 @@ impl IntentPersister for IntentPersisterImpl { .get_commit_statuses_by_id(message_id) } - fn get_pending_commit_statuses( - &self, - min_created_at: u64, - max_last_retried_at: u64, - ) -> CommitPersistResult> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .get_pending_commit_statuses(min_created_at, max_last_retried_at) - } - /// Returns pending bundles satisfying predicate /// NOTE: this constructs `ScheduleIntentBundle` only from existing information /// As persister doesn't save `ScheduleIntentBundle::payer` info, Pubkey::default is used @@ -279,12 +265,19 @@ impl IntentPersister for IntentPersisterImpl { /// It is responsibility of calling site to refresh data in intent fn pending_intent_bundles( &self, + min_created_at: u64, + max_last_retried_at: u64, predicate: F, ) -> CommitPersistResult> where F: Fn(&CommitStatusRow) -> bool, { - let rows = self.get_pending_commit_statuses()?; + let rows = self + .commits_db + .lock() + .expect(POISONED_MUTEX_MSG) + .get_pending_commit_statuses(min_created_at, max_last_retried_at)?; + Ok(pending_rows_to_scheduled_intent_bundles(rows, predicate)) } @@ -434,29 +427,21 @@ impl IntentPersister for Option { } } - fn get_pending_commit_statuses( + fn pending_intent_bundles( &self, min_created_at: u64, max_last_retried_at: u64, - ) -> CommitPersistResult> { - match self { - Some(persister) => persister.get_pending_commit_statuses( - min_created_at, - max_last_retried_at, - ), - None => Ok(Vec::new()), - } - } - - fn pending_intent_bundles( - &self, predicate: F, ) -> CommitPersistResult> where F: Fn(&CommitStatusRow) -> bool, { match self { - Some(persister) => persister.pending_intent_bundles(predicate), + Some(persister) => persister.pending_intent_bundles( + min_created_at, + max_last_retried_at, + predicate, + ), None => Ok(Vec::new()), } } From da243aa36ad88e6433a14719b8d3555ebad5cdde Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 16:43:09 +0700 Subject: [PATCH 23/97] fix: tests --- .../src/committor_processor.rs | 10 +-- .../src/persist/commit_persister.rs | 77 ++++++++++++++++++- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 0f942e1e7..0e18fdd72 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -37,6 +37,9 @@ use crate::{ }; const POISONED_MUTEX_MSG: &str = "CommittorProcessor pending messages mutex poisoned!"; +pub(crate) const RECOVERY_MIN_AGE_SECS: u64 = 5 * 60; +pub(crate) const RECOVERY_MAX_AGE_SECS: u64 = 14 * 24 * 60 * 60; + type BundleResultListener = oneshot::Sender; pub struct CommittorProcessor { @@ -156,9 +159,6 @@ impl CommittorProcessor { pub async fn pending_intent_bundles( &self, ) -> CommittorServiceResult> { - const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; - const RECOVERY_MAX_AGE_SECS: u64 = 14 * 24 * 60 * 60; - // Extract pending bundles satisfying predicate let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -169,8 +169,8 @@ impl CommittorProcessor { now.saturating_sub(RECOVERY_MIN_AGE_SECS), |row| { now.saturating_sub(row.last_retried_at) <= RECOVERY_MIN_AGE_SECS - && now.saturating_sub(row.last_retried_at) - >= RECOVERY_MAX_AGE_SECS + || now.saturating_sub(row.created_at) + > RECOVERY_MAX_AGE_SECS }, )?; diff --git a/magicblock-committor-service/src/persist/commit_persister.rs b/magicblock-committor-service/src/persist/commit_persister.rs index 1f7af7dca..c97ac34d0 100644 --- a/magicblock-committor-service/src/persist/commit_persister.rs +++ b/magicblock-committor-service/src/persist/commit_persister.rs @@ -67,7 +67,7 @@ pub trait IntentPersister: Send + Sync + Clone + 'static { fn pending_intent_bundles( &self, min_created_at: u64, - max_created_at: u64, + max_last_retried_at: u64, predicate: F, ) -> CommitPersistResult> where @@ -633,9 +633,8 @@ mod tests { use tempfile::NamedTempFile; use super::*; - - const RECOVERY_MIN_AGE_SECS: u64 = 30 * 60; use crate::{ + committor_processor::{RECOVERY_MAX_AGE_SECS, RECOVERY_MIN_AGE_SECS}, persist::{types, CommitStatusSignatures}, test_utils, }; @@ -990,7 +989,7 @@ mod tests { let undelegate_pubkey = Pubkey::new_unique(); let recovery_time = RECOVERY_MIN_AGE_SECS + 2; - // filter returns true = skip; rows with last_retried_at=1 are old enough → don't skip + // filter returns true = skip; default rows (last_retried_at=1, created_at=1) are in recovery window → don't skip let bundles = pending_rows_to_scheduled_intent_bundles( vec![ pending_row( @@ -1006,6 +1005,8 @@ mod tests { |row| { recovery_time.saturating_sub(row.last_retried_at) <= RECOVERY_MIN_AGE_SECS + || recovery_time.saturating_sub(row.created_at) + > RECOVERY_MAX_AGE_SECS }, ); @@ -1030,4 +1031,72 @@ mod tests { assert_eq!(undelegate_accounts[0].account.owner, owner); assert!(bundle.has_undelegate_intent()); } + + fn make_predicate(recovery_time: u64) -> impl Fn(&CommitStatusRow) -> bool { + move |row| { + recovery_time.saturating_sub(row.last_retried_at) + <= RECOVERY_MIN_AGE_SECS + || recovery_time.saturating_sub(row.created_at) + > RECOVERY_MAX_AGE_SECS + } + } + + #[test] + fn pending_rows_skip_intents_outside_recovery_window() { + let owner = Pubkey::new_unique(); + let blockhash = Hash::new_unique(); + let recovery_time = RECOVERY_MAX_AGE_SECS + RECOVERY_MIN_AGE_SECS; + + // Within window: last_retried_at old enough, created_at not too old → included + let mut in_window = pending_row( + 1, + Pubkey::new_unique(), + owner, + blockhash, + false, + Some(vec![1]), + ); + in_window.created_at = recovery_time - RECOVERY_MAX_AGE_SECS; + in_window.last_retried_at = recovery_time - RECOVERY_MIN_AGE_SECS - 1; + let included = pending_rows_to_scheduled_intent_bundles( + vec![in_window], + make_predicate(recovery_time), + ); + assert_eq!(included.len(), 1); + assert_eq!(included[0].id, 1); + + // Too recently retried → excluded + let mut too_recent = pending_row( + 2, + Pubkey::new_unique(), + owner, + blockhash, + false, + Some(vec![1]), + ); + too_recent.created_at = recovery_time - RECOVERY_MAX_AGE_SECS; + too_recent.last_retried_at = recovery_time - RECOVERY_MIN_AGE_SECS; + let excluded_recent = pending_rows_to_scheduled_intent_bundles( + vec![too_recent], + make_predicate(recovery_time), + ); + assert!(excluded_recent.is_empty()); + + // Too old by created_at → excluded + let mut too_old = pending_row( + 3, + Pubkey::new_unique(), + owner, + blockhash, + false, + Some(vec![1]), + ); + too_old.created_at = recovery_time - RECOVERY_MAX_AGE_SECS - 1; + too_old.last_retried_at = recovery_time - RECOVERY_MIN_AGE_SECS - 1; + let excluded_old = pending_rows_to_scheduled_intent_bundles( + vec![too_old], + make_predicate(recovery_time), + ); + assert!(excluded_old.is_empty()); + } } From 0e5872021ff2ecd12dcc4a6b61460bec14d31944 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 17:02:02 +0700 Subject: [PATCH 24/97] refactor: change magicblock-program file structure for better separation --- mimd-0025.md | 293 ++++++++++++++++++ programs/magicblock/src/intent_bundles/mod.rs | 2 + .../src/intent_bundles/outbox/mod.rs | 0 .../schedule}/mod.rs | 0 .../process_accept_scheduled_commits.rs | 0 .../schedule}/process_add_action_callback.rs | 0 .../schedule}/process_execute_callback.rs | 0 .../schedule}/process_schedule_commit.rs | 0 .../process_schedule_commit_tests.rs | 0 .../process_schedule_intent_bundle.rs | 0 .../process_scheduled_commit_sent.rs | 0 .../schedule}/transaction_scheduler.rs | 0 programs/magicblock/src/lib.rs | 3 +- 13 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 mimd-0025.md create mode 100644 programs/magicblock/src/intent_bundles/mod.rs create mode 100644 programs/magicblock/src/intent_bundles/outbox/mod.rs rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/mod.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_accept_scheduled_commits.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_add_action_callback.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_execute_callback.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_schedule_commit.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_schedule_commit_tests.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_schedule_intent_bundle.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/process_scheduled_commit_sent.rs (100%) rename programs/magicblock/src/{schedule_transactions => intent_bundles/schedule}/transaction_scheduler.rs (100%) diff --git a/mimd-0025.md b/mimd-0025.md new file mode 100644 index 000000000..6416e6ff5 --- /dev/null +++ b/mimd-0025.md @@ -0,0 +1,293 @@ +# Intent Execution Durability + +## Motivation + +When a user schedules a commit, they are delegating an account to the ER with the expectation that its state will be persisted to the base chain. The validator is obligated to fulfill this — an unexecuted commit is a protocol-level breach of trust. + +Today, that obligation is not enforced past the accept boundary. Once an Intent is accepted it leaves `AccountsDB` and lives exclusively in process memory. A validator crash anywhere between acceptance and L1 confirmation silently drops the Intent. There is no recovery, no retry, no report — the commitment is gone. + +This proposal closes that gap by anchoring accepted Intents back in `AccountsDB` for their entire execution lifecycle. + +--- + +## Problem + +`Accepted` intents leave `AccountsDB` and exist only in the in-memory `TransactionScheduler` global and the `CommittorService` mpsc channel. Neither survives a process restart. + +### Current flow + +``` +1. User calls ScheduleCommit / ScheduleIntentBundle + → Intent stored in MagicContext (AccountsDB, survives restart) +2. Validator calls AcceptScheduledCommits + → Intents popped from MagicContext + → Moved into TransactionScheduler (in-memory global, lost on restart) +3. CommittorService picks up intents, executes them on L1 +4. ScheduledCommitSent ER tx logs the result +``` + +The window between step 2 and step 4 is unprotected. Any crash drops all in-flight intents. + +--- + +## Proposed Solution + +This proposal is similar to the **outbox pattern**, where `AccountsDB` itself serves as the durable outbox. Rather than handing an Intent directly to the `CommittorService` and hoping for the best, the validator first writes the Intent into `AccountsDB` as a dedicated ephemeral account. The `CommittorService` then processes it from there and only closes the account once L1 execution is confirmed. A crash at any point leaves the account in `AccountsDB`; on restart the validator calls `getProgramAccounts` with a discriminator filter, reads each account's status, and resumes accordingly. + +Additionally, just before sending a transaction to L1, the validator records the transaction signature back into the `MagicIntentAccount` via `SetIntentExecutionStage`. This lets the restart logic distinguish "prepared but not yet sent" from "sent — check the signature on L1." + +**No stage is executed twice.** Because `SetIntentExecutionStage` is committed to `AccountsDB` before the L1 transaction is submitted, any restart that finds `status = Executing(...)` will always query L1 for the stored signature first. A confirmed & successful tx signature advances or closes the intent. The write-before-send ordering is the invariant that makes this hold. + +### New execution flow + +``` +1. ScheduleCommit / ScheduleIntentBundle +2. AcceptScheduledCommits — moves Intent from MagicContext into a PDA with status Accepted +3. IntentExecutionService takes Intent for execution +4. SetIntentExecutionStage(signature) — updates Intent status with the constructed L1 signature +5. Send TX to Base +6. Close Intent account with report +``` + +#### Crash 1→2 + +Intent is still in `MagicContext`. The next `AcceptScheduledCommits` call picks it up and retries. No state is lost. + +#### Crash 2→3 + +`MagicIntentAccount` exists with `status = Accepted`. On restart: discovered via `getProgramAccounts`, rescheduled for full execution. + +#### Crash 3→4 + +`MagicIntentAccount` exists with `status = Accepted` — indistinguishable from crash 2→3. The L1 transaction was never sent, so the Intent can safely be re-prepared and re-executed. + +#### Crash 4→5 + +`MagicIntentAccount` exists with `status = Executing(signature)`. On restart: check `signature` on L1. +- Confirmed & successful → close Intent account +- !(Confirmed & successful) → reschedule + +#### Crash 5→6 + +Same as crash 4→5: `status = Executing(signature)`. Check signature, close or reschedule accordingly. + +> **Note:** Two-stage execution (commit + finalize) is a complication of step 5. Each stage has its own signature stored separately in the `MagicIntentAccount`. This is expanded in [Instruction changes](#new-setintentexecutionstage--intent_id-stage-executionstage-). + +--- + +## Account Design + +### `MagicIntentAccount` ephemeral account + +**Seeds:** `["magic-intent", intent_id.to_le_bytes()]` +**Owner:** MagicBlock builtin program + +```rust +pub struct MagicIntentAccount { + pub bundle: ScheduledIntentBundle, + pub status: MagicIntentStatus, +} + +pub enum MagicIntentStatus { + Accepted, + Executing(ExecutionStage), +} + +pub enum ExecutionStage { + SingleStage(Signature), + TwoStage(TwoStageProgress), +} + +pub enum TwoStageProgress { + Committing(Signature), + Finalizing { commit: Signature, finalize: Signature }, +} +``` + +--- + +## Instruction Changes + +### Modified: `AcceptScheduledCommits` + +Removes up to N intents from the front of `MagicContext.scheduled_base_intents`. For each intent a `MagicIntentAccount` ephemeral account is created with `status = Accepted`. N is bounded by the transaction account limit: every new account must be passed as writable in the same transaction. + +**Accounts:** +``` +[validator_auth (signer), magic_context (writable), + new_pda_0 (writable), new_pda_1 (writable), ...] +``` + +**Logic (after existing pop from MagicContext):** for each accepted intent, create a `MagicIntentAccount` ephemeral account with `status = Accepted`. + +--- + +### New: `SetIntentExecutionStage { intent_id, stage: ExecutionStage }` + +Sets or advances the execution stage of an intent. Handles the initial transition from `Accepted`, signature replacement on retry, and advancement from `Committing` to `Finalizing` — all through a single instruction with a unified set of transition rules. + +**Accounts:** `[validator_auth (signer), intent_pda (writable)]` + +**Transitions:** + +| From | To | Notes | +|---|---|---| +| `Accepted` | `Executing(SingleStage(sig))` | | +| `Accepted` | `Executing(TwoStage(Committing(sig)))` | | +| `Executing(SingleStage(_))` | `Executing(SingleStage(new_sig))` | Signature replacement on retry | +| `Executing(TwoStage(Committing(_)))` | `Executing(TwoStage(Committing(new_sig)))` | Signature replacement on retry | +| `Executing(TwoStage(Committing(commit)))` | `Executing(TwoStage(Finalizing { commit, finalize: sig }))` | Advance to finalize phase | +| `Executing(TwoStage(Finalizing { commit, _ }))` | `Executing(TwoStage(Finalizing { commit, finalize: new_sig }))` | Signature replacement on retry; `commit` must match stored value | + +**Checks:** +- From `Accepted`: rejects `TwoStage(Finalizing { .. })` — cannot skip the commit phase. +- From `Executing(_)`: the execution type (`SingleStage` vs `TwoStage`) cannot change. +- From `Executing(TwoStage(Finalizing { .. }))`: rejects `TwoStage(Committing(_))` — **downgrade not permitted**. Reaching `Finalizing` means the commit transaction was confirmed on L1; going back to `Committing` would re-commit an already-committed account. + +--- + +### New: `CloseMagicIntentAccount { intent_id }` + +Deallocates the `MagicIntentAccount` ephemeral account. Called after L1 execution is confirmed. + +**Accounts:** +``` +[validator_auth (signer), closing_pda (writable)] +``` + +**Logic:** +1. Wipe and deallocate `closing_pda` + +--- + +## Restart Recovery + +### Discovery + +``` +1. getProgramAccounts(magic_program_id, filters=[discriminator(MagicIntentAccount)]) + → returns all open intent accounts +2. Deserialize each account, sort by bundle.id ascending +``` + +Cost: one RPC call returning the set of intents that were accepted and not yet closed. No scanning of closed IDs, no range guessing, no dependency on total historical intent count. + +### Classification and recovery + +```rust +match intent.status { + Accepted => + reschedule_full(intent.bundle), + + Executing(SingleStage(sig)) => + match get_signature_status(sig).await { + Succeeded => close_intent_account(intent.bundle.id).await, + _ => reschedule_full(intent.bundle), + }, + + Executing(TwoStage(Committing(sig))) => + match get_signature_status(sig).await { + Succeeded => reschedule_finalize_only(intent.bundle), + _ => reschedule_full(intent.bundle), + }, + + Executing(TwoStage(Finalizing { finalize, .. })) => + match get_signature_status(finalize).await { + Succeeded => close_intent_account(intent.bundle.id).await, + _ => reschedule_finalize_only(intent.bundle), + }, +} +``` + +--- + +## Intent Discovery: `getProgramAccounts` + +The MagicBlock builtin program owns `MagicContext` (one singleton) and one `MagicIntentAccount` per open intent. A `getProgramAccounts` call filtered by the `MagicIntentAccount` discriminator returns all intent accounts that were accepted and have not yet been closed. After deserialization, accounts are sorted by `bundle.id` to restore processing order. + +Each account's `status` field determines how recovery proceeds — an account with `status = Accepted` means the intent has not yet been sent to L1 and needs full execution; an account with `status = Executing(...)` requires checking the stored signature(s) on L1 before deciding whether to close or reschedule. + +--- + +## Considered Alternatives + +### Option A: Linked list (`MagicIntentList` + `prev_id`/`next_id`) + +Maintain a `MagicIntentList` singleton (head + tail) and embed `prev_id`/`next_id` pointers in each `MagicIntentAccount`. On restart, traverse from head to tail. + +**Problem — unnecessary complexity:** `getProgramAccounts` already returns the same set of open intents in one call. The linked list duplicates this with extra fields per account, a shared singleton written on every accept and every close, and more complex close logic (relinking neighbors). + +**Verdict:** Superseded by `getProgramAccounts`. + +--- + +### Option B: Watermark + range scan + +Store `accepted_intent_watermark: u64` in `MagicContext`. On restart, scan intent PDAs for IDs in the range `[watermark, intent_id)`. + +**Mechanism:** +- Watermark advances only when the intent at the watermark position closes +- On restart: batch-fetch PDAs via `getMultipleAccounts` for the range + +**Problem — stuck watermark:** If a single intent fails persistently (e.g., an integrator bug causes repeated execution failure), the watermark is stuck at that intent's ID. Every subsequent restart must scan from that ID through the entire current range. Other intents in the range are discoverable, but the scan grows unboundedly until the stuck intent is resolved. + +Furthermore, the watermark advancement logic requires careful coordination: advancing past gaps means scanning forward for the next live PDA, which requires either additional instructions or off-chain bookkeeping. + +**Verdict:** Functionally correct but operationally fragile. A single stuck intent degrades restart performance for all subsequent restarts. + +--- + +### Option C: Bitmap pages + +Group intent IDs into pages of fixed size (e.g., 256). Each page is a PDA storing a bitmap of which intent IDs within its range are still active. + +```rust +// PDA: ["magic-intent-page", page_id.to_le_bytes()] +struct ActiveIntentPage { + page_id: u64, + base_intent_id: u64, + active_bitmap: [u64; 4], // 256 bits + active_count: u16, +} +``` + +On restart: scan page PDAs from `min_active_page` to `current_page`, extract set bits, fetch the corresponding intent PDAs. + +**Problem — hot shared account:** Every `CloseMagicIntentAccount` for any intent in page range [0, 255] writes to the same page 0 PDA. Every `AcceptScheduledCommits` that allocates into page 0 also writes to page 0. All operations in the same page range serialize on a single shared account. + +**Problem — page watermark inherits the stuck-intent problem:** `min_active_page` cannot advance past page N until all 256 intents in page N close. One stuck intent holds up the entire page's retirement. The restart scan cost is O(N/256) page fetches — better than a plain watermark, but the underlying problem remains. + +**Problem — two-level indirection on restart:** Pages tell you which IDs were accepted; you still need a second round of PDA fetches to get the actual intent data. + +**Verdict:** Better than a plain watermark for restart scan efficiency (coarser granularity), but introduces a shared hot-account bottleneck and does not fully eliminate the stuck-intent scan problem. + +--- + +### Comparison + +| | `getProgramAccounts` | Linked list | Bitmap pages | Watermark scan | +|---|---|---|---|---| +| Restart discovery | One filtered RPC call | O(open) traversal | O(N/PAGE\_SIZE) + O(open) | O(gap from watermark) | +| Stuck-intent impact on restart | None | None | Blocks page retirement | Blocks watermark advancement | +| Close instruction complexity | Wipe account (2 accounts) | Relink 2 neighbors (3–5 accounts) | Flip 1 bit (2 accounts) | Wipe account (2 accounts) | +| Shared hot account | None | `MagicIntentList` singleton | Page PDA (up to 256 intents) | None | +| Extra state per intent | 0 | +16 bytes (prev\_id, next\_id) | ~0.2 bytes in page bitmap | 0 | + +**`getProgramAccounts` is chosen** for its minimal account structure, no shared hot accounts, simplest close logic, and correct stuck-intent behavior. + +--- + +## Implementation Scope + +| Component | Change | +|---|---| +| `magicblock-magic-program-api/src/pda.rs` | Add `intent_account_pda(id)` | +| `magicblock-magic-program-api/src/instruction.rs` | Add `SetIntentExecutionStage`, `CloseMagicIntentAccount` variants | +| `programs/magicblock/src/` | Add `MagicIntentAccount`, `MagicIntentStatus` types | +| `programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs` | Create intent ephemeral accounts | +| `programs/magicblock/src/schedule_transactions/` (new files) | `process_set_intent_execution_stage.rs`, `process_close_intent_account.rs` | +| `programs/magicblock/src/magicblock_processor.rs` | Wire new instruction variants | +| `magicblock-committor-service/src/` (new file) | `restart_loader.rs` — `getProgramAccounts` fetch, sort by id, classification, recovery dispatch | +| `magicblock-committor-service/src/intent_executor/` | Call `SetIntentExecutionStage` before L1 submission | +| `magicblock-accounts/src/scheduled_commits_processor.rs` | Replace `register_scheduled_commit_sent` with `CloseMagicIntentAccount` call | + diff --git a/programs/magicblock/src/intent_bundles/mod.rs b/programs/magicblock/src/intent_bundles/mod.rs new file mode 100644 index 000000000..6866833f8 --- /dev/null +++ b/programs/magicblock/src/intent_bundles/mod.rs @@ -0,0 +1,2 @@ +pub mod schedule; +pub mod outbox; \ No newline at end of file diff --git a/programs/magicblock/src/intent_bundles/outbox/mod.rs b/programs/magicblock/src/intent_bundles/outbox/mod.rs new file mode 100644 index 000000000..e69de29bb diff --git a/programs/magicblock/src/schedule_transactions/mod.rs b/programs/magicblock/src/intent_bundles/schedule/mod.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/mod.rs rename to programs/magicblock/src/intent_bundles/schedule/mod.rs diff --git a/programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/schedule/process_accept_scheduled_commits.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs rename to programs/magicblock/src/intent_bundles/schedule/process_accept_scheduled_commits.rs diff --git a/programs/magicblock/src/schedule_transactions/process_add_action_callback.rs b/programs/magicblock/src/intent_bundles/schedule/process_add_action_callback.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_add_action_callback.rs rename to programs/magicblock/src/intent_bundles/schedule/process_add_action_callback.rs diff --git a/programs/magicblock/src/schedule_transactions/process_execute_callback.rs b/programs/magicblock/src/intent_bundles/schedule/process_execute_callback.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_execute_callback.rs rename to programs/magicblock/src/intent_bundles/schedule/process_execute_callback.rs diff --git a/programs/magicblock/src/schedule_transactions/process_schedule_commit.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_schedule_commit.rs rename to programs/magicblock/src/intent_bundles/schedule/process_schedule_commit.rs diff --git a/programs/magicblock/src/schedule_transactions/process_schedule_commit_tests.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_schedule_commit_tests.rs rename to programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs diff --git a/programs/magicblock/src/schedule_transactions/process_schedule_intent_bundle.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_intent_bundle.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_schedule_intent_bundle.rs rename to programs/magicblock/src/intent_bundles/schedule/process_schedule_intent_bundle.rs diff --git a/programs/magicblock/src/schedule_transactions/process_scheduled_commit_sent.rs b/programs/magicblock/src/intent_bundles/schedule/process_scheduled_commit_sent.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/process_scheduled_commit_sent.rs rename to programs/magicblock/src/intent_bundles/schedule/process_scheduled_commit_sent.rs diff --git a/programs/magicblock/src/schedule_transactions/transaction_scheduler.rs b/programs/magicblock/src/intent_bundles/schedule/transaction_scheduler.rs similarity index 100% rename from programs/magicblock/src/schedule_transactions/transaction_scheduler.rs rename to programs/magicblock/src/intent_bundles/schedule/transaction_scheduler.rs diff --git a/programs/magicblock/src/lib.rs b/programs/magicblock/src/lib.rs index f3ae44268..4d66df032 100644 --- a/programs/magicblock/src/lib.rs +++ b/programs/magicblock/src/lib.rs @@ -5,7 +5,6 @@ mod magic_context; pub mod magic_sys; mod mutate_accounts; mod schedule_task; -mod schedule_transactions; pub use magic_context::MagicContext; pub mod magic_scheduled_base_intent; pub mod magicblock_processor; @@ -13,6 +12,8 @@ pub mod outbox_intent_bundles; pub mod test_utils; mod utils; pub mod validator; +mod intent_bundles; +use intent_bundles::schedule as schedule_transactions; pub use magic_sys::init_magic_sys; pub use magicblock_magic_program_api::*; From a4e4eea6bdc74a51c02e4894b9eeb3ae5bcbbeb5 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 17:05:18 +0700 Subject: [PATCH 25/97] refactor: file structure --- programs/magicblock/src/intent_bundles/mod.rs | 3 ++- .../magicblock/src/intent_bundles/outbox/mod.rs | 2 ++ .../process_accept_scheduled_commits.rs | 0 .../process_scheduled_commit_sent.rs | 0 .../{schedule => }/process_execute_callback.rs | 0 .../magicblock/src/intent_bundles/schedule/mod.rs | 14 ++++++-------- programs/magicblock/src/lib.rs | 3 +-- 7 files changed, 11 insertions(+), 11 deletions(-) rename programs/magicblock/src/intent_bundles/{schedule => outbox}/process_accept_scheduled_commits.rs (100%) rename programs/magicblock/src/intent_bundles/{schedule => outbox}/process_scheduled_commit_sent.rs (100%) rename programs/magicblock/src/intent_bundles/{schedule => }/process_execute_callback.rs (100%) diff --git a/programs/magicblock/src/intent_bundles/mod.rs b/programs/magicblock/src/intent_bundles/mod.rs index 6866833f8..f43ada336 100644 --- a/programs/magicblock/src/intent_bundles/mod.rs +++ b/programs/magicblock/src/intent_bundles/mod.rs @@ -1,2 +1,3 @@ +pub mod outbox; +mod process_execute_callback; pub mod schedule; -pub mod outbox; \ No newline at end of file diff --git a/programs/magicblock/src/intent_bundles/outbox/mod.rs b/programs/magicblock/src/intent_bundles/outbox/mod.rs index e69de29bb..eae59110c 100644 --- a/programs/magicblock/src/intent_bundles/outbox/mod.rs +++ b/programs/magicblock/src/intent_bundles/outbox/mod.rs @@ -0,0 +1,2 @@ +pub mod process_accept_scheduled_commits; +pub mod process_scheduled_commit_sent; diff --git a/programs/magicblock/src/intent_bundles/schedule/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs similarity index 100% rename from programs/magicblock/src/intent_bundles/schedule/process_accept_scheduled_commits.rs rename to programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs diff --git a/programs/magicblock/src/intent_bundles/schedule/process_scheduled_commit_sent.rs b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs similarity index 100% rename from programs/magicblock/src/intent_bundles/schedule/process_scheduled_commit_sent.rs rename to programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs diff --git a/programs/magicblock/src/intent_bundles/schedule/process_execute_callback.rs b/programs/magicblock/src/intent_bundles/process_execute_callback.rs similarity index 100% rename from programs/magicblock/src/intent_bundles/schedule/process_execute_callback.rs rename to programs/magicblock/src/intent_bundles/process_execute_callback.rs diff --git a/programs/magicblock/src/intent_bundles/schedule/mod.rs b/programs/magicblock/src/intent_bundles/schedule/mod.rs index 574446361..732b17c74 100644 --- a/programs/magicblock/src/intent_bundles/schedule/mod.rs +++ b/programs/magicblock/src/intent_bundles/schedule/mod.rs @@ -1,11 +1,8 @@ -mod process_accept_scheduled_commits; mod process_add_action_callback; -mod process_execute_callback; mod process_schedule_commit; #[cfg(test)] mod process_schedule_commit_tests; mod process_schedule_intent_bundle; -mod process_scheduled_commit_sent; pub(crate) mod transaction_scheduler; use std::sync::Arc; @@ -14,14 +11,9 @@ use magicblock_core::intent::CommittedAccount; use magicblock_magic_program_api::{ pda::CALLBACK_SIGNER, MAGIC_CONTEXT_PUBKEY, }; -pub(crate) use process_accept_scheduled_commits::*; pub(crate) use process_add_action_callback::process_add_action_callback; -pub(crate) use process_execute_callback::*; pub(crate) use process_schedule_commit::*; pub(crate) use process_schedule_intent_bundle::process_schedule_intent_bundle; -pub use process_scheduled_commit_sent::{ - process_scheduled_commit_sent, register_scheduled_commit_sent, SentCommit, -}; use solana_clock::Clock; use solana_instruction::{error::InstructionError, AccountMeta}; use solana_log_collector::ic_msg; @@ -29,6 +21,12 @@ use solana_program_runtime::invoke_context::InvokeContext; use solana_pubkey::Pubkey; use solana_transaction_context::TransactionContext; +pub use crate::intent_bundles::outbox::process_scheduled_commit_sent::{ + process_scheduled_commit_sent, register_scheduled_commit_sent, SentCommit, +}; +pub(crate) use crate::intent_bundles::{ + outbox::process_accept_scheduled_commits::*, process_execute_callback::*, +}; use crate::{ magic_sys::{ fetch_current_commit_nonces, COMMIT_LIMIT, COMMIT_LIMIT_ERR, diff --git a/programs/magicblock/src/lib.rs b/programs/magicblock/src/lib.rs index 4d66df032..cdbf3ba1e 100644 --- a/programs/magicblock/src/lib.rs +++ b/programs/magicblock/src/lib.rs @@ -6,15 +6,14 @@ pub mod magic_sys; mod mutate_accounts; mod schedule_task; pub use magic_context::MagicContext; +mod intent_bundles; pub mod magic_scheduled_base_intent; pub mod magicblock_processor; pub mod outbox_intent_bundles; pub mod test_utils; mod utils; pub mod validator; -mod intent_bundles; use intent_bundles::schedule as schedule_transactions; - pub use magic_sys::init_magic_sys; pub use magicblock_magic_program_api::*; pub use schedule_transactions::{ From bbcc11391722bf3fe0c589c1411f9bbba85a69e4 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 17:09:49 +0700 Subject: [PATCH 26/97] refactor: file structure --- .../src/service/outbox_intent_bundles_reader.rs | 2 +- .../{ => intent_bundles}/magic_scheduled_base_intent.rs | 0 programs/magicblock/src/intent_bundles/mod.rs | 2 ++ .../src/{ => intent_bundles}/outbox_intent_bundles.rs | 0 programs/magicblock/src/lib.rs | 8 +++++--- 5 files changed, 8 insertions(+), 4 deletions(-) rename programs/magicblock/src/{ => intent_bundles}/magic_scheduled_base_intent.rs (100%) rename programs/magicblock/src/{ => intent_bundles}/outbox_intent_bundles.rs (100%) diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index f7cd5baf6..a3b41641f 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -2,9 +2,9 @@ use std::{num::NonZeroUsize, sync::Arc}; use async_trait::async_trait; use magicblock_accounts_db::AccountsDb; -use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::ReadableAccount; use solana_pubkey::Pubkey; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; // TODO(edwin): name - OutboxIntentAccount/PendingIntentAccount // TODO(edwin): name - PendingIntentBundlesReader? diff --git a/programs/magicblock/src/magic_scheduled_base_intent.rs b/programs/magicblock/src/intent_bundles/magic_scheduled_base_intent.rs similarity index 100% rename from programs/magicblock/src/magic_scheduled_base_intent.rs rename to programs/magicblock/src/intent_bundles/magic_scheduled_base_intent.rs diff --git a/programs/magicblock/src/intent_bundles/mod.rs b/programs/magicblock/src/intent_bundles/mod.rs index f43ada336..01c902946 100644 --- a/programs/magicblock/src/intent_bundles/mod.rs +++ b/programs/magicblock/src/intent_bundles/mod.rs @@ -1,3 +1,5 @@ +pub mod magic_scheduled_base_intent; pub mod outbox; +pub mod outbox_intent_bundles; mod process_execute_callback; pub mod schedule; diff --git a/programs/magicblock/src/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs similarity index 100% rename from programs/magicblock/src/outbox_intent_bundles.rs rename to programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs diff --git a/programs/magicblock/src/lib.rs b/programs/magicblock/src/lib.rs index cdbf3ba1e..a2de797e7 100644 --- a/programs/magicblock/src/lib.rs +++ b/programs/magicblock/src/lib.rs @@ -7,13 +7,15 @@ mod mutate_accounts; mod schedule_task; pub use magic_context::MagicContext; mod intent_bundles; -pub mod magic_scheduled_base_intent; pub mod magicblock_processor; -pub mod outbox_intent_bundles; pub mod test_utils; mod utils; pub mod validator; -use intent_bundles::schedule as schedule_transactions; +// TODO(edwin): safe to just rename or will break integrations? +pub use intent_bundles::{ + magic_scheduled_base_intent, outbox_intent_bundles, + schedule as schedule_transactions, +}; pub use magic_sys::init_magic_sys; pub use magicblock_magic_program_api::*; pub use schedule_transactions::{ From a0e4da1203127f88195ccd835e51b1c7d6523a8e Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 3 Jun 2026 17:11:45 +0700 Subject: [PATCH 27/97] wip --- programs/magicblock/src/intent_bundles/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/magicblock/src/intent_bundles/mod.rs b/programs/magicblock/src/intent_bundles/mod.rs index 01c902946..bb21a1a21 100644 --- a/programs/magicblock/src/intent_bundles/mod.rs +++ b/programs/magicblock/src/intent_bundles/mod.rs @@ -1,5 +1,7 @@ pub mod magic_scheduled_base_intent; pub mod outbox; +// TODO(edwin): rename? pub mod outbox_intent_bundles; +// TODO(edwin): rename? mod process_execute_callback; pub mod schedule; From 9b402bf2f0a91379a0066c1930ae06892d4e56f7 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Thu, 4 Jun 2026 12:37:05 +0700 Subject: [PATCH 28/97] wip --- .../service/outbox_intent_bundles_reader.rs | 2 +- magicblock-core/src/intent.rs | 2 + magicblock-core/src/intent/outbox.rs | 15 ++ .../src/instruction.rs | 15 +- programs/magicblock/src/intent_bundles/mod.rs | 2 +- .../process_accept_scheduled_commits.rs | 232 ++++++++++++++---- .../intent_bundles/outbox_intent_bundles.rs | 39 ++- programs/magicblock/src/magic_context.rs | 8 + 8 files changed, 257 insertions(+), 58 deletions(-) create mode 100644 magicblock-core/src/intent/outbox.rs diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index a3b41641f..f7cd5baf6 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -2,9 +2,9 @@ use std::{num::NonZeroUsize, sync::Arc}; use async_trait::async_trait; use magicblock_accounts_db::AccountsDb; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::ReadableAccount; use solana_pubkey::Pubkey; -use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; // TODO(edwin): name - OutboxIntentAccount/PendingIntentAccount // TODO(edwin): name - PendingIntentBundlesReader? diff --git a/magicblock-core/src/intent.rs b/magicblock-core/src/intent.rs index d66b0b1fd..8dd85dc19 100644 --- a/magicblock-core/src/intent.rs +++ b/magicblock-core/src/intent.rs @@ -3,6 +3,8 @@ use serde::{Deserialize, Serialize}; use solana_account::{Account, AccountSharedData}; use solana_pubkey::Pubkey; +pub mod outbox; + use crate::token_programs::try_remap_ata_to_eata; pub type CommittedAccountRef = (Pubkey, AccountSharedData); diff --git a/magicblock-core/src/intent/outbox.rs b/magicblock-core/src/intent/outbox.rs new file mode 100644 index 000000000..c6529f8d3 --- /dev/null +++ b/magicblock-core/src/intent/outbox.rs @@ -0,0 +1,15 @@ +use solana_pubkey::Pubkey; + +pub const OUTBOX_INTENT_SEED: &[u8] = b"outbox-intent"; + +pub const OUTBOX_INTENT_DISCRIMINATOR: [u8; 8] = *b"obintent"; + +/// Derives the outbox intent PDA for a given intent ID. +/// Seeds: `["outbox-intent", intent_id.to_le_bytes()]` +pub fn outbox_intent_pda(id: u64) -> Pubkey { + Pubkey::find_program_address( + &[OUTBOX_INTENT_SEED, &id.to_le_bytes()], + &magicblock_magic_program_api::ID, + ) + .0 +} diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index 6b3655ff5..380c46578 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -57,15 +57,18 @@ pub enum MagicBlockInstruction { /// - **2..n** `[]` Accounts to be committed and undelegated ScheduleCommitAndUndelegate, - /// Moves the scheduled commit from the MagicContext to the global scheduled commits - /// map. This is the second part of scheduling a commit. + /// Pops up to N intents from the front of `MagicContext.scheduled_base_intents` + /// and for each creates an outbox intent ephemeral account (`MagicIntentAccount`) + /// with `status = Accepted`. This is the second part of scheduling a commit. /// - /// It is run at the start of the slot to update the global scheduled commits map just - /// in time for the validator to realize the commits right after. + /// N is determined by the number of writable PDA accounts provided beyond + /// the two fixed accounts. It is run at the start of the slot. /// /// # Account references - /// - **0.** `[SIGNER]` Validator Authority - /// - **1.** `[WRITE]` Magic Context Account containing the initially scheduled commits + /// - **0.** `[SIGNER]` Validator Authority + /// - **1.** `[WRITE]` Magic Context Account containing the initially scheduled commits + /// - **2..n** `[WRITE]` Outbox intent PDAs, one per accepted intent, + /// seeds: `["outbox-intent", intent_id.to_le_bytes()]` AcceptScheduleCommits, /// Records the attempt to realize a scheduled commit on chain. diff --git a/programs/magicblock/src/intent_bundles/mod.rs b/programs/magicblock/src/intent_bundles/mod.rs index bb21a1a21..06991aa80 100644 --- a/programs/magicblock/src/intent_bundles/mod.rs +++ b/programs/magicblock/src/intent_bundles/mod.rs @@ -1,6 +1,6 @@ pub mod magic_scheduled_base_intent; pub mod outbox; -// TODO(edwin): rename? +// TODO(edwin): rename? pub mod outbox_intent_bundles; // TODO(edwin): rename? mod process_execute_callback; diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index cc4cc844f..1cad2c14b 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -1,36 +1,154 @@ use std::collections::HashSet; +use magicblock_core::intent::outbox::outbox_intent_pda; +use magicblock_magic_program_api::{ + instruction::MagicBlockInstruction, EPHEMERAL_VAULT_PUBKEY, +}; use solana_account::{ReadableAccount, WritableAccount}; -use solana_instruction::error::InstructionError; +use solana_instruction::{error::InstructionError, AccountMeta, Instruction}; use solana_log_collector::ic_msg; use solana_program_runtime::invoke_context::InvokeContext; use solana_pubkey::Pubkey; use crate::{ + intent_bundles::outbox_intent_bundles::OutboxIntentBundle, schedule_transactions, utils::accounts::{ get_instruction_account_with_idx, get_instruction_pubkey_with_idx, }, validator::effective_validator_authority_id, - MagicContext, TransactionScheduler, + MagicContext, }; +const VALIDATOR_AUTHORITY_IDX: u16 = 0; +const MAGIC_CONTEXT_IDX: u16 = VALIDATOR_AUTHORITY_IDX + 1; +const VAULT_IDX: u16 = MAGIC_CONTEXT_IDX + 1; +const INTENT_PDAS_OFFSET: u16 = VAULT_IDX + 1; + pub fn process_accept_scheduled_commits( signers: HashSet, invoke_context: &mut InvokeContext, ) -> Result<(), InstructionError> { - const VALIDATOR_AUTHORITY_IDX: u16 = 0; - const MAGIC_CONTEXT_IDX: u16 = VALIDATOR_AUTHORITY_IDX + 1; + // Common conditions verification + let validator_auth = effective_validator_authority_id(); + verify(&signers, invoke_context, &validator_auth)?; - let transaction_context = &*invoke_context.transaction_context; + // pop first n intents + // n - is number of OutboxIntentBundle PDAs passed + let intents = pop_scheduled_intents(invoke_context)?; + if intents.is_empty() { + // NOTE: we should have not been called if no commits are scheduled + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: no scheduled commits to accept" + ); + return Ok(()); + } + + for (i, intent) in intents.into_iter().enumerate() { + let pda_idx = INTENT_PDAS_OFFSET + i as u16; + let pda = verify_intent_pda(invoke_context, intent.id, pda_idx)?; + + // Create outbox ephemeral account + create_outbox_account_cpi( + invoke_context, + validator_auth, + pda, + OutboxIntentBundle::accepted(intent), + )?; + } + + Ok(()) +} - // 1. Read all scheduled commits from the `MagicContext` account - // We do this first so we can skip all checks in case there is nothing - // to be processed +fn verify( + signers: &HashSet, + invoke_context: &InvokeContext, + validator_auth: &Pubkey, +) -> Result<(), InstructionError> { + // Check magic context schedule_transactions::check_magic_context_id( invoke_context, MAGIC_CONTEXT_IDX, )?; + + // Validate authority + let transaction_context = &*invoke_context.transaction_context; + let provided_validator_auth = get_instruction_pubkey_with_idx( + transaction_context, + VALIDATOR_AUTHORITY_IDX, + )?; + if provided_validator_auth != validator_auth { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: invalid validator authority {}, should be {}", + provided_validator_auth, + validator_auth + ); + return Err(InstructionError::InvalidArgument); + } + + // Validate authority is a signer + if !signers.contains(&validator_auth) { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: validator authority pubkey {} not in signers", + validator_auth + ); + return Err(InstructionError::MissingRequiredSignature); + } + + Ok(()) +} + +fn verify_intent_pda( + invoke_context: &InvokeContext, + intent_id: u64, + pda_idx: u16, +) -> Result { + // TODO(edwin): add check that acount doesn't exist? + let transaction_context = &*invoke_context.transaction_context; + let provided = + get_instruction_pubkey_with_idx(transaction_context, pda_idx)?; + let expected = outbox_intent_pda(intent_id); + if *provided != expected { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: account at idx {} is {}, expected PDA {} for intent {}", + pda_idx, + provided, + expected, + intent_id + ); + return Err(InstructionError::InvalidArgument); + } + Ok(expected) +} + +fn pop_scheduled_intents( + invoke_context: &InvokeContext, +) -> Result< + Vec, + InstructionError, +> { + let transaction_context = &*invoke_context.transaction_context; + let num_ix_accounts = transaction_context + .get_current_instruction_context()? + .get_number_of_instruction_accounts() + as usize; + + // Assert enough accounts + let Some(num_accept_intents) = + num_ix_accounts.checked_sub(INTENT_PDAS_OFFSET as usize) + else { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: not enough accounts to accept intents ({}), need validator authority, magic context, vault, and at least one outbox intent PDA", + num_ix_accounts + ); + return Err(InstructionError::MissingAccount); + }; + let magic_context_acc = get_instruction_account_with_idx( transaction_context, MAGIC_CONTEXT_IDX, @@ -46,52 +164,72 @@ pub fn process_accept_scheduled_commits( ); InstructionError::InvalidAccountData })?; - if magic_context.scheduled_base_intents.is_empty() { - ic_msg!( - invoke_context, - "AcceptScheduledCommits: no scheduled commits to accept" - ); - // NOTE: we should have not been called if no commits are scheduled - return Ok(()); - } - // 2. Check that the validator authority (first account) is correct and signer - let provided_validator_auth = get_instruction_pubkey_with_idx( - transaction_context, - VALIDATOR_AUTHORITY_IDX, - )?; - let validator_auth = effective_validator_authority_id(); - if !provided_validator_auth.eq(&validator_auth) { - ic_msg!( - invoke_context, - "AcceptScheduledCommits ERR: invalid validator authority {}, should be {}", - provided_validator_auth, - validator_auth - ); - return Err(InstructionError::InvalidArgument); - } - if !signers.contains(&validator_auth) { + let intents = + magic_context.take_front_scheduled_commits(num_accept_intents); + magic_context + .write_to(magic_context_acc.borrow_mut()?.data_as_mut_slice())?; + + Ok(intents) +} + +fn create_outbox_account_cpi( + invoke_context: &mut InvokeContext, + validator_auth: Pubkey, + pda: Pubkey, + outbox_account: OutboxIntentBundle, +) -> Result<(), InstructionError> { + let intent_id = outbox_account.intent_bundle.id; + let data = outbox_account.try_to_bytes().map_err(|_| { ic_msg!( invoke_context, - "AcceptScheduledCommits ERR: validator authority pubkey {} not in signers", - validator_auth + "AcceptScheduledCommits ERR: failed to serialize intent {}", + intent_id ); - return Err(InstructionError::MissingRequiredSignature); - } + InstructionError::InvalidAccountData + })?; - // 3. Move scheduled commits (without copying) - let scheduled_commits = magic_context.take_scheduled_commits(); - ic_msg!( + create_ephemeral_account_cpi( invoke_context, - "AcceptScheduledCommits: accepted {} scheduled commit(s)", - scheduled_commits.len() - ); - TransactionScheduler::default() - .accept_scheduled_base_intent(scheduled_commits); + validator_auth, + pda, + data.len() as u32, // TODO(edwin): fix cast + )?; - // 4. Serialize and store the updated `MagicContext` account - magic_context - .write_to(magic_context_acc.borrow_mut()?.data_as_mut_slice())?; + // TODO(edwin): simplify/ + // Move intent data in new account + let transaction_context = &*invoke_context.transaction_context; + let tx_idx = transaction_context + .find_index_of_account(&pda) + .ok_or(InstructionError::MissingAccount)?; + transaction_context + .accounts() + .try_borrow_mut(tx_idx) + .map_err(|_| InstructionError::AccountBorrowFailed)? + .data_as_mut_slice() + .copy_from_slice(&data); Ok(()) } + +fn create_ephemeral_account_cpi( + invoke_context: &mut InvokeContext, + sponsor: Pubkey, + pda: Pubkey, + data_len: u32, +) -> Result<(), InstructionError> { + invoke_context.native_invoke( + Instruction { + program_id: crate::id(), + accounts: vec![ + AccountMeta::new(sponsor, true), + AccountMeta::new(pda, true), + AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), + ], + data: MagicBlockInstruction::CreateEphemeralAccount { data_len } + .try_to_vec() + .map_err(|_| InstructionError::InvalidInstructionData)?, + }, + &[pda], + ) +} diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index bdf12a48c..12ba58364 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,25 +1,28 @@ +use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; +use serde::{Deserialize, Serialize}; use solana_signature::Signature; use crate::magic_scheduled_base_intent::ScheduledIntentBundle; -// TODO(edwin): naming Outbox/Pending +#[derive(Debug, Serialize, Deserialize)] pub struct OutboxIntentBundle { - // TODO(edwin): define visability pub intent_bundle: ScheduledIntentBundle, - // TODO(edwin): define visability pub status: OutboxIntentBundleStatus, } +#[derive(Debug, Serialize, Deserialize)] pub enum OutboxIntentBundleStatus { Accepted, Executing(ExecutionStage), } +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum ExecutionStage { SingleStage(Signature), TwoStage(TwoStageProgress), } +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum TwoStageProgress { Committing(Signature), Finalizing { @@ -27,3 +30,33 @@ pub enum TwoStageProgress { finalize: Signature, }, } + +impl OutboxIntentBundle { + pub fn accepted(intent_bundle: ScheduledIntentBundle) -> Self { + Self { + intent_bundle, + status: OutboxIntentBundleStatus::Accepted, + } + } + + pub fn try_to_bytes(&self) -> Result, bincode::Error> { + let body = bincode::serialize(self)?; + let mut out = + Vec::with_capacity(OUTBOX_INTENT_DISCRIMINATOR.len() + body.len()); + out.extend_from_slice(&OUTBOX_INTENT_DISCRIMINATOR); + out.extend_from_slice(&body); + Ok(out) + } + + pub fn try_from_bytes(data: &[u8]) -> Result { + let disc_len = OUTBOX_INTENT_DISCRIMINATOR.len(); + if data.len() < disc_len + || data[..disc_len] != OUTBOX_INTENT_DISCRIMINATOR + { + return Err(Box::new(bincode::ErrorKind::Custom( + "invalid discriminator".into(), + ))); + } + bincode::deserialize(&data[disc_len..]) + } +} diff --git a/programs/magicblock/src/magic_context.rs b/programs/magicblock/src/magic_context.rs index 51d819db0..bf9a54a20 100644 --- a/programs/magicblock/src/magic_context.rs +++ b/programs/magicblock/src/magic_context.rs @@ -59,6 +59,14 @@ impl MagicContext { mem::take(&mut self.scheduled_base_intents) } + pub(crate) fn take_front_scheduled_commits( + &mut self, + n: usize, + ) -> Vec { + let n = n.min(self.scheduled_base_intents.len()); + self.scheduled_base_intents.drain(..n).collect() + } + pub fn has_scheduled_commits(data: &[u8]) -> bool { const LEN_OFF: usize = mem::size_of::(); const LEN_END: usize = LEN_OFF + mem::size_of::(); From 76602c63b0d2632d0e6dab82cd5805f73451a641 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Thu, 4 Jun 2026 18:37:21 +0700 Subject: [PATCH 29/97] fix: bool passing --- .../intent_execution_engine.rs | 24 ++++------------- .../src/intent_executor/mod.rs | 9 ++++++- .../delivery_preparator.rs | 27 ++++++++++++++----- .../src/transaction_preparator/mod.rs | 12 +++++++-- .../src/transactions.rs | 2 +- .../tests/test_intent_executor.rs | 2 ++ 6 files changed, 47 insertions(+), 29 deletions(-) diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 7b435a85a..593504f7e 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -283,7 +283,6 @@ where // Execute an Intent let result = executor.execute(intent.clone(), persister).await; - let execution_succeeded = result.inner.is_ok(); let _ = result.inner.as_ref().inspect_err(|err| { error!(intent_id = intent.id, error = ?err, "Failed to execute intent bundle"); }); @@ -308,24 +307,11 @@ where .complete(&intent) .expect("Valid completion of previously scheduled message"); - // Cleaning buffers on failure isn't safe due to potential race condition: - // Assume pubkey set A being committed - // Intent1 fails and cleans up, another Intent2 with set A executes right away - // That could lead for Intent2 init of buffers executing prior of Intent1 buffer cleanup - // With same set A buffers will have same address - // - // To avoid this race on buffers we cleanup only succesfully executed intents - // With intent retries all buffers will be eventually closed once intent succeeds - // - // Note: not sure this is safa for TableMania as it susceptible to same race condition - // unless it does handle it internally somehow - if execution_succeeded { - tokio::spawn(async move { - if let Err(err) = executor.cleanup().await { - error!(error = ?err, "Failed to cleanup after intent"); - } - }); - } + tokio::spawn(async move { + if let Err(err) = executor.cleanup().await { + error!(error = ?err, "Failed to cleanup after intent"); + } + }); // Free worker drop(execution_permit); diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index f7c8744bb..3a11bd45a 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -158,6 +158,10 @@ pub struct IntentExecutorImpl { pub started_at: Instant, /// Junk that needs to be cleaned up junk: Vec, + /// Set to false on execution failure so cleanup only releases ALT + /// reservations without closing buffer PDAs (see race condition note in + /// intent_execution_engine) + close_buffers: bool, } impl IntentExecutorImpl @@ -185,6 +189,7 @@ where started_at: Instant::now(), junk: vec![], + close_buffers: true, } } @@ -567,6 +572,7 @@ where }); }); + self.close_buffers = result.is_ok(); self.junk = execution_report.junk; IntentExecutionResult { inner: result, @@ -575,13 +581,14 @@ where } } - /// Cleanup after intent using junk async fn cleanup(mut self) -> Result<(), BufferExecutionError> { + let close_buffers = self.close_buffers; let cleanup_futs = self.junk.iter().map(|to_cleanup| { self.transaction_preparator.cleanup_for_strategy( &self.authority, &to_cleanup.optimized_tasks, &to_cleanup.lookup_tables_keys, + close_buffers, ) }); diff --git a/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs b/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs index fc4e47e84..f5d7a8c9e 100644 --- a/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs +++ b/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs @@ -192,7 +192,7 @@ impl DeliveryPreparator { let preparation_task = preparation_task.clone(); let cleanup_task = preparation_task.cleanup_task(); - self.cleanup(authority, &[cleanup_task], &[]).await?; + self.cleanup_buffers(authority, &[cleanup_task]).await?; self.rpc_client.invalidate_cached_blockhash().await; // Restore preparation stage for retry @@ -439,20 +439,35 @@ impl DeliveryPreparator { Ok(alts) } - /// Releases pubkeys from TableMania and - /// cleans up after buffer tasks + /// Releases pubkeys from TableMania and cleans up after buffer tasks. + // + // Cleaning buffers on failure isn't safe due to potential race condition: + // Assume pubkey set A being committed + // Intent1 fails and cleans up, another Intent2 with set A executes right away + // That could lead for Intent2 init of buffers executing prior of Intent1 buffer cleanup + // With same set A buffers will have same address + // + // To avoid this race on buffers we cleanup only succesfully executed intents + // With intent retries all buffers will be eventually closed once intent succeeds pub async fn cleanup( &self, authority: &Keypair, cleanup_tasks: &[CleanupTask], lookup_table_keys: &[Pubkey], + close_buffers: bool, ) -> DeliveryPreparatorResult<(), BufferExecutionError> { - let fut1 = self.cleanup_buffers(authority, cleanup_tasks); let alt_set = HashSet::from_iter(lookup_table_keys.iter().cloned()); let fut2 = self.table_mania.release_pubkeys(&alt_set); - let (res, ()) = join(fut1, fut2).await; - res + if close_buffers { + let (res, ()) = + join(self.cleanup_buffers(authority, cleanup_tasks), fut2) + .await; + res + } else { + fut2.await; + Ok(()) + } } async fn cleanup_buffers( diff --git a/magicblock-committor-service/src/transaction_preparator/mod.rs b/magicblock-committor-service/src/transaction_preparator/mod.rs index 1f5b514fc..08f3af024 100644 --- a/magicblock-committor-service/src/transaction_preparator/mod.rs +++ b/magicblock-committor-service/src/transaction_preparator/mod.rs @@ -34,12 +34,14 @@ pub trait TransactionPreparator: Send + Sync + 'static { intent_persister: &Option

, ) -> PreparatorResult; - /// Cleans up after strategy + /// Cleans up after strategy. + /// `close_buffers`: if false, only ALT reservations are released. async fn cleanup_for_strategy( &self, authority: &Keypair, tasks: &[BaseTaskImpl], lookup_table_keys: &[Pubkey], + close_buffers: bool, ) -> DeliveryPreparatorResult<(), BufferExecutionError>; } @@ -118,6 +120,7 @@ impl TransactionPreparator for TransactionPreparatorImpl { authority: &Keypair, tasks: &[BaseTaskImpl], lookup_table_keys: &[Pubkey], + close_buffers: bool, ) -> DeliveryPreparatorResult<(), BufferExecutionError> { let cleanup_tasks: Vec<_> = tasks .iter() @@ -136,7 +139,12 @@ impl TransactionPreparator for TransactionPreparatorImpl { }) .collect(); self.delivery_preparator - .cleanup(authority, &cleanup_tasks, lookup_table_keys) + .cleanup( + authority, + &cleanup_tasks, + lookup_table_keys, + close_buffers, + ) .await } } diff --git a/magicblock-committor-service/src/transactions.rs b/magicblock-committor-service/src/transactions.rs index 112455331..2c25c7d42 100644 --- a/magicblock-committor-service/src/transactions.rs +++ b/magicblock-committor-service/src/transactions.rs @@ -9,4 +9,4 @@ pub fn serialized_transaction_size( ) -> usize { // SAFETY: runs on transactions we already serialize before sending. usize::try_from(bincode::serialized_size(transaction).unwrap()).unwrap() -} \ No newline at end of file +} diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 5355379d3..26acbc58a 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -834,6 +834,7 @@ async fn test_cpi_limits_error_recovery() { &fixture.authority, &to_cleanup.optimized_tasks, &to_cleanup.lookup_tables_keys, + true, ) }); assert!(try_join_all(cleanup_futs).await.is_ok()); @@ -981,6 +982,7 @@ async fn test_commit_id_actions_cpi_limit_errors_recovery() { &fixture.authority, &to_cleanup.optimized_tasks, &to_cleanup.lookup_tables_keys, + true, ) }); assert!(try_join_all(cleanup_futs).await.is_ok()); From 119e715e6d6f5710502187433f812667a9606642 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Thu, 4 Jun 2026 19:20:54 +0700 Subject: [PATCH 30/97] fix: accept --- magicblock-committor-service/src/service.rs | 2 +- mimd-0025.md | 70 +++++++++---------- .../process_accept_scheduled_commits.rs | 14 +++- .../intent_bundles/outbox_intent_bundles.rs | 4 +- 4 files changed, 51 insertions(+), 39 deletions(-) diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index f437f1fbc..4c06b9dc4 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -225,7 +225,7 @@ where let read_len = intent_bundles_chunk.len(); let intent_bundles = intent_bundles_chunk .into_iter() - .map(|el| el.intent_bundle) + .map(|el| el.inner) .collect(); // Schedule without initial persistence as bundle already exists in db diff --git a/mimd-0025.md b/mimd-0025.md index 6416e6ff5..8a3bf2f5a 100644 --- a/mimd-0025.md +++ b/mimd-0025.md @@ -34,7 +34,7 @@ The window between step 2 and step 4 is unprotected. Any crash drops all in-flig This proposal is similar to the **outbox pattern**, where `AccountsDB` itself serves as the durable outbox. Rather than handing an Intent directly to the `CommittorService` and hoping for the best, the validator first writes the Intent into `AccountsDB` as a dedicated ephemeral account. The `CommittorService` then processes it from there and only closes the account once L1 execution is confirmed. A crash at any point leaves the account in `AccountsDB`; on restart the validator calls `getProgramAccounts` with a discriminator filter, reads each account's status, and resumes accordingly. -Additionally, just before sending a transaction to L1, the validator records the transaction signature back into the `MagicIntentAccount` via `SetIntentExecutionStage`. This lets the restart logic distinguish "prepared but not yet sent" from "sent — check the signature on L1." +Additionally, just before sending a transaction to L1, the validator records the transaction signature back into the `OutboxIntentBundle` via `SetIntentExecutionStage`. This lets the restart logic distinguish "prepared but not yet sent" from "sent — check the signature on L1." **No stage is executed twice.** Because `SetIntentExecutionStage` is committed to `AccountsDB` before the L1 transaction is submitted, any restart that finds `status = Executing(...)` will always query L1 for the stored signature first. A confirmed & successful tx signature advances or closes the intent. The write-before-send ordering is the invariant that makes this hold. @@ -55,15 +55,15 @@ Intent is still in `MagicContext`. The next `AcceptScheduledCommits` call picks #### Crash 2→3 -`MagicIntentAccount` exists with `status = Accepted`. On restart: discovered via `getProgramAccounts`, rescheduled for full execution. +`OutboxIntentBundle` exists with `status = Accepted`. On restart: discovered via `getProgramAccounts`, rescheduled for full execution. #### Crash 3→4 -`MagicIntentAccount` exists with `status = Accepted` — indistinguishable from crash 2→3. The L1 transaction was never sent, so the Intent can safely be re-prepared and re-executed. +`OutboxIntentBundle` exists with `status = Accepted` — indistinguishable from crash 2→3. The L1 transaction was never sent, so the Intent can safely be re-prepared and re-executed. #### Crash 4→5 -`MagicIntentAccount` exists with `status = Executing(signature)`. On restart: check `signature` on L1. +`OutboxIntentBundle` exists with `status = Executing(signature)`. On restart: check `signature` on L1. - Confirmed & successful → close Intent account - !(Confirmed & successful) → reschedule @@ -71,24 +71,24 @@ Intent is still in `MagicContext`. The next `AcceptScheduledCommits` call picks Same as crash 4→5: `status = Executing(signature)`. Check signature, close or reschedule accordingly. -> **Note:** Two-stage execution (commit + finalize) is a complication of step 5. Each stage has its own signature stored separately in the `MagicIntentAccount`. This is expanded in [Instruction changes](#new-setintentexecutionstage--intent_id-stage-executionstage-). +> **Note:** Two-stage execution (commit + finalize) is a complication of step 5. Each stage has its own signature stored separately in the `OutboxIntentBundle`. This is expanded in [Instruction changes](#new-setintentexecutionstage--intent_id-stage-executionstage-). --- ## Account Design -### `MagicIntentAccount` ephemeral account +### `OutboxIntentBundle` ephemeral account -**Seeds:** `["magic-intent", intent_id.to_le_bytes()]` +**Seeds:** `["outbox-intent", intent_id.to_le_bytes()]` **Owner:** MagicBlock builtin program ```rust -pub struct MagicIntentAccount { - pub bundle: ScheduledIntentBundle, - pub status: MagicIntentStatus, +pub struct OutboxIntentBundle { + pub inner: ScheduledIntentBundle, + pub status: OutboxIntentBundleStatus, } -pub enum MagicIntentStatus { +pub enum OutboxIntentBundleStatus { Accepted, Executing(ExecutionStage), } @@ -110,7 +110,7 @@ pub enum TwoStageProgress { ### Modified: `AcceptScheduledCommits` -Removes up to N intents from the front of `MagicContext.scheduled_base_intents`. For each intent a `MagicIntentAccount` ephemeral account is created with `status = Accepted`. N is bounded by the transaction account limit: every new account must be passed as writable in the same transaction. +Removes up to N intents from the front of `MagicContext.scheduled_base_intents`. For each intent an `OutboxIntentBundle` ephemeral account is created with `status = Accepted`. N is bounded by the transaction account limit: every new account must be passed as writable in the same transaction. **Accounts:** ``` @@ -118,7 +118,7 @@ Removes up to N intents from the front of `MagicContext.scheduled_base_intents`. new_pda_0 (writable), new_pda_1 (writable), ...] ``` -**Logic (after existing pop from MagicContext):** for each accepted intent, create a `MagicIntentAccount` ephemeral account with `status = Accepted`. +**Logic (after existing pop from MagicContext):** for each accepted intent, create an `OutboxIntentBundle` ephemeral account with `status = Accepted`. --- @@ -146,9 +146,9 @@ Sets or advances the execution stage of an intent. Handles the initial transitio --- -### New: `CloseMagicIntentAccount { intent_id }` +### New: `CloseOutboxIntent { intent_id }` -Deallocates the `MagicIntentAccount` ephemeral account. Called after L1 execution is confirmed. +Deallocates the `OutboxIntentBundle` ephemeral account. Called after L1 execution is confirmed. **Accounts:** ``` @@ -165,9 +165,9 @@ Deallocates the `MagicIntentAccount` ephemeral account. Called after L1 executio ### Discovery ``` -1. getProgramAccounts(magic_program_id, filters=[discriminator(MagicIntentAccount)]) +1. getProgramAccounts(magic_program_id, filters=[discriminator(OutboxIntentBundle)]) → returns all open intent accounts -2. Deserialize each account, sort by bundle.id ascending +2. Deserialize each account, sort by inner.id ascending ``` Cost: one RPC call returning the set of intents that were accepted and not yet closed. No scanning of closed IDs, no range guessing, no dependency on total historical intent count. @@ -177,24 +177,24 @@ Cost: one RPC call returning the set of intents that were accepted and not yet c ```rust match intent.status { Accepted => - reschedule_full(intent.bundle), + reschedule_full(intent.inner), Executing(SingleStage(sig)) => match get_signature_status(sig).await { - Succeeded => close_intent_account(intent.bundle.id).await, - _ => reschedule_full(intent.bundle), + Succeeded => close_outbox_intent(intent.inner.id).await, + _ => reschedule_full(intent.inner), }, Executing(TwoStage(Committing(sig))) => match get_signature_status(sig).await { - Succeeded => reschedule_finalize_only(intent.bundle), - _ => reschedule_full(intent.bundle), + Succeeded => reschedule_finalize_only(intent.inner), + _ => reschedule_full(intent.inner), }, Executing(TwoStage(Finalizing { finalize, .. })) => match get_signature_status(finalize).await { - Succeeded => close_intent_account(intent.bundle.id).await, - _ => reschedule_finalize_only(intent.bundle), + Succeeded => close_outbox_intent(intent.inner.id).await, + _ => reschedule_finalize_only(intent.inner), }, } ``` @@ -203,7 +203,7 @@ match intent.status { ## Intent Discovery: `getProgramAccounts` -The MagicBlock builtin program owns `MagicContext` (one singleton) and one `MagicIntentAccount` per open intent. A `getProgramAccounts` call filtered by the `MagicIntentAccount` discriminator returns all intent accounts that were accepted and have not yet been closed. After deserialization, accounts are sorted by `bundle.id` to restore processing order. +The MagicBlock builtin program owns `MagicContext` (one singleton) and one `OutboxIntentBundle` per open intent. A `getProgramAccounts` call filtered by the `OutboxIntentBundle` discriminator returns all intent accounts that were accepted and have not yet been closed. After deserialization, accounts are sorted by `inner.id` to restore processing order. Each account's `status` field determines how recovery proceeds — an account with `status = Accepted` means the intent has not yet been sent to L1 and needs full execution; an account with `status = Executing(...)` requires checking the stored signature(s) on L1 before deciding whether to close or reschedule. @@ -211,9 +211,9 @@ Each account's `status` field determines how recovery proceeds — an account wi ## Considered Alternatives -### Option A: Linked list (`MagicIntentList` + `prev_id`/`next_id`) +### Option A: Linked list (`OutboxIntentList` + `prev_id`/`next_id`) -Maintain a `MagicIntentList` singleton (head + tail) and embed `prev_id`/`next_id` pointers in each `MagicIntentAccount`. On restart, traverse from head to tail. +Maintain an `OutboxIntentList` singleton (head + tail) and embed `prev_id`/`next_id` pointers in each `OutboxIntentBundle`. On restart, traverse from head to tail. **Problem — unnecessary complexity:** `getProgramAccounts` already returns the same set of open intents in one call. The linked list duplicates this with extra fields per account, a shared singleton written on every accept and every close, and more complex close logic (relinking neighbors). @@ -242,7 +242,7 @@ Furthermore, the watermark advancement logic requires careful coordination: adva Group intent IDs into pages of fixed size (e.g., 256). Each page is a PDA storing a bitmap of which intent IDs within its range are still active. ```rust -// PDA: ["magic-intent-page", page_id.to_le_bytes()] +// PDA: ["outbox-intent-page", page_id.to_le_bytes()] struct ActiveIntentPage { page_id: u64, base_intent_id: u64, @@ -253,7 +253,7 @@ struct ActiveIntentPage { On restart: scan page PDAs from `min_active_page` to `current_page`, extract set bits, fetch the corresponding intent PDAs. -**Problem — hot shared account:** Every `CloseMagicIntentAccount` for any intent in page range [0, 255] writes to the same page 0 PDA. Every `AcceptScheduledCommits` that allocates into page 0 also writes to page 0. All operations in the same page range serialize on a single shared account. +**Problem — hot shared account:** Every `CloseOutboxIntent` for any intent in page range [0, 255] writes to the same page 0 PDA. Every `AcceptScheduledCommits` that allocates into page 0 also writes to page 0. All operations in the same page range serialize on a single shared account. **Problem — page watermark inherits the stuck-intent problem:** `min_active_page` cannot advance past page N until all 256 intents in page N close. One stuck intent holds up the entire page's retirement. The restart scan cost is O(N/256) page fetches — better than a plain watermark, but the underlying problem remains. @@ -270,7 +270,7 @@ On restart: scan page PDAs from `min_active_page` to `current_page`, extract set | Restart discovery | One filtered RPC call | O(open) traversal | O(N/PAGE\_SIZE) + O(open) | O(gap from watermark) | | Stuck-intent impact on restart | None | None | Blocks page retirement | Blocks watermark advancement | | Close instruction complexity | Wipe account (2 accounts) | Relink 2 neighbors (3–5 accounts) | Flip 1 bit (2 accounts) | Wipe account (2 accounts) | -| Shared hot account | None | `MagicIntentList` singleton | Page PDA (up to 256 intents) | None | +| Shared hot account | None | `OutboxIntentList` singleton | Page PDA (up to 256 intents) | None | | Extra state per intent | 0 | +16 bytes (prev\_id, next\_id) | ~0.2 bytes in page bitmap | 0 | **`getProgramAccounts` is chosen** for its minimal account structure, no shared hot accounts, simplest close logic, and correct stuck-intent behavior. @@ -281,13 +281,13 @@ On restart: scan page PDAs from `min_active_page` to `current_page`, extract set | Component | Change | |---|---| -| `magicblock-magic-program-api/src/pda.rs` | Add `intent_account_pda(id)` | -| `magicblock-magic-program-api/src/instruction.rs` | Add `SetIntentExecutionStage`, `CloseMagicIntentAccount` variants | -| `programs/magicblock/src/` | Add `MagicIntentAccount`, `MagicIntentStatus` types | +| `magicblock-magic-program-api/src/pda.rs` | Add `outbox_intent_pda(id)` | +| `magicblock-magic-program-api/src/instruction.rs` | Add `SetIntentExecutionStage`, `CloseOutboxIntent` variants | +| `programs/magicblock/src/` | Add `OutboxIntentBundle`, `OutboxIntentBundleStatus` types | | `programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs` | Create intent ephemeral accounts | -| `programs/magicblock/src/schedule_transactions/` (new files) | `process_set_intent_execution_stage.rs`, `process_close_intent_account.rs` | +| `programs/magicblock/src/schedule_transactions/` (new files) | `process_set_intent_execution_stage.rs`, `process_close_outbox_intent.rs` | | `programs/magicblock/src/magicblock_processor.rs` | Wire new instruction variants | | `magicblock-committor-service/src/` (new file) | `restart_loader.rs` — `getProgramAccounts` fetch, sort by id, classification, recovery dispatch | | `magicblock-committor-service/src/intent_executor/` | Call `SetIntentExecutionStage` before L1 submission | -| `magicblock-accounts/src/scheduled_commits_processor.rs` | Replace `register_scheduled_commit_sent` with `CloseMagicIntentAccount` call | +| `magicblock-accounts/src/scheduled_commits_processor.rs` | Replace `register_scheduled_commit_sent` with `CloseOutboxIntent` call | diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index 1cad2c14b..a03c53b37 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -167,6 +167,18 @@ fn pop_scheduled_intents( let intents = magic_context.take_front_scheduled_commits(num_accept_intents); + if intents.len() != num_accept_intents { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: requested {} intents but only {} available", + num_accept_intents, + intents.len() + ); + + return Err(InstructionError::InvalidArgument); + } + + // Write updated account data magic_context .write_to(magic_context_acc.borrow_mut()?.data_as_mut_slice())?; @@ -179,7 +191,7 @@ fn create_outbox_account_cpi( pda: Pubkey, outbox_account: OutboxIntentBundle, ) -> Result<(), InstructionError> { - let intent_id = outbox_account.intent_bundle.id; + let intent_id = outbox_account.inner.id; let data = outbox_account.try_to_bytes().map_err(|_| { ic_msg!( invoke_context, diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 12ba58364..5fdd9fa32 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -6,7 +6,7 @@ use crate::magic_scheduled_base_intent::ScheduledIntentBundle; #[derive(Debug, Serialize, Deserialize)] pub struct OutboxIntentBundle { - pub intent_bundle: ScheduledIntentBundle, + pub inner: ScheduledIntentBundle, pub status: OutboxIntentBundleStatus, } @@ -34,7 +34,7 @@ pub enum TwoStageProgress { impl OutboxIntentBundle { pub fn accepted(intent_bundle: ScheduledIntentBundle) -> Self { Self { - intent_bundle, + inner: intent_bundle, status: OutboxIntentBundleStatus::Accepted, } } From 558a083edd41d7aa1a0f494340ebfc05df44d295 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 5 Jun 2026 12:14:17 +0700 Subject: [PATCH 31/97] feat: set execution statis --- .../src/instruction.rs | 12 ++ magicblock-magic-program-api/src/lib.rs | 1 + magicblock-magic-program-api/src/outbox.rs | 90 +++++++++++++ .../src/intent_bundles/outbox/mod.rs | 1 + .../process_set_intent_execution_stage.rs | 118 ++++++++++++++++++ .../intent_bundles/outbox_intent_bundles.rs | 59 +++++---- .../src/intent_bundles/schedule/mod.rs | 6 +- .../magicblock/src/magicblock_processor.rs | 11 +- 8 files changed, 274 insertions(+), 24 deletions(-) create mode 100644 magicblock-magic-program-api/src/outbox.rs create mode 100644 programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index 38568bf22..8f84136c2 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -8,6 +8,7 @@ use crate::{ ScheduleTaskArgs, }, compat::Instruction, + outbox::ExecutionStage, Pubkey, }; @@ -71,6 +72,17 @@ pub enum MagicBlockInstruction { /// seeds: `["outbox-intent", intent_id.to_le_bytes()]` AcceptScheduleCommits, + /// Sets or advances the execution stage of an outbox intent. + /// Must be called before sending the L1 transaction. + /// + /// # Account references + /// - **0.** `[SIGNER]` Validator Authority + /// - **1.** `[WRITE]` Outbox intent PDA, seeds: `["outbox-intent", intent_id.to_le_bytes()]` + SetIntentExecutionStage { + intent_id: u64, + stage: ExecutionStage, + }, + /// Records the attempt to realize a scheduled commit on chain. /// /// The signature of this transaction can be pre-calculated since we pass the diff --git a/magicblock-magic-program-api/src/lib.rs b/magicblock-magic-program-api/src/lib.rs index 25b614b7e..506ae7ccb 100644 --- a/magicblock-magic-program-api/src/lib.rs +++ b/magicblock-magic-program-api/src/lib.rs @@ -1,6 +1,7 @@ pub mod args; pub mod compat; pub mod instruction; +pub mod outbox; pub mod pda; pub mod response; diff --git a/magicblock-magic-program-api/src/outbox.rs b/magicblock-magic-program-api/src/outbox.rs new file mode 100644 index 000000000..335830bad --- /dev/null +++ b/magicblock-magic-program-api/src/outbox.rs @@ -0,0 +1,90 @@ +use serde::{Deserialize, Serialize}; +use solana_signature::Signature; + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub enum ExecutionStage { + SingleStage(Signature), + TwoStage(TwoStageProgress), +} + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub enum TwoStageProgress { + Committing(Signature), + Finalizing { + commit: Signature, + finalize: Signature, + }, +} + +impl ExecutionStage { + pub fn apply_stage_transition( + &mut self, + stage: ExecutionStage, + ) -> Result<(), &'static str> { + match (self, stage) { + (Self::SingleStage(ref mut this_sig), Self::SingleStage(sig)) => { + *this_sig = sig; + } + // TODO(edwin): validate this case, + ( + this @ Self::SingleStage(_), + val @ Self::TwoStage(TwoStageProgress::Committing(_)), + ) => { + *this = val; + } + ( + Self::SingleStage(_), + Self::TwoStage(TwoStageProgress::Finalizing { .. }), + ) => { + return Err("cannot transition from SingleStage to Finalizing"); + } + (Self::TwoStage(ref mut this), Self::TwoStage(value)) => { + this.apply_stage_transition(value)?; + } + (Self::TwoStage(_), Self::SingleStage(_)) => { + return Err( + "cannot change execution type from TwoStage to SingleStage", + ); + } + } + Ok(()) + } +} + +impl TwoStageProgress { + fn apply_stage_transition( + &mut self, + stage: TwoStageProgress, + ) -> Result<(), &'static str> { + match (self, stage) { + (Self::Committing(this_sig), Self::Committing(value)) => { + *this_sig = value; + } + ( + this @ Self::Committing(this_sig), + Self::Finalizing { commit, finalize }, + ) => { + if this_sig != &commit { + return Err( + "commit signature mismatch on advance to Finalizing", + ); + } + *this = Self::Finalizing { commit, finalize }; + } + ( + Self::Finalizing { commit: this_commit, .. }, + Self::Finalizing { + commit, .. + }, + ) => { + *this_commit = commit; + } + (Self::Finalizing { .. }, Self::Committing(_)) => { + return Err( + "downgrade from Finalizing to Committing not permitted", + ); + } + } + Ok(()) + } +} diff --git a/programs/magicblock/src/intent_bundles/outbox/mod.rs b/programs/magicblock/src/intent_bundles/outbox/mod.rs index eae59110c..5f5624246 100644 --- a/programs/magicblock/src/intent_bundles/outbox/mod.rs +++ b/programs/magicblock/src/intent_bundles/outbox/mod.rs @@ -1,2 +1,3 @@ pub mod process_accept_scheduled_commits; pub mod process_scheduled_commit_sent; +pub mod process_set_intent_execution_stage; diff --git a/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs b/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs new file mode 100644 index 000000000..1c7547f0e --- /dev/null +++ b/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs @@ -0,0 +1,118 @@ +use std::collections::HashSet; + +use magicblock_core::intent::outbox::outbox_intent_pda; +use magicblock_magic_program_api::outbox::ExecutionStage; +use solana_account::{ReadableAccount, WritableAccount}; +use solana_instruction::error::InstructionError; +use solana_log_collector::ic_msg; +use solana_program_runtime::invoke_context::InvokeContext; +use solana_pubkey::Pubkey; + +use crate::{ + intent_bundles::outbox_intent_bundles::OutboxIntentBundle, + utils::accounts::{ + get_instruction_account_with_idx, get_instruction_pubkey_with_idx, + }, + validator::effective_validator_authority_id, +}; + +const VALIDATOR_AUTHORITY_IDX: u16 = 0; +const INTENT_PDA_IDX: u16 = 1; + +pub fn process_set_intent_execution_stage( + signers: HashSet, + invoke_context: &mut InvokeContext, + intent_id: u64, + stage: ExecutionStage, +) -> Result<(), InstructionError> { + let validator_auth = effective_validator_authority_id(); + verify(&signers, invoke_context, &validator_auth, intent_id)?; + + let transaction_context = &*invoke_context.transaction_context; + let intent_acc = + get_instruction_account_with_idx(transaction_context, INTENT_PDA_IDX)?; + + let mut bundle = + OutboxIntentBundle::try_from_bytes(intent_acc.borrow()?.data()) + .map_err(|_| { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: failed to deserialize outbox intent {}", + intent_id + ); + InstructionError::InvalidAccountData + })?; + + bundle.apply_stage_transition(stage).map_err(|reason| { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: intent {}: invalid transition: {}", + intent_id, + reason + ); + InstructionError::InvalidArgument + })?; + + let data = bundle.try_to_bytes().map_err(|_| { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: failed to serialize outbox intent {}", + intent_id + ); + InstructionError::InvalidAccountData + })?; + intent_acc + .borrow_mut()? + .data_as_mut_slice() + .copy_from_slice(&data); + + Ok(()) +} + +fn verify( + signers: &HashSet, + invoke_context: &InvokeContext, + validator_auth: &Pubkey, + intent_id: u64, +) -> Result<(), InstructionError> { + let transaction_context = &*invoke_context.transaction_context; + + let provided_validator_auth = get_instruction_pubkey_with_idx( + transaction_context, + VALIDATOR_AUTHORITY_IDX, + )?; + if provided_validator_auth != validator_auth { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: invalid validator authority {}, should be {}", + provided_validator_auth, + validator_auth + ); + return Err(InstructionError::InvalidArgument); + } + if !signers.contains(validator_auth) { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: validator authority {} not in signers", + validator_auth + ); + return Err(InstructionError::MissingRequiredSignature); + } + + let provided_pda = + get_instruction_pubkey_with_idx(transaction_context, INTENT_PDA_IDX)?; + let expected_pda = outbox_intent_pda(intent_id); + if *provided_pda != expected_pda { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: account at idx {} is {}, expected PDA {} for intent {}", + INTENT_PDA_IDX, + provided_pda, + expected_pda, + intent_id + ); + return Err(InstructionError::InvalidArgument); + } + + Ok(()) +} diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 5fdd9fa32..2ed4c4d21 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,6 +1,6 @@ use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; +use magicblock_magic_program_api::outbox::{ExecutionStage, TwoStageProgress}; use serde::{Deserialize, Serialize}; -use solana_signature::Signature; use crate::magic_scheduled_base_intent::ScheduledIntentBundle; @@ -10,27 +10,6 @@ pub struct OutboxIntentBundle { pub status: OutboxIntentBundleStatus, } -#[derive(Debug, Serialize, Deserialize)] -pub enum OutboxIntentBundleStatus { - Accepted, - Executing(ExecutionStage), -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum ExecutionStage { - SingleStage(Signature), - TwoStage(TwoStageProgress), -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum TwoStageProgress { - Committing(Signature), - Finalizing { - commit: Signature, - finalize: Signature, - }, -} - impl OutboxIntentBundle { pub fn accepted(intent_bundle: ScheduledIntentBundle) -> Self { Self { @@ -39,6 +18,13 @@ impl OutboxIntentBundle { } } + pub fn apply_stage_transition( + &mut self, + stage: ExecutionStage, + ) -> Result<(), &'static str> { + self.status.apply_stage_transition(stage) + } + pub fn try_to_bytes(&self) -> Result, bincode::Error> { let body = bincode::serialize(self)?; let mut out = @@ -60,3 +46,32 @@ impl OutboxIntentBundle { bincode::deserialize(&data[disc_len..]) } } + +#[derive(Debug, Serialize, Deserialize)] +pub enum OutboxIntentBundleStatus { + Accepted, + Executing(ExecutionStage), +} + +impl OutboxIntentBundleStatus { + fn apply_stage_transition( + &mut self, + stage: ExecutionStage, + ) -> Result<(), &'static str> { + match (self, stage) { + ( + Self::Accepted, + ExecutionStage::TwoStage(TwoStageProgress::Finalizing { + .. + }), + ) => Err("cannot transition from Accepted to Finalizing"), + (this @ Self::Accepted, stage) => { + *this = Self::Executing(stage); + Ok(()) + } + (Self::Executing(ref mut this), stage) => { + this.apply_stage_transition(stage) + } + } + } +} diff --git a/programs/magicblock/src/intent_bundles/schedule/mod.rs b/programs/magicblock/src/intent_bundles/schedule/mod.rs index 732b17c74..70ca2ea57 100644 --- a/programs/magicblock/src/intent_bundles/schedule/mod.rs +++ b/programs/magicblock/src/intent_bundles/schedule/mod.rs @@ -25,7 +25,11 @@ pub use crate::intent_bundles::outbox::process_scheduled_commit_sent::{ process_scheduled_commit_sent, register_scheduled_commit_sent, SentCommit, }; pub(crate) use crate::intent_bundles::{ - outbox::process_accept_scheduled_commits::*, process_execute_callback::*, + outbox::{ + process_accept_scheduled_commits::*, + process_set_intent_execution_stage::process_set_intent_execution_stage, + }, + process_execute_callback::*, }; use crate::{ magic_sys::{ diff --git a/programs/magicblock/src/magicblock_processor.rs b/programs/magicblock/src/magicblock_processor.rs index 2bd91df52..5e8a5d3da 100644 --- a/programs/magicblock/src/magicblock_processor.rs +++ b/programs/magicblock/src/magicblock_processor.rs @@ -27,7 +27,8 @@ use crate::{ schedule_transactions::{ process_accept_scheduled_commits, process_add_action_callback, process_execute_callback, process_schedule_commit, - process_schedule_intent_bundle, ProcessScheduleCommitOptions, + process_schedule_intent_bundle, process_set_intent_execution_stage, + ProcessScheduleCommitOptions, }, }; @@ -95,6 +96,14 @@ declare_process_instruction!( AcceptScheduleCommits => { process_accept_scheduled_commits(signers, invoke_context) } + SetIntentExecutionStage { intent_id, stage } => { + process_set_intent_execution_stage( + signers, + invoke_context, + intent_id, + stage, + ) + } ScheduledCommitSent((id, _bump)) => process_scheduled_commit_sent( signers, invoke_context, From fb1f86bca18314fefff6e8df310fc5ca0ae0d6c2 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 5 Jun 2026 12:35:06 +0700 Subject: [PATCH 32/97] feat: simplify query --- .../src/committor_processor.rs | 4 - .../src/persist/commit_persister.rs | 115 +++++++----------- 2 files changed, 42 insertions(+), 77 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 2f1b3a3d0..b610480f7 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -165,10 +165,6 @@ impl CommittorProcessor { .as_secs(); let mut bundles = self.persister.pending_intent_bundles( now.saturating_sub(RECOVERY_MAX_AGE_SECS), - |row| { - now.saturating_sub(row.created_at) - > RECOVERY_MAX_AGE_SECS - }, )?; if bundles.is_empty() { diff --git a/magicblock-committor-service/src/persist/commit_persister.rs b/magicblock-committor-service/src/persist/commit_persister.rs index 7a0126d74..375dd7f59 100644 --- a/magicblock-committor-service/src/persist/commit_persister.rs +++ b/magicblock-committor-service/src/persist/commit_persister.rs @@ -64,13 +64,10 @@ pub trait IntentPersister: Send + Sync + Clone + 'static { &self, message_id: u64, ) -> CommitPersistResult>; - fn pending_intent_bundles( + fn pending_intent_bundles( &self, min_created_at: u64, - predicate: F, - ) -> CommitPersistResult> - where - F: Fn(&CommitStatusRow) -> bool; + ) -> CommitPersistResult>; fn get_commit_status_by_message( &self, message_id: u64, @@ -257,26 +254,25 @@ impl IntentPersister for IntentPersisterImpl { .get_commit_statuses_by_id(message_id) } - /// Returns pending bundles satisfying predicate - /// NOTE: this constructs `ScheduleIntentBundle` only from existing information - /// As persister doesn't save `ScheduleIntentBundle::payer` info, Pubkey::default is used - /// `CommittedAccount` information like slot maybe also outdated - /// It is responsibility of calling site to refresh data in intent - fn pending_intent_bundles( + /// Returns pending bundles created at or after `min_created_at`. + /// NOTE: this constructs `ScheduleIntentBundle` only from existing information. + /// As persister doesn't save `ScheduleIntentBundle::payer` info, Pubkey::default is used. + /// `CommittedAccount` information like slot may also be outdated. + /// It is the responsibility of the calling site to refresh data in intent. + fn pending_intent_bundles( &self, min_created_at: u64, - predicate: F, - ) -> CommitPersistResult> - where - F: Fn(&CommitStatusRow) -> bool, - { + ) -> CommitPersistResult> { let rows = self .commits_db .lock() - .expect(POISONED_MUTEX_MSG) + .expect(POISONED_MUTEX_MSG) .get_pending_commit_statuses(min_created_at)?; - Ok(pending_rows_to_scheduled_intent_bundles(rows, predicate)) + Ok(pending_rows_to_scheduled_intent_bundles( + rows, + min_created_at, + )) } fn get_commit_status_by_message( @@ -425,19 +421,12 @@ impl IntentPersister for Option { } } - fn pending_intent_bundles( + fn pending_intent_bundles( &self, min_created_at: u64, - predicate: F, - ) -> CommitPersistResult> - where - F: Fn(&CommitStatusRow) -> bool, - { + ) -> CommitPersistResult> { match self { - Some(persister) => persister.pending_intent_bundles( - min_created_at, - predicate, - ), + Some(persister) => persister.pending_intent_bundles(min_created_at), None => Ok(Vec::new()), } } @@ -492,13 +481,10 @@ impl IntentPersister for Option { } } -fn pending_rows_to_scheduled_intent_bundles( +fn pending_rows_to_scheduled_intent_bundles( rows: Vec, - predicate: F, -) -> Vec -where - F: Fn(&CommitStatusRow) -> bool, -{ + min_created_at: u64, +) -> Vec { let mut grouped_rows = BTreeMap::>::new(); for row in rows { grouped_rows.entry(row.message_id).or_default().push(row); @@ -506,12 +492,12 @@ where grouped_rows .into_iter() - // Filter row if any item doesn't satisfy predicate + // Filter bundles where any row is older than the recovery window .filter(|(message_id, rows)| { - if rows.iter().any(&predicate) { + if rows.iter().any(|row| row.created_at < min_created_at) { warn!( intent_id = message_id, - "Skipping pending commit intent because it is not old enough to recover" + "Skipping pending commit intent: too old to recover" ); false } else { @@ -630,7 +616,7 @@ mod tests { use super::*; use crate::{ - committor_processor::{RECOVERY_MAX_AGE_SECS, RECOVERY_MIN_AGE_SECS}, + committor_processor::RECOVERY_MAX_AGE_SECS, persist::{types, CommitStatusSignatures}, test_utils, }; @@ -984,8 +970,7 @@ mod tests { let commit_pubkey = Pubkey::new_unique(); let undelegate_pubkey = Pubkey::new_unique(); - let recovery_time = RECOVERY_MIN_AGE_SECS + 2; - // filter returns true = skip; default rows (last_retried_at=1, created_at=1) are in recovery window → don't skip + // default rows have created_at=1; use min_created_at=0 so nothing is filtered let bundles = pending_rows_to_scheduled_intent_bundles( vec![ pending_row( @@ -998,12 +983,7 @@ mod tests { ), pending_row(9, undelegate_pubkey, owner, blockhash, true, None), ], - |row| { - recovery_time.saturating_sub(row.last_retried_at) - <= RECOVERY_MIN_AGE_SECS - || recovery_time.saturating_sub(row.created_at) - > RECOVERY_MAX_AGE_SECS - }, + 0, ); assert_eq!(bundles.len(), 1); @@ -1028,22 +1008,14 @@ mod tests { assert!(bundle.has_undelegate_intent()); } - fn make_predicate(recovery_time: u64) -> impl Fn(&CommitStatusRow) -> bool { - move |row| { - recovery_time.saturating_sub(row.last_retried_at) - <= RECOVERY_MIN_AGE_SECS - || recovery_time.saturating_sub(row.created_at) - > RECOVERY_MAX_AGE_SECS - } - } - #[test] - fn pending_rows_skip_intents_outside_recovery_window() { + fn pending_rows_skip_intents_older_than_recovery_window() { let owner = Pubkey::new_unique(); let blockhash = Hash::new_unique(); - let recovery_time = RECOVERY_MAX_AGE_SECS + RECOVERY_MIN_AGE_SECS; + let recovery_time = RECOVERY_MAX_AGE_SECS + 1; + let min_created_at = recovery_time - RECOVERY_MAX_AGE_SECS; // = 1 - // Within window: last_retried_at old enough, created_at not too old → included + // At the boundary → included let mut in_window = pending_row( 1, Pubkey::new_unique(), @@ -1052,17 +1024,16 @@ mod tests { false, Some(vec![1]), ); - in_window.created_at = recovery_time - RECOVERY_MAX_AGE_SECS; - in_window.last_retried_at = recovery_time - RECOVERY_MIN_AGE_SECS - 1; + in_window.created_at = min_created_at; let included = pending_rows_to_scheduled_intent_bundles( vec![in_window], - make_predicate(recovery_time), + min_created_at, ); assert_eq!(included.len(), 1); assert_eq!(included[0].id, 1); - // Too recently retried → excluded - let mut too_recent = pending_row( + // Recent (just created) → included + let mut recent = pending_row( 2, Pubkey::new_unique(), owner, @@ -1070,15 +1041,14 @@ mod tests { false, Some(vec![1]), ); - too_recent.created_at = recovery_time - RECOVERY_MAX_AGE_SECS; - too_recent.last_retried_at = recovery_time - RECOVERY_MIN_AGE_SECS; - let excluded_recent = pending_rows_to_scheduled_intent_bundles( - vec![too_recent], - make_predicate(recovery_time), + recent.created_at = recovery_time; + let recent_result = pending_rows_to_scheduled_intent_bundles( + vec![recent], + min_created_at, ); - assert!(excluded_recent.is_empty()); + assert_eq!(recent_result.len(), 1); - // Too old by created_at → excluded + // Too old → excluded let mut too_old = pending_row( 3, Pubkey::new_unique(), @@ -1087,11 +1057,10 @@ mod tests { false, Some(vec![1]), ); - too_old.created_at = recovery_time - RECOVERY_MAX_AGE_SECS - 1; - too_old.last_retried_at = recovery_time - RECOVERY_MIN_AGE_SECS - 1; + too_old.created_at = min_created_at - 1; // = 0 let excluded_old = pending_rows_to_scheduled_intent_bundles( vec![too_old], - make_predicate(recovery_time), + min_created_at, ); assert!(excluded_old.is_empty()); } From f218ed4f0274fffa3005fd816f66c0be234426ec Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 8 Jun 2026 13:36:32 +0700 Subject: [PATCH 33/97] wip --- .../src/intent_execution_manager.rs | 1 + .../src/intent_execution_manager/db.rs | 89 +++++++++++++++--- .../intent_channerl.rs | 9 ++ magicblock-committor-service/src/service.rs | 1 + .../src/service/acceptor.rs | 0 magicblock-magic-program-api/src/outbox.rs | 39 ++++++-- .../process_accept_scheduled_commits.rs | 4 +- .../process_set_intent_execution_stage.rs | 92 ++++++++++--------- .../intent_bundles/outbox_intent_bundles.rs | 31 ++++--- 9 files changed, 191 insertions(+), 75 deletions(-) create mode 100644 magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs create mode 100644 magicblock-committor-service/src/service/acceptor.rs diff --git a/magicblock-committor-service/src/intent_execution_manager.rs b/magicblock-committor-service/src/intent_execution_manager.rs index 725224386..4a4d55bea 100644 --- a/magicblock-committor-service/src/intent_execution_manager.rs +++ b/magicblock-committor-service/src/intent_execution_manager.rs @@ -1,6 +1,7 @@ pub(crate) mod db; mod intent_execution_engine; pub mod intent_scheduler; +pub mod intent_channerl; use std::sync::Arc; diff --git a/magicblock-committor-service/src/intent_execution_manager/db.rs b/magicblock-committor-service/src/intent_execution_manager/db.rs index 8c58e2b3e..0afc4a461 100644 --- a/magicblock-committor-service/src/intent_execution_manager/db.rs +++ b/magicblock-committor-service/src/intent_execution_manager/db.rs @@ -1,9 +1,14 @@ use std::{collections::VecDeque, sync::Mutex}; - +use std::sync::Arc; /// DB for storing intents that overflow committor channel use async_trait::async_trait; +use solana_account::ReadableAccount; +use magicblock_accounts_db::AccountsDb; +use magicblock_accounts_db::traits::AccountsBank; +use magicblock_core::intent::outbox::outbox_intent_pda; use magicblock_metrics::metrics; use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; const POISONED_MUTEX_MSG: &str = "Dummy db mutex poisoned"; @@ -11,22 +16,22 @@ const POISONED_MUTEX_MSG: &str = "Dummy db mutex poisoned"; pub trait DB: Send + Sync + 'static { async fn store_intent_bundle( &self, - intent_bundle: ScheduledIntentBundle, + intent_bundle: OutboxIntentBundle, ) -> DBResult<()>; async fn store_intent_bundles( &self, - intent_bundles: Vec, + intent_bundles: Vec, ) -> DBResult<()>; /// Returns the oldest (first stored) intent bundle async fn pop_intent_bundle( &self, - ) -> DBResult>; + ) -> DBResult>; fn is_empty(&self) -> bool; } pub(crate) struct DummyDB { - db: Mutex>, + db: Mutex>, } impl DummyDB { @@ -41,7 +46,7 @@ impl DummyDB { impl DB for DummyDB { async fn store_intent_bundle( &self, - intent_bundle: ScheduledIntentBundle, + intent_bundle: OutboxIntentBundle, ) -> DBResult<()> { let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); db.push_back(intent_bundle); @@ -52,7 +57,7 @@ impl DB for DummyDB { async fn store_intent_bundles( &self, - intent_bundles: Vec, + intent_bundles: Vec, ) -> DBResult<()> { let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); db.extend(intent_bundles); @@ -63,7 +68,7 @@ impl DB for DummyDB { async fn pop_intent_bundle( &self, - ) -> DBResult> { + ) -> DBResult> { let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); let res = db.pop_front(); @@ -76,12 +81,72 @@ impl DB for DummyDB { } } +pub struct DumberDB { + accounts_db: Arc, + queue: Mutex> +} + +impl DumberDB { + pub fn new(accounts_db: Arc) -> Self { + Self { + accounts_db, + queue: Mutex::new(VecDeque::new()), + } + } +} + +#[async_trait] +impl DB for DumberDB { + async fn store_intent_bundle( + &self, + intent_bundle: OutboxIntentBundle, + ) -> DBResult<()> { + let mut queue = self.queue.lock().expect(POISONED_MUTEX_MSG); + queue.push_back(intent_bundle.inner.id); + + metrics::set_committor_intents_backlog_count(queue.len() as i64); + Ok(()) + } + + async fn store_intent_bundles( + &self, + intent_bundles: Vec, + ) -> DBResult<()> { + let mut queue = self.queue.lock().expect(POISONED_MUTEX_MSG); + queue.extend(intent_bundles.into_iter().map(|el| el.inner.id)); + + metrics::set_committor_intents_backlog_count(queue.len() as i64); + Ok(()) + } + + async fn pop_intent_bundle( + &self, + ) -> DBResult> { + let mut queue = self.queue.lock().expect(POISONED_MUTEX_MSG); + let Some(id) = queue.pop_front() else { + return Ok(None); + }; + metrics::set_committor_intents_backlog_count(queue.len() as i64); + drop(queue); + + let intent_pda = outbox_intent_pda(id); + let account = self.accounts_db.get_account(&intent_pda).ok_or(Error::IntentNotFoundError(id))?; + + let outbox_intent = OutboxIntentBundle::try_from_bytes(account.data())?; + Ok(Some(outbox_intent)) + } + + fn is_empty(&self) -> bool { + self.queue.lock().expect(POISONED_MUTEX_MSG).is_empty() + } +} + #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("StoreError")] - StoreError, - #[error("FetchError")] - FetchError, + #[error("Failed to find intent in AccountsDB, id: {0}")] + IntentNotFoundError(u64), + #[error("Failed to deserialize Outbox Account")] + DeserializeError(#[from] bincode::Error) } pub type DBResult = Result; diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs new file mode 100644 index 000000000..3df52c638 --- /dev/null +++ b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs @@ -0,0 +1,9 @@ +use std::sync::Arc; +use tokio::sync::mpsc; +use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use crate::intent_execution_manager::db::DB; + +pub struct Sender { + db: Arc, + sender: mpsc::Sender +} \ No newline at end of file diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 4c06b9dc4..a95971e78 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,5 +1,6 @@ pub mod intent_client; pub mod outbox_intent_bundles_reader; +pub mod acceptor; use std::{ collections::{HashMap, HashSet}, diff --git a/magicblock-committor-service/src/service/acceptor.rs b/magicblock-committor-service/src/service/acceptor.rs new file mode 100644 index 000000000..e69de29bb diff --git a/magicblock-magic-program-api/src/outbox.rs b/magicblock-magic-program-api/src/outbox.rs index 335830bad..dd10716ee 100644 --- a/magicblock-magic-program-api/src/outbox.rs +++ b/magicblock-magic-program-api/src/outbox.rs @@ -22,31 +22,38 @@ impl ExecutionStage { stage: ExecutionStage, ) -> Result<(), &'static str> { match (self, stage) { + // Current sig wasn't confirmed, we replace it with new attempt (Self::SingleStage(ref mut this_sig), Self::SingleStage(sig)) => { *this_sig = sig; } // TODO(edwin): validate this case, + // We tried SingleStage execution, but failed (CpiLimit, etc) + // We patch it moving to TwoStage flow ( this @ Self::SingleStage(_), val @ Self::TwoStage(TwoStageProgress::Committing(_)), ) => { *this = val; } + // Only transition to TwoStageProgress::Committing is valid from SingleStage ( Self::SingleStage(_), Self::TwoStage(TwoStageProgress::Finalizing { .. }), ) => { return Err("cannot transition from SingleStage to Finalizing"); } + // Transitions within TwoStage states (Self::TwoStage(ref mut this), Self::TwoStage(value)) => { this.apply_stage_transition(value)?; } + // TwoStage can't be downgraded into SingleStage (Self::TwoStage(_), Self::SingleStage(_)) => { return Err( "cannot change execution type from TwoStage to SingleStage", ); } } + Ok(()) } } @@ -56,12 +63,14 @@ impl TwoStageProgress { &mut self, stage: TwoStageProgress, ) -> Result<(), &'static str> { - match (self, stage) { - (Self::Committing(this_sig), Self::Committing(value)) => { - *this_sig = value; + let new_state = match (&self, stage) { + // Current sig didn't succeed on Base, we replace it with new attempt + (Self::Committing(_), Self::Committing(new_sig)) => { + Self::Committing(new_sig) } + // Commit was successfully executed and now we move on to Finalizing ( - this @ Self::Committing(this_sig), + Self::Committing(this_sig), Self::Finalizing { commit, finalize }, ) => { if this_sig != &commit { @@ -69,22 +78,34 @@ impl TwoStageProgress { "commit signature mismatch on advance to Finalizing", ); } - *this = Self::Finalizing { commit, finalize }; + + Self::Finalizing { commit, finalize } } + // Current finalize sig wasn't confirmed, we replace it with new attempt ( - Self::Finalizing { commit: this_commit, .. }, Self::Finalizing { - commit, .. + commit: this_commit, + .. }, + Self::Finalizing { commit, finalize }, ) => { - *this_commit = commit; + if this_commit != &commit { + return Err( + "commit signature can't be replaced in Finalize stage", + ); + } + + Self::Finalizing { commit, finalize } } + // Incorrect state transition (Self::Finalizing { .. }, Self::Committing(_)) => { return Err( "downgrade from Finalizing to Committing not permitted", ); } - } + }; + + *self = new_state; Ok(()) } } diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index a03c53b37..af4be4759 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -31,7 +31,7 @@ pub fn process_accept_scheduled_commits( ) -> Result<(), InstructionError> { // Common conditions verification let validator_auth = effective_validator_authority_id(); - verify(&signers, invoke_context, &validator_auth)?; + validate(&signers, invoke_context, &validator_auth)?; // pop first n intents // n - is number of OutboxIntentBundle PDAs passed @@ -61,7 +61,7 @@ pub fn process_accept_scheduled_commits( Ok(()) } -fn verify( +fn validate( signers: &HashSet, invoke_context: &InvokeContext, validator_auth: &Pubkey, diff --git a/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs b/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs index 1c7547f0e..340cd771b 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_set_intent_execution_stage.rs @@ -26,50 +26,12 @@ pub fn process_set_intent_execution_stage( stage: ExecutionStage, ) -> Result<(), InstructionError> { let validator_auth = effective_validator_authority_id(); - verify(&signers, invoke_context, &validator_auth, intent_id)?; + validate(&signers, invoke_context, &validator_auth, intent_id)?; - let transaction_context = &*invoke_context.transaction_context; - let intent_acc = - get_instruction_account_with_idx(transaction_context, INTENT_PDA_IDX)?; - - let mut bundle = - OutboxIntentBundle::try_from_bytes(intent_acc.borrow()?.data()) - .map_err(|_| { - ic_msg!( - invoke_context, - "SetIntentExecutionStage ERR: failed to deserialize outbox intent {}", - intent_id - ); - InstructionError::InvalidAccountData - })?; - - bundle.apply_stage_transition(stage).map_err(|reason| { - ic_msg!( - invoke_context, - "SetIntentExecutionStage ERR: intent {}: invalid transition: {}", - intent_id, - reason - ); - InstructionError::InvalidArgument - })?; - - let data = bundle.try_to_bytes().map_err(|_| { - ic_msg!( - invoke_context, - "SetIntentExecutionStage ERR: failed to serialize outbox intent {}", - intent_id - ); - InstructionError::InvalidAccountData - })?; - intent_acc - .borrow_mut()? - .data_as_mut_slice() - .copy_from_slice(&data); - - Ok(()) + set_new_execution_stage(invoke_context, intent_id, stage) } -fn verify( +fn validate( signers: &HashSet, invoke_context: &InvokeContext, validator_auth: &Pubkey, @@ -77,6 +39,7 @@ fn verify( ) -> Result<(), InstructionError> { let transaction_context = &*invoke_context.transaction_context; + // Check that validator authority signed the tx let provided_validator_auth = get_instruction_pubkey_with_idx( transaction_context, VALIDATOR_AUTHORITY_IDX, @@ -99,6 +62,7 @@ fn verify( return Err(InstructionError::MissingRequiredSignature); } + // Validate pda we about to apply transition to let provided_pda = get_instruction_pubkey_with_idx(transaction_context, INTENT_PDA_IDX)?; let expected_pda = outbox_intent_pda(intent_id); @@ -116,3 +80,49 @@ fn verify( Ok(()) } + +fn set_new_execution_stage( + invoke_context: &InvokeContext, + intent_id: u64, + stage: ExecutionStage, +) -> Result<(), InstructionError> { + let transaction_context = &*invoke_context.transaction_context; + let intent_acc = + get_instruction_account_with_idx(transaction_context, INTENT_PDA_IDX)?; + + let mut bundle = + OutboxIntentBundle::try_from_bytes(intent_acc.borrow()?.data()) + .map_err(|_| { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: failed to deserialize outbox intent {}", + intent_id + ); + InstructionError::InvalidAccountData + })?; + + bundle.apply_stage_transition(stage).map_err(|reason| { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: intent {}: invalid transition: {}", + intent_id, + reason + ); + InstructionError::InvalidArgument + })?; + + let data = bundle.try_to_bytes().map_err(|_| { + ic_msg!( + invoke_context, + "SetIntentExecutionStage ERR: failed to serialize outbox intent {}", + intent_id + ); + InstructionError::InvalidAccountData + })?; + intent_acc + .borrow_mut()? + .data_as_mut_slice() + .copy_from_slice(&data); + + Ok(()) +} diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 2ed4c4d21..ba498a0bf 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -54,24 +54,33 @@ pub enum OutboxIntentBundleStatus { } impl OutboxIntentBundleStatus { + // TODO(edwin): split into is_valid_transition and apply_transaction fn apply_stage_transition( &mut self, stage: ExecutionStage, ) -> Result<(), &'static str> { match (self, stage) { - ( - Self::Accepted, - ExecutionStage::TwoStage(TwoStageProgress::Finalizing { - .. - }), - ) => Err("cannot transition from Accepted to Finalizing"), - (this @ Self::Accepted, stage) => { - *this = Self::Executing(stage); - Ok(()) + (this @ Self::Accepted, ExecutionStage::TwoStage(stage)) => { + match stage { + // Transition from Accepted to Committing stage + val @ TwoStageProgress::Committing(_) => { + *this = Self::Executing(ExecutionStage::TwoStage(val)); + } + // Transition from Accepted state to TwoStage::Finalizing is invalid + TwoStageProgress::Finalizing { .. } => { + return Err( + "cannot transition from Accepted to Finalizing", + ) + } + } + } + (this @ Self::Accepted, val @ ExecutionStage::SingleStage(_)) => { + *this = Self::Executing(val); } (Self::Executing(ref mut this), stage) => { - this.apply_stage_transition(stage) + this.apply_stage_transition(stage)?; } - } + }; + Ok(()) } } From f50bb77d62856a266d54571e93a3645e8013954a Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 8 Jun 2026 17:25:23 +0700 Subject: [PATCH 34/97] wip --- Cargo.lock | 18 +-- Cargo.toml | 1 + magicblock-committor-service/Cargo.toml | 2 + .../src/intent_execution_manager.rs | 5 +- .../src/intent_execution_manager/db.rs | 67 +++++----- .../intent_channerl.rs | 123 ++++++++++++++++-- magicblock-committor-service/src/service.rs | 2 +- .../src/service/acceptor.rs | 1 + test-integration/Cargo.lock | 20 +-- 9 files changed, 175 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8cef276d3..eb2cdef0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1949,7 +1949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3903,6 +3903,7 @@ dependencies = [ "magicblock-program", "magicblock-rpc-client", "magicblock-table-mania", + "pin-project", "rand 0.9.4", "rusqlite", "solana-account 3.4.0", @@ -3927,6 +3928,7 @@ dependencies = [ "test-kit", "thiserror 2.0.18", "tokio", + "tokio-stream", "tokio-util", "tracing", ] @@ -4945,18 +4947,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -5939,7 +5941,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9840,7 +9842,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10891,7 +10893,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f1088bc60..0e892f827 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,6 +131,7 @@ num-traits = "0.2" num_cpus = "1.16.0" parking_lot = "0.12" paste = "1.0" +pin-project = "1.1.13" prometheus = "0.13.4" # Keep in sync with `solana-storage-proto` codegen. prost = "0.14" diff --git a/magicblock-committor-service/Cargo.toml b/magicblock-committor-service/Cargo.toml index d9728d739..ea8dce441 100644 --- a/magicblock-committor-service/Cargo.toml +++ b/magicblock-committor-service/Cargo.toml @@ -32,6 +32,7 @@ magicblock-metrics = { workspace = true } magicblock-program = { workspace = true } magicblock-rpc-client = { workspace = true } magicblock-table-mania = { workspace = true } +pin-project = { workspace = true } rusqlite = { workspace = true } solana-account = { workspace = true } solana-account-decoder = { workspace = true } @@ -54,6 +55,7 @@ solana-transaction-status-client-types = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } +tokio-stream = { workspace = true } [dev-dependencies] solana-signature = { workspace = true, features = ["rand"] } diff --git a/magicblock-committor-service/src/intent_execution_manager.rs b/magicblock-committor-service/src/intent_execution_manager.rs index 4a4d55bea..c484041f4 100644 --- a/magicblock-committor-service/src/intent_execution_manager.rs +++ b/magicblock-committor-service/src/intent_execution_manager.rs @@ -1,7 +1,7 @@ pub(crate) mod db; +pub mod intent_channerl; mod intent_execution_engine; pub mod intent_scheduler; -pub mod intent_channerl; use std::sync::Arc; @@ -82,7 +82,7 @@ impl IntentExecutionManager { // Worker first will clean-up channel, and then DB. // Pushing into channel would break order of commits if !self.db.is_empty() { - self.db.store_intent_bundles(intent_bundles).await?; + self.db.store_intent_bundles(intent_bundles)?; return Ok(()); } @@ -99,7 +99,6 @@ impl IntentExecutionManager { let leftovers = std::iter::once(el).chain(iter).collect(); self.db .store_intent_bundles(leftovers) - .await .map_err(IntentExecutionManagerError::from) } } diff --git a/magicblock-committor-service/src/intent_execution_manager/db.rs b/magicblock-committor-service/src/intent_execution_manager/db.rs index 0afc4a461..63dfcc9a2 100644 --- a/magicblock-committor-service/src/intent_execution_manager/db.rs +++ b/magicblock-committor-service/src/intent_execution_manager/db.rs @@ -1,32 +1,34 @@ -use std::{collections::VecDeque, sync::Mutex}; -use std::sync::Arc; +use std::{ + cell::RefCell, + collections::VecDeque, + sync::{Arc, Mutex}, +}; + /// DB for storing intents that overflow committor channel use async_trait::async_trait; -use solana_account::ReadableAccount; -use magicblock_accounts_db::AccountsDb; -use magicblock_accounts_db::traits::AccountsBank; +use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_core::intent::outbox::outbox_intent_pda; use magicblock_metrics::metrics; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; -use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, +}; +use solana_account::ReadableAccount; const POISONED_MUTEX_MSG: &str = "Dummy db mutex poisoned"; -#[async_trait] -pub trait DB: Send + Sync + 'static { - async fn store_intent_bundle( +pub trait DB { + fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, ) -> DBResult<()>; - async fn store_intent_bundles( + fn store_intent_bundles( &self, intent_bundles: Vec, ) -> DBResult<()>; /// Returns the oldest (first stored) intent bundle - async fn pop_intent_bundle( - &self, - ) -> DBResult>; + fn pop_intent_bundle(&self) -> DBResult>; fn is_empty(&self) -> bool; } @@ -42,9 +44,8 @@ impl DummyDB { } } -#[async_trait] impl DB for DummyDB { - async fn store_intent_bundle( + fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, ) -> DBResult<()> { @@ -55,7 +56,7 @@ impl DB for DummyDB { Ok(()) } - async fn store_intent_bundles( + fn store_intent_bundles( &self, intent_bundles: Vec, ) -> DBResult<()> { @@ -66,9 +67,7 @@ impl DB for DummyDB { Ok(()) } - async fn pop_intent_bundle( - &self, - ) -> DBResult> { + fn pop_intent_bundle(&self) -> DBResult> { let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); let res = db.pop_front(); @@ -83,46 +82,43 @@ impl DB for DummyDB { pub struct DumberDB { accounts_db: Arc, - queue: Mutex> + queue: RefCell>, } impl DumberDB { pub fn new(accounts_db: Arc) -> Self { Self { accounts_db, - queue: Mutex::new(VecDeque::new()), + queue: RefCell::new(VecDeque::new()), } } } -#[async_trait] impl DB for DumberDB { - async fn store_intent_bundle( + fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, ) -> DBResult<()> { - let mut queue = self.queue.lock().expect(POISONED_MUTEX_MSG); + let mut queue = self.queue.borrow_mut(); queue.push_back(intent_bundle.inner.id); metrics::set_committor_intents_backlog_count(queue.len() as i64); Ok(()) } - async fn store_intent_bundles( + fn store_intent_bundles( &self, intent_bundles: Vec, ) -> DBResult<()> { - let mut queue = self.queue.lock().expect(POISONED_MUTEX_MSG); + let mut queue = self.queue.borrow_mut(); queue.extend(intent_bundles.into_iter().map(|el| el.inner.id)); metrics::set_committor_intents_backlog_count(queue.len() as i64); Ok(()) } - async fn pop_intent_bundle( - &self, - ) -> DBResult> { - let mut queue = self.queue.lock().expect(POISONED_MUTEX_MSG); + fn pop_intent_bundle(&self) -> DBResult> { + let mut queue = self.queue.borrow_mut(); let Some(id) = queue.pop_front() else { return Ok(None); }; @@ -130,14 +126,17 @@ impl DB for DumberDB { drop(queue); let intent_pda = outbox_intent_pda(id); - let account = self.accounts_db.get_account(&intent_pda).ok_or(Error::IntentNotFoundError(id))?; + let account = self + .accounts_db + .get_account(&intent_pda) + .ok_or(Error::IntentNotFoundError(id))?; let outbox_intent = OutboxIntentBundle::try_from_bytes(account.data())?; Ok(Some(outbox_intent)) } fn is_empty(&self) -> bool { - self.queue.lock().expect(POISONED_MUTEX_MSG).is_empty() + self.queue.is_empty() } } @@ -146,7 +145,7 @@ pub enum Error { #[error("Failed to find intent in AccountsDB, id: {0}")] IntentNotFoundError(u64), #[error("Failed to deserialize Outbox Account")] - DeserializeError(#[from] bincode::Error) + DeserializeError(#[from] bincode::Error), } pub type DBResult = Result; diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs index 3df52c638..d0be1ec11 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs @@ -1,9 +1,114 @@ -use std::sync::Arc; -use tokio::sync::mpsc; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; -use crate::intent_execution_manager::db::DB; - -pub struct Sender { - db: Arc, - sender: mpsc::Sender -} \ No newline at end of file +use std::{ + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll}, +}; + +use futures_util::ready; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, +}; +use pin_project::pin_project; +use tokio::sync::mpsc::{error::TrySendError, Receiver, Sender}; +use tokio_stream::{wrappers::ReceiverStream, Stream}; + +use crate::intent_execution_manager::{ + db, db::DB, IntentExecutionManagerError, +}; + +const POISONED_MSG: &str = "Dummy DB mutex poisoned"; + +pub struct IntentScheduleHandle { + db: Arc>, + sender: Sender, +} + +impl IntentScheduleHandle { + pub fn new(db: Arc>, sender: Sender) -> Self { + Self { db, sender } + } + + pub fn schedule( + &self, + intent_bundles: Vec, + ) -> Result<(), IntentScheduleError> { + // If db not empty push el-t there + // This means that at some point channel got full + // Worker first will clean-up channel, and then DB. + // Pushing into channel would break order of commits + // Lock shall be held across to avoid races + let db = self.db.lock().expect(POISONED_MSG); + if !db.is_empty() { + db.store_intent_bundles(intent_bundles)?; + return Ok(()); + } + + let mut iter = intent_bundles.into_iter(); + // Treated as regular value not propagated lower + #[allow(clippy::result_large_err)] + let res = iter.try_for_each(|el| self.sender.try_send(el)); + match res { + Ok(_) => Ok(()), + Err(TrySendError::Closed(_)) => { + Err(IntentScheduleError::ChannelClosed) + } + Err(TrySendError::Full(el)) => { + let leftovers = std::iter::once(el).chain(iter).collect(); + db.store_intent_bundles(leftovers) + .map_err(IntentScheduleError::from) + } + } + } +} + +#[pin_project] +pub struct IntentStreamer { + #[pin] + db: Arc>, + #[pin] + stream: ReceiverStream, +} + +impl IntentStreamer { + pub fn new( + db: Arc>, + receiver: Receiver, + ) -> Self { + Self { + db, + stream: ReceiverStream::new(receiver), + } + } +} + +impl Stream for IntentStreamer { + type Item = Result; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + // TODO(edwin): could be relaxed and used after receive or not? + let this = self.project(); + let db = this.db.lock().expect(POISONED_MSG); + // That means we have backlog + // prior to using channel again we have to clean it all first + if !db.is_empty() { + // Some(T) always will be returned here as per check above + let el = db.pop_intent_bundle(); + return Poll::Ready(el.transpose()); + } + + let item = ready!(this.stream.poll_next(cx)); + Poll::Ready(item.map(Ok)) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum IntentScheduleError { + #[error("Channel was closed")] + ChannelClosed, + #[error("DBError: {0}")] + DBError(#[from] db::Error), +} diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index a95971e78..323ed55f2 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,6 +1,6 @@ +pub mod acceptor; pub mod intent_client; pub mod outbox_intent_bundles_reader; -pub mod acceptor; use std::{ collections::{HashMap, HashSet}, diff --git a/magicblock-committor-service/src/service/acceptor.rs b/magicblock-committor-service/src/service/acceptor.rs index e69de29bb..8b1378917 100644 --- a/magicblock-committor-service/src/service/acceptor.rs +++ b/magicblock-committor-service/src/service/acceptor.rs @@ -0,0 +1 @@ + diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index f80dff0dc..ffb15be56 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -2375,7 +2375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4449,6 +4449,7 @@ dependencies = [ "magicblock-program", "magicblock-rpc-client", "magicblock-table-mania", + "pin-project", "rusqlite", "solana-account", "solana-account-decoder", @@ -4470,6 +4471,7 @@ dependencies = [ "solana-transaction-status-client-types", "thiserror 2.0.18", "tokio", + "tokio-stream", "tokio-util 0.7.17", "tracing", ] @@ -5606,18 +5608,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -6734,7 +6736,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6835,7 +6837,7 @@ dependencies = [ "security-framework 3.5.1", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -11098,7 +11100,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12375,7 +12377,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] From 4c97135e77e6e6fe7b3861573ecbc9bc5dfa8a4d Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 9 Jun 2026 12:26:18 +0700 Subject: [PATCH 35/97] wip removing persister --- .../src/committor_processor.rs | 4 +- .../src/intent_execution_manager.rs | 66 +- .../src/intent_execution_manager/db.rs | 10 +- .../intent_channerl.rs | 34 +- .../intent_execution_engine.rs | 129 +- .../intent_scheduler.rs | 26 +- .../src/intent_executor/mod.rs | 158 +-- .../intent_executor/single_stage_executor.rs | 9 +- .../src/intent_executor/two_stage_executor.rs | 15 +- .../src/intent_executor/utils.rs | 40 +- .../src/persist/commit_persister.rs | 1067 --------------- .../src/persist/db.rs | 1216 ----------------- .../src/persist/error.rs | 41 - .../src/persist/mod.rs | 11 - .../src/persist/types/commit_status.rs | 161 --- .../src/persist/types/commit_strategy.rs | 72 - .../src/persist/types/commit_type.rs | 28 - .../src/persist/types/mod.rs | 7 - .../src/persist/utils.rs | 65 - .../src/tasks/task_builder.rs | 27 +- .../src/tasks/task_strategist.rs | 127 +- .../delivery_preparator.rs | 43 +- .../src/transaction_preparator/mod.rs | 9 +- magicblock-committor-service/src/utils.rs | 40 - .../intent_bundles/outbox_intent_bundles.rs | 13 +- 25 files changed, 169 insertions(+), 3249 deletions(-) delete mode 100644 magicblock-committor-service/src/persist/commit_persister.rs delete mode 100644 magicblock-committor-service/src/persist/db.rs delete mode 100644 magicblock-committor-service/src/persist/error.rs delete mode 100644 magicblock-committor-service/src/persist/mod.rs delete mode 100644 magicblock-committor-service/src/persist/types/commit_status.rs delete mode 100644 magicblock-committor-service/src/persist/types/commit_strategy.rs delete mode 100644 magicblock-committor-service/src/persist/types/commit_type.rs delete mode 100644 magicblock-committor-service/src/persist/types/mod.rs delete mode 100644 magicblock-committor-service/src/persist/utils.rs diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index b610480f7..cb589185f 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -7,7 +7,7 @@ use std::{ use futures_util::future::join_all; use magicblock_core::traits::ActionsCallbackScheduler; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use solana_keypair::Keypair; @@ -35,6 +35,8 @@ use crate::{ MessageSignatures, }, }; +use crate::intent_execution_manager::db::DumberDB; + const POISONED_MUTEX_MSG: &str = "CommittorProcessor pending messages mutex poisoned!"; pub(crate) const RECOVERY_MAX_AGE_SECS: u64 = 14 * 24 * 60 * 60; diff --git a/magicblock-committor-service/src/intent_execution_manager.rs b/magicblock-committor-service/src/intent_execution_manager.rs index c484041f4..13b8f121b 100644 --- a/magicblock-committor-service/src/intent_execution_manager.rs +++ b/magicblock-committor-service/src/intent_execution_manager.rs @@ -3,11 +3,13 @@ pub mod intent_channerl; mod intent_execution_engine; pub mod intent_scheduler; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; pub use intent_execution_engine::BroadcastedIntentExecutionResult; use magicblock_core::traits::ActionsCallbackScheduler; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use magicblock_program::{ + outbox_intent_bundles::OutboxIntentBundle, +}; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::TableMania; use tokio::sync::{broadcast, mpsc, mpsc::error::TrySendError}; @@ -15,36 +17,35 @@ use tokio::sync::{broadcast, mpsc, mpsc::error::TrySendError}; use crate::{ intent_execution_manager::{ db::DB, + intent_channerl::{channel, IntentScheduleError, IntentScheduleHandle}, intent_execution_engine::{IntentExecutionEngine, ResultSubscriber}, }, intent_executor::{ intent_executor_factory::{ExecutorConfig, IntentExecutorFactoryImpl}, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }, - persist::IntentPersister, }; +// TODO(edwin): rename pub struct IntentExecutionManager { - db: Arc, + // TODO(edwin): add JoinHandle of + intent_schedule_handle: IntentScheduleHandle, result_subscriber: ResultSubscriber, - intent_sender: mpsc::Sender, } impl IntentExecutionManager { - pub fn new( + pub fn new( rpc_client: MagicblockRpcClient, db: D, task_info_fetcher: Arc>, - intent_persister: Option

, table_mania: TableMania, executor_config: ExecutorConfig, actions_callback_executor: A, ) -> Self where A: ActionsCallbackScheduler, - P: IntentPersister, { - let db = Arc::new(db); + let db = Arc::new(Mutex::new(db)); let executor_factory = IntentExecutorFactoryImpl { rpc_client, @@ -54,18 +55,15 @@ impl IntentExecutionManager { actions_callback_executor, }; - let (sender, receiver) = mpsc::channel(1000); + let (handle, intent_stream) = channel(&db, 1000); let worker = IntentExecutionEngine::new( - db.clone(), + intent_stream, executor_factory, - intent_persister, - receiver, ); let result_subscriber = worker.spawn(); Self { - db, - intent_sender: sender, + intent_schedule_handle: handle, result_subscriber, } } @@ -75,33 +73,9 @@ impl IntentExecutionManager { /// Intents will be extracted and handled in the [`IntentExecutionEngine`] pub async fn schedule( &self, - intent_bundles: Vec, - ) -> Result<(), IntentExecutionManagerError> { - // If db not empty push el-t there - // This means that at some point channel got full - // Worker first will clean-up channel, and then DB. - // Pushing into channel would break order of commits - if !self.db.is_empty() { - self.db.store_intent_bundles(intent_bundles)?; - return Ok(()); - } - - let mut iter = intent_bundles.into_iter(); - // Treated as regular value not propagated lower - #[allow(clippy::result_large_err)] - let res = iter.try_for_each(|el| self.intent_sender.try_send(el)); - match res { - Ok(_) => Ok(()), - Err(TrySendError::Closed(_)) => { - Err(IntentExecutionManagerError::ChannelClosed) - } - Err(TrySendError::Full(el)) => { - let leftovers = std::iter::once(el).chain(iter).collect(); - self.db - .store_intent_bundles(leftovers) - .map_err(IntentExecutionManagerError::from) - } - } + intent_bundles: Vec, + ) -> Result<(), IntentScheduleError> { + self.intent_schedule_handle.schedule(intent_bundles) } /// Creates a subscription for results of BaseIntent execution @@ -111,11 +85,3 @@ impl IntentExecutionManager { self.result_subscriber.subscribe() } } - -#[derive(thiserror::Error, Debug)] -pub enum IntentExecutionManagerError { - #[error("Channel was closed")] - ChannelClosed, - #[error("DBError: {0}")] - DBError(#[from] db::Error), -} diff --git a/magicblock-committor-service/src/intent_execution_manager/db.rs b/magicblock-committor-service/src/intent_execution_manager/db.rs index 63dfcc9a2..70084fb94 100644 --- a/magicblock-committor-service/src/intent_execution_manager/db.rs +++ b/magicblock-committor-service/src/intent_execution_manager/db.rs @@ -5,19 +5,15 @@ use std::{ }; /// DB for storing intents that overflow committor channel -use async_trait::async_trait; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_core::intent::outbox::outbox_intent_pda; use magicblock_metrics::metrics; -use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, -}; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::ReadableAccount; const POISONED_MUTEX_MSG: &str = "Dummy db mutex poisoned"; -pub trait DB { +pub trait DB: Send + 'static { fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, @@ -136,7 +132,7 @@ impl DB for DumberDB { } fn is_empty(&self) -> bool { - self.queue.is_empty() + self.queue.borrow().is_empty() } } diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs index d0be1ec11..d8730b53b 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs @@ -5,17 +5,15 @@ use std::{ }; use futures_util::ready; -use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, -}; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use pin_project::pin_project; -use tokio::sync::mpsc::{error::TrySendError, Receiver, Sender}; +use tokio::sync::{ + mpsc, + mpsc::{error::TrySendError, Receiver, Sender}, +}; use tokio_stream::{wrappers::ReceiverStream, Stream}; -use crate::intent_execution_manager::{ - db, db::DB, IntentExecutionManagerError, -}; +use crate::intent_execution_manager::{db, db::DB}; const POISONED_MSG: &str = "Dummy DB mutex poisoned"; @@ -63,14 +61,13 @@ impl IntentScheduleHandle { } #[pin_project] -pub struct IntentStreamer { - #[pin] +pub struct IntentStream { db: Arc>, #[pin] stream: ReceiverStream, } -impl IntentStreamer { +impl IntentStream { pub fn new( db: Arc>, receiver: Receiver, @@ -82,14 +79,13 @@ impl IntentStreamer { } } -impl Stream for IntentStreamer { +impl Stream for IntentStream { type Item = Result; fn poll_next( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll> { - // TODO(edwin): could be relaxed and used after receive or not? let this = self.project(); let db = this.db.lock().expect(POISONED_MSG); // That means we have backlog @@ -105,6 +101,18 @@ impl Stream for IntentStreamer { } } +pub(crate) fn channel( + db: &Arc>, + buffer: usize, +) -> (IntentScheduleHandle, IntentStream) { + let (sender, receiver) = mpsc::channel(buffer); + + let handle = IntentScheduleHandle::new(db.clone(), sender); + let stream = IntentStream::new(db.clone(), receiver); + + (handle, stream) +} + #[derive(thiserror::Error, Debug)] pub enum IntentScheduleError { #[error("Channel was closed")] diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 593504f7e..379a6af12 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -4,10 +4,10 @@ use std::{ time::{Duration, Instant}, }; -use futures_util::{stream::FuturesUnordered, StreamExt}; +use futures_util::stream::FuturesUnordered; use magicblock_core::traits::CallbackScheduleError; use magicblock_metrics::metrics; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_signature::Signature; use tokio::{ sync::{ @@ -16,13 +16,14 @@ use tokio::{ }, task::JoinHandle, }; +use tokio_stream::StreamExt; use tracing::{error, info, instrument, trace, warn}; use crate::{ intent_execution_manager::{ db::DB, + intent_channerl::{IntentScheduleError, IntentStream}, intent_scheduler::{IntentScheduler, POISONED_INNER_MSG}, - IntentExecutionManagerError, }, intent_executor::{ error::{ @@ -32,7 +33,6 @@ use crate::{ intent_executor_factory::IntentExecutorFactory, ExecutionOutput, IntentExecutionResult, IntentExecutor, }, - persist::IntentPersister, }; const SEMAPHORE_CLOSED_MSG: &str = "Executors semaphore closed!"; @@ -88,35 +88,28 @@ impl ResultSubscriber { } } -pub(crate) struct IntentExecutionEngine { - db: Arc, +pub(crate) struct IntentExecutionEngine { + intent_stream: IntentStream, executor_factory: F, - intents_persister: Option

, - receiver: mpsc::Receiver, inner: Arc>, running_executors: FuturesUnordered>, executors_semaphore: Arc, } -impl IntentExecutionEngine +impl IntentExecutionEngine where D: DB, - P: IntentPersister, F: IntentExecutorFactory + Send + Sync + 'static, E: IntentExecutor, { pub fn new( - db: Arc, + intent_stream: IntentStream, executor_factory: F, - intents_persister: Option

, - receiver: mpsc::Receiver, ) -> Self { Self { - db, - intents_persister, + intent_stream, executor_factory, - receiver, running_executors: FuturesUnordered::new(), executors_semaphore: Arc::new(Semaphore::new( MAX_EXECUTORS as usize, @@ -145,13 +138,14 @@ where loop { let intent = match self.next_scheduled_intent().await { Ok(value) => value, - Err(IntentExecutionManagerError::ChannelClosed) => { + Err(IntentScheduleError::ChannelClosed) => { info!("Channel closed, exiting"); break; } - Err(IntentExecutionManagerError::DBError(err)) => { + Err(IntentScheduleError::DBError(err)) => { + // TODO(edwin): add to alert as this is critical error error!(error = ?err, "Failed to fetch intent"); - break; + continue; } }; let Some(intent) = intent else { @@ -172,12 +166,10 @@ where // Spawn executor let executor = self.executor_factory.create_instance(); - let persister = self.intents_persister.clone(); let inner = self.inner.clone(); let handle = tokio::spawn(Self::execute( executor, - persister, intent, inner, permit, @@ -191,12 +183,11 @@ where } } - /// Returns [`ScheduledIntentBundle`] or None if all intents are blocked + /// Returns [`OutboxIntentBundle`] or None if all intents are blocked #[instrument(skip(self))] async fn next_scheduled_intent( &mut self, - ) -> Result, IntentExecutionManagerError> - { + ) -> Result, IntentScheduleError> { // Limit on number of intents that can be stored in scheduler const SCHEDULER_CAPACITY: usize = 1000; @@ -215,8 +206,7 @@ where }; let running_executors = &mut self.running_executors; - let receiver = &mut self.receiver; - let db = &self.db; + let intent_stream = &mut self.intent_stream; let intent = tokio::select! { // Notify polled first to prioritize unblocked intents over new one biased; @@ -227,7 +217,7 @@ where trace!("Worker executed intent bundle, fetching new available one"); self.inner.lock().expect(POISONED_INNER_MSG).pop_next_scheduled_intent() }, - result = Self::get_new_intent(receiver, db), if can_receive() => { + result = Self::get_new_intent(intent_stream), if can_receive() => { let intent = result?; self.inner.lock().expect(POISONED_INNER_MSG).schedule(intent) }, @@ -246,35 +236,20 @@ where /// Returns [`ScheduledIntentBundle`] from external channel async fn get_new_intent( - receiver: &mut mpsc::Receiver, - db: &Arc, - ) -> Result { - match receiver.try_recv() { - Ok(val) => Ok(val), - Err(TryRecvError::Empty) => { - // Worker either cleaned-up congested channel and now need to clean-up DB - // or we're just waiting on empty channel - if let Some(intent_bundle) = db.pop_intent_bundle().await? { - Ok(intent_bundle) - } else { - receiver - .recv() - .await - .ok_or(IntentExecutionManagerError::ChannelClosed) - } - } - Err(TryRecvError::Disconnected) => { - Err(IntentExecutionManagerError::ChannelClosed) - } - } + intent_stream: &mut IntentStream, + ) -> Result { + intent_stream + .next() + .await + .ok_or(IntentScheduleError::ChannelClosed)? + .map_err(IntentScheduleError::DBError) } /// Wrapper on [`IntentExecutor`] that handles its results and drops execution permit - #[instrument(skip(executor, persister, intent, inner_scheduler, execution_permit, result_sender), fields(intent_id = intent.id))] + #[instrument(skip(executor, intent, inner_scheduler, execution_permit, result_sender), fields(intent_id = intent.id))] async fn execute( mut executor: E, - persister: Option

, - intent: ScheduledIntentBundle, + intent: OutboxIntentBundle, inner_scheduler: Arc>, execution_permit: OwnedSemaphorePermit, result_sender: broadcast::Sender, @@ -282,7 +257,7 @@ where let instant = Instant::now(); // Execute an Intent - let result = executor.execute(intent.clone(), persister).await; + let result = executor.execute(intent.clone()).await; let _ = result.inner.as_ref().inspect_err(|err| { error!(intent_id = intent.id, error = ?err, "Failed to execute intent bundle"); }); @@ -320,7 +295,7 @@ where /// Records metrics related to intent execution fn execution_metrics( execution_time: Duration, - intent: &ScheduledIntentBundle, + intent: &OutboxIntentBundle, result: &IntentExecutorResult, ) { const EXECUTION_TIME_THRESHOLD: f64 = 5.0; @@ -366,57 +341,44 @@ mod tests { }; use async_trait::async_trait; - use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use solana_pubkey::{pubkey, Pubkey}; use solana_signature::Signature; use solana_signer::SignerError; use solana_transaction_error::TransactionError; - use tokio::{sync::mpsc, time::sleep}; + use tokio::time::sleep; use super::*; use crate::{ intent_execution_manager::{ db::{DummyDB, DB}, + intent_channerl::{channel, IntentScheduleHandle}, intent_scheduler::{create_test_intent, create_test_intent_bundle}, }, intent_executor::{ error::{IntentExecutorError as ExecutorError, InternalError}, IntentExecutionResult, }, - persist::IntentPersisterImpl, test_utils, transaction_preparator::delivery_preparator::BufferExecutionError, }; - type MockIntentExecutionEngine = IntentExecutionEngine< - DummyDB, - IntentPersisterImpl, - MockIntentExecutorFactory, - >; + type MockIntentExecutionEngine = + IntentExecutionEngine; fn setup_engine( should_fail: bool, - ) -> ( - mpsc::Sender, - MockIntentExecutionEngine, - ) { + ) -> (IntentScheduleHandle, MockIntentExecutionEngine) { test_utils::init_test_logger(); - let (sender, receiver) = mpsc::channel(1000); - - let db = Arc::new(DummyDB::new()); + let db = Arc::new(Mutex::new(DummyDB::new())); + let (handle, intent_stream) = channel(&db, 1000); let executor_factory = if !should_fail { MockIntentExecutorFactory::new() } else { MockIntentExecutorFactory::new_failing() }; - let worker = IntentExecutionEngine::new( - db.clone(), - executor_factory, - None::, - receiver, - ); + let worker = IntentExecutionEngine::new(intent_stream, executor_factory); - (sender, worker) + (handle, worker) } #[tokio::test] @@ -431,7 +393,7 @@ mod tests { &[pubkey!("1111111111111111111111111111111111111111111")], false, ); - sender.send(msg.clone()).await.unwrap(); + sender.schedule(vec![msg.clone()]).unwrap(); // Verify the message was processed let result = result_receiver.recv().await.unwrap(); @@ -502,7 +464,7 @@ mod tests { &[pubkey!("1111111111111111111111111111111111111111111")], false, ); - sender.send(msg.clone()).await.unwrap(); + sender.schedule(vec![msg.clone()]).unwrap(); // Verify the failure was properly reported let result = result_receiver.recv().await.unwrap(); @@ -563,7 +525,7 @@ mod tests { &[pubkey!("1111111111111111111111111111111111111111111")], false, ); - sender.send(msg).await.unwrap(); + sender.schedule(vec![msg]).unwrap(); } // Process results and verify constraints @@ -599,7 +561,7 @@ mod tests { &[pubkey!("1111111111111111111111111111111111111111111")], false, ); - sender.send(msg).await.unwrap(); + sender.schedule(vec![msg]).unwrap(); } // Verify all failures are processed and semaphore slots released @@ -631,7 +593,7 @@ mod tests { let msg = create_test_intent(i, &[unique_pubkey], false); received_ids.insert(i); - sender.send(msg).await.unwrap(); + sender.schedule(vec![msg]).unwrap(); } // Process results @@ -695,7 +657,7 @@ mod tests { }; let msg = create_test_intent(i as u64, &pubkeys, false); - sender.send(msg).await.unwrap(); + sender.schedule(vec![msg]).unwrap(); } // Process results @@ -800,10 +762,9 @@ mod tests { #[async_trait] impl IntentExecutor for MockIntentExecutor { - async fn execute( + async fn execute( &mut self, - _base_intent: ScheduledIntentBundle, - _persister: Option

, + _base_intent: OutboxIntentBundle, ) -> IntentExecutionResult { self.on_task_started(); diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs b/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs index 580acebf6..61ad72a51 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs @@ -4,6 +4,7 @@ use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use solana_pubkey::Pubkey; use thiserror::Error; use tracing::error; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; pub(crate) const POISONED_INNER_MSG: &str = "Mutex on CommitSchedulerInner is poisoned."; @@ -11,7 +12,7 @@ pub(crate) const POISONED_INNER_MSG: &str = type IntentID = u64; struct IntentMeta { num_keys: usize, - intent: ScheduledIntentBundle, + intent: OutboxIntentBundle, } /// A scheduler that ensures mutually exclusive access to pubkeys across intents @@ -78,8 +79,8 @@ impl IntentScheduler { /// otherwise consumes it and enqueues pub fn schedule( &mut self, - intent_bundle: ScheduledIntentBundle, - ) -> Option { + intent_bundle: OutboxIntentBundle, + ) -> Option { let intent_id = intent_bundle.id; // To check duplicate scheduling its enough to check: @@ -148,7 +149,7 @@ impl IntentScheduler { /// NOTE: this shall be called on executing intents to finilize their execution. pub fn complete( &mut self, - intent_bundle: &ScheduledIntentBundle, + intent_bundle: &OutboxIntentBundle, ) -> IntentSchedulerResult<()> { // Release data for completed intent let intent_id = intent_bundle.id; @@ -235,7 +236,7 @@ impl IntentScheduler { // Returns [`ScheduledBaseIntent`] that can be executed pub fn pop_next_scheduled_intent( &mut self, - ) -> Option { + ) -> Option { // TODO(edwin): optimize. Create counter im IntentMeta & update let mut execute_candidates: HashMap = HashMap::new(); self.blocked_keys.iter().for_each(|(_, queue)| { @@ -627,7 +628,7 @@ mod edge_cases_test { setup(); let mut scheduler = IntentScheduler::new(); let mut msg = create_test_intent(1, &[], false); - msg.intent_bundle = MagicIntentBundle::default(); + msg.inner.intent_bundle = MagicIntentBundle::default(); // Should execute immediately since it has no pubkeys assert!(scheduler.schedule(msg.clone()).is_some()); @@ -681,7 +682,7 @@ mod complete_error_test { let msg2 = create_test_intent(2, &[pubkey1], false); assert!(scheduler.schedule(msg2.clone()).is_none()); - msg1.get_commit_intent_accounts_mut().unwrap().pop(); + msg1.inner.get_commit_intent_accounts_mut().unwrap().pop(); // Attempt to complete msg1 - should detect corrupted state let result = scheduler.complete(&msg1); @@ -703,7 +704,7 @@ mod complete_error_test { let mut msg1 = create_test_intent(1, &[pubkey1, pubkey2], false); assert!(scheduler.schedule(msg1.clone()).is_some()); - msg1.intent_bundle + msg1.inner.intent_bundle .get_commit_intent_accounts_mut() .unwrap() .push(CommittedAccount { @@ -853,7 +854,7 @@ pub(crate) fn create_test_intent( id: u64, pubkeys: &[Pubkey], is_undelegate: bool, -) -> ScheduledIntentBundle { +) -> OutboxIntentBundle { use magicblock_core::intent::CommittedAccount; use magicblock_program::magic_scheduled_base_intent::{ CommitAndUndelegate, CommitType, MagicIntentBundle, @@ -872,7 +873,6 @@ pub(crate) fn create_test_intent( intent_bundle: MagicIntentBundle::default(), }; - // Only set pubkeys if provided if !pubkeys.is_empty() { let committed_accounts = pubkeys .iter() @@ -895,7 +895,7 @@ pub(crate) fn create_test_intent( } } - intent + OutboxIntentBundle::accepted(intent) } #[cfg(test)] @@ -903,7 +903,7 @@ pub(crate) fn create_test_intent_bundle( id: u64, commit_pubkeys: &[Pubkey], commit_and_undelegate_pubkeys: &[Pubkey], -) -> ScheduledIntentBundle { +) -> OutboxIntentBundle { use magicblock_core::intent::CommittedAccount; use magicblock_program::magic_scheduled_base_intent::{ CommitAndUndelegate, CommitType, MagicIntentBundle, @@ -948,5 +948,5 @@ pub(crate) fn create_test_intent_bundle( }); } - intent + OutboxIntentBundle::accepted(intent) } diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 3a11bd45a..a4f0742d4 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -19,6 +19,7 @@ use magicblock_core::traits::{ use magicblock_metrics::metrics; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, validator::validator_authority, }; use magicblock_rpc_client::MagicblockRpcClient; @@ -43,7 +44,6 @@ use crate::{ FinalizeStage, SingleStage, }, }, - persist::{CommitStatus, CommitStatusSignatures, IntentPersister}, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_strategist::{ @@ -54,7 +54,6 @@ use crate::{ delivery_preparator::BufferExecutionError, error::TransactionPreparatorError, TransactionPreparator, }, - utils::persist_status_update_by_message_set, }; #[derive(Clone, Copy, Debug)] @@ -97,10 +96,9 @@ pub struct IntentExecutionResult { pub trait IntentExecutor: Send + Sync + 'static { /// Executes Message on Base layer /// Returns `ExecutionOutput` or an `Error` - async fn execute( + async fn execute( &mut self, - base_intent: ScheduledIntentBundle, - persister: Option

, + base_intent: OutboxIntentBundle, ) -> IntentExecutionResult; /// Cleans up after intent @@ -193,28 +191,16 @@ where } } - async fn execute_inner( + async fn execute_inner( &mut self, intent_bundle: ScheduledIntentBundle, execution_report: &mut IntentExecutionReport, - persister: &Option

, ) -> IntentExecutorResult { if intent_bundle.is_empty() { return Err(IntentExecutorError::EmptyIntentError); } let all_committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); - // Update tasks status to Pending - { - let update_status = CommitStatus::Pending; - persist_status_update_by_message_set( - persister, - intent_bundle.id, - &all_committed_pubkeys, - update_status, - ); - } - if all_committed_pubkeys.is_empty() { // Build tasks for commit stage // TODO (snawaz): it's actually MagicBaseIntent::BaseActions scenario, not Commit @@ -222,7 +208,6 @@ where let commit_tasks = TaskBuilderImpl::commit_tasks( &self.task_info_fetcher, &intent_bundle, - persister, ) .await?; @@ -230,7 +215,6 @@ where let mut strategy = TaskStrategist::build_strategy( commit_tasks, &self.authority.pubkey(), - persister, )?; strategy.standalone_action_nonce = Some(intent_bundle.id); return self @@ -238,7 +222,6 @@ where intent_bundle, strategy, execution_report, - persister, ) .await; }; @@ -248,7 +231,6 @@ where let commit_tasks_fut = TaskBuilderImpl::commit_tasks( &self.task_info_fetcher, &intent_bundle, - persister, ); let finalize_tasks_fut = TaskBuilderImpl::finalize_tasks( &self.task_info_fetcher, @@ -265,7 +247,6 @@ where commit_tasks, finalize_tasks, &self.authority.pubkey(), - persister, )? { StrategyExecutionMode::SingleStage(strategy) => { trace!("Single stage execution"); @@ -273,7 +254,6 @@ where intent_bundle, strategy, execution_report, - persister, ) .await } @@ -287,7 +267,6 @@ where commit_stage, finalize_stage, execution_report, - persister, ) .await } @@ -299,12 +278,11 @@ where } /// Starting execution from single stage - pub async fn single_stage_execution_flow( + pub async fn single_stage_execution_flow( &mut self, base_intent: ScheduledIntentBundle, transaction_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, - persister: &Option

, ) -> IntentExecutorResult { let committed_pubkeys = base_intent.get_all_committed_pubkeys(); @@ -323,7 +301,6 @@ where transaction_preparator: &self.transaction_preparator, committed_pubkeys: &committed_pubkeys, }, - persister, ) .await; @@ -366,18 +343,16 @@ where commit_strategy, finalize_strategy, execution_report, - persister, ) .await } - pub async fn two_stage_execution_flow( + pub async fn two_stage_execution_flow( &mut self, committed_pubkeys: &[Pubkey], commit_strategy: TransactionStrategy, finalize_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, - persister: &Option

, ) -> IntentExecutorResult { let mut executor = TwoStageExecutor::new( self.authority.insecure_clone(), @@ -396,7 +371,6 @@ where task_info_fetcher: &self.task_info_fetcher, committed_pubkeys, }, - persister, ) .await?; @@ -407,7 +381,6 @@ where inner: &mut finalize_executor, transaction_preparator: &self.transaction_preparator, }, - persister, ) .await?; @@ -418,114 +391,6 @@ where }) } - /// Flushes result into presistor - /// The result will be propagated down to callers - fn persist_result( - persistor: &P, - result: &IntentExecutorResult, - message_id: u64, - pubkeys: &[Pubkey], - ) { - let update_status = match result { - Ok(value) => { - let signatures = match *value { - ExecutionOutput::SingleStage(signature) => { - CommitStatusSignatures { - commit_stage_signature: signature, - finalize_stage_signature: Some(signature), - } - } - ExecutionOutput::TwoStage { - commit_signature, - finalize_signature, - } => CommitStatusSignatures { - commit_stage_signature: commit_signature, - finalize_stage_signature: Some(finalize_signature), - }, - }; - let update_status = CommitStatus::Succeeded(signatures); - persist_status_update_by_message_set( - persistor, - message_id, - pubkeys, - update_status, - ); - - if let Err(err) = - persistor.finalize_base_intent(message_id, *value) - { - tracing::error!(error = ?err, "Failed to persist ExecutionOutput"); - } - - return; - } - Err(IntentExecutorError::EmptyIntentError) - | Err(IntentExecutorError::FailedToFitError) - | Err(IntentExecutorError::TaskBuilderError(_)) - | Err(IntentExecutorError::FailedCommitPreparationError( - TransactionPreparatorError::SignerError(_), - )) - | Err(IntentExecutorError::FailedFinalizePreparationError( - TransactionPreparatorError::SignerError(_), - )) => Some(CommitStatus::Failed), - Err(IntentExecutorError::FailedCommitPreparationError( - TransactionPreparatorError::FailedToFitError, - )) => Some(CommitStatus::PartOfTooLargeBundleToProcess), - Err(IntentExecutorError::FailedCommitPreparationError( - TransactionPreparatorError::DeliveryPreparationError(_), - )) => { - // Intermediate commit preparation progress recorded by DeliveryPreparator - None - } - Err(IntentExecutorError::FailedToCommitError { - err: _, - signature, - }) => { - // Commit is a single TX, so if it fails, all of commited accounts marked FailedProcess - let status_signature = - signature.map(|sig| CommitStatusSignatures { - commit_stage_signature: sig, - finalize_stage_signature: None, - }); - Some(CommitStatus::FailedProcess(status_signature)) - } - Err(IntentExecutorError::FailedFinalizePreparationError(_)) => { - // Not supported in persistor - None - } - Err(IntentExecutorError::FailedToFinalizeError { - err: _, - commit_signature, - finalize_signature, - }) => { - // Finalize is a single TX, so if it fails, all of commited accounts marked FailedFinalize - let update_status = - if let Some(commit_signature) = commit_signature { - let signatures = CommitStatusSignatures { - commit_stage_signature: *commit_signature, - finalize_stage_signature: *finalize_signature, - }; - CommitStatus::FailedFinalize(signatures) - } else { - CommitStatus::FailedProcess(None) - }; - - Some(update_status) - } - Err(IntentExecutorError::SignerError(_)) => { - Some(CommitStatus::Failed) - } - }; - - if let Some(update_status) = update_status { - persist_status_update_by_message_set( - persistor, - message_id, - pubkeys, - update_status, - ); - } - } } #[async_trait] @@ -537,19 +402,17 @@ where { /// Executes Message on Base layer /// Returns `ExecutionOutput` or an `Error` - async fn execute( + async fn execute( &mut self, - base_intent: ScheduledIntentBundle, - persister: Option

, + base_intent: OutboxIntentBundle, ) -> IntentExecutionResult { self.started_at = Instant::now(); - let message_id = base_intent.id; let is_undelegate = base_intent.has_undelegate_intent(); let pubkeys = base_intent.get_all_committed_pubkeys(); let mut execution_report = IntentExecutionReport::default(); let result = self - .execute_inner(base_intent, &mut execution_report, &persister) + .execute_inner(base_intent.inner, &mut execution_report) .await; if !pubkeys.is_empty() { // Reset TaskInfoFetcher, as cache could become invalid @@ -558,9 +421,6 @@ where if result.is_err() || is_undelegate { self.task_info_fetcher.reset(ResetType::Specific(&pubkeys)); } - - // Write result of intent into Persister - Self::persist_result(&persister, &result, message_id, &pubkeys); } // Gather metrics in separate task diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs index 44ca65884..803c06757 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs @@ -21,7 +21,6 @@ use crate::{ }, IntentExecutionReport, }, - persist::{IntentPersister, IntentPersisterImpl}, tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, transaction_preparator::TransactionPreparator, }; @@ -62,18 +61,16 @@ where } #[instrument( - skip(self, committed_pubkeys, transaction_preparator, persister), + skip(self, committed_pubkeys, transaction_preparator), fields(stage = "single_stage") )] - pub async fn execute( + pub async fn execute( &mut self, committed_pubkeys: &[Pubkey], transaction_preparator: &T, - persister: &Option

, ) -> IntentExecutorResult where T: TransactionPreparator, - P: IntentPersister, { const RECURSION_CEILING: u8 = 10; @@ -86,7 +83,6 @@ where &self.authority, transaction_preparator, &mut self.transaction_strategy, - persister, ) .await .map_err(IntentExecutorError::FailedFinalizePreparationError)?; @@ -285,7 +281,6 @@ where lookup_tables_keys: vec![], standalone_action_nonce: None, }, - &None::, ) .await .map_err(IntentExecutorError::FailedFinalizePreparationError)? diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index f5cd00703..d761a39ba 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -22,7 +22,6 @@ use crate::{ }, IntentExecutionReport, }, - persist::{IntentPersister, IntentPersisterImpl}, tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, transaction_preparator::TransactionPreparator, }; @@ -93,21 +92,18 @@ where committed_pubkeys, transaction_preparator, task_info_fetcher, - persister ), fields(stage = "commit") )] - pub async fn commit( + pub async fn commit( &mut self, committed_pubkeys: &[Pubkey], transaction_preparator: &T, task_info_fetcher: &CacheTaskInfoFetcher, - persister: &Option

, ) -> IntentExecutorResult where T: TransactionPreparator, F: TaskInfoFetcher, - P: IntentPersister, { let commit_result = loop { self.state.current_attempt += 1; @@ -118,7 +114,6 @@ where &self.authority, transaction_preparator, &mut self.state.commit_strategy, - persister, ) .await .map_err(IntentExecutorError::FailedCommitPreparationError)?; @@ -286,7 +281,6 @@ where lookup_tables_keys: vec![], standalone_action_nonce: None, }, - &None::, ) .await .map_err(IntentExecutorError::FailedCommitPreparationError)? @@ -366,17 +360,15 @@ where const RECURSION_CEILING: u8 = 10; #[instrument( - skip(self, transaction_preparator, persister), + skip(self, transaction_preparator), fields(stage = "finalize") )] - pub async fn finalize( + pub async fn finalize( &mut self, transaction_preparator: &T, - persister: &Option

, ) -> IntentExecutorResult where T: TransactionPreparator, - P: IntentPersister, { let finalize_result = loop { self.state.current_attempt += 1; @@ -387,7 +379,6 @@ where &self.authority, transaction_preparator, &mut self.state.finalize_strategy, - persister, ) .await .map_err(IntentExecutorError::FailedFinalizePreparationError)?; diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index b4f83517c..0cda26363 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -19,7 +19,6 @@ use crate::{ two_stage_executor::{Committed, Initialized, TwoStageExecutor}, IntentExecutionReport, }, - persist::IntentPersister, tasks::{ task_builder::TaskBuilderError, task_strategist::{TaskStrategist, TransactionStrategy}, @@ -30,22 +29,20 @@ use crate::{ }, }; -pub async fn prepare_and_execute_strategy( +pub async fn prepare_and_execute_strategy( client: &IntentExecutionClient, authority: &Keypair, transaction_preparator: &T, transaction_strategy: &mut TransactionStrategy, - persister: &Option

, ) -> IntentExecutorResult< IntentExecutorResult, TransactionPreparatorError, > where T: TransactionPreparator, - P: IntentPersister, { let prepared_message = transaction_preparator - .prepare_for_strategy(authority, transaction_strategy, persister) + .prepare_for_strategy(authority, transaction_strategy) .await?; let execution_result = client @@ -261,16 +258,13 @@ where junk } -pub(in crate::intent_executor) async fn execute_with_timeout< - P: IntentPersister, ->( +pub(in crate::intent_executor) async fn execute_with_timeout( time_left: Option, mut executor: impl StageExecutor, - persister: &Option

, ) -> IntentExecutorResult { if executor.has_callbacks() { if let Some(time_left) = time_left { - match timeout(time_left, executor.execute(persister)).await { + match timeout(time_left, executor.execute()).await { Ok(res) => return res, Err(_) => { // The race between callback and intent txn is handled @@ -289,16 +283,13 @@ pub(in crate::intent_executor) async fn execute_with_timeout< } } - executor.execute(persister).await + executor.execute().await } #[async_trait] pub(in crate::intent_executor) trait StageExecutor { fn has_callbacks(&self) -> bool; - async fn execute( - &mut self, - persister: &Option

, - ) -> IntentExecutorResult; + async fn execute(&mut self) -> IntentExecutorResult; fn execute_callbacks( &mut self, signature: Option, @@ -323,15 +314,11 @@ where self.inner.has_callbacks() } - async fn execute( - &mut self, - persister: &Option

, - ) -> IntentExecutorResult { + async fn execute(&mut self) -> IntentExecutorResult { self.inner .execute( self.committed_pubkeys, self.transaction_preparator, - persister, ) .await } @@ -365,16 +352,12 @@ where self.inner.has_callbacks() } - async fn execute( - &mut self, - persister: &Option

, - ) -> IntentExecutorResult { + async fn execute(&mut self) -> IntentExecutorResult { self.inner .commit( self.committed_pubkeys, self.transaction_preparator, self.task_info_fetcher, - persister, ) .await } @@ -404,12 +387,9 @@ where self.inner.has_callbacks() } - async fn execute( - &mut self, - persister: &Option

, - ) -> IntentExecutorResult { + async fn execute(&mut self) -> IntentExecutorResult { self.inner - .finalize(self.transaction_preparator, persister) + .finalize(self.transaction_preparator) .await } diff --git a/magicblock-committor-service/src/persist/commit_persister.rs b/magicblock-committor-service/src/persist/commit_persister.rs deleted file mode 100644 index 375dd7f59..000000000 --- a/magicblock-committor-service/src/persist/commit_persister.rs +++ /dev/null @@ -1,1067 +0,0 @@ -use std::{ - collections::BTreeMap, - path::Path, - sync::{Arc, Mutex}, -}; - -use magicblock_core::intent::CommittedAccount; -use magicblock_program::magic_scheduled_base_intent::{ - CommitAndUndelegate, CommitType as IntentCommitType, MagicIntentBundle, - ScheduledIntentBundle, UndelegateType, -}; -use solana_account::Account; -use solana_pubkey::Pubkey; -use solana_transaction::Transaction; -use tracing::warn; - -use super::{ - db::CommitStatusRow, error::CommitPersistResult, utils::now, CommitStatus, - CommitStrategy, CommitType, CommittsDb, MessageSignatures, -}; -use crate::{ - intent_executor::ExecutionOutput, persist::db::BundleSignatureRow, -}; - -const POISONED_MUTEX_MSG: &str = "Commitor Persister lock poisoned"; - -/// Records lifespan pf BaseIntent -pub trait IntentPersister: Send + Sync + Clone + 'static { - /// Starts persisting BaseIntents - fn start_base_intents( - &self, - base_intent: &[ScheduledIntentBundle], - ) -> CommitPersistResult<()>; - /// Starts persisting BaseIntent - fn start_base_intent( - &self, - base_intent: &ScheduledIntentBundle, - ) -> CommitPersistResult<()>; - fn set_commit_id( - &self, - message_id: u64, - pubkey: &Pubkey, - commit_id: u64, - ) -> CommitPersistResult<()>; - fn set_commit_strategy( - &self, - commit_id: u64, - pubkey: &Pubkey, - value: CommitStrategy, - ) -> CommitPersistResult<()>; - fn update_status_by_message( - &self, - message_id: u64, - pubkey: &Pubkey, - status: CommitStatus, - ) -> CommitPersistResult<()>; - fn update_status_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - status: CommitStatus, - ) -> CommitPersistResult<()>; - fn get_commit_statuses_by_message( - &self, - message_id: u64, - ) -> CommitPersistResult>; - fn pending_intent_bundles( - &self, - min_created_at: u64, - ) -> CommitPersistResult>; - fn get_commit_status_by_message( - &self, - message_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult>; - fn get_signatures_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult>; - fn get_bundle_signatures( - &self, - message_id: u64, - ) -> CommitPersistResult>; - fn finalize_base_intent( - &self, - message_id: u64, - execution_output: ExecutionOutput, - ) -> CommitPersistResult<()>; -} - -#[derive(Clone)] -pub struct IntentPersisterImpl { - // DB that tracks lifespan of Commit intents - commits_db: Arc>, - // TODO(edwin): add something like - // actions_db: Arc> -} - -impl IntentPersisterImpl { - pub fn try_new

(db_file: P) -> CommitPersistResult - where - P: AsRef, - { - let db = CommittsDb::new(db_file)?; - db.create_commit_status_table()?; - db.create_bundle_signature_table()?; - - Ok(Self { - commits_db: Arc::new(Mutex::new(db)), - }) - } - - pub fn create_commit_rows( - intent_bundle: &ScheduledIntentBundle, - ) -> Vec { - let created_at = now(); - - let create_row = |undelegate: bool, account: &CommittedAccount| { - let data = &account.account.data; - let (commit_type, data) = if data.is_empty() { - (CommitType::EmptyAccount, None) - } else { - (CommitType::DataAccount, Some(data.clone())) - }; - - CommitStatusRow { - message_id: intent_bundle.id, - commit_id: 0, // Not known at creation, set later - pubkey: account.pubkey, - delegated_account_owner: account.account.owner, - slot: intent_bundle.slot, - ephemeral_blockhash: intent_bundle.blockhash, - undelegate, - lamports: account.account.lamports, - data, - commit_type, - created_at, - commit_strategy: CommitStrategy::default(), - commit_status: CommitStatus::Pending, - last_retried_at: created_at, - retries_count: 0, - } - }; - - [ - (false, intent_bundle.get_commit_intent_accounts()), - (true, intent_bundle.get_undelegate_intent_accounts()), - (false, intent_bundle.get_commit_finalize_intent_accounts()), - ( - true, - intent_bundle - .get_commit_finalize_and_undelegate_intent_accounts(), - ), - ] - .into_iter() - .filter_map(|(undelegate, accounts)| { - accounts.map(|accounts| (undelegate, accounts)) - }) - .flat_map(|(undelegate, accounts)| { - accounts - .iter() - .map(move |account| create_row(undelegate, account)) - }) - .collect() - } -} - -impl IntentPersister for IntentPersisterImpl { - fn start_base_intents( - &self, - base_intents: &[ScheduledIntentBundle], - ) -> CommitPersistResult<()> { - let commit_rows = base_intents - .iter() - .flat_map(Self::create_commit_rows) - .collect::>(); - // Insert all commit rows into the database - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .insert_commit_status_rows(&commit_rows)?; - Ok(()) - } - - fn start_base_intent( - &self, - base_intents: &ScheduledIntentBundle, - ) -> CommitPersistResult<()> { - let commit_row = Self::create_commit_rows(base_intents); - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .insert_commit_status_rows(&commit_row)?; - - Ok(()) - } - - fn set_commit_id( - &self, - message_id: u64, - pubkey: &Pubkey, - commit_id: u64, - ) -> CommitPersistResult<()> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .set_commit_id(message_id, pubkey, commit_id) - } - - fn set_commit_strategy( - &self, - commit_id: u64, - pubkey: &Pubkey, - value: CommitStrategy, - ) -> CommitPersistResult<()> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .set_commit_strategy(commit_id, pubkey, value) - } - - fn update_status_by_message( - &self, - message_id: u64, - pubkey: &Pubkey, - status: CommitStatus, - ) -> CommitPersistResult<()> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .update_status_by_message(message_id, pubkey, &status) - } - - fn update_status_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - status: CommitStatus, - ) -> CommitPersistResult<()> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .update_status_by_commit(commit_id, pubkey, &status) - } - - fn get_commit_statuses_by_message( - &self, - message_id: u64, - ) -> CommitPersistResult> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .get_commit_statuses_by_id(message_id) - } - - /// Returns pending bundles created at or after `min_created_at`. - /// NOTE: this constructs `ScheduleIntentBundle` only from existing information. - /// As persister doesn't save `ScheduleIntentBundle::payer` info, Pubkey::default is used. - /// `CommittedAccount` information like slot may also be outdated. - /// It is the responsibility of the calling site to refresh data in intent. - fn pending_intent_bundles( - &self, - min_created_at: u64, - ) -> CommitPersistResult> { - let rows = self - .commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .get_pending_commit_statuses(min_created_at)?; - - Ok(pending_rows_to_scheduled_intent_bundles( - rows, - min_created_at, - )) - } - - fn get_commit_status_by_message( - &self, - message_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .get_commit_status(message_id, pubkey) - } - - fn get_signatures_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .get_signatures_by_commit(commit_id, pubkey) - } - - fn get_bundle_signatures( - &self, - message_id: u64, - ) -> CommitPersistResult> { - self.commits_db - .lock() - .expect(POISONED_MUTEX_MSG) - .get_bundle_signature_by_bundle_id(message_id) - } - - fn finalize_base_intent( - &self, - message_id: u64, - execution_output: ExecutionOutput, - ) -> CommitPersistResult<()> { - let (commit_signature, finalize_signature) = match execution_output { - ExecutionOutput::SingleStage(signature) => (signature, signature), - ExecutionOutput::TwoStage { - commit_signature, - finalize_signature, - } => (commit_signature, finalize_signature), - }; - - let bundle_signature_row = BundleSignatureRow::new( - message_id, - commit_signature, - finalize_signature, - ); - let commits_db = self.commits_db.lock().expect(POISONED_MUTEX_MSG); - commits_db.insert_bundle_signature_row(&bundle_signature_row)?; - Ok(()) - } -} - -/// Blanket implementation for Option -impl IntentPersister for Option { - fn start_base_intents( - &self, - base_intents: &[ScheduledIntentBundle], - ) -> CommitPersistResult<()> { - match self { - Some(persister) => persister.start_base_intents(base_intents), - None => Ok(()), - } - } - - fn start_base_intent( - &self, - base_intents: &ScheduledIntentBundle, - ) -> CommitPersistResult<()> { - match self { - Some(persister) => persister.start_base_intent(base_intents), - None => Ok(()), - } - } - - fn set_commit_id( - &self, - message_id: u64, - pubkey: &Pubkey, - commit_id: u64, - ) -> CommitPersistResult<()> { - match self { - Some(persister) => { - persister.set_commit_id(message_id, pubkey, commit_id) - } - None => Ok(()), - } - } - - fn set_commit_strategy( - &self, - commit_id: u64, - pubkey: &Pubkey, - value: CommitStrategy, - ) -> CommitPersistResult<()> { - match self { - Some(persister) => { - persister.set_commit_strategy(commit_id, pubkey, value) - } - None => Ok(()), - } - } - - fn update_status_by_message( - &self, - message_id: u64, - pubkey: &Pubkey, - status: CommitStatus, - ) -> CommitPersistResult<()> { - match self { - Some(persister) => { - persister.update_status_by_message(message_id, pubkey, status) - } - None => Ok(()), - } - } - - fn update_status_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - status: CommitStatus, - ) -> CommitPersistResult<()> { - match self { - Some(persister) => { - persister.update_status_by_commit(commit_id, pubkey, status) - } - None => Ok(()), - } - } - - fn get_commit_statuses_by_message( - &self, - message_id: u64, - ) -> CommitPersistResult> { - match self { - Some(persister) => { - persister.get_commit_statuses_by_message(message_id) - } - None => Ok(Vec::new()), - } - } - - fn pending_intent_bundles( - &self, - min_created_at: u64, - ) -> CommitPersistResult> { - match self { - Some(persister) => persister.pending_intent_bundles(min_created_at), - None => Ok(Vec::new()), - } - } - - fn get_commit_status_by_message( - &self, - message_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - match self { - Some(persister) => { - persister.get_commit_status_by_message(message_id, pubkey) - } - None => Ok(None), - } - } - - fn get_signatures_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - match self { - Some(persister) => { - persister.get_signatures_by_commit(commit_id, pubkey) - } - None => Ok(None), - } - } - - fn get_bundle_signatures( - &self, - message_id: u64, - ) -> CommitPersistResult> { - match self { - Some(persister) => persister.get_bundle_signatures(message_id), - None => Ok(None), - } - } - - fn finalize_base_intent( - &self, - message_id: u64, - execution_output: ExecutionOutput, - ) -> CommitPersistResult<()> { - match self { - Some(persister) => { - persister.finalize_base_intent(message_id, execution_output) - } - None => Ok(()), - } - } -} - -fn pending_rows_to_scheduled_intent_bundles( - rows: Vec, - min_created_at: u64, -) -> Vec { - let mut grouped_rows = BTreeMap::>::new(); - for row in rows { - grouped_rows.entry(row.message_id).or_default().push(row); - } - - grouped_rows - .into_iter() - // Filter bundles where any row is older than the recovery window - .filter(|(message_id, rows)| { - if rows.iter().any(|row| row.created_at < min_created_at) { - warn!( - intent_id = message_id, - "Skipping pending commit intent: too old to recover" - ); - false - } else { - true - } - }) - // Filter malformed items - .filter_map(|(message_id, rows)| { - let first = rows.first()?; - let slot = first.slot; - let blockhash = first.ephemeral_blockhash; - if rows.iter().any(|r| r.slot != slot || r.ephemeral_blockhash != blockhash) { - warn!( - intent_id = message_id, - "Skipping pending commit intent: rows disagree on slot or ephemeral_blockhash" - ); - None - } else { - Some(rows) - } - }) - .filter_map(intent_bundle_from_rows) - .collect() -} - -fn intent_bundle_from_rows( - rows: Vec, -) -> Option { - let first = rows.first()?; - let message_id = first.message_id; - let slot = first.slot; - let blockhash = first.ephemeral_blockhash; - - let mut commit_finalize_accounts = Vec::new(); - let mut commit_finalize_and_undelegate_accounts = Vec::new(); - for row in rows { - let Some((account, undelegate)) = committed_account_from_row(row) - else { - continue; - }; - if undelegate { - commit_finalize_and_undelegate_accounts.push(account); - } else { - commit_finalize_accounts.push(account); - } - } - - let mut intent_bundle = MagicIntentBundle::default(); - if !commit_finalize_accounts.is_empty() { - intent_bundle.commit_finalize = - Some(IntentCommitType::Standalone(commit_finalize_accounts)); - } - if !commit_finalize_and_undelegate_accounts.is_empty() { - intent_bundle.commit_finalize_and_undelegate = - Some(CommitAndUndelegate { - commit_action: IntentCommitType::Standalone( - commit_finalize_and_undelegate_accounts, - ), - undelegate_action: UndelegateType::Standalone, - }); - } - if intent_bundle.is_empty() { - return None; - } - - Some(ScheduledIntentBundle { - id: message_id, - slot, - blockhash, - sent_transaction: Transaction::default(), - payer: Pubkey::default(), - intent_bundle, - }) -} - -fn committed_account_from_row( - row: CommitStatusRow, -) -> Option<(CommittedAccount, bool)> { - if row.commit_type == CommitType::DataAccount && row.data.is_none() { - warn!( - intent_id = row.message_id, - pubkey = %row.pubkey, - "Skipping pending data-account commit row without account data" - ); - return None; - } - - Some(( - CommittedAccount { - pubkey: row.pubkey, - account: Account { - lamports: row.lamports, - data: row.data.unwrap_or_default(), - owner: row.delegated_account_owner, - executable: false, - rent_epoch: 0, - }, - remote_slot: 0, - }, - row.undelegate, - )) -} - -#[cfg(test)] -mod tests { - use magicblock_core::intent::CommittedAccount; - use magicblock_program::magic_scheduled_base_intent::{ - CommitAndUndelegate, CommitType, MagicIntentBundle, UndelegateType, - }; - use solana_account::Account; - use solana_hash::Hash; - use solana_pubkey::Pubkey; - use solana_signature::Signature; - use solana_transaction::Transaction; - use tempfile::NamedTempFile; - - use super::*; - use crate::{ - committor_processor::RECOVERY_MAX_AGE_SECS, - persist::{types, CommitStatusSignatures}, - test_utils, - }; - - fn create_test_persister() -> (IntentPersisterImpl, NamedTempFile) { - test_utils::init_test_logger(); - let temp_file = NamedTempFile::new().unwrap(); - let persister = IntentPersisterImpl::try_new(temp_file.path()).unwrap(); - (persister, temp_file) - } - - fn test_accounts() -> Vec { - let account1 = Account { - lamports: 1000, - owner: Pubkey::new_unique(), - data: vec![], - executable: false, - rent_epoch: 0, - }; - let account2 = Account { - lamports: 2000, - owner: Pubkey::new_unique(), - data: vec![1, 2, 3], - executable: false, - rent_epoch: 0, - }; - - vec![ - CommittedAccount { - pubkey: Pubkey::new_unique(), - account: account1, - remote_slot: 1, - }, - CommittedAccount { - pubkey: Pubkey::new_unique(), - account: account2, - remote_slot: 2, - }, - ] - } - - fn commit_only_bundle() -> MagicIntentBundle { - MagicIntentBundle { - commit: Some(CommitType::Standalone(test_accounts())), - commit_and_undelegate: None, - commit_finalize: None, - commit_finalize_and_undelegate: None, - standalone_actions: vec![], - } - } - - fn undelegate_only_bundle() -> MagicIntentBundle { - MagicIntentBundle { - commit: None, - commit_and_undelegate: Some(CommitAndUndelegate { - commit_action: CommitType::Standalone(test_accounts()), - undelegate_action: UndelegateType::Standalone, - }), - commit_finalize: None, - commit_finalize_and_undelegate: None, - standalone_actions: vec![], - } - } - - fn bundle_both() -> MagicIntentBundle { - MagicIntentBundle { - commit: Some(CommitType::Standalone(test_accounts())), - commit_and_undelegate: Some(CommitAndUndelegate { - commit_action: CommitType::Standalone(test_accounts()), - undelegate_action: UndelegateType::Standalone, - }), - commit_finalize: None, - commit_finalize_and_undelegate: None, - standalone_actions: vec![], - } - } - - fn bundle_none() -> MagicIntentBundle { - MagicIntentBundle { - commit: None, - commit_and_undelegate: None, - commit_finalize: None, - commit_finalize_and_undelegate: None, - standalone_actions: vec![], - } - } - - fn create_test_message( - id: u64, - intent_bundle: MagicIntentBundle, - ) -> ScheduledIntentBundle { - ScheduledIntentBundle { - id, - slot: 100, - blockhash: Hash::new_unique(), - sent_transaction: Transaction::default(), - payer: Pubkey::new_unique(), - intent_bundle, - } - } - - #[test] - fn test_create_commit_rows_commit_only() { - let message = create_test_message(1, commit_only_bundle()); - let rows = IntentPersisterImpl::create_commit_rows(&message); - - assert_eq!(rows.len(), 2); - assert!(rows.iter().all(|r| !r.undelegate)); - - let empty_account = rows.iter().find(|r| r.data.is_none()).unwrap(); - assert_eq!(empty_account.commit_type, types::CommitType::EmptyAccount); - assert_eq!(empty_account.lamports, 1000); - - let data_account = rows.iter().find(|r| r.data.is_some()).unwrap(); - assert_eq!(data_account.commit_type, types::CommitType::DataAccount); - assert_eq!(data_account.lamports, 2000); - assert_eq!(data_account.data, Some(vec![1, 2, 3])); - } - - #[test] - fn test_create_commit_rows_undelegate_only() { - let message = create_test_message(1, undelegate_only_bundle()); - let rows = IntentPersisterImpl::create_commit_rows(&message); - - assert_eq!(rows.len(), 2); - assert!(rows.iter().all(|r| r.undelegate)); - } - - #[test] - fn test_create_commit_rows_both_intents() { - let message = create_test_message(1, bundle_both()); - let rows = IntentPersisterImpl::create_commit_rows(&message); - - // 2 from commit + 2 from commit_and_undelegate - assert_eq!(rows.len(), 4); - - let commit_rows = rows.iter().filter(|r| !r.undelegate).count(); - let undelegate_rows = rows.iter().filter(|r| r.undelegate).count(); - - assert_eq!(commit_rows, 2); - assert_eq!(undelegate_rows, 2); - } - - #[test] - fn test_start_base_message_commit_only() { - let (persister, _temp_file) = create_test_persister(); - let message = create_test_message(1, commit_only_bundle()); - - persister.start_base_intent(&message).unwrap(); - - let expected_statuses = - IntentPersisterImpl::create_commit_rows(&message); - let statuses = persister.get_commit_statuses_by_message(1).unwrap(); - - assert_eq!(statuses.len(), expected_statuses.len()); - assert_eq!(expected_statuses[0], statuses[0]); - assert_eq!(expected_statuses[1], statuses[1]); - } - - #[test] - fn test_start_base_messages_mixed() { - let (persister, _temp_file) = create_test_persister(); - let message1 = create_test_message(1, commit_only_bundle()); // 2 rows - let message2 = create_test_message(2, undelegate_only_bundle()); // 2 rows - let message3 = create_test_message(3, bundle_both()); // 4 rows - let message4 = create_test_message(4, bundle_none()); // 0 rows - - persister - .start_base_intents(&[message1, message2, message3, message4]) - .unwrap(); - - assert_eq!( - persister.get_commit_statuses_by_message(1).unwrap().len(), - 2 - ); - assert_eq!( - persister.get_commit_statuses_by_message(2).unwrap().len(), - 2 - ); - assert_eq!( - persister.get_commit_statuses_by_message(3).unwrap().len(), - 4 - ); - assert_eq!( - persister.get_commit_statuses_by_message(4).unwrap().len(), - 0 - ); - } - - #[test] - fn test_update_status() { - let (persister, _temp_file) = create_test_persister(); - let message = create_test_message(1, commit_only_bundle()); - persister.start_base_intent(&message).unwrap(); - - let pubkey = message.get_all_committed_pubkeys()[0]; - - // Update by message - persister - .update_status_by_message(1, &pubkey, CommitStatus::Pending) - .unwrap(); - - let updated = persister - .get_commit_status_by_message(1, &pubkey) - .unwrap() - .unwrap(); - assert_eq!(updated.commit_status, CommitStatus::Pending); - - // Set commit ID and update by commit - persister.set_commit_id(1, &pubkey, 100).unwrap(); - persister - .update_status_by_commit( - 100, - &pubkey, - CommitStatus::BufferAndChunkInitialized, - ) - .unwrap(); - - let updated = persister - .get_commit_status_by_message(1, &pubkey) - .unwrap() - .unwrap(); - assert_eq!( - updated.commit_status, - CommitStatus::BufferAndChunkInitialized - ); - } - - #[test] - fn test_set_commit_strategy() { - let (persister, _temp_file) = create_test_persister(); - let message = create_test_message(1, commit_only_bundle()); - persister.start_base_intent(&message).unwrap(); - - let pubkey = message.get_all_committed_pubkeys()[0]; - persister.set_commit_id(1, &pubkey, 100).unwrap(); - - persister - .set_commit_strategy(100, &pubkey, CommitStrategy::StateArgs) - .unwrap(); - - let updated = persister - .get_commit_status_by_message(1, &pubkey) - .unwrap() - .unwrap(); - assert_eq!(updated.commit_strategy, CommitStrategy::StateArgs); - } - - #[test] - fn test_get_signatures() { - let (persister, _temp_file) = create_test_persister(); - let message = create_test_message(1, commit_only_bundle()); - persister.start_base_intent(&message).unwrap(); - - let statuses = persister.get_commit_statuses_by_message(1).unwrap(); - let pubkey = statuses[0].pubkey; - persister.set_commit_id(1, &pubkey, 100).unwrap(); - - let process_sig = Signature::new_unique(); - let finalize_sig = Signature::new_unique(); - let status = CommitStatus::Succeeded(CommitStatusSignatures { - commit_stage_signature: process_sig, - finalize_stage_signature: Some(finalize_sig), - }); - - persister - .update_status_by_commit(100, &pubkey, status) - .unwrap(); - - let sigs = persister - .get_signatures_by_commit(100, &pubkey) - .unwrap() - .unwrap(); - assert_eq!(sigs.commit_stage_signature, process_sig); - assert_eq!(sigs.finalize_stage_signature, Some(finalize_sig)); - } - - #[test] - fn test_finalize_base_intent() { - let (persister, _temp_file) = create_test_persister(); - let message_id = 1; - - let commit_sig = Signature::new_unique(); - let finalize_sig = Signature::new_unique(); - let execution_output = ExecutionOutput::TwoStage { - commit_signature: commit_sig, - finalize_signature: finalize_sig, - }; - - persister - .finalize_base_intent(message_id, execution_output) - .unwrap(); - - let bundle_sig = persister - .get_bundle_signatures(message_id) - .unwrap() - .unwrap(); - assert_eq!(bundle_sig.bundle_id, message_id); - assert_eq!(bundle_sig.commit_stage_signature, commit_sig); - assert_eq!(bundle_sig.finalize_stage_signature, finalize_sig); - } - - #[test] - fn test_empty_accounts_not_persisted() { - let (persister, _temp_file) = create_test_persister(); - let message = create_test_message(1, bundle_none()); - - persister.start_base_intent(&message).unwrap(); - - let statuses = persister.get_commit_statuses_by_message(1).unwrap(); - assert_eq!(statuses.len(), 0); - } - - fn pending_row( - message_id: u64, - pubkey: Pubkey, - owner: Pubkey, - blockhash: Hash, - undelegate: bool, - data: Option>, - ) -> CommitStatusRow { - let commit_type = - if data.as_ref().map(|data| data.is_empty()) == Some(false) { - types::CommitType::DataAccount - } else { - types::CommitType::EmptyAccount - }; - CommitStatusRow { - message_id, - pubkey, - commit_id: 0, - delegated_account_owner: owner, - slot: 42, - ephemeral_blockhash: blockhash, - undelegate, - lamports: 1_000, - data, - commit_type, - created_at: 1, - commit_status: CommitStatus::Pending, - commit_strategy: Default::default(), - last_retried_at: 1, - retries_count: 0, - } - } - - #[test] - fn pending_rows_reconstruct_commit_finalize_bundle() { - let owner = Pubkey::new_unique(); - let blockhash = Hash::new_unique(); - let commit_pubkey = Pubkey::new_unique(); - let undelegate_pubkey = Pubkey::new_unique(); - - // default rows have created_at=1; use min_created_at=0 so nothing is filtered - let bundles = pending_rows_to_scheduled_intent_bundles( - vec![ - pending_row( - 9, - commit_pubkey, - owner, - blockhash, - false, - Some(vec![1, 2, 3]), - ), - pending_row(9, undelegate_pubkey, owner, blockhash, true, None), - ], - 0, - ); - - assert_eq!(bundles.len(), 1); - let bundle = &bundles[0]; - assert_eq!(bundle.id, 9); - assert_eq!(bundle.slot, 42); - assert_eq!(bundle.blockhash, blockhash); - - let commit_accounts = - bundle.get_commit_finalize_intent_accounts().unwrap(); - assert_eq!(commit_accounts.len(), 1); - assert_eq!(commit_accounts[0].pubkey, commit_pubkey); - assert_eq!(commit_accounts[0].account.data, vec![1, 2, 3]); - assert_eq!(commit_accounts[0].remote_slot, 0); - - let undelegate_accounts = bundle - .get_commit_finalize_and_undelegate_intent_accounts() - .unwrap(); - assert_eq!(undelegate_accounts.len(), 1); - assert_eq!(undelegate_accounts[0].pubkey, undelegate_pubkey); - assert_eq!(undelegate_accounts[0].account.owner, owner); - assert!(bundle.has_undelegate_intent()); - } - - #[test] - fn pending_rows_skip_intents_older_than_recovery_window() { - let owner = Pubkey::new_unique(); - let blockhash = Hash::new_unique(); - let recovery_time = RECOVERY_MAX_AGE_SECS + 1; - let min_created_at = recovery_time - RECOVERY_MAX_AGE_SECS; // = 1 - - // At the boundary → included - let mut in_window = pending_row( - 1, - Pubkey::new_unique(), - owner, - blockhash, - false, - Some(vec![1]), - ); - in_window.created_at = min_created_at; - let included = pending_rows_to_scheduled_intent_bundles( - vec![in_window], - min_created_at, - ); - assert_eq!(included.len(), 1); - assert_eq!(included[0].id, 1); - - // Recent (just created) → included - let mut recent = pending_row( - 2, - Pubkey::new_unique(), - owner, - blockhash, - false, - Some(vec![1]), - ); - recent.created_at = recovery_time; - let recent_result = pending_rows_to_scheduled_intent_bundles( - vec![recent], - min_created_at, - ); - assert_eq!(recent_result.len(), 1); - - // Too old → excluded - let mut too_old = pending_row( - 3, - Pubkey::new_unique(), - owner, - blockhash, - false, - Some(vec![1]), - ); - too_old.created_at = min_created_at - 1; // = 0 - let excluded_old = pending_rows_to_scheduled_intent_bundles( - vec![too_old], - min_created_at, - ); - assert!(excluded_old.is_empty()); - } -} diff --git a/magicblock-committor-service/src/persist/db.rs b/magicblock-committor-service/src/persist/db.rs deleted file mode 100644 index 314c627ff..000000000 --- a/magicblock-committor-service/src/persist/db.rs +++ /dev/null @@ -1,1216 +0,0 @@ -use std::{fmt, path::Path, str::FromStr}; - -use rusqlite::{params, Connection, Result, Transaction}; -use solana_hash::Hash; -use solana_program::clock::Slot; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use tracing::error; - -use super::{ - error::CommitPersistResult, - utils::{i64_into_u64, u64_into_i64}, - CommitStatus, CommitStatusSignatures, CommitStrategy, CommitType, -}; -use crate::persist::{error::CommitPersistError, utils::now}; - -// ----------------- -// CommitStatusRow -// ----------------- -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CommitStatusRow { - /// ID of the messages within which this account is committed - pub message_id: u64, - /// The on chain address of the delegated account - pub pubkey: Pubkey, - /// Commit ID of an account - /// Determined and set during runtime - pub commit_id: u64, - /// The original owner of the delegated account on chain - pub delegated_account_owner: Pubkey, - /// The ephemeral slot at which those changes were requested - pub slot: Slot, - /// The ephemeral blockhash at which those changes were requested - pub ephemeral_blockhash: Hash, - /// If we also undelegate the account after committing it - pub undelegate: bool, - /// Lamports of the account in the ephemeral - pub lamports: u64, - /// The account data in the ephemeral (only set if the commit is for a data account) - pub data: Option>, - /// The type of commit that was requested, i.e. lamports only or including data - pub commit_type: CommitType, - /// Time since epoch at which the commit was requested - pub created_at: u64, - /// The current status of the commit - /// Includes the bundle_id which will be the same for accounts whose commits - /// need to be applied atomically in a single transaction - /// For single accounts a bundle_id will be generated as well for consistency - /// For Pending commits the bundle_id is not set - pub commit_status: CommitStatus, - /// Strategy defined for Commit of a particular account - pub commit_strategy: CommitStrategy, - /// Time since epoch at which the commit was last retried - pub last_retried_at: u64, - /// Number of times the commit was retried - pub retries_count: u16, -} - -#[derive(Debug)] -pub struct MessageSignatures { - /// The signature of the transaction on chain that executed Commit Stage - pub commit_stage_signature: Signature, - /// The signature of the transaction on chain that executed Finalize Stage - /// if they were executed in 1 tx it is the same as [Self::commit_stage_signature]. - pub finalize_stage_signature: Option, - /// Time since epoch at which the bundle signature was created - pub created_at: u64, -} - -impl fmt::Display for CommitStatusRow { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "CommitStatusRow {{ - message_id: {} - pubkey: {}, - commit_id: {}, - delegated_account_owner: {}, - slot: {}, - ephemeral_blockhash: {}, - undelegate: {}, - lamports: {}, - data.len: {}, - commit_type: {}, - created_at: {}, - commit_status: {}, - commit_strategy: {}, - last_retried_at: {}, - retries_count: {} -}}", - self.message_id, - self.pubkey, - self.commit_id, - self.delegated_account_owner, - self.slot, - self.ephemeral_blockhash, - self.undelegate, - self.lamports, - self.data.as_ref().map(|x| x.len()).unwrap_or_default(), - self.commit_type.as_str(), - self.created_at, - self.commit_status, - self.commit_strategy.as_str(), - self.last_retried_at, - self.retries_count - ) - } -} - -const ALL_COMMIT_STATUS_COLUMNS: &str = " - message_id, - pubkey, - commit_id, - delegated_account_owner, - slot, - ephemeral_blockhash, - undelegate, - lamports, - data, - commit_type, - created_at, - commit_strategy, - commit_status, - commit_stage_signature, - finalize_stage_signature, - last_retried_at, - retries_count -"; - -const SELECT_ALL_COMMIT_STATUS_COLUMNS: &str = r#" -SELECT - message_id, - pubkey, - commit_id, - delegated_account_owner, - slot, - ephemeral_blockhash, - undelegate, - lamports, - data, - commit_type, - created_at, - commit_strategy, - commit_status, - commit_stage_signature, - finalize_stage_signature, - last_retried_at, - retries_count -FROM commit_status -"#; - -// ----------------- -// Bundle Signature -// ----------------- -// The BundleSignature table exists to store mappings from bundle_id to the signatures used -// to commit/finalize these bundles. -// The signatures are repeated in the commit_status table, however the rows in there have a -// different lifetime than the bundle signature rows. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BundleSignatureRow { - /// The id of the bundle that was commited - /// If an account was not part of a bundle it is treated as a single account bundle - /// for consistency. - /// The bundle_id is unique - pub bundle_id: u64, - /// The signature of the transaction that executed Commit stage - pub commit_stage_signature: Signature, - /// The signature of the transaction that executed Finalize stage - /// if Commit & Finalize stages this will equal [Self::commit_stage_signature] - pub finalize_stage_signature: Signature, - /// Time since epoch at which the bundle signature was created - pub created_at: u64, -} - -impl BundleSignatureRow { - pub fn new( - bundle_id: u64, - commit_stage_signature: Signature, - finalize_stage_signature: Signature, - ) -> Self { - let created_at = now(); - Self { - bundle_id, - commit_stage_signature, - finalize_stage_signature, - created_at, - } - } -} - -const ALL_BUNDLE_SIGNATURE_COLUMNS: &str = r#" - bundle_id, - commit_stage_signature, - finalize_stage_signature, - created_at -"#; - -const SELECT_ALL_BUNDLE_SIGNATURE_COLUMNS: &str = r#" -SELECT - bundle_id, - commit_stage_signature, - finalize_stage_signature, - created_at -FROM bundle_signature -"#; - -// ----------------- -// CommittorDb -// ----------------- -pub struct CommittsDb { - conn: Connection, -} - -impl CommittsDb { - pub fn new

(db_file: P) -> Result - where - P: AsRef, - { - let conn = Connection::open(db_file)?; - Ok(Self { conn }) - } - - pub fn path(&self) -> Option<&str> { - self.conn.path() - } - - // ----------------- - // Methods affecting both tables - // ----------------- - pub fn update_status_by_message( - &mut self, - message_id: u64, - pubkey: &Pubkey, - status: &CommitStatus, - ) -> CommitPersistResult<()> { - let query = "UPDATE commit_status - SET - commit_status = ?1, - commit_stage_signature = ?2, - finalize_stage_signature = ?3 - WHERE - pubkey = ?4 AND message_id = ?5"; - - self.conn.execute( - query, - params![ - status.as_str(), - status - .signatures() - .map(|s| s.commit_stage_signature.to_string()), - status - .signatures() - .and_then(|s| s.finalize_stage_signature) - .map(|s| s.to_string()), - pubkey.to_string(), - u64_into_i64(message_id) - ], - )?; - - Ok(()) - } - - pub fn update_status_by_commit( - &mut self, - commit_id: u64, - pubkey: &Pubkey, - status: &CommitStatus, - ) -> CommitPersistResult<()> { - let query = "UPDATE commit_status - SET - commit_status = ?1, - commit_stage_signature = ?2, - finalize_stage_signature = ?3 - WHERE - pubkey = ?4 AND commit_id = ?5"; - - self.conn.execute( - query, - params![ - status.as_str(), - status - .signatures() - .map(|s| s.commit_stage_signature.to_string()), - status - .signatures() - .and_then(|s| s.finalize_stage_signature) - .map(|s| s.to_string()), - pubkey.to_string(), - u64_into_i64(commit_id) - ], - )?; - - Ok(()) - } - - pub fn set_commit_strategy( - &mut self, - commit_id: u64, - pubkey: &Pubkey, - value: CommitStrategy, - ) -> CommitPersistResult<()> { - let query = "UPDATE commit_status - SET - commit_strategy = ?1 - WHERE - pubkey = ?2 AND commit_id = ?3"; - - self.conn.execute( - query, - params![ - value.as_str(), - pubkey.to_string(), - u64_into_i64(commit_id) - ], - )?; - - Ok(()) - } - - pub fn set_commit_id( - &mut self, - message_id: u64, - pubkey: &Pubkey, - commit_id: u64, - ) -> CommitPersistResult<()> { - let query = "UPDATE commit_status - SET - commit_id = ?1 - WHERE - pubkey = ?2 AND message_id = ?3"; - - self.conn.execute( - query, - params![ - u64_into_i64(commit_id), - pubkey.to_string(), - u64_into_i64(message_id) - ], - )?; - - Ok(()) - } - - // ----------------- - // Commit Status - // ----------------- - pub fn create_commit_status_table(&self) -> Result<()> { - match self.conn.execute_batch( - " - BEGIN; - CREATE TABLE IF NOT EXISTS commit_status ( - message_id INTEGER NOT NULL, - pubkey TEXT NOT NULL, - commit_id INTEGER NOT NULL, - delegated_account_owner TEXT NOT NULL, - slot INTEGER NOT NULL, - ephemeral_blockhash TEXT NOT NULL, - undelegate INTEGER NOT NULL, - lamports INTEGER NOT NULL, - data BLOB, - commit_type TEXT NOT NULL, - created_at INTEGER NOT NULL, - commit_strategy TEXT NOT NULL, - commit_status TEXT NOT NULL, - commit_stage_signature TEXT, - finalize_stage_signature TEXT, - last_retried_at INTEGER NOT NULL, - retries_count INTEGER NOT NULL, - PRIMARY KEY (message_id, commit_id, pubkey) - ); - CREATE INDEX IF NOT EXISTS idx_commits_pubkey ON commit_status (pubkey); - CREATE INDEX IF NOT EXISTS idx_commits_message_id ON commit_status (message_id); - CREATE INDEX IF NOT EXISTS idx_commits_commit_id ON commit_status (commit_id); - COMMIT;", - ) { - Ok(_) => Ok(()), - Err(err) => { - error!(error = ?err, "Failed to create commit_status table"); - Err(err) - } - } - } - - // ----------------- - // Bundle Signature - // ----------------- - pub fn create_bundle_signature_table(&self) -> Result<()> { - match self.conn.execute_batch( - " - BEGIN; - CREATE TABLE IF NOT EXISTS bundle_signature ( - bundle_id INTEGER NOT NULL PRIMARY KEY, - commit_stage_signature TEXT NOT NULL, - finalize_stage_signature TEXT, - created_at INTEGER NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_bundle_signature ON bundle_signature (bundle_id); - COMMIT;", - ) { - Ok(_) => Ok(()), - Err(err) => { - error!(error = ?err, "Failed to create bundle_signature table"); - Err(err) - } - } - } - - pub fn insert_bundle_signature_row( - &self, - bundle_signature: &BundleSignatureRow, - ) -> CommitPersistResult<()> { - let query = format!("INSERT OR REPLACE INTO bundle_signature ({ALL_BUNDLE_SIGNATURE_COLUMNS}) - VALUES (?1, ?2, ?3, ?4)"); - - self.conn.execute( - &query, - params![ - bundle_signature.bundle_id, - bundle_signature.commit_stage_signature.to_string(), - bundle_signature.finalize_stage_signature.to_string(), - u64_into_i64(bundle_signature.created_at) - ], - )?; - - Ok(()) - } - - pub fn get_bundle_signature_by_bundle_id( - &self, - bundle_id: u64, - ) -> CommitPersistResult> { - let query = format!( - "{SELECT_ALL_BUNDLE_SIGNATURE_COLUMNS} WHERE bundle_id = ?1" - ); - let stmt = &mut self.conn.prepare(&query)?; - let mut rows = stmt.query(params![bundle_id])?; - - if let Some(row) = rows.next()? { - let bundle_signature_row = Self::extract_bundle_signature_row(row)?; - Ok(Some(bundle_signature_row)) - } else { - Ok(None) - } - } - - fn extract_bundle_signature_row( - row: &rusqlite::Row, - ) -> CommitPersistResult { - let bundle_id: u64 = { - let bundle_id: i64 = row.get(0)?; - i64_into_u64(bundle_id) - }; - let commit_stage_signature = { - let processed_signature: String = row.get(1)?; - Signature::from_str(processed_signature.as_str())? - }; - let finalize_stage_signature = { - let finalized_signature: String = row.get(2)?; - Signature::from_str(finalized_signature.as_str())? - }; - let created_at: u64 = { - let created_at: i64 = row.get(3)?; - i64_into_u64(created_at) - }; - - Ok(BundleSignatureRow { - bundle_id, - commit_stage_signature, - finalize_stage_signature, - created_at, - }) - } - - pub fn insert_commit_status_rows( - &mut self, - commit_rows: &[CommitStatusRow], - ) -> CommitPersistResult<()> { - if commit_rows.is_empty() { - return Ok(()); - } - - let tx = self.conn.transaction()?; - for commit in commit_rows { - Self::insert_commit_status_row(&tx, commit)?; - } - tx.commit()?; - Ok(()) - } - - fn insert_commit_status_row( - tx: &Transaction<'_>, - commit: &CommitStatusRow, - ) -> CommitPersistResult<()> { - let (commit_stage_signature, finalize_stage_signature) = - match commit.commit_status.signatures() { - Some(sigs) => ( - Some(sigs.commit_stage_signature), - sigs.finalize_stage_signature, - ), - None => (None, None), - }; - tx.execute( - &format!( - "INSERT INTO commit_status ({ALL_COMMIT_STATUS_COLUMNS}) VALUES - (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)", - ), - params![ - u64_into_i64(commit.message_id), - commit.pubkey.to_string(), - u64_into_i64(commit.commit_id), - commit.delegated_account_owner.to_string(), - u64_into_i64(commit.slot), - commit.ephemeral_blockhash.to_string(), - if commit.undelegate { 1 } else { 0 }, - u64_into_i64(commit.lamports), - commit.data.as_deref(), - commit.commit_type.as_str(), - u64_into_i64(commit.created_at), - commit.commit_strategy.as_str(), - commit.commit_status.as_str(), - commit_stage_signature - .as_ref() - .map(|s| s.to_string()), - finalize_stage_signature - .as_ref() - .map(|s| s.to_string()), - u64_into_i64(commit.last_retried_at), - commit.retries_count, - ], - )?; - Ok(()) - } - - #[cfg(test)] - #[allow(dead_code)] - fn get_commit_statuses_by_pubkey( - &self, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - let query = - format!("{SELECT_ALL_COMMIT_STATUS_COLUMNS} WHERE pubkey = ?1"); - let stmt = &mut self.conn.prepare(&query)?; - let mut rows = stmt.query(params![pubkey.to_string()])?; - - extract_committor_rows(&mut rows) - } - - pub(crate) fn get_commit_statuses_by_id( - &self, - message_id: u64, - ) -> CommitPersistResult> { - let query = - format!("{SELECT_ALL_COMMIT_STATUS_COLUMNS} WHERE message_id = ?1"); - let stmt = &mut self.conn.prepare(&query)?; - let mut rows = stmt.query(params![u64_into_i64(message_id)])?; - - extract_committor_rows(&mut rows) - } - - pub(crate) fn get_pending_commit_statuses( - &self, - min_created_at: u64, - ) -> CommitPersistResult> { - let query = format!( - "WITH eligible_messages AS ( - SELECT message_id - FROM commit_status - WHERE commit_status = ?1 - GROUP BY message_id - HAVING MIN(created_at) >= ?2 - ) - {SELECT_ALL_COMMIT_STATUS_COLUMNS} - WHERE commit_status = ?1 - AND message_id IN ( - SELECT message_id FROM eligible_messages - ) - ORDER BY message_id, created_at, pubkey" - ); - let stmt = &mut self.conn.prepare(&query)?; - let mut rows = stmt.query(params![ - CommitStatus::Pending.as_str(), - u64_into_i64(min_created_at) - ])?; - - extract_committor_rows(&mut rows) - } - - pub(crate) fn get_commit_status( - &self, - message_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - let query = format!( - "{SELECT_ALL_COMMIT_STATUS_COLUMNS} WHERE message_id = ?1 AND pubkey = ?2" - ); - let stmt = &mut self.conn.prepare(&query)?; - let mut rows = - stmt.query(params![u64_into_i64(message_id), pubkey.to_string()])?; - - extract_committor_rows(&mut rows).map(|mut rows| rows.pop()) - } - - pub fn remove_commit_statuses_with_id( - &self, - message_id: u64, - ) -> CommitPersistResult<()> { - let query = "DELETE FROM commit_status WHERE message_id = ?1"; - let stmt = &mut self.conn.prepare(query)?; - stmt.execute(params![u64_into_i64(message_id)])?; - Ok(()) - } - - pub fn get_signatures_by_commit( - &self, - commit_id: u64, - pubkey: &Pubkey, - ) -> CommitPersistResult> { - let query = " - SELECT - commit_stage_signature, finalize_stage_signature, created_at - FROM commit_status - WHERE commit_id = ?1 AND pubkey = ?2 - LIMIT 1"; - - let mut stmt = self.conn.prepare(query)?; - let mut rows = - stmt.query(params![u64_into_i64(commit_id), pubkey.to_string()])?; - - let result = rows - .next()? - .map(|row| { - let commit_stage_signature: String = row.get(0)?; - let finalize_stage_signature: Option = row.get(1)?; - let created_at: i64 = row.get(2)?; - - Ok::<_, CommitPersistError>(MessageSignatures { - commit_stage_signature: Signature::from_str( - &commit_stage_signature, - )?, - finalize_stage_signature: finalize_stage_signature - .map(|s| Signature::from_str(&s)) - .transpose()?, - created_at: i64_into_u64(created_at), - }) - }) - .transpose()?; - - Ok(result) - } -} - -// ----------------- -// Commit Status Helpers -// ----------------- -fn extract_committor_rows( - rows: &mut rusqlite::Rows, -) -> CommitPersistResult> { - let mut commits = Vec::new(); - while let Some(row) = rows.next()? { - let commit_row = extract_committor_row(row)?; - commits.push(commit_row); - } - Ok(commits) -} - -fn extract_committor_row( - row: &rusqlite::Row, -) -> CommitPersistResult { - let message_id: u64 = { - let message_id: i64 = row.get(0)?; - i64_into_u64(message_id) - }; - - let pubkey = { - let pubkey: String = row.get(1)?; - Pubkey::try_from(pubkey.as_str())? - }; - let commit_id = { - let message_id: i64 = row.get(2)?; - i64_into_u64(message_id) - }; - let delegated_account_owner = { - let delegated_account_owner: String = row.get(3)?; - Pubkey::try_from(delegated_account_owner.as_str())? - }; - let slot: Slot = { - let slot: i64 = row.get(4)?; - i64_into_u64(slot) - }; - - let ephemeral_blockhash = { - let ephemeral_blockhash: String = row.get(5)?; - Hash::from_str(ephemeral_blockhash.as_str())? - }; - - let undelegate: bool = { - let undelegate: u8 = row.get(6)?; - undelegate == 1 - }; - - let lamports: u64 = { - let lamports: i64 = row.get(7)?; - i64_into_u64(lamports) - }; - - let data: Option> = row.get(8)?; - - let commit_type = { - let commit_type: String = row.get(9)?; - CommitType::try_from(commit_type.as_str())? - }; - let created_at: u64 = { - let created_at: i64 = row.get(10)?; - i64_into_u64(created_at) - }; - - let commit_strategy = { - let commit_strategy: String = row.get(11)?; - CommitStrategy::try_from(commit_strategy.as_str())? - }; - - let commit_status = { - let commit_status: String = row.get(12)?; - let commit_stage_signature = { - let commit_stage_signature: Option = row.get(13)?; - commit_stage_signature - .map(|s| Signature::from_str(s.as_str())) - .transpose()? - }; - let finalize_stage_signature = { - let finalize_stage_signature: Option = row.get(14)?; - finalize_stage_signature - .map(|s| Signature::from_str(s.as_str())) - .transpose()? - }; - let sigs = commit_stage_signature.map(|s| CommitStatusSignatures { - commit_stage_signature: s, - finalize_stage_signature, - }); - CommitStatus::try_from((commit_status.as_str(), sigs))? - }; - - let last_retried_at: u64 = { - let last_retried_at: i64 = row.get(15)?; - i64_into_u64(last_retried_at) - }; - let retries_count: u16 = { - let retries_count: i64 = row.get(16)?; - retries_count.try_into().unwrap_or_default() - }; - - Ok(CommitStatusRow { - message_id, - pubkey, - commit_id, - delegated_account_owner, - slot, - ephemeral_blockhash, - undelegate, - lamports, - data, - commit_type, - created_at, - commit_strategy, - commit_status, - last_retried_at, - retries_count, - }) -} - -#[cfg(test)] -mod tests { - use solana_hash::Hash; - use solana_signature::Signature; - use tempfile::NamedTempFile; - - use super::*; - use crate::test_utils; - - // Helper to create a test database - fn setup_test_db() -> (CommittsDb, NamedTempFile) { - test_utils::init_test_logger(); - let temp_file = NamedTempFile::new().unwrap(); - let db = CommittsDb::new(temp_file.path()).unwrap(); - db.create_commit_status_table().unwrap(); - db.create_bundle_signature_table().unwrap(); - - (db, temp_file) - } - - // Helper to create a test CommitStatusRow - fn create_test_row(message_id: u64, commit_id: u64) -> CommitStatusRow { - CommitStatusRow { - message_id, - commit_id, - pubkey: Pubkey::new_unique(), - delegated_account_owner: Pubkey::new_unique(), - slot: 100, - ephemeral_blockhash: Hash::new_unique(), - undelegate: false, - lamports: 1000, - data: Some(vec![1, 2, 3]), - commit_type: CommitType::DataAccount, - created_at: 1000, - commit_status: CommitStatus::Pending, - commit_strategy: CommitStrategy::StateArgs, - last_retried_at: 1000, - retries_count: 0, - } - } - - #[test] - fn test_table_creation() { - let (db, _) = setup_test_db(); - - // Verify table exists - let table_exists: bool = db.conn.query_row( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='commit_status')", - [], - |row| row.get(0), - ).unwrap(); - assert!(table_exists); - } - - #[test] - fn test_bundle_signature_table_creation() { - let (db, _) = setup_test_db(); - - // Verify bundle_signature table exists - let table_exists: bool = db.conn.query_row( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='bundle_signature')", - [], - |row| row.get(0), - ).unwrap(); - assert!(table_exists); - } - - #[test] - fn test_insert_and_retrieve_rows() { - let (mut db, _file) = setup_test_db(); - let row1 = create_test_row(1, 0); - let row2 = create_test_row(1, 0); // Same message_id, different pubkey - - // Insert rows - db.insert_commit_status_rows(&[row1.clone(), row2.clone()]) - .unwrap(); - - // Retrieve by message_id - let rows = db.get_commit_statuses_by_id(1).unwrap(); - assert_eq!(rows.len(), 2); - assert!(rows.contains(&row1)); - assert!(rows.contains(&row2)); - - // Retrieve individual row - let retrieved = db.get_commit_status(1, &row1.pubkey).unwrap().unwrap(); - assert_eq!(retrieved, row1); - } - - #[test] - fn test_get_pending_commit_statuses() { - let (mut db, _file) = setup_test_db(); - let pending = create_test_row(1, 0); - let mut succeeded = create_test_row(2, 0); - succeeded.commit_status = - CommitStatus::Succeeded(CommitStatusSignatures { - commit_stage_signature: Signature::new_unique(), - finalize_stage_signature: Some(Signature::new_unique()), - }); - - db.insert_commit_status_rows(&[pending.clone(), succeeded]) - .unwrap(); - - let rows = db.get_pending_commit_statuses(0).unwrap(); - assert_eq!(rows, vec![pending]); - } - - #[test] - fn test_get_pending_commit_statuses_filters_recovery_window() { - let (mut db, _file) = setup_test_db(); - let mut pending = create_test_row(1, 0); - pending.created_at = 10; - pending.last_retried_at = 19; - let mut pending_same_message = create_test_row(1, 0); - pending_same_message.created_at = 11; - pending_same_message.last_retried_at = 18; - let mut too_old = create_test_row(2, 0); - too_old.created_at = 9; - too_old.last_retried_at = 9; - let mut too_recent = create_test_row(3, 0); - too_recent.created_at = 20; - too_recent.last_retried_at = 20; - let mut partially_eligible = create_test_row(4, 0); - partially_eligible.created_at = 10; - partially_eligible.last_retried_at = 19; - let mut partially_too_recent = create_test_row(4, 0); - partially_too_recent.created_at = 10; - partially_too_recent.last_retried_at = 20; - - db.insert_commit_status_rows(&[ - pending.clone(), - pending_same_message.clone(), - too_old, - too_recent.clone(), - partially_eligible.clone(), - partially_too_recent.clone(), - ]) - .unwrap(); - - let rows = db.get_pending_commit_statuses(10).unwrap(); - assert_eq!( - rows, - vec![ - pending, - pending_same_message, - too_recent, - partially_eligible, - partially_too_recent - ] - ); - } - - #[test] - fn test_insert_and_retrieve_bundle_signature() { - let (db, _file) = setup_test_db(); - let bundle_id = 123; - let commit_sig = Signature::new_unique(); - let finalize_sig = Signature::new_unique(); - - let bundle_row = - BundleSignatureRow::new(bundle_id, commit_sig, finalize_sig); - db.insert_bundle_signature_row(&bundle_row).unwrap(); - - let retrieved = db - .get_bundle_signature_by_bundle_id(bundle_id) - .unwrap() - .unwrap(); - assert_eq!(retrieved.bundle_id, bundle_id); - assert_eq!(retrieved.commit_stage_signature, commit_sig); - assert_eq!(retrieved.finalize_stage_signature, finalize_sig); - } - - #[test] - fn test_get_nonexistent_bundle_signature() { - let (db, _file) = setup_test_db(); - let result = db.get_bundle_signature_by_bundle_id(999).unwrap(); - assert!(result.is_none()); - } - - #[test] - fn test_set_commit_id() { - let (mut db, _file) = setup_test_db(); - let row = create_test_row(1, 0); - db.insert_commit_status_rows(std::slice::from_ref(&row)) - .unwrap(); - - // Update commit_id - db.set_commit_id(1, &row.pubkey, 100).unwrap(); - - // Verify update - let updated = db.get_commit_status(1, &row.pubkey).unwrap().unwrap(); - assert_eq!(updated.commit_id, 100); - } - - #[test] - fn test_update_status_by_message() { - let (mut db, _file) = setup_test_db(); - let row = create_test_row(1, 0); - db.insert_commit_status_rows(std::slice::from_ref(&row)) - .unwrap(); - - let new_status = CommitStatus::Pending; - db.update_status_by_message(1, &row.pubkey, &new_status) - .unwrap(); - - let updated = db.get_commit_status(1, &row.pubkey).unwrap().unwrap(); - assert_eq!(updated.commit_status, new_status); - } - - #[test] - fn test_update_status_by_commit() { - let (mut db, _file) = setup_test_db(); - let row = create_test_row(1, 100); // Set commit_id to 100 - db.insert_commit_status_rows(std::slice::from_ref(&row)) - .unwrap(); - - let new_status = CommitStatus::Succeeded(CommitStatusSignatures { - commit_stage_signature: Signature::new_unique(), - finalize_stage_signature: None, - }); - db.update_status_by_commit(100, &row.pubkey, &new_status) - .unwrap(); - - let updated = db.get_commit_status(1, &row.pubkey).unwrap().unwrap(); - assert_eq!(updated.commit_status, new_status); - } - - #[test] - fn test_set_commit_strategy() { - let (mut db, _file) = setup_test_db(); - let row = create_test_row(1, 100); - db.insert_commit_status_rows(std::slice::from_ref(&row)) - .unwrap(); - - let new_strategy = CommitStrategy::StateBuffer; - db.set_commit_strategy(100, &row.pubkey, new_strategy) - .unwrap(); - - let updated = db.get_commit_status(1, &row.pubkey).unwrap().unwrap(); - assert_eq!(updated.commit_strategy, new_strategy); - } - - #[test] - fn test_get_signatures_by_commit() { - let (mut db, _file) = setup_test_db(); - let commit_stage_signature = Signature::new_unique(); - let finalize_sig = Signature::new_unique(); - - let mut row = create_test_row(1, 100); - row.commit_status = CommitStatus::Succeeded(CommitStatusSignatures { - commit_stage_signature, - finalize_stage_signature: Some(finalize_sig), - }); - db.insert_commit_status_rows(&[row.clone()]).unwrap(); - - let sigs = db - .get_signatures_by_commit(100, &row.pubkey) - .unwrap() - .unwrap(); - assert_eq!(sigs.commit_stage_signature, commit_stage_signature); - assert_eq!(sigs.finalize_stage_signature, Some(finalize_sig)); - } - - #[test] - fn test_remove_commit_statuses() { - let (mut db, _file) = setup_test_db(); - let row1 = create_test_row(1, 0); - let row2 = create_test_row(2, 0); - db.insert_commit_status_rows(&[row1.clone(), row2.clone()]) - .unwrap(); - - // Remove one message - db.remove_commit_statuses_with_id(1).unwrap(); - - // Verify removal - assert!(db.get_commit_statuses_by_id(1).unwrap().is_empty()); - assert_eq!(db.get_commit_statuses_by_id(2).unwrap().len(), 1); - } - - #[test] - fn test_empty_data_handling() { - let (mut db, _file) = setup_test_db(); - let mut row = create_test_row(1, 0); - row.data = None; - row.commit_type = CommitType::EmptyAccount; - - db.insert_commit_status_rows(&[row.clone()]).unwrap(); - - let retrieved = db.get_commit_status(1, &row.pubkey).unwrap().unwrap(); - assert!(retrieved.data.is_none()); - assert_eq!(retrieved.commit_type, CommitType::EmptyAccount); - } - - #[test] - fn test_undelegate_flag() { - let (mut db, _file) = setup_test_db(); - let mut row = create_test_row(1, 0); - row.undelegate = true; - - db.insert_commit_status_rows(&[row.clone()]).unwrap(); - - let retrieved = db.get_commit_status(1, &row.pubkey).unwrap().unwrap(); - assert!(retrieved.undelegate); - } - - #[test] - fn test_flow_message_and_commit_id_roundtrip_boundaries() { - let (mut db, _file) = setup_test_db(); - - // Boundary IDs we care about for storage and retrieval. - // Skip u64::MAX due to the known overflow behavior in u64_into_i64. - let ids: [u64; 4] = - [0, i64::MAX as u64, (i64::MAX as u64) + 1, u64::MAX - 1]; - - // Prepare rows with message_id == commit_id == each boundary value. - let mut rows = Vec::new(); - for &id in &ids { - let mut row = create_test_row(id, id); - // give each row a distinct status/signatures pattern to make sure - // we don't accidentally mix them up in queries/updates - if id % 2 == 0 { - row.commit_status = CommitStatus::Pending; - } else { - row.commit_status = - CommitStatus::Succeeded(CommitStatusSignatures { - commit_stage_signature: Signature::new_unique(), - finalize_stage_signature: None, - }); - } - rows.push(row); - } - - // Insert all rows - db.insert_commit_status_rows(&rows).unwrap(); - - // 1) Retrieval by message_id should give back exactly the rows with that ID, - // and the internal IDs should be round-tripped correctly. - for row in &rows { - let fetched = db.get_commit_statuses_by_id(row.message_id).unwrap(); - // We inserted a single row for each (message_id, pubkey, commit_id) triple, - // so count should be 1 for this message_id. - assert_eq!( - fetched.len(), - 1, - "unexpected count for message_id={}", - row.message_id - ); - let got = &fetched[0]; - assert_eq!(got.message_id, row.message_id, "message_id mismatch"); - assert_eq!(got.commit_id, row.commit_id, "commit_id mismatch"); - assert_eq!(got.pubkey, row.pubkey, "pubkey mismatch"); - assert_eq!(got.commit_status, row.commit_status, "status mismatch"); - } - - // 2) Retrieval by (message_id, pubkey) via get_commit_status should match exactly. - for row in &rows { - let got = db - .get_commit_status(row.message_id, &row.pubkey) - .unwrap() - .unwrap(); - assert_eq!(got.message_id, row.message_id); - assert_eq!(got.commit_id, row.commit_id); - assert_eq!(got.pubkey, row.pubkey); - } - - // 3) Verify update by message_id preserves IDs and writes the new status. - // Use the smallest and largest IDs to stress the conversion. - let smallest_id = ids[0]; - let largest_id = ids[ids.len() - 1]; - - { - let target = - rows.iter().find(|r| r.message_id == smallest_id).unwrap(); - let new_status = CommitStatus::Failed; - db.update_status_by_message( - target.message_id, - &target.pubkey, - &new_status, - ) - .unwrap(); - - let got = db - .get_commit_status(target.message_id, &target.pubkey) - .unwrap() - .unwrap(); - assert_eq!(got.message_id, target.message_id); - assert_eq!(got.commit_id, target.commit_id); - assert_eq!(got.commit_status, new_status); - } - - // 4) Verify update by commit_id also works for large negative-mapped i64. - { - let target = - rows.iter().find(|r| r.commit_id == largest_id).unwrap(); - let new_status = CommitStatus::Succeeded(CommitStatusSignatures { - commit_stage_signature: Signature::new_unique(), - finalize_stage_signature: Some(Signature::new_unique()), - }); - db.update_status_by_commit( - target.commit_id, - &target.pubkey, - &new_status, - ) - .unwrap(); - - let got = db - .get_commit_status(target.message_id, &target.pubkey) - .unwrap() - .unwrap(); - assert_eq!(got.message_id, target.message_id); - assert_eq!(got.commit_id, target.commit_id); - assert_eq!(got.commit_status, new_status); - - // also verify get_signatures_by_commit returns the same signatures - let sigs = db - .get_signatures_by_commit(target.commit_id, &target.pubkey) - .unwrap() - .unwrap(); - if let CommitStatus::Succeeded(ss) = new_status { - assert_eq!( - sigs.commit_stage_signature, - ss.commit_stage_signature - ); - assert_eq!( - sigs.finalize_stage_signature, - ss.finalize_stage_signature - ); - } else { - panic!("unexpected status shape"); - } - } - - // 5) set_commit_id should update correctly across the boundary values - // (use XOR to create a different but valid u64). - for row in &rows { - let new_commit_id = row.commit_id ^ 0xDEAD_BEEF_DEAD_BEEF; - db.set_commit_id(row.message_id, &row.pubkey, new_commit_id) - .unwrap(); - let got = db - .get_commit_status(row.message_id, &row.pubkey) - .unwrap() - .unwrap(); - assert_eq!(got.message_id, row.message_id); - assert_eq!(got.commit_id, new_commit_id); - } - } -} diff --git a/magicblock-committor-service/src/persist/error.rs b/magicblock-committor-service/src/persist/error.rs deleted file mode 100644 index 49e0df299..000000000 --- a/magicblock-committor-service/src/persist/error.rs +++ /dev/null @@ -1,41 +0,0 @@ -use thiserror::Error; - -pub type CommitPersistResult = Result; - -#[derive(Error, Debug)] -pub enum CommitPersistError { - #[error("RusqliteError: '{0}' ({0:?})")] - RusqliteError(#[from] rusqlite::Error), - - #[error("ParsePubkeyError: '{0}' ({0:?})")] - ParsePubkeyError(#[from] solana_pubkey::ParsePubkeyError), - - #[error("ParseSignatureError: '{0}' ({0:?})")] - ParseSignatureError(#[from] solana_signature::ParseSignatureError), - - #[error("ParseHashError: '{0}' ({0:?})")] - ParseHashError(#[from] solana_hash::ParseHashError), - - #[error("Invalid Commit Type: '{0}' ({0:?})")] - InvalidCommitType(String), - - #[error("Invalid Commit Status: '{0}' ({0:?})")] - InvalidCommitStatus(String), - - #[error("Invalid Commit Strategy: '{0}' ({0:?})")] - InvalidCommitStrategy(String), - - #[error( - "Commit Status update requires status with bundle id: '{0}' ({0:?})" - )] - CommitStatusUpdateRequiresStatusWithBundleId(String), - - #[error("Commit Status needs bundle id: '{0}' ({0:?})")] - CommitStatusNeedsBundleId(String), - - #[error("Commit Status needs signatures: '{0}' ({0:?})")] - CommitStatusNeedsSignatures(String), - - #[error("Commit Status needs commit strategy: '{0}' ({0:?})")] - CommitStatusNeedsStrategy(String), -} diff --git a/magicblock-committor-service/src/persist/mod.rs b/magicblock-committor-service/src/persist/mod.rs deleted file mode 100644 index 6d7083055..000000000 --- a/magicblock-committor-service/src/persist/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod commit_persister; -mod db; -pub mod error; -mod types; -mod utils; - -pub use commit_persister::{IntentPersister, IntentPersisterImpl}; -pub use db::{CommitStatusRow, CommittsDb, MessageSignatures}; -pub use types::{ - CommitStatus, CommitStatusSignatures, CommitStrategy, CommitType, -}; diff --git a/magicblock-committor-service/src/persist/types/commit_status.rs b/magicblock-committor-service/src/persist/types/commit_status.rs deleted file mode 100644 index 9942dd296..000000000 --- a/magicblock-committor-service/src/persist/types/commit_status.rs +++ /dev/null @@ -1,161 +0,0 @@ -use std::fmt; - -use solana_signature::Signature; - -use crate::persist::error::CommitPersistError; - -/// The status of a committed account. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CommitStatus { - /// We sent the request to commit this account, but haven't received a result yet. - Pending, - /// No part of the commit pipeline succeeded. - /// The commit for this account needs to be restarted from scratch. - Failed, - /// The buffer and chunks account were initialized, but could either not - /// be retrieved or deserialized. It is recommended to fully re-initialize - /// them on retry. - BufferAndChunkPartiallyInitialized, - /// The buffer and chunks accounts were initialized and could be - /// deserialized, however we did not complete writing to them - /// We can reuse them on retry, but need to rewrite all chunks. - BufferAndChunkInitialized, - /// The buffer and chunks accounts were initialized and all data was - /// written to them (for data accounts). - /// This means on retry we can skip that step and just try to process - /// these buffers to complete the commit. - BufferAndChunkFullyInitialized, - /// The commit is part of a bundle that contains too many commits to be included - /// in a single transaction. Thus we cannot commit any of them. - PartOfTooLargeBundleToProcess, - /// The commit was properly initialized and added to a chunk of instructions to process - /// commits via a transaction. For large commits the buffer and chunk accounts were properly - /// prepared and haven't been closed. - FailedProcess(Option), - /// The commit was properly processed but the requested finalize transaction failed. - FailedFinalize(CommitStatusSignatures), - /// The commit was successfully processed and finalized. - Succeeded(CommitStatusSignatures), -} - -impl fmt::Display for CommitStatus { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - CommitStatus::Pending => write!(f, "Pending"), - CommitStatus::Failed => { - write!(f, "Failed") - } - CommitStatus::BufferAndChunkPartiallyInitialized => { - write!(f, "BufferAndChunkPartiallyInitialized") - } - CommitStatus::BufferAndChunkInitialized => { - write!(f, "BufferAndChunkInitialized") - } - CommitStatus::BufferAndChunkFullyInitialized => { - write!(f, "BufferAndChunkFullyInitialized") - } - CommitStatus::PartOfTooLargeBundleToProcess => { - write!(f, "PartOfTooLargeBundleToProcess") - } - CommitStatus::FailedProcess(sigs) => { - write!(f, "FailedProcess({:?})", sigs) - } - CommitStatus::FailedFinalize(sigs) => { - write!(f, "FailedFinalize({:?})", sigs) - } - CommitStatus::Succeeded(sigs) => { - write!(f, "Succeeded({:?})", sigs) - } - } - } -} - -impl TryFrom<(&str, Option)> for CommitStatus { - type Error = CommitPersistError; - - fn try_from( - (status, sigs): (&str, Option), - ) -> Result { - let get_sigs = || { - if let Some(sigs) = sigs.clone() { - Ok(sigs) - } else { - Err(CommitPersistError::CommitStatusNeedsSignatures( - status.to_string(), - )) - } - }; - - use CommitStatus::*; - match status { - "Pending" => Ok(Pending), - "Failed" => Ok(Failed), - "BufferAndChunkPartiallyInitialized" => { - Ok(BufferAndChunkPartiallyInitialized) - } - "BufferAndChunkInitialized" => Ok(BufferAndChunkInitialized), - "BufferAndChunkFullyInitialized" => { - Ok(BufferAndChunkFullyInitialized) - } - "PartOfTooLargeBundleToProcess" => { - Ok(PartOfTooLargeBundleToProcess) - } - "FailedProcess" => Ok(FailedProcess(sigs)), - "FailedFinalize" => Ok(FailedFinalize(get_sigs()?)), - "Succeeded" => Ok(Succeeded(get_sigs()?)), - _ => { - Err(CommitPersistError::InvalidCommitStatus(status.to_string())) - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CommitStatusSignatures { - /// The signature of the transaction processing Commit stage - pub commit_stage_signature: Signature, - /// The signature of the transaction processing Finalize Stage. - /// If the account was not finalized or it failed then this is `None`. - /// If the finalize instruction was part of the Commit stage transaction then - /// this signature is the same as [Self::commit_stage_signature]. - pub finalize_stage_signature: Option, -} - -impl CommitStatus { - pub fn as_str(&self) -> &str { - use CommitStatus::*; - match self { - Pending => "Pending", - Failed => "Failed", - BufferAndChunkPartiallyInitialized => { - "BufferAndChunkPartiallyInitialized" - } - BufferAndChunkInitialized => "BufferAndChunkInitialized", - BufferAndChunkFullyInitialized => "BufferAndChunkFullyInitialized", - PartOfTooLargeBundleToProcess => "PartOfTooLargeBundleToProcess", - FailedProcess(_) => "FailedProcess", - FailedFinalize(_) => "FailedFinalize", - Succeeded(_) => "Succeeded", - } - } - - pub fn signatures(&self) -> Option { - use CommitStatus::*; - match self { - FailedProcess(sigs) => sigs.as_ref().cloned(), - FailedFinalize(sigs) => Some(sigs.clone()), - Succeeded(sigs) => Some(sigs.clone()), - _ => None, - } - } - - /// The commit fully succeeded and no retry is necessary. - pub fn is_complete(&self) -> bool { - use CommitStatus::*; - matches!(self, Succeeded(_)) - } - - pub fn all_completed(stages: &[Self]) -> bool { - stages.iter().all(Self::is_complete) - } -} diff --git a/magicblock-committor-service/src/persist/types/commit_strategy.rs b/magicblock-committor-service/src/persist/types/commit_strategy.rs deleted file mode 100644 index 3bf007333..000000000 --- a/magicblock-committor-service/src/persist/types/commit_strategy.rs +++ /dev/null @@ -1,72 +0,0 @@ -use crate::persist::error::CommitPersistError; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum CommitStrategy { - /// Args without the use of a lookup table - #[default] - StateArgs, - /// Args with the use of a lookup table - StateArgsWithLookupTable, - /// Buffer and chunks which has the most overhead - StateBuffer, - /// Buffer and chunks with the use of a lookup table - StateBufferWithLookupTable, - - /// Args without the use of a lookup table - DiffArgs, - /// Args with the use of a lookup table - DiffArgsWithLookupTable, - /// Buffer and chunks which has the most overhead - DiffBuffer, - /// Buffer and chunks with the use of a lookup table - DiffBufferWithLookupTable, -} - -impl CommitStrategy { - pub fn as_str(&self) -> &str { - use CommitStrategy::*; - match self { - StateArgs => "StateArgs", - StateArgsWithLookupTable => "StateArgsWithLookupTable", - StateBuffer => "StateBuffer", - StateBufferWithLookupTable => "StateBufferWithLookupTable", - DiffArgs => "DiffArgs", - DiffArgsWithLookupTable => "DiffArgsWithLookupTable", - DiffBuffer => "DiffBuffer", - DiffBufferWithLookupTable => "DiffBufferWithLookupTable", - } - } - - pub fn uses_lookup(&self) -> bool { - matches!( - self, - CommitStrategy::StateArgsWithLookupTable - | CommitStrategy::StateBufferWithLookupTable - | CommitStrategy::DiffArgsWithLookupTable - | CommitStrategy::DiffBufferWithLookupTable - ) - } -} - -impl TryFrom<&str> for CommitStrategy { - type Error = CommitPersistError; - fn try_from(value: &str) -> Result { - match value { - "Args" | "StateArgs" => Ok(Self::StateArgs), - "ArgsWithLookupTable" | "StateArgsWithLookupTable" => { - Ok(Self::StateArgsWithLookupTable) - } - "FromBuffer" | "StateBuffer" => Ok(Self::StateBuffer), - "FromBufferWithLookupTable" | "StateBufferWithLookupTable" => { - Ok(Self::StateBufferWithLookupTable) - } - "DiffArgs" => Ok(Self::DiffArgs), - "DiffArgsWithLookupTable" => Ok(Self::DiffArgsWithLookupTable), - "DiffBuffer" => Ok(Self::DiffBuffer), - "DiffBufferWithLookupTable" => Ok(Self::DiffBufferWithLookupTable), - _ => Err(CommitPersistError::InvalidCommitStrategy( - value.to_string(), - )), - } - } -} diff --git a/magicblock-committor-service/src/persist/types/commit_type.rs b/magicblock-committor-service/src/persist/types/commit_type.rs deleted file mode 100644 index 96324456b..000000000 --- a/magicblock-committor-service/src/persist/types/commit_type.rs +++ /dev/null @@ -1,28 +0,0 @@ -use crate::persist::error::CommitPersistError; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CommitType { - EmptyAccount, - DataAccount, -} - -impl TryFrom<&str> for CommitType { - type Error = CommitPersistError; - - fn try_from(value: &str) -> Result { - match value { - "EmptyAccount" => Ok(CommitType::EmptyAccount), - "DataAccount" => Ok(CommitType::DataAccount), - _ => Err(CommitPersistError::InvalidCommitType(value.to_string())), - } - } -} - -impl CommitType { - pub fn as_str(&self) -> &str { - match self { - CommitType::EmptyAccount => "EmptyAccount", - CommitType::DataAccount => "DataAccount", - } - } -} diff --git a/magicblock-committor-service/src/persist/types/mod.rs b/magicblock-committor-service/src/persist/types/mod.rs deleted file mode 100644 index b0c68fa59..000000000 --- a/magicblock-committor-service/src/persist/types/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod commit_status; -mod commit_strategy; -mod commit_type; - -pub use commit_status::*; -pub use commit_strategy::*; -pub use commit_type::*; diff --git a/magicblock-committor-service/src/persist/utils.rs b/magicblock-committor-service/src/persist/utils.rs deleted file mode 100644 index 3bdbda592..000000000 --- a/magicblock-committor-service/src/persist/utils.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - -/// Fits a u64 into an i64, by mapping the range [0, i64::MAX] to itself, and -/// mapping the range [i64::MAX + 1, u64::MAX - 1] into the negative range of i64. -/// NOTE: this fails for u64::MAX -/// TODO(edwin): just store bit-copy in i64 -pub(crate) fn u64_into_i64(n: u64) -> i64 { - if n > i64::MAX as u64 { - -((n - i64::MAX as u64) as i64) - } else { - n as i64 - } -} - -/// Extracts a u64 that was fitted into an i64 by `u64_into_i64`. -pub(crate) fn i64_into_u64(n: i64) -> u64 { - if n < 0 { - n.unsigned_abs() + i64::MAX as u64 - } else { - n as u64 - } -} - -/// Gets the current timestamp in seconds since the Unix epoch -pub(crate) fn now() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils; - - fn setup() { - test_utils::init_test_logger(); - } - - fn round_trip(u: u64) { - let i = u64_into_i64(u); - let u2 = i64_into_u64(i); - assert_eq!(u, u2); - } - - #[test] - fn test_u64_i64_conversion_via_round_trip() { - setup(); - round_trip(0); - round_trip(1); - round_trip(i64::MAX as u64); - round_trip(i64::MAX as u64 + 1); - - // NOTE: the below which points out that we cannot round trip u64::MAX, - assert_eq!(i64::MAX as u64 * 2 + 1, u64::MAX); - - // This is the largest we can roundtrip - round_trip(u64::MAX - 1); - round_trip(i64::MAX as u64 * 2); - - // This would fail: - // round_trip(u64::MAX); - } -} diff --git a/magicblock-committor-service/src/tasks/task_builder.rs b/magicblock-committor-service/src/tasks/task_builder.rs index 515e925f5..87d743e20 100644 --- a/magicblock-committor-service/src/tasks/task_builder.rs +++ b/magicblock-committor-service/src/tasks/task_builder.rs @@ -16,7 +16,6 @@ use crate::{ intent_executor::task_info_fetcher::{ TaskInfoFetcher, TaskInfoFetcherError, TaskInfoFetcherResult, }, - persist::IntentPersister, tasks::{ commit_task::{CommitDelivery, CommitTask}, BaseActionTask, BaseActionTaskV1, BaseActionTaskV2, BaseTaskImpl, @@ -27,10 +26,9 @@ use crate::{ #[async_trait] pub trait TasksBuilder { // Creates tasks for commit stage - async fn commit_tasks( + async fn commit_tasks( commit_id_fetcher: &Arc, base_intent: &ScheduledIntentBundle, - persister: &Option

, ) -> TaskBuilderResult>; // Create tasks for finalize stage @@ -137,10 +135,9 @@ impl TaskBuilderImpl { .await } - async fn fetch_commit_stage_info( + async fn fetch_commit_stage_info( intent_bundle: &ScheduledIntentBundle, task_info_fetcher: &Arc, - persister: &Option

, ) -> TaskBuilderResult { // Fetch necessary data for BaseTasks creation let all_committed_accounts = intent_bundle.get_all_committed_accounts(); @@ -169,15 +166,6 @@ impl TaskBuilderImpl { Default::default() }); - // Persist commit ids for commitees - commit_nonces - .iter() - .for_each(|(pubkey, commit_id) | { - if let Err(err) = persister.set_commit_id(intent_bundle.id, pubkey, *commit_id) { - error!(intent_id = intent_bundle.id, pubkey = %pubkey, error = ?err, "Failed to persist commit id"); - } - }); - Ok(CommitStageTaskInfo { commit_nonces, base_accounts, @@ -215,10 +203,9 @@ impl TaskBuilderImpl { #[async_trait] impl TasksBuilder for TaskBuilderImpl { /// Returns [`BaseTaskImpl`]s for Commit stage - async fn commit_tasks( + async fn commit_tasks( task_info_fetcher: &Arc, intent_bundle: &ScheduledIntentBundle, - persister: &Option

, ) -> TaskBuilderResult> { let mut tasks = Vec::new(); // Add standalone actions first @@ -230,12 +217,8 @@ impl TasksBuilder for TaskBuilderImpl { let CommitStageTaskInfo { mut commit_nonces, mut base_accounts, - } = Self::fetch_commit_stage_info( - intent_bundle, - task_info_fetcher, - persister, - ) - .await?; + } = Self::fetch_commit_stage_info(intent_bundle, task_info_fetcher) + .await?; // Create tasks per intent type if let Some(ref value) = intent_bundle.intent_bundle.commit { diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index b55cbd7dc..9156f22b7 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -7,7 +7,6 @@ use solana_signer::{Signer, SignerError}; use tracing::error; use crate::{ - persist::{CommitStrategy, IntentPersister}, tasks::{ commit_task::CommitDelivery, utils::TransactionUtils, BaseActionTask, BaseTask, BaseTaskImpl, @@ -123,11 +122,10 @@ impl TaskStrategist { /// Builds execution strategy from [`BaseTask`]s /// 1. Optimizes tasks to fit in TX /// 2. Chooses the fastest execution mode for Tasks - pub fn build_execution_strategy( + pub fn build_execution_strategy( commit_tasks: Vec, finalize_tasks: Vec, authority: &Pubkey, - persister: &Option

, ) -> TaskStrategistResult { const MAX_UNITED_TASKS_LEN: usize = 22; @@ -138,37 +136,28 @@ impl TaskStrategist { // In case this fails as well, it will be retried with TwoStage approach // on retry, once retries are introduced if commit_tasks.len() + finalize_tasks.len() > MAX_UNITED_TASKS_LEN { - return Self::build_two_stage( - commit_tasks, - finalize_tasks, - authority, - persister, - ); + return Self::build_two_stage(commit_tasks, finalize_tasks, authority); } // Clone tasks since strategies applied to united case maybe suboptimal for regular one // Unite tasks to attempt running as single tx let single_stage_tasks = [commit_tasks.clone(), finalize_tasks.clone()].concat(); - let single_stage_strategy = match TaskStrategist::build_strategy( - single_stage_tasks, - authority, - persister, - ) { - Ok(strategy) => StrategyExecutionMode::SingleStage(strategy), - Err(TaskStrategistError::FailedToFitError) => { - // If Tasks can't fit in SingleStage - use TwpStage execution - return Self::build_two_stage( - commit_tasks, - finalize_tasks, - authority, - persister, - ); - } - Err(TaskStrategistError::SignerError(err)) => { - return Err(err.into()) - } - }; + let single_stage_strategy = + match TaskStrategist::build_strategy(single_stage_tasks, authority) { + Ok(strategy) => StrategyExecutionMode::SingleStage(strategy), + Err(TaskStrategistError::FailedToFitError) => { + // If Tasks can't fit in SingleStage - use TwpStage execution + return Self::build_two_stage( + commit_tasks, + finalize_tasks, + authority, + ); + } + Err(TaskStrategistError::SignerError(err)) => { + return Err(err.into()) + } + }; // If ALTs aren't used then we sure this will be optimal - return if !single_stage_strategy.uses_alts() { @@ -178,12 +167,8 @@ impl TaskStrategist { // As ALTs take a very long time to activate // it is actually faster to execute in TwoStage mode // unless TwoStage also uses ALTs - let two_stage = Self::build_two_stage( - commit_tasks, - finalize_tasks, - authority, - persister, - )?; + let two_stage = + Self::build_two_stage(commit_tasks, finalize_tasks, authority)?; if two_stage.uses_alts() { Ok(single_stage_strategy) } else { @@ -226,10 +211,6 @@ impl TaskStrategist { <= MAX_TRANSACTION_WIRE_SIZE { // Persist tasks strategy - if let Some(persistor) = persistor { - Self::persist_tasks_strategy(persistor, &tasks, false); - } - Ok(TransactionStrategy { optimized_tasks: tasks, lookup_tables_keys: vec![], @@ -315,76 +296,6 @@ impl TaskStrategist { ) } - fn persist_tasks_strategy( - persistor: &P, - tasks: &[BaseTaskImpl], - uses_lookup_tables: bool, - ) { - let commit_strategy_from_delivery = - |delivery: &CommitDelivery| match delivery { - CommitDelivery::StateInArgs => { - if uses_lookup_tables { - CommitStrategy::StateArgsWithLookupTable - } else { - CommitStrategy::StateArgs - } - } - CommitDelivery::DiffInArgs { .. } => { - if uses_lookup_tables { - CommitStrategy::DiffArgsWithLookupTable - } else { - CommitStrategy::DiffArgs - } - } - CommitDelivery::StateInBuffer { .. } => { - if uses_lookup_tables { - CommitStrategy::StateBufferWithLookupTable - } else { - CommitStrategy::StateBuffer - } - } - CommitDelivery::DiffInBuffer { .. } => { - if uses_lookup_tables { - CommitStrategy::DiffBufferWithLookupTable - } else { - CommitStrategy::DiffBuffer - } - } - }; - - for task in tasks { - let (commit_id, pubkey, commit_strategy) = match task { - BaseTaskImpl::Commit(commit_task) => ( - commit_task.commit_id, - commit_task.committed_account.pubkey, - commit_strategy_from_delivery( - &commit_task.delivery_details, - ), - ), - BaseTaskImpl::CommitFinalize(commit_finalize_task) => ( - commit_finalize_task.commit_id, - commit_finalize_task.committed_account.pubkey, - commit_strategy_from_delivery( - &commit_finalize_task.delivery, - ), - ), - _ => continue, - }; - if let Err(err) = persistor.set_commit_strategy( - commit_id, - &pubkey, - commit_strategy, - ) { - error!( - commit_id = %commit_id, - pubkey = %pubkey, - strategy = commit_strategy.as_str(), - error = ?err, - "Failed to persist commit strategy" - ); - } - } - } /// Optimizes tasks so as to bring the transaction size within the limit [`MAX_TRANSACTION_WIRE_SIZE`] /// Returns Ok(size of tx after optimizations) else Err(SignerError). diff --git a/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs b/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs index f5d7a8c9e..e245b38ff 100644 --- a/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs +++ b/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs @@ -28,12 +28,10 @@ use solana_transaction_error::TransactionError; use tracing::{error, info}; use crate::{ - persist::{CommitStatus, IntentPersister}, tasks::{ commit_task::CommitBufferStage, task_strategist::TransactionStrategy, BaseTaskImpl, CleanupTask, PreparationTask, }, - utils::persist_status_update, ComputeBudgetConfig, }; @@ -59,11 +57,10 @@ impl DeliveryPreparator { } /// Prepares buffers and necessary pieces for optimized TX - pub async fn prepare_for_delivery( + pub async fn prepare_for_delivery( &self, authority: &Keypair, strategy: &mut TransactionStrategy, - persister: &Option

, ) -> DeliveryPreparatorResult> { let preparation_futures = strategy.optimized_tasks.iter_mut().map(|task| async move { @@ -71,7 +68,7 @@ impl DeliveryPreparator { metrics::observe_committor_intent_task_preparation_time( &*task, ); - self.prepare_task_handling_errors(authority, task, persister) + self.prepare_task_handling_errors(authority, task) .await }); @@ -94,11 +91,10 @@ impl DeliveryPreparator { } /// Prepares necessary parts for TX if needed, otherwise returns immediately - pub async fn prepare_task( + pub async fn prepare_task( &self, authority: &Keypair, task: &mut BaseTaskImpl, - persister: &Option

, ) -> DeliveryPreparatorResult<(), InternalError> { let stage = match task { BaseTaskImpl::Commit(commit_task) => commit_task.stage_mut(), @@ -114,39 +110,13 @@ impl DeliveryPreparator { return Ok(()); }; - // Persist as failed until rewritten - let update_status = CommitStatus::BufferAndChunkPartiallyInitialized; - persist_status_update( - persister, - &preparation_task.pubkey, - preparation_task.commit_id, - update_status, - ); - // Initialize buffer account. Init + reallocs self.initialize_buffer_account(authority, preparation_task) .await?; - // Persist initialization success - let update_status = CommitStatus::BufferAndChunkInitialized; - persist_status_update( - persister, - &preparation_task.pubkey, - preparation_task.commit_id, - update_status, - ); - // Writing chunks with some retries self.write_buffer_with_retries(authority, preparation_task) .await?; - // Persist that buffer account initiated successfully - let update_status = CommitStatus::BufferAndChunkFullyInitialized; - persist_status_update( - persister, - &preparation_task.pubkey, - preparation_task.commit_id, - update_status, - ); let cleanup_task = preparation_task.cleanup_task(); *stage = CommitBufferStage::Cleanup(cleanup_task); @@ -155,13 +125,12 @@ impl DeliveryPreparator { /// Runs `prepare_task` and, if the buffer was already initialized, /// performs cleanup and retries once. - pub async fn prepare_task_handling_errors( + pub async fn prepare_task_handling_errors( &self, authority: &Keypair, task: &mut BaseTaskImpl, - persister: &Option

, ) -> Result<(), InternalError> { - let res = self.prepare_task(authority, task, persister).await; + let res = self.prepare_task(authority, task).await; match res { Err(InternalError::BufferExecutionError( BufferExecutionError::AccountAlreadyInitializedError( @@ -198,7 +167,7 @@ impl DeliveryPreparator { // Restore preparation stage for retry *stage = CommitBufferStage::Preparation(preparation_task); - self.prepare_task(authority, task, persister).await + self.prepare_task(authority, task).await } /// Initializes buffer account for future writes diff --git a/magicblock-committor-service/src/transaction_preparator/mod.rs b/magicblock-committor-service/src/transaction_preparator/mod.rs index 08f3af024..2b6ab9dde 100644 --- a/magicblock-committor-service/src/transaction_preparator/mod.rs +++ b/magicblock-committor-service/src/transaction_preparator/mod.rs @@ -6,7 +6,6 @@ use solana_message::VersionedMessage; use solana_pubkey::Pubkey; use crate::{ - persist::IntentPersister, tasks::{ commit_task::CommitBufferStage, task_strategist::TransactionStrategy, utils::TransactionUtils, BaseTaskImpl, @@ -27,11 +26,10 @@ pub mod error; pub trait TransactionPreparator: Send + Sync + 'static { /// Return [`VersionedMessage`] corresponding to [`TransactionStrategy`] /// Handles all necessary preparation needed for successful [`BaseTask`] execution - async fn prepare_for_strategy( + async fn prepare_for_strategy( &self, authority: &Keypair, transaction_strategy: &mut TransactionStrategy, - intent_persister: &Option

, ) -> PreparatorResult; /// Cleans up after strategy. @@ -74,11 +72,10 @@ impl TransactionPreparatorImpl { #[async_trait] impl TransactionPreparator for TransactionPreparatorImpl { - async fn prepare_for_strategy( + async fn prepare_for_strategy( &self, authority: &Keypair, tx_strategy: &mut TransactionStrategy, - intent_persister: &Option

, ) -> PreparatorResult { // If message won't fit, there's no reason to prepare anything // Fail early @@ -98,7 +95,7 @@ impl TransactionPreparator for TransactionPreparatorImpl { // Pre tx preparations. Create buffer accs + lookup tables let lookup_tables = self .delivery_preparator - .prepare_for_delivery(authority, tx_strategy, intent_persister) + .prepare_for_delivery(authority, tx_strategy) .await?; let message = diff --git a/magicblock-committor-service/src/utils.rs b/magicblock-committor-service/src/utils.rs index 51b2c18b2..e69de29bb 100644 --- a/magicblock-committor-service/src/utils.rs +++ b/magicblock-committor-service/src/utils.rs @@ -1,40 +0,0 @@ -use solana_pubkey::Pubkey; -use tracing::error; - -use crate::persist::{CommitStatus, IntentPersister}; - -pub(crate) fn persist_status_update( - persister: &Option

, - pubkey: &Pubkey, - commit_id: u64, - update_status: CommitStatus, -) { - let Some(persister) = persister else { - return; - }; - if let Err(err) = persister.update_status_by_message( - commit_id, - pubkey, - update_status.clone(), - ) { - error!(pubkey = %pubkey, error = ?err, "Failed to persist status"); - } -} - -/// Persists update by message/intent id -pub(crate) fn persist_status_update_by_message_set( - persister: &P, - message_id: u64, - pubkeys: &[Pubkey], - update_status: CommitStatus, -) { - pubkeys.iter().for_each(|pubkey| { - if let Err(err) = persister.update_status_by_message( - message_id, - pubkey, - update_status.clone(), - ) { - error!(pubkey = %pubkey, error = ?err, "Failed to persist status"); - } - }); -} diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index ba498a0bf..17ff28d63 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,10 +1,11 @@ +use std::ops::Deref; use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; use magicblock_magic_program_api::outbox::{ExecutionStage, TwoStageProgress}; use serde::{Deserialize, Serialize}; use crate::magic_scheduled_base_intent::ScheduledIntentBundle; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OutboxIntentBundle { pub inner: ScheduledIntentBundle, pub status: OutboxIntentBundleStatus, @@ -47,7 +48,7 @@ impl OutboxIntentBundle { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum OutboxIntentBundleStatus { Accepted, Executing(ExecutionStage), @@ -84,3 +85,11 @@ impl OutboxIntentBundleStatus { Ok(()) } } + +impl Deref for OutboxIntentBundle { + type Target = ScheduledIntentBundle; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} \ No newline at end of file From 08954c8d834f18cedbcb9f80c620117e73095e8c Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 9 Jun 2026 18:37:57 +0700 Subject: [PATCH 36/97] wip intent executor adoptation --- magicblock-api/src/magic_validator.rs | 9 +- .../src/committor_processor.rs | 122 ++------------ magicblock-committor-service/src/error.rs | 11 +- .../src/intent_execution_manager.rs | 18 +- .../intent_channerl.rs | 4 + .../intent_execution_engine.rs | 10 +- .../intent_scheduler.rs | 13 +- .../intent_execution_client.rs | 156 +++++++++++++----- .../intent_executor_factory.rs | 10 +- .../intent_executor_gateway.rs | 35 ++++ .../src/intent_executor/mod.rs | 18 +- .../intent_executor/single_stage_executor.rs | 9 +- .../src/intent_executor/two_stage_executor.rs | 32 +++- .../src/intent_executor/utils.rs | 13 +- magicblock-committor-service/src/lib.rs | 2 +- .../intent_client.rs => outbox_client.rs} | 36 +++- magicblock-committor-service/src/service.rs | 27 ++- .../src/tasks/task_strategist.rs | 108 ++++-------- .../delivery_preparator.rs | 3 +- magicblock-committor-service/src/utils.rs | 1 + magicblock-rpc-client/src/utils.rs | 39 ++++- .../intent_bundles/outbox_intent_bundles.rs | 3 +- .../tests/test_intent_executor.rs | 4 - 23 files changed, 366 insertions(+), 317 deletions(-) create mode 100644 magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs rename magicblock-committor-service/src/{service/intent_client.rs => outbox_client.rs} (81%) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 50bd8b07c..d0d79af05 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -19,9 +19,8 @@ use magicblock_chainlink::{ ProdInnerChainlink, }; use magicblock_committor_service::{ - committor_processor::CommittorProcessor, - config::ChainConfig, - service::{intent_client::InternalIntentRpcClient, IntentExecutionService}, + committor_processor::CommittorProcessor, config::ChainConfig, + outbox_client::InternalOutboxClient, service::IntentExecutionService, ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, }; use magicblock_config::{ @@ -99,7 +98,7 @@ type InnerChainlinkImpl = ProdInnerChainlink; type ChainlinkImpl = ProdChainlink; type IntentExecutionServiceImpl = - IntentExecutionService>; + IntentExecutionService>; // ----------------- // MagicValidator @@ -493,7 +492,7 @@ impl MagicValidator { slot_interval: Duration, cancellation_token: &CancellationToken, ) -> IntentExecutionServiceImpl { - let intent_client = InternalIntentRpcClient::new( + let intent_client = InternalOutboxClient::new( accounts_db.clone(), transaction_scheduler.clone(), latest_block.clone(), diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index cb589185f..e61d21a78 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -21,7 +21,8 @@ use crate::{ config::ChainConfig, error::{CommittorServiceError, CommittorServiceResult}, intent_execution_manager::{ - db::DummyDB, BroadcastedIntentExecutionResult, IntentExecutionManager, + db::{DumberDB, DummyDB}, + BroadcastedIntentExecutionResult, IntentExecutionManager, }, intent_executor::{ intent_executor_factory::ExecutorConfig, @@ -30,12 +31,8 @@ use crate::{ TaskInfoFetcherResult, }, }, - persist::{ - CommitStatusRow, IntentPersister, IntentPersisterImpl, - MessageSignatures, - }, + outbox_client::OutboxClient, }; -use crate::intent_execution_manager::db::DumberDB; const POISONED_MUTEX_MSG: &str = "CommittorProcessor pending messages mutex poisoned!"; @@ -47,23 +44,22 @@ pub struct CommittorProcessor { authority: Keypair, _table_mania: TableMania, magic_rpc_client: MagicblockRpcClient, - persister: IntentPersisterImpl, commits_scheduler: IntentExecutionManager, task_info_fetcher: Arc>, pending_result_listeners: Arc>>, } impl CommittorProcessor { - pub fn try_new( + pub fn try_new( authority: Keypair, - persist_file: P, chain_config: ChainConfig, chain_slot: Option>, + outbox_client: Arc, actions_callback_executor: A, ) -> CommittorServiceResult where - P: AsRef, A: ActionsCallbackScheduler, + O: OutboxClient, { let rpc_client = RpcClient::new_with_commitment( chain_config.rpc_uri.to_string(), @@ -96,9 +92,6 @@ impl CommittorProcessor { Some(gc_config), ); - // Create commit persister - let persister = IntentPersisterImpl::try_new(persist_file)?; - // Create commit scheduler let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( RpcTaskInfoFetcher::new(magic_block_rpc_client.clone()), @@ -107,7 +100,7 @@ impl CommittorProcessor { magic_block_rpc_client.clone(), DummyDB::new(), task_info_fetcher.clone(), - Some(persister.clone()), + outbox_client, table_mania.clone(), ExecutorConfig { compute_budget_config: chain_config @@ -130,111 +123,16 @@ impl CommittorProcessor { _table_mania: table_mania, magic_rpc_client: magic_block_rpc_client, commits_scheduler, - persister, task_info_fetcher, pending_result_listeners, }) } - pub fn get_commit_statuses( - &self, - message_id: u64, - ) -> CommittorServiceResult> { - let commit_statuses = - self.persister.get_commit_statuses_by_message(message_id)?; - Ok(commit_statuses) - } - - pub fn get_commit_signature( - &self, - commit_id: u64, - pubkey: Pubkey, - ) -> CommittorServiceResult> { - let signatures = self - .persister - .get_signatures_by_commit(commit_id, &pubkey)?; - Ok(signatures) - } - - /// Fetches pending bundles from DB - pub async fn pending_intent_bundles( - &self, - ) -> CommittorServiceResult> { - // Extract pending bundles satisfying predicate - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - let mut bundles = self.persister.pending_intent_bundles( - now.saturating_sub(RECOVERY_MAX_AGE_SECS), - )?; - - if bundles.is_empty() { - return Ok(bundles); - } - - // Log info about extracted bundles - { - let accounts_count: usize = bundles - .iter() - .map(|bundle| bundle.get_all_committed_pubkeys().len()) - .sum(); - info!( - intent_count = bundles.len(), - accounts_count, - "Loaded pending commit intents from persistence for recovery" - ); - } - - // Extracted bundles are out of data and missing some of the info - self.refresh_intent_bundles(&mut bundles).await?; - - Ok(bundles) - } - - async fn refresh_intent_bundles( - &self, - intent_bundles: &mut [ScheduledIntentBundle], - ) -> CommittorServiceResult<()> { - let payer = self.authority.pubkey(); - let slot = self.magic_rpc_client.get_slot().await?; - - macro_rules! set_remote_slot { - ($field:expr, $slot:expr) => { - if let Some(ref mut v) = $field { - v.get_committed_accounts_mut() - .iter_mut() - .for_each(|a| a.remote_slot = slot); - } - }; - } - - intent_bundles.iter_mut().for_each(|b| { - b.payer = payer; - set_remote_slot!(b.intent_bundle.commit, slot); - set_remote_slot!(b.intent_bundle.commit_finalize, slot); - set_remote_slot!(b.intent_bundle.commit_and_undelegate, slot); - set_remote_slot!( - b.intent_bundle.commit_finalize_and_undelegate, - slot - ); - }); - - Ok(()) - } - #[instrument(skip(self, intent_bundles))] pub async fn schedule_intent_bundles( &self, - intent_bundles: Vec, + intent_bundles: Vec, ) -> CommittorServiceResult<()> { - if let Err(err) = self.persister.start_base_intents(&intent_bundles) { - // We will still try to perform the commits, but the fact that we cannot - // persist the intent is very serious and we should probably restart the - // valiator - error!(error = ?err, "DB EXCEPTION: Failed to persist changeset"); - }; - self.commits_scheduler .schedule(intent_bundles) .await @@ -247,7 +145,7 @@ impl CommittorProcessor { pub async fn execute_intent_bundles( &self, - intent_bundles: Vec, + intent_bundles: Vec, ) -> CommittorServiceResult> { // Critical section let (receivers, inserted_ids) = { @@ -304,7 +202,7 @@ impl CommittorProcessor { #[instrument(skip(self, intent_bundles))] pub async fn schedule_recovered_intent_bundles( &self, - intent_bundles: Vec, + intent_bundles: Vec, ) -> CommittorServiceResult<()> { self.commits_scheduler .schedule(intent_bundles) diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index a44977104..7e59fd0d4 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,17 +1,16 @@ use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::intent_execution_manager::IntentExecutionManagerError; +use crate::intent_execution_manager::{ + intent_channerl::IntentScheduleError, IntentExecutionManagerError, +}; pub type CommittorServiceResult = Result; #[derive(Error, Debug)] pub enum CommittorServiceError { - #[error("CommitPersistError: {0} ({0:?})")] - CommitPersistError(#[from] crate::persist::error::CommitPersistError), - - #[error("IntentExecutionManagerError: {0} ({0:?})")] - IntentExecutionManagerError(#[from] IntentExecutionManagerError), + #[error("IntentScheduleError: {0} ({0:?})")] + IntentScheduleError(#[from] IntentScheduleError), #[error("RecvError: {0}")] IntentResultRecvError(#[from] RecvError), diff --git a/magicblock-committor-service/src/intent_execution_manager.rs b/magicblock-committor-service/src/intent_execution_manager.rs index 13b8f121b..72001505b 100644 --- a/magicblock-committor-service/src/intent_execution_manager.rs +++ b/magicblock-committor-service/src/intent_execution_manager.rs @@ -7,12 +7,10 @@ use std::sync::{Arc, Mutex}; pub use intent_execution_engine::BroadcastedIntentExecutionResult; use magicblock_core::traits::ActionsCallbackScheduler; -use magicblock_program::{ - outbox_intent_bundles::OutboxIntentBundle, -}; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::TableMania; -use tokio::sync::{broadcast, mpsc, mpsc::error::TrySendError}; +use tokio::sync::broadcast; use crate::{ intent_execution_manager::{ @@ -24,6 +22,7 @@ use crate::{ intent_executor_factory::{ExecutorConfig, IntentExecutorFactoryImpl}, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }, + outbox_client::OutboxClient, }; // TODO(edwin): rename @@ -34,16 +33,18 @@ pub struct IntentExecutionManager { } impl IntentExecutionManager { - pub fn new( + pub fn new( rpc_client: MagicblockRpcClient, db: D, task_info_fetcher: Arc>, + outbox_client: Arc, table_mania: TableMania, executor_config: ExecutorConfig, actions_callback_executor: A, ) -> Self where A: ActionsCallbackScheduler, + O: OutboxClient, { let db = Arc::new(Mutex::new(db)); @@ -51,15 +52,14 @@ impl IntentExecutionManager { rpc_client, table_mania, executor_config, + outbox_client, task_info_fetcher, actions_callback_executor, }; let (handle, intent_stream) = channel(&db, 1000); - let worker = IntentExecutionEngine::new( - intent_stream, - executor_factory, - ); + let worker = + IntentExecutionEngine::new(intent_stream, executor_factory); let result_subscriber = worker.spawn(); Self { diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs index d8730b53b..94eb8e655 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs @@ -17,6 +17,7 @@ use crate::intent_execution_manager::{db, db::DB}; const POISONED_MSG: &str = "Dummy DB mutex poisoned"; +/// Handle for scheduling intents in ExecutionEngine pub struct IntentScheduleHandle { db: Arc>, sender: Sender, @@ -60,6 +61,9 @@ impl IntentScheduleHandle { } } +/// Stream of Intents that also handles backlog +/// If backlog is not empty we switch to reading from it until it is depleted +/// Once it is depleted we switch to polling `ReceiverStream` #[pin_project] pub struct IntentStream { db: Arc>, diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 379a6af12..bd305d860 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -103,10 +103,7 @@ where F: IntentExecutorFactory + Send + Sync + 'static, E: IntentExecutor, { - pub fn new( - intent_stream: IntentStream, - executor_factory: F, - ) -> Self { + pub fn new(intent_stream: IntentStream, executor_factory: F) -> Self { Self { intent_stream, executor_factory, @@ -272,6 +269,8 @@ where warn!(error = ?err, "No result listeners"); } + // TODO(edwin): if intent is Err(T) we need to "lock" pubkeys somehow + // That would // Remove executed task from Scheduler to unblock other intents // SAFETY: Self::execute is called ONLY after IntentScheduler // successfully is able to schedule execution of some Intent @@ -376,7 +375,8 @@ mod tests { } else { MockIntentExecutorFactory::new_failing() }; - let worker = IntentExecutionEngine::new(intent_stream, executor_factory); + let worker = + IntentExecutionEngine::new(intent_stream, executor_factory); (handle, worker) } diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs b/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs index 61ad72a51..9916a6704 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs @@ -1,10 +1,12 @@ use std::collections::{hash_map::Entry, HashMap, VecDeque}; -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, +}; use solana_pubkey::Pubkey; use thiserror::Error; use tracing::error; -use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; pub(crate) const POISONED_INNER_MSG: &str = "Mutex on CommitSchedulerInner is poisoned."; @@ -234,9 +236,7 @@ impl IntentScheduler { } // Returns [`ScheduledBaseIntent`] that can be executed - pub fn pop_next_scheduled_intent( - &mut self, - ) -> Option { + pub fn pop_next_scheduled_intent(&mut self) -> Option { // TODO(edwin): optimize. Create counter im IntentMeta & update let mut execute_candidates: HashMap = HashMap::new(); self.blocked_keys.iter().for_each(|(_, queue)| { @@ -704,7 +704,8 @@ mod complete_error_test { let mut msg1 = create_test_intent(1, &[pubkey1, pubkey2], false); assert!(scheduler.schedule(msg1.clone()).is_some()); - msg1.inner.intent_bundle + msg1.inner + .intent_bundle .get_commit_intent_accounts_mut() .unwrap() .push(CommittedAccount { diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 34d771a1c..83226dbd8 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -3,12 +3,14 @@ use std::{ops::ControlFlow, time::Duration}; use magicblock_metrics::metrics; use magicblock_rpc_client::{ utils::{ - decide_rpc_error_flow, map_magicblock_client_error, + decide_rpc_error_flow, get_with_retries, map_magicblock_client_error, send_transaction_with_retries, SendErrorMapper, TransactionErrorMapper, }, - MagicBlockRpcClientError, MagicBlockSendTransactionConfig, - MagicBlockSendTransactionOutcome, MagicblockRpcClient, + MagicBlockRpcClientError, MagicBlockRpcClientResult, + MagicBlockSendTransactionConfig, MagicBlockSendTransactionOutcome, + MagicblockRpcClient, }; +use solana_hash::Hash; use solana_keypair::Keypair; use solana_message::VersionedMessage; use solana_rpc_client_api::config::RpcTransactionConfig; @@ -48,42 +50,32 @@ impl IntentExecutionClient { tasks: &[BaseTaskImpl], ) -> IntentExecutorResult { - struct IntentErrorMapper { - transaction_error_mapper: TxMap, - } - impl SendErrorMapper for IntentErrorMapper - where - TxMap: TransactionErrorMapper< - ExecutionError = TransactionStrategyExecutionError, - >, - { - type ExecutionError = TransactionStrategyExecutionError; - fn map(&self, error: InternalError) -> Self::ExecutionError { - match error { - InternalError::MagicBlockRpcClientError(err) => { - map_magicblock_client_error( - &self.transaction_error_mapper, - *err, - ) - } - err => { - TransactionStrategyExecutionError::InternalError(err) - } - } - } + const RETRY_FOR: Duration = Duration::from_secs(2 * 60); + const MIN_ATTEMPTS: usize = 3; - fn decide_flow( - err: &Self::ExecutionError, - ) -> ControlFlow<(), Duration> { - match err { - TransactionStrategyExecutionError::InternalError( - InternalError::MagicBlockRpcClientError(err), - ) => decide_rpc_error_flow(err), - _ => ControlFlow::Break(()), - } - } - } + // Send with retries + let send_error_mapper = IntentErrorMapper { + transaction_error_mapper: IntentTransactionErrorMapper { tasks }, + }; + let attempt = || async { + self.send_prepared_message(authority, prepared_message.clone()) + .await + }; + send_transaction_with_retries( + attempt, + send_error_mapper, + |i, elapsed| !(elapsed < RETRY_FOR || i < MIN_ATTEMPTS), + ) + .await + } + // Sends signed tx + // Returns success on successfully sent tx, signature can be extracted from tx itself + pub(in crate::intent_executor) async fn send_signed_tx_with_retries( + &self, + transaction: &VersionedTransaction, + tasks: &[BaseTaskImpl], + ) -> Result<(), TransactionStrategyExecutionError> { const RETRY_FOR: Duration = Duration::from_secs(2 * 60); const MIN_ATTEMPTS: usize = 3; @@ -92,14 +84,31 @@ impl IntentExecutionClient { transaction_error_mapper: IntentTransactionErrorMapper { tasks }, }; let attempt = || async { - self.send_prepared_message(authority, prepared_message.clone()) + self.rpc_client + .send_transaction( + transaction, + &MagicBlockSendTransactionConfig::ensure_committed(), + ) .await }; + send_transaction_with_retries( attempt, send_error_mapper, |i, elapsed| !(elapsed < RETRY_FOR || i < MIN_ATTEMPTS), ) + .await?; + Ok(()) + } + + async fn get_latest_blockhash(&self) -> MagicBlockRpcClientResult { + const RETRY_FOR: Duration = Duration::from_secs(60); + const MIN_ATTEMPTS: usize = 5; + + let attempt = || async { self.rpc_client.get_latest_blockhash().await }; + get_with_retries(attempt, DefaultGetErrorMapper, |i, elapsed| { + !(elapsed < RETRY_FOR || i < MIN_ATTEMPTS) + }) .await } @@ -190,3 +199,74 @@ impl IntentExecutionClient { } } } + +struct IntentErrorMapper { + transaction_error_mapper: TxMap, +} + +// TODO(edwin): probably could be removed +impl SendErrorMapper for IntentErrorMapper +where + TxMap: TransactionErrorMapper< + ExecutionError = TransactionStrategyExecutionError, + >, +{ + type ExecutionError = TransactionStrategyExecutionError; + fn map(&self, error: InternalError) -> Self::ExecutionError { + match error { + InternalError::MagicBlockRpcClientError(err) => { + map_magicblock_client_error( + &self.transaction_error_mapper, + *err, + ) + } + err => TransactionStrategyExecutionError::InternalError(err), + } + } + + fn decide_flow(err: &Self::ExecutionError) -> ControlFlow<(), Duration> { + match err { + TransactionStrategyExecutionError::InternalError( + InternalError::MagicBlockRpcClientError(err), + ) => decide_rpc_error_flow(err), + _ => ControlFlow::Break(()), + } + } +} + +impl SendErrorMapper + for IntentErrorMapper +where + TxMap: TransactionErrorMapper< + ExecutionError = TransactionStrategyExecutionError, + >, +{ + type ExecutionError = TransactionStrategyExecutionError; + fn map(&self, error: MagicBlockRpcClientError) -> Self::ExecutionError { + map_magicblock_client_error(&self.transaction_error_mapper, error) + } + + fn decide_flow(err: &Self::ExecutionError) -> ControlFlow<(), Duration> { + match err { + TransactionStrategyExecutionError::InternalError( + InternalError::MagicBlockRpcClientError(err), + ) => decide_rpc_error_flow(err), + _ => ControlFlow::Break(()), + } + } +} + +struct DefaultGetErrorMapper; +impl SendErrorMapper for DefaultGetErrorMapper { + type ExecutionError = MagicBlockRpcClientError; + + fn map(&self, error: MagicBlockRpcClientError) -> Self::ExecutionError { + error + } + + fn decide_flow( + mapped_error: &Self::ExecutionError, + ) -> ControlFlow<(), Duration> { + decide_rpc_error_flow(mapped_error) + } +} diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index b6da1104a..96f68ddcf 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -9,6 +9,7 @@ use crate::{ task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, IntentExecutor, IntentExecutorImpl, }, + outbox_client::{InternalOutboxClient, OutboxClient}, transaction_preparator::TransactionPreparatorImpl, ComputeBudgetConfig, }; @@ -25,20 +26,22 @@ pub struct ExecutorConfig { } /// Dummy struct to simplify signature of CommitSchedulerWorker -pub struct IntentExecutorFactoryImpl { +pub struct IntentExecutorFactoryImpl { pub rpc_client: MagicblockRpcClient, pub table_mania: TableMania, pub executor_config: ExecutorConfig, pub task_info_fetcher: Arc>, + pub outbox_client: Arc, pub actions_callback_executor: A, } -impl IntentExecutorFactory for IntentExecutorFactoryImpl +impl IntentExecutorFactory for IntentExecutorFactoryImpl where A: ActionsCallbackScheduler, + O: OutboxClient, { type Executor = - IntentExecutorImpl; + IntentExecutorImpl; fn create_instance(&self) -> Self::Executor { let transaction_preparator = TransactionPreparatorImpl::new( @@ -50,6 +53,7 @@ where self.rpc_client.clone(), transaction_preparator, self.task_info_fetcher.clone(), + self.outbox_client.clone(), self.actions_callback_executor.clone(), self.executor_config.actions_timeout, ) diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs b/magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs new file mode 100644 index 000000000..0945ac37e --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs @@ -0,0 +1,35 @@ +use std::{sync::Arc, time::Duration}; + +use magicblock_core::traits::ActionsCallbackScheduler; +use solana_keypair::Keypair; + +use crate::{ + intent_executor::{ + intent_execution_client::IntentExecutionClient, + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + }, + transaction_preparator::TransactionPreparator, +}; + +pub struct IntentExecutorGateway { + authority: Keypair, + intent_client: IntentExecutionClient, + transaction_preparator: T, + task_info_fetcher: Arc>, + actions_callback_executor: A, + /// Timeout for Intent's actions + actions_timeout: Duration, +} + +impl IntentExecutorGateway +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, +{ + pub fn new() -> Self { + Self {} + } +} + +async fn intent_executor_gateway() {} diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index a4f0742d4..987052b4f 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -1,6 +1,7 @@ pub mod error; pub mod intent_execution_client; pub(crate) mod intent_executor_factory; +pub mod intent_executor_gateway; pub mod single_stage_executor; pub mod task_info_fetcher; pub mod two_stage_executor; @@ -19,8 +20,7 @@ use magicblock_core::traits::{ use magicblock_metrics::metrics; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, - validator::validator_authority, + outbox_intent_bundles::OutboxIntentBundle, validator::validator_authority, }; use magicblock_rpc_client::MagicblockRpcClient; use solana_keypair::Keypair; @@ -44,6 +44,7 @@ use crate::{ FinalizeStage, SingleStage, }, }, + outbox_client::OutboxClient, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_strategist::{ @@ -143,12 +144,13 @@ impl IntentExecutionReport { } } -pub struct IntentExecutorImpl { +pub struct IntentExecutorImpl { authority: Keypair, intent_client: IntentExecutionClient, transaction_preparator: T, task_info_fetcher: Arc>, actions_callback_executor: A, + outbox_client: Arc, /// Timeout for Intent's actions actions_timeout: Duration, @@ -162,16 +164,18 @@ pub struct IntentExecutorImpl { close_buffers: bool, } -impl IntentExecutorImpl +impl IntentExecutorImpl where T: TransactionPreparator, F: TaskInfoFetcher, A: ActionsCallbackScheduler, + O: OutboxClient, { pub fn new( rpc_client: MagicblockRpcClient, transaction_preparator: T, task_info_fetcher: Arc>, + outbox_client: Arc, actions_callback_executor: A, actions_timeout: Duration, ) -> Self { @@ -182,6 +186,7 @@ where intent_client, transaction_preparator, task_info_fetcher, + outbox_client, actions_callback_executor, actions_timeout, @@ -359,6 +364,7 @@ where commit_strategy, finalize_strategy, self.intent_client.clone(), + self.outbox_client.clone(), self.actions_callback_executor.clone(), execution_report, ); @@ -390,15 +396,15 @@ where finalize_signature: finalized_stage.finalize_signature, }) } - } #[async_trait] -impl IntentExecutor for IntentExecutorImpl +impl IntentExecutor for IntentExecutorImpl where T: TransactionPreparator, C: TaskInfoFetcher, A: ActionsCallbackScheduler, + O: OutboxClient, { /// Executes Message on Base layer /// Returns `ExecutionOutput` or an `Error` diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs index 803c06757..596f0cb4b 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs @@ -21,30 +21,34 @@ use crate::{ }, IntentExecutionReport, }, + outbox_client::OutboxClient, tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, transaction_preparator::TransactionPreparator, }; -pub struct SingleStageExecutor<'a, F, A> { +pub struct SingleStageExecutor<'a, F, A, O> { current_attempt: u8, execution_report: &'a mut IntentExecutionReport, authority: Keypair, intent_client: IntentExecutionClient, task_info_fetcher: Arc>, + outbox_client: Arc, callback_scheduler: A, transaction_strategy: TransactionStrategy, } -impl<'a, F, A> SingleStageExecutor<'a, F, A> +impl<'a, F, A, O> SingleStageExecutor<'a, F, A, O> where F: TaskInfoFetcher, A: ActionsCallbackScheduler, + O: OutboxClient, { pub fn new( authority: Keypair, intent_client: IntentExecutionClient, task_info_fetcher: Arc>, + outbox_client: Arc, transaction_strategy: TransactionStrategy, callback_scheduler: A, execution_report: &'a mut IntentExecutionReport, @@ -54,6 +58,7 @@ where authority, intent_client, task_info_fetcher, + outbox_client, transaction_strategy, callback_scheduler, execution_report, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index d761a39ba..5df7ca934 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -1,4 +1,4 @@ -use std::{mem, ops::ControlFlow}; +use std::{mem, ops::ControlFlow, sync::Arc}; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; use solana_keypair::Keypair; @@ -22,6 +22,7 @@ use crate::{ }, IntentExecutionReport, }, + outbox_client::OutboxClient, tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, transaction_preparator::TransactionPreparator, }; @@ -31,7 +32,8 @@ pub struct Initialized { commit_strategy: TransactionStrategy, /// Finalize stage strategy finalize_strategy: TransactionStrategy, - + // /// + // commit_signature: Option, current_attempt: u8, } @@ -51,17 +53,19 @@ pub struct Finalized { pub finalize_signature: Signature, } -pub struct TwoStageExecutor<'a, A, S: Sealed> { +pub struct TwoStageExecutor<'a, A, O, S: Sealed> { state: S, authority: Keypair, intent_client: IntentExecutionClient, + outbox_client: Arc, callback_scheduler: A, execution_report: &'a mut IntentExecutionReport, } -impl<'a, A> TwoStageExecutor<'a, A, Initialized> +impl<'a, A, O> TwoStageExecutor<'a, A, O, Initialized> where A: ActionsCallbackScheduler, + O: OutboxClient, { const RECURSION_CEILING: u8 = 10; @@ -70,6 +74,7 @@ where commit_strategy: TransactionStrategy, finalize_strategy: TransactionStrategy, intent_client: IntentExecutionClient, + outbox_client: Arc, callback_scheduler: A, execution_report: &'a mut IntentExecutionReport, ) -> Self { @@ -78,6 +83,7 @@ where intent_client, execution_report, callback_scheduler, + outbox_client, state: Initialized { commit_strategy, finalize_strategy, @@ -86,6 +92,18 @@ where } } + // We need to: + // 1. Prepare everything for tx delivery + // 2. Sign Tx + // 3. Update outbox with Committing(signature) + // 4, Send Tx + // 5. Confirm signature + // a. Success - terminate + // b. jump to 1 + + // We can start from 5: + // Committing(sig) was loaded from outbox + // confirm signature #[instrument( skip( self, @@ -338,10 +356,11 @@ where pub fn done( self, commit_signature: Signature, - ) -> TwoStageExecutor<'a, A, Committed> { + ) -> TwoStageExecutor<'a, A, O, Committed> { TwoStageExecutor { authority: self.authority, intent_client: self.intent_client, + outbox_client: self.outbox_client, callback_scheduler: self.callback_scheduler, execution_report: self.execution_report, state: Committed { @@ -353,9 +372,10 @@ where } } -impl<'a, A> TwoStageExecutor<'a, A, Committed> +impl<'a, A, O> TwoStageExecutor<'a, A, O, Committed> where A: ActionsCallbackScheduler, + O: OutboxClient, { const RECURSION_CEILING: u8 = 10; diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 0cda26363..492dccfd8 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -34,8 +34,8 @@ pub async fn prepare_and_execute_strategy( authority: &Keypair, transaction_preparator: &T, transaction_strategy: &mut TransactionStrategy, -) -> IntentExecutorResult< - IntentExecutorResult, +) -> Result< + Result, TransactionPreparatorError, > where @@ -316,10 +316,7 @@ where async fn execute(&mut self) -> IntentExecutorResult { self.inner - .execute( - self.committed_pubkeys, - self.transaction_preparator, - ) + .execute(self.committed_pubkeys, self.transaction_preparator) .await } @@ -388,9 +385,7 @@ where } async fn execute(&mut self) -> IntentExecutorResult { - self.inner - .finalize(self.transaction_preparator) - .await + self.inner.finalize(self.transaction_preparator).await } fn execute_callbacks( diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 48d90930f..3eef2a410 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -5,12 +5,12 @@ mod consts; pub mod error; pub mod intent_execution_manager; pub mod intent_executor; -pub mod persist; pub mod tasks; pub mod transaction_preparator; pub mod transactions; pub(crate) mod utils; +pub mod outbox_client; pub mod service; #[cfg(test)] pub mod test_utils; diff --git a/magicblock-committor-service/src/service/intent_client.rs b/magicblock-committor-service/src/outbox_client.rs similarity index 81% rename from magicblock-committor-service/src/service/intent_client.rs rename to magicblock-committor-service/src/outbox_client.rs index f89d4d515..098fd098f 100644 --- a/magicblock-committor-service/src/service/intent_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -8,7 +8,7 @@ use magicblock_core::{ }; use magicblock_program::{ instruction_utils::InstructionUtils, - magic_scheduled_base_intent::ScheduledIntentBundle, + magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, register_scheduled_commit_sent, MagicContext, SentCommit, TransactionScheduler, MAGIC_CONTEXT_PUBKEY, }; @@ -22,17 +22,26 @@ use crate::service::outbox_intent_bundles_reader::{ }; #[async_trait] -pub trait ERIntentClient: Send + Sync + 'static { +pub trait OutboxClient: Send + Sync + 'static { + type Error: std::error::Error + Send; /// Type that is able to read IntentBundles from Outbox /// Can be via AccountsDB, RpcClient or any other means type OutboxReader: OutboxIntentBundlesReader; - type Error: std::error::Error + Send; /// Executes `Accept` tx and returns accepted intents async fn accept_scheduled_intents( &self, ) -> Result, Self::Error>; + /// Sets execution stage for outbox intent + /// Note: intent has to be accepted prior + /// Calling with invalid state transitions will lead to `TransactionError` + async fn set_intent_execution_stage( + &self, + intent_id: u64, + stage: ExecutionStage, + ) -> Result<(), Self::Error>; + /// Processes intent results, submitting them on chain(ER) async fn notify_commit_sent( &self, @@ -42,12 +51,11 @@ pub trait ERIntentClient: Send + Sync + 'static { /// Returns reader capable of reading IntentBundles from Outbox fn outbox_reader(&self) -> Self::OutboxReader; - - // TODO(edwin): probably more proper place to load pending intent - // CommittorProcessor::pending_intent_bundles could be moved here in the future } -pub struct InternalIntentRpcClient { +/// Implementation of `OutboxClient` that uses ER internals +/// Potentially could be replaced with RPC base Client +pub struct InternalOutboxClient { /// Provides access to MagicContext accounts_db: Arc, /// Internal endpoint for scheduling ER TXs @@ -56,7 +64,7 @@ pub struct InternalIntentRpcClient { latest_block_provider: L, } -impl InternalIntentRpcClient { +impl InternalOutboxClient { pub fn new( accounts_db: Arc, transaction_scheduler: TransactionSchedulerHandle, @@ -90,7 +98,7 @@ impl InternalIntentRpcClient { } #[async_trait] -impl ERIntentClient for InternalIntentRpcClient { +impl OutboxClient for InternalOutboxClient { type Error = InternalIntentClientError; type OutboxReader = InternalOutboxIntentBundlesReader; @@ -117,6 +125,7 @@ impl ERIntentClient for InternalIntentRpcClient { sent_tx: Transaction, sent_commit: SentCommit, ) -> Result<(), Self::Error> { + // TODO(edwin): is using handle directly here ok? This could require Chainlink mechanics register_scheduled_commit_sent(sent_commit); let txn = with_encoded(sent_tx).inspect_err(|err| { // Unreachable case, all intent transactions are smaller than 64KB by construction @@ -133,6 +142,15 @@ impl ERIntentClient for InternalIntentRpcClient { Ok(()) } + fn set_intent_execution_stage( + &self, + intent_id: u64, + stage: ExecutionStage, + ) -> Result<(), Self::Error> { + // TODO(edwin): use rpc of scheduler + todo!() + } + fn outbox_reader(&self) -> Self::OutboxReader { InternalOutboxIntentBundlesReader::new(self.accounts_db.clone()) } diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 323ed55f2..dfe1fafc9 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,5 +1,4 @@ pub mod acceptor; -pub mod intent_client; pub mod outbox_intent_bundles_reader; use std::{ @@ -12,7 +11,6 @@ use std::{ }; use futures_util::future::join_all; -use intent_client::{ERIntentClient, InternalIntentClientError}; use magicblock_account_cloner::ChainlinkCloner; use magicblock_chainlink::{ProdChainlink, ProdInnerChainlink}; use magicblock_metrics::metrics::{self, AccountFetchOrigin}; @@ -34,6 +32,7 @@ use crate::{ error::CommittorServiceResult, intent_execution_manager::BroadcastedIntentExecutionResult, intent_executor::ExecutionOutput, + outbox_client::{InternalIntentClientError, OutboxClient}, service::outbox_intent_bundles_reader::{ InternalOutboxIntentBundlesReaderError, OutboxIntentBundlesReader, }, @@ -53,7 +52,7 @@ pub enum IntentExecutionService { impl IntentExecutionService where - R: ERIntentClient, + R: OutboxClient, // ERIntentClient errors should be convertible to Service errors R::Error: Into, // OutboxReader errors should be convertible to Service errors @@ -105,11 +104,11 @@ where } } -pub struct ServiceInner { +pub struct ServiceInner { /// Chainlink for notifying of undelegations chainlink: Arc, /// ER client specific for Intent needs. Could be switched to RpcClient - intent_client: Arc, + outbox_client: Arc, /// Processor of accepted intents processor: Arc, /// Time interval to scrape MagicContext(ER slot interval) @@ -120,25 +119,25 @@ pub struct ServiceInner { intents_meta_map: Arc>>, } -impl ServiceInner +impl ServiceInner where - R: ERIntentClient, + O: OutboxClient, // ERIntentClient errors should be convertible to Service errors - R::Error: Into, + O::Error: Into, // OutboxReader errors should be convertible to Service errors - ::Error: + ::Error: Into, { pub fn new( chainlink: Arc, - intent_rpc_client: R, + outbox_client: Arc, processor: Arc, slot_interval: Duration, cancellation_token: CancellationToken, ) -> Self { Self { chainlink, - intent_client: Arc::new(intent_rpc_client), + outbox_client, processor, slot_interval, cancellation_token, @@ -154,7 +153,7 @@ where result_subscriber, cancellation_token, self.intents_meta_map.clone(), - self.intent_client.clone(), + self.outbox_client.clone(), )); tokio::task::spawn(self.accept_worker()) @@ -185,7 +184,7 @@ where } _ = interval.tick() => { let accept_result = self - .intent_client + .outbox_client .accept_scheduled_intents() .await; let intent_bundles = match accept_result { @@ -211,7 +210,7 @@ where const RESCHEDULE_CHUNK_SIZE: NonZeroUsize = NonZeroUsize::new(1000).unwrap(); - let mut outbox_bundles_reader = self.intent_client.outbox_reader(); + let mut outbox_bundles_reader = self.outbox_client.outbox_reader(); loop { // Read by chunks in order not to overload `IntentExecutionEngine` let intent_bundles_chunk = outbox_bundles_reader diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 9156f22b7..93bd16f7a 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -136,7 +136,11 @@ impl TaskStrategist { // In case this fails as well, it will be retried with TwoStage approach // on retry, once retries are introduced if commit_tasks.len() + finalize_tasks.len() > MAX_UNITED_TASKS_LEN { - return Self::build_two_stage(commit_tasks, finalize_tasks, authority); + return Self::build_two_stage( + commit_tasks, + finalize_tasks, + authority, + ); } // Clone tasks since strategies applied to united case maybe suboptimal for regular one @@ -144,7 +148,8 @@ impl TaskStrategist { let single_stage_tasks = [commit_tasks.clone(), finalize_tasks.clone()].concat(); let single_stage_strategy = - match TaskStrategist::build_strategy(single_stage_tasks, authority) { + match TaskStrategist::build_strategy(single_stage_tasks, authority) + { Ok(strategy) => StrategyExecutionMode::SingleStage(strategy), Err(TaskStrategistError::FailedToFitError) => { // If Tasks can't fit in SingleStage - use TwpStage execution @@ -176,22 +181,18 @@ impl TaskStrategist { } } - fn build_two_stage( + pub fn build_two_stage( commit_tasks: Vec, finalize_tasks: Vec, authority: &Pubkey, - persister: &Option

, ) -> TaskStrategistResult { // Build strategy for Commit stage let commit_strategy = - TaskStrategist::build_strategy(commit_tasks, authority, persister)?; + TaskStrategist::build_strategy(commit_tasks, authority)?; // Build strategy for Finalize stage - let finalize_strategy = TaskStrategist::build_strategy( - finalize_tasks, - authority, - persister, - )?; + let finalize_strategy = + TaskStrategist::build_strategy(finalize_tasks, authority)?; Ok(StrategyExecutionMode::TwoStage { commit_stage: commit_strategy, @@ -201,10 +202,9 @@ impl TaskStrategist { /// Returns [`TransactionStrategy`] for tasks /// Returns Error if all optimizations weren't enough - pub fn build_strategy( + pub fn build_strategy( mut tasks: Vec, validator: &Pubkey, - persistor: &Option

, ) -> TaskStrategistResult { // Attempt optimizing tasks themselves(using buffers) if Self::try_optimize_tx_size_if_needed(&mut tasks)? @@ -220,11 +220,6 @@ impl TaskStrategist { // In case task optimization didn't work // attempt using lookup tables for all keys involved in tasks else if Self::attempt_lookup_tables(&tasks) { - // Persist tasks strategy - if let Some(persistor) = persistor { - Self::persist_tasks_strategy(persistor, &tasks, true); - } - // Get lookup table keys let lookup_tables_keys = Self::collect_lookup_table_keys(validator, &tasks); @@ -296,7 +291,6 @@ impl TaskStrategist { ) } - /// Optimizes tasks so as to bring the transaction size within the limit [`MAX_TRANSACTION_WIRE_SIZE`] /// Returns Ok(size of tx after optimizations) else Err(SignerError). /// Note that the returned size, though possibly optimized one, may still not be under @@ -528,12 +522,8 @@ mod tests { let task = create_test_commit_task(1, 100, 0); let tasks = vec![task.into()]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy"); assert_eq!(strategy.optimized_tasks.len(), 1); assert!(strategy.lookup_tables_keys.is_empty()); @@ -546,12 +536,8 @@ mod tests { let task = create_test_commit_task(1, 1000, 0); // Large task let tasks = vec![task.into()]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); assert_eq!(strategy.optimized_tasks.len(), 1); assert!(matches!( @@ -567,12 +553,8 @@ mod tests { let task = create_test_commit_task(1, 66_000, 0); // Large task let tasks = vec![task.into()]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); assert_eq!(strategy.optimized_tasks.len(), 1); assert!(matches!( @@ -589,12 +571,8 @@ mod tests { create_test_commit_task(1, 66_000, COMMIT_STATE_SIZE_THRESHOLD); // large account but small diff let tasks = vec![task.into()]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); assert_eq!(strategy.optimized_tasks.len(), 1); assert_eq!(strategy.optimized_tasks[0].strategy(), TaskStrategy::Args); @@ -609,12 +587,8 @@ mod tests { create_test_commit_task(1, 66_000, COMMIT_STATE_SIZE_THRESHOLD + 1); // large account but small diff let tasks = vec![task.into()]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); assert_eq!(strategy.optimized_tasks.len(), 1); assert_eq!(strategy.optimized_tasks[0].strategy(), TaskStrategy::Args); @@ -628,12 +602,8 @@ mod tests { create_test_commit_task(1, 66_000, COMMIT_STATE_SIZE_THRESHOLD * 4); // large account but small diff let tasks = vec![task.into()]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); assert_eq!(strategy.optimized_tasks.len(), 1); assert_eq!( @@ -656,12 +626,8 @@ mod tests { }) .collect(); - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); for optimized_task in strategy.optimized_tasks { assert!(matches!(optimized_task.strategy(), TaskStrategy::Buffer)); @@ -684,12 +650,8 @@ mod tests { }) .collect(); - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy with buffer optimization"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy with buffer optimization"); for optimized_task in strategy.optimized_tasks { assert!(matches!(optimized_task.strategy(), TaskStrategy::Buffer)); @@ -711,11 +673,7 @@ mod tests { }) .collect(); - let result = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ); + let result = TaskStrategist::build_strategy(tasks, &validator); assert!(matches!(result, Err(TaskStrategistError::FailedToFitError))); } @@ -743,12 +701,8 @@ mod tests { create_test_undelegate_task().into(), ]; - let strategy = TaskStrategist::build_strategy( - tasks, - &validator, - &None::, - ) - .expect("Should build strategy"); + let strategy = TaskStrategist::build_strategy(tasks, &validator) + .expect("Should build strategy"); assert_eq!(strategy.optimized_tasks.len(), 4); diff --git a/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs b/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs index e245b38ff..8a3c920f0 100644 --- a/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs +++ b/magicblock-committor-service/src/transaction_preparator/delivery_preparator.rs @@ -68,8 +68,7 @@ impl DeliveryPreparator { metrics::observe_committor_intent_task_preparation_time( &*task, ); - self.prepare_task_handling_errors(authority, task) - .await + self.prepare_task_handling_errors(authority, task).await }); let task_preparations = join_all(preparation_futures); diff --git a/magicblock-committor-service/src/utils.rs b/magicblock-committor-service/src/utils.rs index e69de29bb..8b1378917 100644 --- a/magicblock-committor-service/src/utils.rs +++ b/magicblock-committor-service/src/utils.rs @@ -0,0 +1 @@ + diff --git a/magicblock-rpc-client/src/utils.rs b/magicblock-rpc-client/src/utils.rs index 93acbb6e8..4fe44415d 100644 --- a/magicblock-rpc-client/src/utils.rs +++ b/magicblock-rpc-client/src/utils.rs @@ -33,7 +33,7 @@ pub trait SendErrorMapper { /// - `ExecErr`: The unified execution error type returned to the caller with mapped errors pub async fn send_transaction_with_retries( make_send_fut: F, - send_result_mapper: Map, + send_error_mapper: Map, stop_predicate: impl Fn(usize, Duration) -> bool, ) -> Result where @@ -56,7 +56,42 @@ where }, Err(err) => err, }; - let mapped_error = send_result_mapper.map(err); + let mapped_error = send_error_mapper.map(err); + let sleep_duration = match Map::decide_flow(&mapped_error) { + ControlFlow::Continue(value) => value, + ControlFlow::Break(()) => return Err(mapped_error), + }; + + if stop_predicate(i, start.elapsed()) { + return Err(mapped_error); + } + + sleep(sleep_duration).await + } +} + +pub async fn get_with_retries( + make_get_fut: F, + error_mapper: Map, + stop_predicate: impl Fn(usize, Duration) -> bool, +) -> Result +where + F: Fn() -> Fut, + Fut: Future>, + Map: SendErrorMapper, +{ + let start = Instant::now(); + let mut i = 0; + + loop { + i += 1; + + let result = make_get_fut().await; + let err = match result { + Ok(value) => return Ok(value), + Err(err) => err, + }; + let mapped_error = error_mapper.map(err); let sleep_duration = match Map::decide_flow(&mapped_error) { ControlFlow::Continue(value) => value, ControlFlow::Break(()) => return Err(mapped_error), diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 17ff28d63..4026dbcc5 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,4 +1,5 @@ use std::ops::Deref; + use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; use magicblock_magic_program_api::outbox::{ExecutionStage, TwoStageProgress}; use serde::{Deserialize, Serialize}; @@ -92,4 +93,4 @@ impl Deref for OutboxIntentBundle { fn deref(&self) -> &Self::Target { &self.inner } -} \ No newline at end of file +} diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 26acbc58a..72cfbcc9f 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -1438,13 +1438,11 @@ async fn create_two_stage_executor<'a>( let commit_strategy = TaskStrategist::build_strategy( commit_tasks, authority, - &None::, ) .unwrap(); let finalize_strategy = TaskStrategist::build_strategy( finalize_tasks, authority, - &None::, ) .unwrap(); TwoStageExecutor::new( @@ -1696,7 +1694,6 @@ async fn single_flow_transaction_strategy( let mut tasks = TaskBuilderImpl::commit_tasks( task_info_fetcher, intent, - &None::, ) .await .unwrap(); @@ -1709,7 +1706,6 @@ async fn single_flow_transaction_strategy( TaskStrategist::build_strategy( tasks, authority, - &None::, ) .unwrap() } From a560353dadb3f82fd2acda5380afa680f1d92673 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 10 Jun 2026 15:17:58 +0700 Subject: [PATCH 37/97] wip intent executors --- .../src/intent_executor/error.rs | 3 ++ .../intent_execution_client.rs | 18 ++++++- .../src/intent_executor/mod.rs | 1 + .../src/intent_executor/two_stage_executor.rs | 54 +++++++++++++++++-- .../src/intent_executor/utils.rs | 30 ++++++++++- .../src/outbox_client.rs | 6 +-- magicblock-committor-service/src/service.rs | 4 +- .../src/transaction_preparator/error.rs | 3 ++ 8 files changed, 109 insertions(+), 10 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index 170087c3a..0f6b15af3 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -14,6 +14,7 @@ use solana_transaction_error::TransactionError; use tracing::error; use crate::{ + outbox_client::InternalOutboxClientError, tasks::{ task_builder::TaskBuilderError, task_strategist::TaskStrategistError, BaseTaskImpl, @@ -75,6 +76,8 @@ pub enum IntentExecutorError { FailedToFitError, #[error("SignerError: {0}")] SignerError(#[from] SignerError), + #[error("OutboxClientError: {0}")] + OutboxClientError(#[from] InternalOutboxClientError), // TODO(edwin): remove once proper retries introduced #[error("TaskBuilderError: {0}")] TaskBuilderError(#[from] TaskBuilderError), diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 83226dbd8..1d4e9e715 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -101,7 +101,23 @@ impl IntentExecutionClient { Ok(()) } - async fn get_latest_blockhash(&self) -> MagicBlockRpcClientResult { + /// Checks whether a previously-sent transaction confirmed. + /// `Err` means the signature was not confirmed within the timeout window. + pub(in crate::intent_executor) async fn wait_for_confirmed_status( + &self, + signature: &Signature, + ) -> MagicBlockRpcClientResult> { + const TIMEOUT: Duration = Duration::from_secs(5); + const INTERVAL: Duration = Duration::from_millis(200); + + self.rpc_client + .wait_for_confirmed_status(signature, &TIMEOUT, &Some(INTERVAL)) + .await + } + + pub(in crate::intent_executor) async fn get_latest_blockhash( + &self, + ) -> MagicBlockRpcClientResult { const RETRY_FOR: Duration = Duration::from_secs(60); const MIN_ATTEMPTS: usize = 5; diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 987052b4f..6d8a25759 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -295,6 +295,7 @@ where self.authority.insecure_clone(), self.intent_client.clone(), self.task_info_fetcher.clone(), + self.outbox_client.clone(), transaction_strategy, self.actions_callback_executor.clone(), execution_report, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index 5df7ca934..583bd25ca 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -1,10 +1,13 @@ use std::{mem, ops::ControlFlow, sync::Arc}; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; +use magicblock_program::outbox::{ExecutionStage, TwoStageProgress}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; +use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_signer::Signer; +use solana_transaction::versioned::VersionedTransaction; use tracing::{error, instrument, warn}; use crate::{ @@ -19,12 +22,15 @@ use crate::{ utils::{ handle_actions_result, handle_commit_id_error, handle_undelegation_error, prepare_and_execute_strategy, + prepare_transaction, }, IntentExecutionReport, }, outbox_client::OutboxClient, tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, - transaction_preparator::TransactionPreparator, + transaction_preparator::{ + error::TransactionPreparatorError, TransactionPreparator, + }, }; pub struct Initialized { @@ -32,8 +38,9 @@ pub struct Initialized { commit_strategy: TransactionStrategy, /// Finalize stage strategy finalize_strategy: TransactionStrategy, - // /// - // commit_signature: Option, + /// Signature pending confirmation + /// Sources: Outbox with status Committing, timeout + pending_signature: Option, current_attempt: u8, } @@ -54,8 +61,11 @@ pub struct Finalized { } pub struct TwoStageExecutor<'a, A, O, S: Sealed> { + /// State per stage state: S, + /// Common state across the stages authority: Keypair, + intent_id: u64, intent_client: IntentExecutionClient, outbox_client: Arc, callback_scheduler: A, @@ -66,11 +76,13 @@ impl<'a, A, O> TwoStageExecutor<'a, A, O, Initialized> where A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { const RECURSION_CEILING: u8 = 10; pub fn new( authority: Keypair, + intent_id: u64, commit_strategy: TransactionStrategy, finalize_strategy: TransactionStrategy, intent_client: IntentExecutionClient, @@ -80,6 +92,7 @@ where ) -> Self { Self { authority, + intent_id, intent_client, execution_report, callback_scheduler, @@ -124,8 +137,39 @@ where F: TaskInfoFetcher, { let commit_result = loop { + if let Some(ref pending_signature) = self.state.pending_signature { + // TODO(edwin): check if tx succeeded here quering by signature + // self.intent_client.wait_for_processed_status / wait_for_confirmed_status?; + } + self.state.current_attempt += 1; + // Prepare message + let prepared_transaction = prepare_transaction( + &self.intent_client, + &self.authority, + transaction_preparator, + &mut self.state.commit_strategy, + ) + .await + .map_err(IntentExecutorError::FailedCommitPreparationError)?; + + // Record in outbox + let signature = prepared_transaction.get_signature(); + self.outbox_client + .set_intent_execution_stage( + self.intent_id, + ExecutionStage::TwoStage(TwoStageProgress::Committing( + *signature, + )), + ) + .await + .map_err(Into::into)?; + + // Now record locally signature of tx we about to send + // Precaution for timeout in between + self.state.pending_signature = Some(*signature); + // Prepare & execute message let execution_result = prepare_and_execute_strategy( &self.intent_client, @@ -135,6 +179,10 @@ where ) .await .map_err(IntentExecutorError::FailedCommitPreparationError)?; + + // Result returned, cleanup pending signature + self.state.pending_signature = None; + let execution_err = match execution_result { Ok(value) => break Ok(value), Err(err) => err, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 492dccfd8..31b57515f 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -7,12 +7,16 @@ use magicblock_core::traits::{ use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_signature::Signature; +use solana_transaction::versioned::VersionedTransaction; use tokio::time::timeout; use tracing::info; use crate::{ intent_executor::{ - error::{IntentExecutorResult, TransactionStrategyExecutionError}, + error::{ + IntentExecutorError, IntentExecutorResult, + TransactionStrategyExecutionError, + }, intent_execution_client::IntentExecutionClient, single_stage_executor::SingleStageExecutor, task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, @@ -29,6 +33,30 @@ use crate::{ }, }; +pub async fn prepare_transaction( + client: &IntentExecutionClient, + authority: &Keypair, + transaction_preparator: &T, + transaction_strategy: &mut TransactionStrategy, +) -> Result { + let mut prepared_message = transaction_preparator + .prepare_for_strategy(&authority, transaction_strategy) + .await?; + + // Get latest blockhash(Part of preparation I guess) + let blockhash = client + .get_latest_blockhash() + .await + .map_err(TransactionPreparatorError::GetLatestBlockhashError)?; + prepared_message.set_recent_blockhash(blockhash); + + // Create and sign transaction(Part of transaction preparation i guess, hence the errors) + let transaction = + VersionedTransaction::try_new(prepared_message, &[&authority]) + .map_err(TransactionPreparatorError::SignerError)?; + Ok(transaction) +} + pub async fn prepare_and_execute_strategy( client: &IntentExecutionClient, authority: &Keypair, diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index 098fd098f..d9c0d81f9 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -79,7 +79,7 @@ impl InternalOutboxClient { /// Sends transaction to move the scheduled commits from the `MagicContext` /// to the global ScheduledCommit store - async fn send_accept_tx(&self) -> Result<(), InternalIntentClientError> { + async fn send_accept_tx(&self) -> Result<(), InternalOutboxClientError> { let tx = InstructionUtils::accept_scheduled_commits( self.latest_block_provider.blockhash(), ); @@ -99,7 +99,7 @@ impl InternalOutboxClient { #[async_trait] impl OutboxClient for InternalOutboxClient { - type Error = InternalIntentClientError; + type Error = InternalOutboxClientError; type OutboxReader = InternalOutboxIntentBundlesReader; async fn accept_scheduled_intents( @@ -157,7 +157,7 @@ impl OutboxClient for InternalOutboxClient { } #[derive(thiserror::Error, Debug)] -pub enum InternalIntentClientError { +pub enum InternalOutboxClientError { #[error("TransactionError: {0}")] TransactionError(#[from] TransactionError), } diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index dfe1fafc9..1e3da7337 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -32,7 +32,7 @@ use crate::{ error::CommittorServiceResult, intent_execution_manager::BroadcastedIntentExecutionResult, intent_executor::ExecutionOutput, - outbox_client::{InternalIntentClientError, OutboxClient}, + outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::{ InternalOutboxIntentBundlesReaderError, OutboxIntentBundlesReader, }, @@ -592,7 +592,7 @@ pub enum IntentExecutionServiceError { #[error("JoinError: {0}")] JoinError(#[from] JoinError), #[error("IntentRpcClientError: {0}")] - IntentRpcClientError(#[from] InternalIntentClientError), + IntentRpcClientError(#[from] InternalOutboxClientError), #[error("OutboxReaderError")] OutboxReaderError(#[from] InternalOutboxIntentBundlesReaderError), } diff --git a/magicblock-committor-service/src/transaction_preparator/error.rs b/magicblock-committor-service/src/transaction_preparator/error.rs index f95c4dbbe..432bdcbda 100644 --- a/magicblock-committor-service/src/transaction_preparator/error.rs +++ b/magicblock-committor-service/src/transaction_preparator/error.rs @@ -1,3 +1,4 @@ +use magicblock_rpc_client::MagicBlockRpcClientError; use solana_signature::Signature; use solana_signer::SignerError; use thiserror::Error; @@ -13,6 +14,8 @@ pub enum TransactionPreparatorError { FailedToFitError, #[error("SignerError: {0}")] SignerError(#[from] SignerError), + #[error("Failed to get latest blockhash: {0}")] + GetLatestBlockhashError(#[from] MagicBlockRpcClientError), #[error("DeliveryPreparationError: {0}")] DeliveryPreparationError(Box), } From 75c02442b17e1a127ad92f3b20fb1f05e277f5c0 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 10 Jun 2026 16:44:55 +0700 Subject: [PATCH 38/97] wip: commit stage adjusted for outbox --- .../src/intent_executor/error.rs | 2 + .../intent_execution_client.rs | 24 ++++++++++++ .../intent_executor/single_stage_executor.rs | 3 +- .../src/intent_executor/two_stage_executor.rs | 39 +++++++++++-------- .../src/intent_executor/utils.rs | 37 +++++++++++++++++- .../src/transaction_preparator/error.rs | 2 +- 6 files changed, 86 insertions(+), 21 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index 0f6b15af3..3b74994c5 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -78,6 +78,8 @@ pub enum IntentExecutorError { SignerError(#[from] SignerError), #[error("OutboxClientError: {0}")] OutboxClientError(#[from] InternalOutboxClientError), + #[error("Failed to get pending signature status: {0}")] + GetPendingSignatureStatusError(#[source] MagicBlockRpcClientError), // TODO(edwin): remove once proper retries introduced #[error("TaskBuilderError: {0}")] TaskBuilderError(#[from] TaskBuilderError), diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 1d4e9e715..ccf29419c 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -16,6 +16,7 @@ use solana_message::VersionedMessage; use solana_rpc_client_api::config::RpcTransactionConfig; use solana_signature::Signature; use solana_transaction::versioned::VersionedTransaction; +use solana_transaction_error::TransactionError; use tracing::warn; use crate::{ @@ -115,6 +116,29 @@ impl IntentExecutionClient { .await } + /// Queries the full transaction history for the given signatures. + /// Each entry is `None` if the signature was never included in a block. + /// Use this for restart recovery where txs may be older than the RPC + /// node's recent signature cache. + pub(in crate::intent_executor) async fn get_signature_statuses_with_history( + &self, + signatures: &[Signature], + ) -> MagicBlockRpcClientResult>>> + { + let response = self + .rpc_client + .get_inner() + .get_signature_statuses_with_history(signatures) + .await + // TODO(edwin): map to more specific error? verify + .map_err(MagicBlockRpcClientError::from)?; + Ok(response + .value + .into_iter() + .map(|s| s.map(|s| s.err.map_or(Ok(()), Err))) + .collect()) + } + pub(in crate::intent_executor) async fn get_latest_blockhash( &self, ) -> MagicBlockRpcClientResult { diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs index 596f0cb4b..8675b8e66 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs @@ -16,8 +16,9 @@ use crate::{ intent_execution_client::IntentExecutionClient, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, utils::{ - handle_actions_result, handle_commit_id_error, + check_pending_signature, handle_actions_result, handle_commit_id_error, handle_undelegation_error, prepare_and_execute_strategy, + prepare_transaction, }, IntentExecutionReport, }, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index 583bd25ca..6cd2f5b8a 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -8,7 +8,7 @@ use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_signer::Signer; use solana_transaction::versioned::VersionedTransaction; -use tracing::{error, instrument, warn}; +use tracing::{error, info, instrument, warn}; use crate::{ intent_executor::{ @@ -20,9 +20,9 @@ use crate::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, two_stage_executor::sealed::Sealed, utils::{ - handle_actions_result, handle_commit_id_error, - handle_undelegation_error, prepare_and_execute_strategy, - prepare_transaction, + check_pending_signature, handle_actions_result, + handle_commit_id_error, handle_undelegation_error, + prepare_and_execute_strategy, prepare_transaction, }, IntentExecutionReport, }, @@ -137,9 +137,15 @@ where F: TaskInfoFetcher, { let commit_result = loop { - if let Some(ref pending_signature) = self.state.pending_signature { - // TODO(edwin): check if tx succeeded here quering by signature - // self.intent_client.wait_for_processed_status / wait_for_confirmed_status?; + if let Some(ref sig) = self.state.pending_signature { + let flow = + check_pending_signature(&self.intent_client, sig).await?; + // Pending signature corresponds to succeeded transaction - break + if let ControlFlow::Break(()) = flow { + break Ok(*sig); + } + + self.state.pending_signature = None; } self.state.current_attempt += 1; @@ -170,21 +176,20 @@ where // Precaution for timeout in between self.state.pending_signature = Some(*signature); - // Prepare & execute message - let execution_result = prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.state.commit_strategy, - ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)?; + // Send signed tx + let execution_result = self + .intent_client + .send_signed_tx_with_retries( + &prepared_transaction, + &self.state.commit_strategy.optimized_tasks, + ) + .await; // Result returned, cleanup pending signature self.state.pending_signature = None; let execution_err = match execution_result { - Ok(value) => break Ok(value), + Ok(()) => break Ok(*signature), Err(err) => err, }; diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 31b57515f..38ba29768 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{ops::ControlFlow, time::Duration}; use async_trait::async_trait; use magicblock_core::traits::{ @@ -9,7 +9,7 @@ use solana_pubkey::Pubkey; use solana_signature::Signature; use solana_transaction::versioned::VersionedTransaction; use tokio::time::timeout; -use tracing::info; +use tracing::{info, warn}; use crate::{ intent_executor::{ @@ -57,6 +57,39 @@ pub async fn prepare_transaction( Ok(transaction) } +/// Checks a `pending_signature` loaded from the outbox against the full +/// transaction history. Returns `Break(sig)` if the tx succeeded — the caller +/// can short-circuit and skip re-execution. Returns `Continue` for every other +/// outcome (tx failed, never sent, or RPC returned no entry), letting the +/// normal execution loop proceed. +pub(in crate::intent_executor) async fn check_pending_signature( + client: &IntentExecutionClient, + sig: &Signature, +) -> IntentExecutorResult> { + let statuses = client + .get_signature_statuses_with_history(std::slice::from_ref(sig)) + .await + .map_err(IntentExecutorError::GetPendingSignatureStatusError)?; + + match statuses.get(0) { + Some(Some(Ok(()))) => Ok(ControlFlow::Break(())), + // TODO(edwin): well, that is bizarre one + None => { + warn!(pending_signature = ?sig, "RPC did not return status for signature"); + Ok(ControlFlow::Continue(())) + } + // Any case below means tx failed in execution we continue attempts + Some(None) => { + info!(pending_signature = ?sig, "Transaction corresponding to pending signature was never sent"); + Ok(ControlFlow::Continue(())) + } + Some(Some(Err(err))) => { + info!(pending_signature = ?sig, error = ?err, "Transaction corresponding to pending signature failed"); + Ok(ControlFlow::Continue(())) + } + } +} + pub async fn prepare_and_execute_strategy( client: &IntentExecutionClient, authority: &Keypair, diff --git a/magicblock-committor-service/src/transaction_preparator/error.rs b/magicblock-committor-service/src/transaction_preparator/error.rs index 432bdcbda..4efafe034 100644 --- a/magicblock-committor-service/src/transaction_preparator/error.rs +++ b/magicblock-committor-service/src/transaction_preparator/error.rs @@ -15,7 +15,7 @@ pub enum TransactionPreparatorError { #[error("SignerError: {0}")] SignerError(#[from] SignerError), #[error("Failed to get latest blockhash: {0}")] - GetLatestBlockhashError(#[from] MagicBlockRpcClientError), + GetLatestBlockhashError(#[source] MagicBlockRpcClientError), #[error("DeliveryPreparationError: {0}")] DeliveryPreparationError(Box), } From c19fc70e7e9fc68bdc28701f600601b10b3a6eff Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 10 Jun 2026 19:02:16 +0700 Subject: [PATCH 39/97] wip: add support of pending signature in all execution stages --- .../intent_executor/single_stage_executor.rs | 61 ++++++++-- .../src/intent_executor/two_stage_executor.rs | 114 ++++++++++++++++-- 2 files changed, 154 insertions(+), 21 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs index 8675b8e66..711732056 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs @@ -1,8 +1,10 @@ use std::{ops::ControlFlow, sync::Arc}; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; +use magicblock_program::outbox::ExecutionStage; use solana_keypair::Keypair; use solana_pubkey::Pubkey; +use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_signer::Signer; use tracing::{error, instrument}; @@ -16,9 +18,9 @@ use crate::{ intent_execution_client::IntentExecutionClient, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, utils::{ - check_pending_signature, handle_actions_result, handle_commit_id_error, - handle_undelegation_error, prepare_and_execute_strategy, - prepare_transaction, + check_pending_signature, handle_actions_result, + handle_commit_id_error, handle_undelegation_error, + prepare_and_execute_strategy, prepare_transaction, }, IntentExecutionReport, }, @@ -29,9 +31,11 @@ use crate::{ pub struct SingleStageExecutor<'a, F, A, O> { current_attempt: u8, + pending_signature: Option, execution_report: &'a mut IntentExecutionReport, authority: Keypair, + intent_id: u64, intent_client: IntentExecutionClient, task_info_fetcher: Arc>, outbox_client: Arc, @@ -44,9 +48,12 @@ where F: TaskInfoFetcher, A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { pub fn new( authority: Keypair, + intent_id: u64, + pending_signature: Option, intent_client: IntentExecutionClient, task_info_fetcher: Arc>, outbox_client: Arc, @@ -55,14 +62,16 @@ where execution_report: &'a mut IntentExecutionReport, ) -> Self { Self { - current_attempt: 0, authority, + intent_id, + pending_signature, intent_client, task_info_fetcher, outbox_client, transaction_strategy, callback_scheduler, execution_report, + current_attempt: 0, } } @@ -81,10 +90,20 @@ where const RECURSION_CEILING: u8 = 10; let result = loop { + if let Some(ref sig) = self.pending_signature { + let flow = + check_pending_signature(&self.intent_client, sig).await?; + // Pending signature corresponds to succeeded transaction - break + if let ControlFlow::Break(()) = flow { + break Ok(*sig); + } + self.pending_signature = None; + } + self.current_attempt += 1; - // Prepare & execute message - let execution_result = prepare_and_execute_strategy( + // Prepare message + let prepared_transaction = prepare_transaction( &self.intent_client, &self.authority, transaction_preparator, @@ -93,12 +112,36 @@ where .await .map_err(IntentExecutorError::FailedFinalizePreparationError)?; + // Record in outbox + let signature = prepared_transaction.get_signature(); + self.outbox_client + .set_intent_execution_stage( + self.intent_id, + ExecutionStage::SingleStage(*signature), + ) + .await + .map_err(Into::into)?; + + // Now record locally signature of tx we about to send + // Precaution for timeout in between + self.pending_signature = Some(*signature); + + // Send signed tx + let execution_result = self + .intent_client + .send_signed_tx_with_retries( + &prepared_transaction, + &self.transaction_strategy.optimized_tasks, + ) + .await; + + // Result returned, cleanup pending signature + self.pending_signature = None; + // Process error: Ok - return, Err - handle further let execution_err = match execution_result { // break with result, strategy that was executed at this point has to be returned for cleanup - Ok(value) => { - break Ok(value); - } + Ok(()) => break Ok(*signature), Err(err) => err, }; diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index 6cd2f5b8a..929ddaba9 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -44,15 +44,47 @@ pub struct Initialized { current_attempt: u8, } +impl Initialized { + pub fn new( + commit_strategy: TransactionStrategy, + finalize_strategy: TransactionStrategy, + pending_signature: Option, + ) -> Self { + Self { + commit_strategy, + finalize_strategy, + pending_signature, + current_attempt: 0, + } + } +} + pub struct Committed { /// Signature of commit stage commit_signature: Signature, /// Finalize stage strategy finalize_strategy: TransactionStrategy, - + /// Signature pending confirmation + /// Sources: Outbox with status Finalizing, timeout + pending_signature: Option, current_attempt: u8, } +impl Committed { + pub fn new( + commit_signature: Signature, + finalize_strategy: TransactionStrategy, + pending_signature: Option, + ) -> Self { + Self { + commit_signature, + finalize_strategy, + pending_signature, + current_attempt: 0, + } + } +} + pub struct Finalized { /// Signature of commit stage pub commit_signature: Signature, @@ -81,10 +113,9 @@ where const RECURSION_CEILING: u8 = 10; pub fn new( + state: Initialized, authority: Keypair, intent_id: u64, - commit_strategy: TransactionStrategy, - finalize_strategy: TransactionStrategy, intent_client: IntentExecutionClient, outbox_client: Arc, callback_scheduler: A, @@ -97,11 +128,7 @@ where execution_report, callback_scheduler, outbox_client, - state: Initialized { - commit_strategy, - finalize_strategy, - current_attempt: 0, - }, + state, } } @@ -412,6 +439,7 @@ where ) -> TwoStageExecutor<'a, A, O, Committed> { TwoStageExecutor { authority: self.authority, + intent_id: self.intent_id, intent_client: self.intent_client, outbox_client: self.outbox_client, callback_scheduler: self.callback_scheduler, @@ -419,6 +447,7 @@ where state: Committed { commit_signature, finalize_strategy: self.state.finalize_strategy, + pending_signature: None, current_attempt: 0, }, } @@ -429,9 +458,30 @@ impl<'a, A, O> TwoStageExecutor<'a, A, O, Committed> where A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { const RECURSION_CEILING: u8 = 10; + pub fn committed( + state: Committed, + authority: Keypair, + intent_id: u64, + intent_client: IntentExecutionClient, + outbox_client: Arc, + callback_scheduler: A, + execution_report: &'a mut IntentExecutionReport, + ) -> Self { + Self { + authority, + intent_id, + intent_client, + execution_report, + callback_scheduler, + outbox_client, + state, + } + } + #[instrument( skip(self, transaction_preparator), fields(stage = "finalize") @@ -444,10 +494,21 @@ where T: TransactionPreparator, { let finalize_result = loop { + if let Some(ref sig) = self.state.pending_signature { + let flow = + check_pending_signature(&self.intent_client, sig).await?; + // Pending signature corresponds to succeeded transaction - break + if let ControlFlow::Break(()) = flow { + break Ok(*sig); + } + + self.state.pending_signature = None; + } + self.state.current_attempt += 1; - // Prepare & execute message - let execution_result = prepare_and_execute_strategy( + // Prepare message + let prepared_transaction = prepare_transaction( &self.intent_client, &self.authority, transaction_preparator, @@ -455,13 +516,42 @@ where ) .await .map_err(IntentExecutorError::FailedFinalizePreparationError)?; + + // Record in outbox + let signature = prepared_transaction.get_signature(); + self.outbox_client + .set_intent_execution_stage( + self.intent_id, + ExecutionStage::TwoStage(TwoStageProgress::Finalizing { + commit: self.state.commit_signature, + finalize: *signature, + }), + ) + .await + .map_err(Into::into)?; + + // Now record locally signature of tx we about to send + // Precaution for timeout in between + self.state.pending_signature = Some(*signature); + + // Send signed tx + let execution_result = self + .intent_client + .send_signed_tx_with_retries( + &prepared_transaction, + &self.state.finalize_strategy.optimized_tasks, + ) + .await; + + // Result returned, cleanup pending signature + self.state.pending_signature = None; + let execution_err = match execution_result { - Ok(value) => break Ok(value), + Ok(()) => break Ok(*signature), Err(err) => err, }; let flow = self.patch_finalize_strategy(&execution_err).await?; - let cleanup = match flow { ControlFlow::Continue(cleanup) => cleanup, ControlFlow::Break(()) => { From cb8eea80280ea471a0477c93711c6f632129b165 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 10 Jun 2026 20:03:05 +0700 Subject: [PATCH 40/97] wip: extracting commom stage execution loop logic --- .../src/intent_executor/mod.rs | 1 + .../src/intent_executor/patcher.rs | 19 ++++ .../src/intent_executor/two_stage_executor.rs | 1 - .../src/intent_executor/utils.rs | 104 +++++++++++++++++- 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 magicblock-committor-service/src/intent_executor/patcher.rs diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 6d8a25759..48e5a1c1a 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -2,6 +2,7 @@ pub mod error; pub mod intent_execution_client; pub(crate) mod intent_executor_factory; pub mod intent_executor_gateway; +pub mod patcher; pub mod single_stage_executor; pub mod task_info_fetcher; pub mod two_stage_executor; diff --git a/magicblock-committor-service/src/intent_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/patcher.rs new file mode 100644 index 000000000..943dfc9c5 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/patcher.rs @@ -0,0 +1,19 @@ +use std::ops::ControlFlow; + +use async_trait::async_trait; + +use crate::{ + intent_executor::error::{ + IntentExecutorResult, TransactionStrategyExecutionError, + }, + tasks::task_strategist::TransactionStrategy, +}; + +#[async_trait] +pub(in crate::intent_executor) trait Patcher { + async fn patch( + &mut self, + err: &TransactionStrategyExecutionError, + strategy: &mut TransactionStrategy, + ) -> IntentExecutorResult>; +} diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index 929ddaba9..f05121769 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -7,7 +7,6 @@ use solana_pubkey::Pubkey; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_signer::Signer; -use solana_transaction::versioned::VersionedTransaction; use tracing::{error, info, instrument, warn}; use crate::{ diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 38ba29768..84f0f1e49 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -4,12 +4,14 @@ use async_trait::async_trait; use magicblock_core::traits::{ ActionError, ActionResult, ActionsCallbackScheduler, }; +use magicblock_program::outbox::ExecutionStage; use solana_keypair::Keypair; use solana_pubkey::Pubkey; +use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_transaction::versioned::VersionedTransaction; use tokio::time::timeout; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use crate::{ intent_executor::{ @@ -18,11 +20,13 @@ use crate::{ TransactionStrategyExecutionError, }, intent_execution_client::IntentExecutionClient, + patcher::Patcher, single_stage_executor::SingleStageExecutor, task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, two_stage_executor::{Committed, Initialized, TwoStageExecutor}, IntentExecutionReport, }, + outbox_client::OutboxClient, tasks::{ task_builder::TaskBuilderError, task_strategist::{TaskStrategist, TransactionStrategy}, @@ -33,6 +37,104 @@ use crate::{ }, }; +const STAGE_LOOP_CEILING: u8 = 10; + +pub(in crate::intent_executor) struct ExecutionState<'a> { + current_attempt: &'a mut u8, + transaction_strategy: &'a mut TransactionStrategy, + pending_signature: &'a mut Option, + execution_report: &'a mut IntentExecutionReport, +} + +pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( + authority: &Keypair, + intent_client: &IntentExecutionClient, + outbox_client: &O, + transaction_preparator: &T, + mut patcher: P, + intent_id: u64, + make_outbox_stage: impl Fn(Signature) -> ExecutionStage, + map_preparation_err: fn(TransactionPreparatorError) -> IntentExecutorError, + mut state: ExecutionState<'a>, +) -> IntentExecutorResult +where + T: TransactionPreparator, + O: OutboxClient, + O::Error: Into, + P: Patcher, +{ + loop { + if let Some(ref sig) = state.pending_signature { + let flow = check_pending_signature(intent_client, sig).await?; + // Pending signature corresponds to succeeded transaction - break + if let ControlFlow::Break(()) = flow { + return Ok(*sig); + } + + *state.pending_signature = None; + } + + *state.current_attempt += 1; + + // Prepare message + let prepared_transaction = prepare_transaction( + intent_client, + authority, + transaction_preparator, + state.transaction_strategy, + ) + .await + .map_err(map_preparation_err)?; + + // Record in outbox + let signature = prepared_transaction.get_signature(); + outbox_client + .set_intent_execution_stage( + intent_id, + make_outbox_stage(*signature), + ) + .await + .map_err(Into::into)?; + + // Now record locally signature of tx we about to send + // Precaution for timeout in between + *state.pending_signature = Some(*signature); + + // Send signed tx + let execution_result = intent_client + .send_signed_tx_with_retries( + &prepared_transaction, + &state.transaction_strategy.optimized_tasks, + ) + .await; + + // Result returned, cleanup pending signature + *state.pending_signature = None; + + let execution_err = match execution_result { + Ok(()) => return Ok(*signature), + Err(err) => err, + }; + + let flow = patcher + .patch(&execution_err, state.transaction_strategy) + .await?; + let cleanup = match flow { + ControlFlow::Continue(value) => value, + ControlFlow::Break(()) => return Err(execution_err.into()), + }; + intent_client.invalidate_cached_blockhash().await; + state.execution_report.dispose(cleanup); + + if *state.current_attempt >= STAGE_LOOP_CEILING { + error!("CRITICAL! Recursion ceiling reached"); + return Err(execution_err.into()); + } else { + state.execution_report.add_patched_error(execution_err); + } + } +} + pub async fn prepare_transaction( client: &IntentExecutionClient, authority: &Keypair, From 8a9cf7c91841cdc6451c249fe36d385d9107f1bf Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 10 Jun 2026 21:23:32 +0700 Subject: [PATCH 41/97] feat: extract common executors logic in single util --- .../src/committor_processor.rs | 3 - magicblock-committor-service/src/error.rs | 4 +- .../intent_execution_engine.rs | 5 +- .../intent_scheduler.rs | 5 +- .../intent_executor_factory.rs | 4 +- .../src/intent_executor/mod.rs | 17 +- .../src/intent_executor/patcher.rs | 393 +++++++++++++++- .../intent_executor/single_stage_executor.rs | 268 ++--------- .../src/intent_executor/two_stage_executor.rs | 433 +++--------------- .../src/intent_executor/utils.rs | 49 +- .../src/outbox_client.rs | 18 +- magicblock-committor-service/src/service.rs | 16 +- .../src/tasks/task_strategist.rs | 6 +- 13 files changed, 548 insertions(+), 673 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index e61d21a78..b2978aedf 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -1,8 +1,6 @@ use std::{ collections::{hash_map::Entry, HashMap}, - path::Path, sync::{atomic::AtomicU64, Arc, Mutex}, - time::{SystemTime, UNIX_EPOCH}, }; use futures_util::future::join_all; @@ -13,7 +11,6 @@ use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_signer::Signer; use tokio::sync::{broadcast, oneshot, oneshot::error::RecvError}; use tracing::{error, info, instrument}; diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index 7e59fd0d4..c447cd42f 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,9 +1,7 @@ use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::intent_execution_manager::{ - intent_channerl::IntentScheduleError, IntentExecutionManagerError, -}; +use crate::intent_execution_manager::intent_channerl::IntentScheduleError; pub type CommittorServiceResult = Result; diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index bd305d860..a3d5d3008 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -10,10 +10,7 @@ use magicblock_metrics::metrics; use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_signature::Signature; use tokio::{ - sync::{ - broadcast, mpsc, mpsc::error::TryRecvError, OwnedSemaphorePermit, - Semaphore, - }, + sync::{broadcast, OwnedSemaphorePermit, Semaphore}, task::JoinHandle, }; use tokio_stream::StreamExt; diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs b/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs index 9916a6704..b8ef05761 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs @@ -1,9 +1,6 @@ use std::collections::{hash_map::Entry, HashMap, VecDeque}; -use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, -}; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_pubkey::Pubkey; use thiserror::Error; use tracing::error; diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index 96f68ddcf..57e6ada61 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -6,10 +6,11 @@ use magicblock_table_mania::TableMania; use crate::{ intent_executor::{ + error::IntentExecutorError, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, IntentExecutor, IntentExecutorImpl, }, - outbox_client::{InternalOutboxClient, OutboxClient}, + outbox_client::OutboxClient, transaction_preparator::TransactionPreparatorImpl, ComputeBudgetConfig, }; @@ -39,6 +40,7 @@ impl IntentExecutorFactory for IntentExecutorFactoryImpl where A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { type Executor = IntentExecutorImpl; diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 48e5a1c1a..76f583f63 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -39,7 +39,7 @@ use crate::{ intent_execution_client::IntentExecutionClient, single_stage_executor::SingleStageExecutor, task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, - two_stage_executor::TwoStageExecutor, + two_stage_executor::{Initialized, TwoStageExecutor}, utils::{ execute_with_timeout, handle_cpi_limit_error, CommitStage, FinalizeStage, SingleStage, @@ -53,8 +53,7 @@ use crate::{ }, }, transaction_preparator::{ - delivery_preparator::BufferExecutionError, - error::TransactionPreparatorError, TransactionPreparator, + delivery_preparator::BufferExecutionError, TransactionPreparator, }, }; @@ -171,6 +170,7 @@ where F: TaskInfoFetcher, A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { pub fn new( rpc_client: MagicblockRpcClient, @@ -269,6 +269,7 @@ where } => { trace!("Two stage execution"); self.two_stage_execution_flow( + intent_bundle.id, &all_committed_pubkeys, commit_stage, finalize_stage, @@ -294,6 +295,8 @@ where let mut single_stage_executor = SingleStageExecutor::new( self.authority.insecure_clone(), + base_intent.id, + None, self.intent_client.clone(), self.task_info_fetcher.clone(), self.outbox_client.clone(), @@ -346,6 +349,7 @@ where execution_report.add_patched_error(execution_err); self.two_stage_execution_flow( + base_intent.id, &committed_pubkeys, commit_strategy, finalize_strategy, @@ -356,15 +360,17 @@ where pub async fn two_stage_execution_flow( &mut self, + intent_id: u64, committed_pubkeys: &[Pubkey], commit_strategy: TransactionStrategy, finalize_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { + let state = Initialized::new(commit_strategy, finalize_strategy, None); let mut executor = TwoStageExecutor::new( + state, self.authority.insecure_clone(), - commit_strategy, - finalize_strategy, + intent_id, self.intent_client.clone(), self.outbox_client.clone(), self.actions_callback_executor.clone(), @@ -407,6 +413,7 @@ where C: TaskInfoFetcher, A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { /// Executes Message on Base layer /// Returns `ExecutionOutput` or an `Error` diff --git a/magicblock-committor-service/src/intent_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/patcher.rs index 943dfc9c5..cde0d9ed1 100644 --- a/magicblock-committor-service/src/intent_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/patcher.rs @@ -1,12 +1,29 @@ use std::ops::ControlFlow; use async_trait::async_trait; +use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_signer::Signer; +use tracing::{error, warn}; use crate::{ - intent_executor::error::{ - IntentExecutorResult, TransactionStrategyExecutionError, + intent_executor::{ + error::{ + IntentExecutorError, IntentExecutorResult, + TransactionStrategyExecutionError, + }, + intent_execution_client::IntentExecutionClient, + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + utils::{ + handle_actions_result, handle_commit_id_error, + handle_undelegation_error, prepare_and_execute_strategy, + }, + IntentExecutionReport, }, - tasks::task_strategist::TransactionStrategy, + tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, + transaction_preparator::TransactionPreparator, }; #[async_trait] @@ -15,5 +32,375 @@ pub(in crate::intent_executor) trait Patcher { &mut self, err: &TransactionStrategyExecutionError, strategy: &mut TransactionStrategy, + report: &mut IntentExecutionReport, ) -> IntentExecutorResult>; } + +// --- SingleStagePatcher --- + +pub(in crate::intent_executor) struct SingleStagePatcher<'a, F, A, T> { + pub authority: &'a Keypair, + pub intent_client: &'a IntentExecutionClient, + pub callback_scheduler: &'a A, + pub task_info_fetcher: &'a CacheTaskInfoFetcher, + pub committed_pubkeys: &'a [Pubkey], + pub transaction_preparator: &'a T, +} + +impl SingleStagePatcher<'_, F, A, T> +where + T: TransactionPreparator, +{ + /// Handles unfinalized account error for single stage execution. + /// Sends a separate tx to finalize account and then continues execution. + async fn handle_unfinalized_account_error( + &self, + failed_signature: &Option, + delegated_account: Pubkey, + ) -> IntentExecutorResult> { + let finalize_task: BaseTaskImpl = + FinalizeTask { delegated_account }.into(); + prepare_and_execute_strategy( + self.intent_client, + self.authority, + self.transaction_preparator, + &mut TransactionStrategy { + optimized_tasks: vec![finalize_task], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + }, + ) + .await + .map_err(IntentExecutorError::FailedFinalizePreparationError)? + .map_err(|err| IntentExecutorError::FailedToFinalizeError { + err, + commit_signature: None, + finalize_signature: *failed_signature, + })?; + + Ok(ControlFlow::Continue(TransactionStrategy { + optimized_tasks: vec![], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + })) + } +} + +#[async_trait] +impl Patcher for SingleStagePatcher<'_, F, A, T> +where + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + T: TransactionPreparator, +{ + /// Patch the current `transaction_strategy` in response to a recoverable + /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data + /// to be applied after a retry. + /// + /// [`TransactionStrategyExecutionError`], returning either: + /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or + /// - `Break(())` when this stage cannot be recovered here. + async fn patch( + &mut self, + err: &TransactionStrategyExecutionError, + strategy: &mut TransactionStrategy, + report: &mut IntentExecutionReport, + ) -> IntentExecutorResult> { + if self.committed_pubkeys.is_empty() { + // No patching is applicable if intent doesn't commit accounts + return Ok(ControlFlow::Break(())); + } + + match err { + TransactionStrategyExecutionError::ActionsError(err, signature) => { + // Here we patch strategy for it to be retried in next iteration + // & we also record data that has to be cleaned up after patch + let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); + let to_cleanup = handle_actions_result( + &self.authority.pubkey(), + self.callback_scheduler, + report, + strategy, + *signature, + action_error, + ); + Ok(ControlFlow::Continue(to_cleanup)) + } + TransactionStrategyExecutionError::CommitIDError(_, _) => { + // Here we patch strategy for it to be retried in next iteration + // & we also record data that has to be cleaned up after patch + let to_cleanup = handle_commit_id_error( + &self.authority.pubkey(), + self.task_info_fetcher, + self.committed_pubkeys, + strategy, + ) + .await?; + Ok(ControlFlow::Continue(to_cleanup)) + } + err @ TransactionStrategyExecutionError::UnfinalizedAccountError( + _, + signature, + ) => { + let optimized_tasks = strategy.optimized_tasks.as_slice(); + if let Some(delegated_account) = err + .task_index() + .and_then(|index| optimized_tasks.get(index as usize)) + .and_then(|task| match task { + BaseTaskImpl::Commit(task) => { + Some(task.committed_account.pubkey) + } + BaseTaskImpl::CommitFinalize(task) => { + Some(task.committed_account.pubkey) + } + _ => None, + }) + { + self.handle_unfinalized_account_error(signature, delegated_account).await + } else { + error!( + task_index = err.task_index(), + optimized_tasks_len = optimized_tasks.len(), + error = ?err, + "RPC returned unexpected task index" + ); + Ok(ControlFlow::Break(())) + } + } + TransactionStrategyExecutionError::UndelegationError(_, _) => { + // Here we patch strategy for it to be retried in next iteration + // & we also record data that has to be cleaned up after patch + let to_cleanup = + handle_undelegation_error(&self.authority.pubkey(), strategy); + Ok(ControlFlow::Continue(to_cleanup)) + } + TransactionStrategyExecutionError::CpiLimitError(_, _) + | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( + _, + _, + ) => { + // Can't be handled in scope of single stage execution + // We signal flow break + Ok(ControlFlow::Break(())) + } + TransactionStrategyExecutionError::InternalError(_) => { + // Error that we can't handle - break with cleanup data + Ok(ControlFlow::Break(())) + } + } + } +} + +// --- CommitStagePatcher --- + +pub(in crate::intent_executor) struct CommitStagePatcher<'a, F, A, T> { + pub authority: &'a Keypair, + pub intent_client: &'a IntentExecutionClient, + pub callback_scheduler: &'a A, + pub task_info_fetcher: &'a CacheTaskInfoFetcher, + pub committed_pubkeys: &'a [Pubkey], + pub transaction_preparator: &'a T, +} + +#[async_trait] +impl Patcher for CommitStagePatcher<'_, F, A, T> +where + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + T: TransactionPreparator, +{ + /// Patches Commit stage `transaction_strategy` in response to a recoverable + /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data + /// to be applied after a retry. + /// + /// [`TransactionStrategyExecutionError`], returning either: + /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or + /// - `Break(())` when this stage cannot be recovered. + async fn patch( + &mut self, + err: &TransactionStrategyExecutionError, + strategy: &mut TransactionStrategy, + report: &mut IntentExecutionReport, + ) -> IntentExecutorResult> { + match err { + TransactionStrategyExecutionError::CommitIDError(_, _) => { + let to_cleanup = handle_commit_id_error( + &self.authority.pubkey(), + self.task_info_fetcher, + self.committed_pubkeys, + strategy, + ) + .await?; + Ok(ControlFlow::Continue(to_cleanup)) + } + err @ TransactionStrategyExecutionError::UnfinalizedAccountError( + _, + signature, + ) => { + let optimized_tasks = strategy.optimized_tasks.as_slice(); + let task_index = err.task_index(); + if let Some(delegated_account) = task_index + .and_then(|index| optimized_tasks.get(index as usize)) + .and_then(|task| match task { + BaseTaskImpl::Commit(task) => { + Some(task.committed_account.pubkey) + } + BaseTaskImpl::CommitFinalize(task) => { + Some(task.committed_account.pubkey) + } + _ => None, + }) + { + self.handle_unfinalized_account_error(signature, delegated_account).await + } else { + error!( + task_index = ?task_index, + optimized_tasks_len = optimized_tasks.len(), + error = ?err, + "RPC returned unexpected task index" + ); + Ok(ControlFlow::Break(())) + } + } + TransactionStrategyExecutionError::ActionsError(err, signature) => { + // Intent bundles allow for actions to be in commit stage + let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); + let to_cleanup = handle_actions_result( + &self.authority.pubkey(), + self.callback_scheduler, + report, + strategy, + *signature, + action_error, + ); + Ok(ControlFlow::Continue(to_cleanup)) + } + TransactionStrategyExecutionError::UndelegationError(_, _) => { + // Unexpected in Two Stage commit + error!(error = ?err, "Unexpected error in two stage commit flow"); + Ok(ControlFlow::Break(())) + } + TransactionStrategyExecutionError::CpiLimitError(_, _) + | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( + _, + _, + ) => { + // Can't be handled + error!(error = ?err, "Commit tasks exceeded execution limit"); + Ok(ControlFlow::Break(())) + } + TransactionStrategyExecutionError::InternalError(_) => { + // Can't be handled + Ok(ControlFlow::Break(())) + } + } + } +} + +impl CommitStagePatcher<'_, F, A, T> +where + T: TransactionPreparator, +{ + /// Handles unfinalized account error for commit stage execution. + /// Sends a separate tx to finalize account and then continues execution. + async fn handle_unfinalized_account_error( + &self, + failed_signature: &Option, + delegated_account: Pubkey, + ) -> IntentExecutorResult> { + let finalize_task: BaseTaskImpl = + FinalizeTask { delegated_account }.into(); + prepare_and_execute_strategy( + self.intent_client, + self.authority, + self.transaction_preparator, + &mut TransactionStrategy { + optimized_tasks: vec![finalize_task], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + }, + ) + .await + .map_err(IntentExecutorError::FailedCommitPreparationError)? + .map_err(|err| IntentExecutorError::FailedToCommitError { + err, + signature: *failed_signature, + })?; + + Ok(ControlFlow::Continue(TransactionStrategy { + optimized_tasks: vec![], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + })) + } +} + +// --- FinalizeStagePatcher --- + +pub(in crate::intent_executor) struct FinalizeStagePatcher<'a, A> { + pub authority: &'a Keypair, + pub callback_scheduler: &'a A, +} + +#[async_trait] +impl Patcher for FinalizeStagePatcher<'_, A> +where + A: ActionsCallbackScheduler, +{ + /// Patches Finalize stage `transaction_strategy` in response to a recoverable + /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data + /// to be applied after a retry. + /// + /// [`TransactionStrategyExecutionError`], returning either: + /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or + /// - `Break(())` when this stage cannot be recovered. + async fn patch( + &mut self, + err: &TransactionStrategyExecutionError, + strategy: &mut TransactionStrategy, + report: &mut IntentExecutionReport, + ) -> IntentExecutorResult> { + match err { + TransactionStrategyExecutionError::CommitIDError(_, _) + | TransactionStrategyExecutionError::UnfinalizedAccountError(_, _) => { + // Unexpected error in Two Stage finalize + error!(error = ?err, "Unexpected error in two stage finalize flow"); + Ok(ControlFlow::Break(())) + } + TransactionStrategyExecutionError::ActionsError(err, signature) => { + // Here we patch strategy for it to be retried in next iteration + // & we also record data that has to be cleaned up after patch + let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); + let to_cleanup = handle_actions_result( + &self.authority.pubkey(), + self.callback_scheduler, + report, + strategy, + *signature, + action_error, + ); + Ok(ControlFlow::Continue(to_cleanup)) + } + TransactionStrategyExecutionError::UndelegationError(_, _) => { + // Here we patch strategy for it to be retried in next iteration + // & we also record data that has to be cleaned up after patch + let to_cleanup = + handle_undelegation_error(&self.authority.pubkey(), strategy); + Ok(ControlFlow::Continue(to_cleanup)) + } + TransactionStrategyExecutionError::CpiLimitError(_, _) + | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( + _, + _, + ) => { + // Can't be handled + warn!(error = ?err, "Finalization tasks exceeded execution limit"); + Ok(ControlFlow::Break(())) + } + TransactionStrategyExecutionError::InternalError(_) => { + // Can't be handled + Ok(ControlFlow::Break(())) + } + } + } +} diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs index 711732056..9e5c973d4 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs @@ -1,31 +1,24 @@ -use std::{ops::ControlFlow, sync::Arc}; +use std::sync::Arc; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; use magicblock_program::outbox::ExecutionStage; use solana_keypair::Keypair; use solana_pubkey::Pubkey; -use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_signer::Signer; -use tracing::{error, instrument}; +use tracing::instrument; use crate::{ intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, - }, + error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, + patcher::SingleStagePatcher, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - utils::{ - check_pending_signature, handle_actions_result, - handle_commit_id_error, handle_undelegation_error, - prepare_and_execute_strategy, prepare_transaction, - }, + utils::{handle_actions_result, stage_execution_loop, ExecutionState}, IntentExecutionReport, }, outbox_client::OutboxClient, - tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, + tasks::task_strategist::TransactionStrategy, transaction_preparator::TransactionPreparator, }; @@ -87,93 +80,32 @@ where where T: TransactionPreparator, { - const RECURSION_CEILING: u8 = 10; - - let result = loop { - if let Some(ref sig) = self.pending_signature { - let flow = - check_pending_signature(&self.intent_client, sig).await?; - // Pending signature corresponds to succeeded transaction - break - if let ControlFlow::Break(()) = flow { - break Ok(*sig); - } - self.pending_signature = None; - } - - self.current_attempt += 1; - - // Prepare message - let prepared_transaction = prepare_transaction( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.transaction_strategy, - ) - .await - .map_err(IntentExecutorError::FailedFinalizePreparationError)?; - - // Record in outbox - let signature = prepared_transaction.get_signature(); - self.outbox_client - .set_intent_execution_stage( - self.intent_id, - ExecutionStage::SingleStage(*signature), - ) - .await - .map_err(Into::into)?; - - // Now record locally signature of tx we about to send - // Precaution for timeout in between - self.pending_signature = Some(*signature); - - // Send signed tx - let execution_result = self - .intent_client - .send_signed_tx_with_retries( - &prepared_transaction, - &self.transaction_strategy.optimized_tasks, - ) - .await; - - // Result returned, cleanup pending signature - self.pending_signature = None; - - // Process error: Ok - return, Err - handle further - let execution_err = match execution_result { - // break with result, strategy that was executed at this point has to be returned for cleanup - Ok(()) => break Ok(*signature), - Err(err) => err, - }; - - // Attempt patching - let flow = self - .patch_strategy( - &execution_err, - committed_pubkeys, - transaction_preparator, - ) - .await?; - let cleanup = match flow { - ControlFlow::Continue(cleanup) => cleanup, - ControlFlow::Break(()) => { - break Err(execution_err); - } - }; - self.intent_client.invalidate_cached_blockhash().await; - self.execution_report.dispose(cleanup); - - if self.current_attempt >= RECURSION_CEILING { - error!( - attempt = self.current_attempt, - ceiling = RECURSION_CEILING, - error = ?execution_err, - "Recursion ceiling exceeded" - ); - break Err(execution_err); - } else { - self.execution_report.add_patched_error(execution_err); - } + let patcher = SingleStagePatcher { + authority: &self.authority, + intent_client: &self.intent_client, + callback_scheduler: &self.callback_scheduler, + task_info_fetcher: &self.task_info_fetcher, + committed_pubkeys, + transaction_preparator, + }; + let execution_state = ExecutionState { + current_attempt: &mut self.current_attempt, + transaction_strategy: &mut self.transaction_strategy, + pending_signature: &mut self.pending_signature, + execution_report: self.execution_report, }; + let result = stage_execution_loop( + &self.authority, + &self.intent_client, + &*self.outbox_client, + transaction_preparator, + patcher, + self.intent_id, + ExecutionStage::SingleStage, + IntentExecutorError::FailedFinalizePreparationError, + execution_state, + ) + .await?; result.map_err(|err| { IntentExecutorError::from_finalize_execution_error( @@ -207,142 +139,4 @@ where pub fn consume_strategy(self) -> TransactionStrategy { self.transaction_strategy } - - /// Patch the current `transaction_strategy` in response to a recoverable - /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data - /// to be applied after a retry. - /// - /// [`TransactionStrategyExecutionError`], returning either: - /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or - /// - `Break(())` when this stage cannot be recovered here. - pub async fn patch_strategy( - &mut self, - err: &TransactionStrategyExecutionError, - committed_pubkeys: &[Pubkey], - transaction_preparator: &T, - ) -> IntentExecutorResult> { - if committed_pubkeys.is_empty() { - // No patching is applicable if intent doesn't commit accounts - return Ok(ControlFlow::Break(())); - } - - match err { - TransactionStrategyExecutionError::ActionsError(err, signature) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); - let to_cleanup = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.transaction_strategy, - *signature, - action_error, - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::CommitIDError(_, _) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let to_cleanup = handle_commit_id_error( - &self.authority.pubkey(), - &self.task_info_fetcher, - committed_pubkeys, - &mut self.transaction_strategy, - ) - .await?; - Ok(ControlFlow::Continue(to_cleanup)) - } - err - @ TransactionStrategyExecutionError::UnfinalizedAccountError( - _, - signature, - ) => { - let optimized_tasks = - self.transaction_strategy.optimized_tasks.as_slice(); - if let Some(delegated_account) = err - .task_index() - .and_then(|index| optimized_tasks.get(index as usize)) - .and_then(|task| match task { - BaseTaskImpl::Commit(task) => { - Some(task.committed_account.pubkey) - } - BaseTaskImpl::CommitFinalize(task) => { - Some(task.committed_account.pubkey) - } - _ => None, - }) - { - self.handle_unfinalized_account_error( - signature, - delegated_account, - transaction_preparator, - ) - .await - } else { - error!( - task_index = err.task_index(), - optimized_tasks_len = optimized_tasks.len(), - error = ?err, - "RPC returned unexpected task index" - ); - Ok(ControlFlow::Break(())) - } - } - TransactionStrategyExecutionError::UndelegationError(_, _) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let to_cleanup = handle_undelegation_error( - &self.authority.pubkey(), - &mut self.transaction_strategy - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::CpiLimitError(_, _) - | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded(_, _) => { - // Can't be handled in scope of single stage execution - // We signal flow break - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::InternalError(_) => { - // Error that we can't handle - break with cleanup data - Ok(ControlFlow::Break(())) - } - } - } - - /// Handles unfinalized account error - /// Sends a separate tx to finalize account and then continues execution - async fn handle_unfinalized_account_error( - &self, - failed_signature: &Option, - delegated_account: Pubkey, - transaction_preparator: &T, - ) -> IntentExecutorResult> { - let finalize_task: BaseTaskImpl = - FinalizeTask { delegated_account }.into(); - prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut TransactionStrategy { - optimized_tasks: vec![finalize_task], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - }, - ) - .await - .map_err(IntentExecutorError::FailedFinalizePreparationError)? - .map_err(|err| IntentExecutorError::FailedToFinalizeError { - err, - commit_signature: None, - finalize_signature: *failed_signature, - })?; - - Ok(ControlFlow::Continue(TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - })) - } } diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index f05121769..553decf1f 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -1,35 +1,26 @@ -use std::{mem, ops::ControlFlow, sync::Arc}; +use std::{mem, sync::Arc}; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; use magicblock_program::outbox::{ExecutionStage, TwoStageProgress}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; -use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_signer::Signer; -use tracing::{error, info, instrument, warn}; +use tracing::instrument; use crate::{ intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, - }, + error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, + patcher::{CommitStagePatcher, FinalizeStagePatcher}, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, two_stage_executor::sealed::Sealed, - utils::{ - check_pending_signature, handle_actions_result, - handle_commit_id_error, handle_undelegation_error, - prepare_and_execute_strategy, prepare_transaction, - }, + utils::{handle_actions_result, stage_execution_loop, ExecutionState}, IntentExecutionReport, }, outbox_client::OutboxClient, - tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, - transaction_preparator::{ - error::TransactionPreparatorError, TransactionPreparator, - }, + tasks::task_strategist::TransactionStrategy, + transaction_preparator::TransactionPreparator, }; pub struct Initialized { @@ -131,18 +122,6 @@ where } } - // We need to: - // 1. Prepare everything for tx delivery - // 2. Sign Tx - // 3. Update outbox with Committing(signature) - // 4, Send Tx - // 5. Confirm signature - // a. Success - terminate - // b. jump to 1 - - // We can start from 5: - // Committing(sig) was loaded from outbox - // confirm signature #[instrument( skip( self, @@ -162,88 +141,34 @@ where T: TransactionPreparator, F: TaskInfoFetcher, { - let commit_result = loop { - if let Some(ref sig) = self.state.pending_signature { - let flow = - check_pending_signature(&self.intent_client, sig).await?; - // Pending signature corresponds to succeeded transaction - break - if let ControlFlow::Break(()) = flow { - break Ok(*sig); - } - - self.state.pending_signature = None; - } - - self.state.current_attempt += 1; - - // Prepare message - let prepared_transaction = prepare_transaction( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.state.commit_strategy, - ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)?; - - // Record in outbox - let signature = prepared_transaction.get_signature(); - self.outbox_client - .set_intent_execution_stage( - self.intent_id, - ExecutionStage::TwoStage(TwoStageProgress::Committing( - *signature, - )), - ) - .await - .map_err(Into::into)?; - - // Now record locally signature of tx we about to send - // Precaution for timeout in between - self.state.pending_signature = Some(*signature); - - // Send signed tx - let execution_result = self - .intent_client - .send_signed_tx_with_retries( - &prepared_transaction, - &self.state.commit_strategy.optimized_tasks, - ) - .await; - - // Result returned, cleanup pending signature - self.state.pending_signature = None; - - let execution_err = match execution_result { - Ok(()) => break Ok(*signature), - Err(err) => err, - }; - - let flow = self - .patch_commit_strategy( - &execution_err, - task_info_fetcher, - transaction_preparator, - committed_pubkeys, - ) - .await?; - let cleanup = match flow { - ControlFlow::Continue(value) => value, - ControlFlow::Break(()) => { - break Err(execution_err); - } - }; - self.intent_client.invalidate_cached_blockhash().await; - self.execution_report.dispose(cleanup); - - if self.state.current_attempt >= Self::RECURSION_CEILING { - error!("CRITICAL! Recursion ceiling reached"); - break Err(execution_err); - } else { - self.execution_report.add_patched_error(execution_err); - } + let execution_state = ExecutionState { + current_attempt: &mut self.state.current_attempt, + transaction_strategy: &mut self.state.commit_strategy, + pending_signature: &mut self.state.pending_signature, + execution_report: self.execution_report, + }; + let commit_stage_patcher = CommitStagePatcher { + authority: &self.authority, + intent_client: &self.intent_client, + callback_scheduler: &self.callback_scheduler, + task_info_fetcher, + committed_pubkeys, + transaction_preparator, }; + // Execute stage + let commit_result = stage_execution_loop( + &self.authority, + &self.intent_client, + &*self.outbox_client, + transaction_preparator, + commit_stage_patcher, + self.intent_id, + |sig| ExecutionStage::TwoStage(TwoStageProgress::Committing(sig)), + IntentExecutorError::FailedCommitPreparationError, + execution_state, + ) + .await?; self.execute_callbacks( commit_result.as_ref().ok().copied(), commit_result.as_ref().map(|_| ()), @@ -259,140 +184,6 @@ where }) } - /// Patches Commit stage `transaction_strategy` in response to a recoverable - /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data - /// to be applied after a retry. - /// - /// [`TransactionStrategyExecutionError`], returning either: - /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or - /// - `Break(())` when this stage cannot be recovered. - pub async fn patch_commit_strategy( - &mut self, - err: &TransactionStrategyExecutionError, - task_info_fetcher: &CacheTaskInfoFetcher, - transaction_preparator: &T, - committed_pubkeys: &[Pubkey], - ) -> IntentExecutorResult> - where - T: TransactionPreparator, - F: TaskInfoFetcher, - { - match err { - TransactionStrategyExecutionError::CommitIDError(_, _) => { - let to_cleanup = handle_commit_id_error( - &self.authority.pubkey(), - task_info_fetcher, - committed_pubkeys, - &mut self.state.commit_strategy - ) - .await?; - Ok(ControlFlow::Continue(to_cleanup)) - } - err - @ TransactionStrategyExecutionError::UnfinalizedAccountError( - _, - signature, - ) => { - let optimized_tasks = - self.state.commit_strategy.optimized_tasks.as_slice(); - let task_index = err.task_index(); - if let Some(delegated_account) = task_index - .and_then(|index| optimized_tasks.get(index as usize)) - .and_then(|task| match task { - BaseTaskImpl::Commit(task) => { - Some(task.committed_account.pubkey) - } - BaseTaskImpl::CommitFinalize(task) => { - Some(task.committed_account.pubkey) - } - _ => None, - }) - { - self.handle_unfinalized_account_error( - signature, - delegated_account, - transaction_preparator, - ) - .await - } else { - error!( - task_index = ?task_index, - optimized_tasks_len = optimized_tasks.len(), - error = ?err, - "RPC returned unexpected task index" - ); - Ok(ControlFlow::Break(())) - } - } - TransactionStrategyExecutionError::ActionsError(err, signature) => { - // Intent bundles allow for actions to be in commit stage - let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); - let to_cleanup = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.commit_strategy, - *signature, - action_error - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::UndelegationError(_, _) => { - // Unexpected in Two Stage commit - // That would mean that Two Stage executes undelegation in commit phase - error!(error = ?err, "Unexpected error in two stage commit flow"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::CpiLimitError(_, _) - | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( - _, - _, - ) => { - // Can't be handled - error!(error = ?err, "Commit tasks exceeded execution limit"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::InternalError(_) => { - // Can't be handled - Ok(ControlFlow::Break(())) - } - } - } - - /// Handles unfinalized account error - /// Sends a separate tx to finalize account and then continues execution - async fn handle_unfinalized_account_error( - &self, - failed_signature: &Option, - delegated_account: Pubkey, - transaction_preparator: &T, - ) -> IntentExecutorResult> { - let finalize_task: BaseTaskImpl = - FinalizeTask { delegated_account }.into(); - prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut TransactionStrategy { - optimized_tasks: vec![finalize_task], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - }, - ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)? - .map_err(|err| IntentExecutorError::FailedToCommitError { - err, - signature: *failed_signature, - })?; - - Ok(ControlFlow::Continue(TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - })) - } - pub fn has_callbacks(&self) -> bool { self.state.commit_strategy.has_actions_callbacks() || self.state.finalize_strategy.has_actions_callbacks() @@ -492,82 +283,35 @@ where where T: TransactionPreparator, { - let finalize_result = loop { - if let Some(ref sig) = self.state.pending_signature { - let flow = - check_pending_signature(&self.intent_client, sig).await?; - // Pending signature corresponds to succeeded transaction - break - if let ControlFlow::Break(()) = flow { - break Ok(*sig); - } - - self.state.pending_signature = None; - } - - self.state.current_attempt += 1; - - // Prepare message - let prepared_transaction = prepare_transaction( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.state.finalize_strategy, - ) - .await - .map_err(IntentExecutorError::FailedFinalizePreparationError)?; - - // Record in outbox - let signature = prepared_transaction.get_signature(); - self.outbox_client - .set_intent_execution_stage( - self.intent_id, - ExecutionStage::TwoStage(TwoStageProgress::Finalizing { - commit: self.state.commit_signature, - finalize: *signature, - }), - ) - .await - .map_err(Into::into)?; - - // Now record locally signature of tx we about to send - // Precaution for timeout in between - self.state.pending_signature = Some(*signature); - - // Send signed tx - let execution_result = self - .intent_client - .send_signed_tx_with_retries( - &prepared_transaction, - &self.state.finalize_strategy.optimized_tasks, - ) - .await; - - // Result returned, cleanup pending signature - self.state.pending_signature = None; - - let execution_err = match execution_result { - Ok(()) => break Ok(*signature), - Err(err) => err, - }; - - let flow = self.patch_finalize_strategy(&execution_err).await?; - let cleanup = match flow { - ControlFlow::Continue(cleanup) => cleanup, - ControlFlow::Break(()) => { - break Err(execution_err); - } - }; - self.intent_client.invalidate_cached_blockhash().await; - self.execution_report.dispose(cleanup); - - if self.state.current_attempt >= Self::RECURSION_CEILING { - error!("CRITICAL! Recursion ceiling reached"); - break Err(execution_err); - } else { - self.execution_report.add_patched_error(execution_err); - } + let commit_signature = self.state.commit_signature; + let execution_state = ExecutionState { + current_attempt: &mut self.state.current_attempt, + transaction_strategy: &mut self.state.finalize_strategy, + pending_signature: &mut self.state.pending_signature, + execution_report: self.execution_report, }; - + let finalize_stage_patcher = FinalizeStagePatcher { + authority: &self.authority, + callback_scheduler: &self.callback_scheduler, + }; + // Execute finalize stage + let finalize_result = stage_execution_loop( + &self.authority, + &self.intent_client, + &*self.outbox_client, + transaction_preparator, + finalize_stage_patcher, + self.intent_id, + |sig| { + ExecutionStage::TwoStage(TwoStageProgress::Finalizing { + commit: commit_signature, + finalize: sig, + }) + }, + IntentExecutorError::FailedFinalizePreparationError, + execution_state, + ) + .await?; // Even if failed - dump finalize into junk self.execute_callbacks( finalize_result.as_ref().ok().copied(), @@ -605,61 +349,6 @@ where self.execution_report.dispose(junk_strategy); } - /// Patches Finalize stage `transaction_strategy` in response to a recoverable - /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data - /// to be applied after a retry. - /// - /// [`TransactionStrategyExecutionError`], returning either: - /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or - /// - `Break(())` when this stage cannot be recovered. - pub async fn patch_finalize_strategy( - &mut self, - err: &TransactionStrategyExecutionError, - ) -> IntentExecutorResult> { - match err { - TransactionStrategyExecutionError::CommitIDError(_, _) - | TransactionStrategyExecutionError::UnfinalizedAccountError( - _, - _, - ) => { - // Unexpected error in Two Stage commit - error!(error = ?err, "Unexpected error in two stage finalize flow"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::ActionsError(err, signature) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); - let to_cleanup = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.finalize_strategy, - *signature, - action_error - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::UndelegationError(_, _) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let to_cleanup = - handle_undelegation_error(&self.authority.pubkey(), &mut self.state.finalize_strategy); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::CpiLimitError(_, _) - | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded(_, _) => { - // Can't be handled - warn!(error = ?err, "Finalization tasks exceeded execution limit"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::InternalError(_) => { - // Can't be handled - Ok(ControlFlow::Break(())) - } - } - } - /// Transitions to next executor state pub fn done(self, finalize_signature: Signature) -> Finalized { Finalized { diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 84f0f1e49..65b34bd35 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -40,10 +40,10 @@ use crate::{ const STAGE_LOOP_CEILING: u8 = 10; pub(in crate::intent_executor) struct ExecutionState<'a> { - current_attempt: &'a mut u8, - transaction_strategy: &'a mut TransactionStrategy, - pending_signature: &'a mut Option, - execution_report: &'a mut IntentExecutionReport, + pub current_attempt: &'a mut u8, + pub transaction_strategy: &'a mut TransactionStrategy, + pub pending_signature: &'a mut Option, + pub execution_report: &'a mut IntentExecutionReport, } pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( @@ -56,7 +56,7 @@ pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( make_outbox_stage: impl Fn(Signature) -> ExecutionStage, map_preparation_err: fn(TransactionPreparatorError) -> IntentExecutorError, mut state: ExecutionState<'a>, -) -> IntentExecutorResult +) -> IntentExecutorResult> where T: TransactionPreparator, O: OutboxClient, @@ -68,7 +68,7 @@ where let flow = check_pending_signature(intent_client, sig).await?; // Pending signature corresponds to succeeded transaction - break if let ControlFlow::Break(()) = flow { - return Ok(*sig); + return Ok(Ok(*sig)); } *state.pending_signature = None; @@ -112,23 +112,27 @@ where *state.pending_signature = None; let execution_err = match execution_result { - Ok(()) => return Ok(*signature), + Ok(()) => return Ok(Ok(*signature)), Err(err) => err, }; let flow = patcher - .patch(&execution_err, state.transaction_strategy) + .patch( + &execution_err, + state.transaction_strategy, + state.execution_report, + ) .await?; let cleanup = match flow { ControlFlow::Continue(value) => value, - ControlFlow::Break(()) => return Err(execution_err.into()), + ControlFlow::Break(()) => return Ok(Err(execution_err)), }; intent_client.invalidate_cached_blockhash().await; state.execution_report.dispose(cleanup); if *state.current_attempt >= STAGE_LOOP_CEILING { error!("CRITICAL! Recursion ceiling reached"); - return Err(execution_err.into()); + return Ok(Err(execution_err)); } else { state.execution_report.add_patched_error(execution_err); } @@ -460,18 +464,21 @@ pub(in crate::intent_executor) trait StageExecutor { ); } -pub(in crate::intent_executor) struct SingleStage<'a, 'e, A, T, F> { - pub(in crate::intent_executor) inner: &'a mut SingleStageExecutor<'e, F, A>, +pub(in crate::intent_executor) struct SingleStage<'a, 'e, A, T, F, O> { + pub(in crate::intent_executor) inner: + &'a mut SingleStageExecutor<'e, F, A, O>, pub(in crate::intent_executor) transaction_preparator: &'a T, pub(in crate::intent_executor) committed_pubkeys: &'a [Pubkey], } #[async_trait] -impl<'a, 'e, A, T, F> StageExecutor for SingleStage<'a, 'e, A, T, F> +impl<'a, 'e, A, T, F, O> StageExecutor for SingleStage<'a, 'e, A, T, F, O> where A: ActionsCallbackScheduler, T: TransactionPreparator, F: TaskInfoFetcher, + O: OutboxClient, + O::Error: Into, { fn has_callbacks(&self) -> bool { self.inner.has_callbacks() @@ -492,9 +499,9 @@ where } } -pub(in crate::intent_executor) struct CommitStage<'a, 'e, A, T, F> { +pub(in crate::intent_executor) struct CommitStage<'a, 'e, A, T, F, O> { pub(in crate::intent_executor) inner: - &'a mut TwoStageExecutor<'e, A, Initialized>, + &'a mut TwoStageExecutor<'e, A, O, Initialized>, pub(in crate::intent_executor) transaction_preparator: &'a T, pub(in crate::intent_executor) task_info_fetcher: &'a CacheTaskInfoFetcher, @@ -502,11 +509,13 @@ pub(in crate::intent_executor) struct CommitStage<'a, 'e, A, T, F> { } #[async_trait] -impl<'a, 'e, A, T, F> StageExecutor for CommitStage<'a, 'e, A, T, F> +impl<'a, 'e, A, T, F, O> StageExecutor for CommitStage<'a, 'e, A, T, F, O> where A: ActionsCallbackScheduler, T: TransactionPreparator, F: TaskInfoFetcher, + O: OutboxClient, + O::Error: Into, { fn has_callbacks(&self) -> bool { self.inner.has_callbacks() @@ -531,17 +540,19 @@ where } } -pub(in crate::intent_executor) struct FinalizeStage<'a, 'e, A, T> { +pub(in crate::intent_executor) struct FinalizeStage<'a, 'e, A, T, O> { pub(in crate::intent_executor) inner: - &'a mut TwoStageExecutor<'e, A, Committed>, + &'a mut TwoStageExecutor<'e, A, O, Committed>, pub(in crate::intent_executor) transaction_preparator: &'a T, } #[async_trait] -impl<'a, 'e, A, T> StageExecutor for FinalizeStage<'a, 'e, A, T> +impl<'a, 'e, A, T, O> StageExecutor for FinalizeStage<'a, 'e, A, T, O> where A: ActionsCallbackScheduler, T: TransactionPreparator, + O: OutboxClient, + O::Error: Into, { fn has_callbacks(&self) -> bool { self.inner.has_callbacks() diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index d9c0d81f9..fb700c569 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -120,6 +120,15 @@ impl OutboxClient for InternalOutboxClient { Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) } + fn set_intent_execution_stage( + &self, + intent_id: u64, + stage: ExecutionStage, + ) -> Result<(), Self::Error> { + // TODO(edwin): use rpc of scheduler + todo!() + } + async fn notify_commit_sent( &self, sent_tx: Transaction, @@ -142,15 +151,6 @@ impl OutboxClient for InternalOutboxClient { Ok(()) } - fn set_intent_execution_stage( - &self, - intent_id: u64, - stage: ExecutionStage, - ) -> Result<(), Self::Error> { - // TODO(edwin): use rpc of scheduler - todo!() - } - fn outbox_reader(&self) -> Self::OutboxReader { InternalOutboxIntentBundlesReader::new(self.accounts_db.clone()) } diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 1e3da7337..5ffe03bdd 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -366,7 +366,7 @@ where result_subscription, cancellation_token, intents_meta_map, - intent_client + outbox_client ))] async fn result_processor( mut result_subscription: broadcast::Receiver< @@ -374,7 +374,7 @@ where >, cancellation_token: CancellationToken, intents_meta_map: Arc>>, - intent_client: Arc, + outbox_client: Arc, ) { loop { let execution_result = tokio::select! { @@ -400,8 +400,8 @@ where } }; - if let Err(err) = ServiceInner::::process_execution_result( - &intent_client, + if let Err(err) = ServiceInner::::process_execution_result( + &outbox_client, execution_result, &intents_meta_map, ) @@ -413,10 +413,10 @@ where } async fn process_execution_result( - intent_client: &Arc, + outbox_client: &Arc, execution_result: BroadcastedIntentExecutionResult, intents_meta_map: &Arc>>, - ) -> Result<(), R::Error> { + ) -> Result<(), O::Error> { // Create IntentMeta let intent_id = execution_result.id; // Remove intent from metas @@ -434,12 +434,12 @@ where let sent_transaction = mem::take(&mut intent_meta.intent_sent_transaction); - let sent_commit = ServiceInner::::build_sent_commit( + let sent_commit = ServiceInner::::build_sent_commit( intent_id, intent_meta, execution_result, ); - intent_client + outbox_client .notify_commit_sent(sent_transaction, sent_commit) .await?; diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 93bd16f7a..42b9beb4b 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -4,13 +4,9 @@ use magicblock_core::intent::BaseActionCallback; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_signer::{Signer, SignerError}; -use tracing::error; use crate::{ - tasks::{ - commit_task::CommitDelivery, utils::TransactionUtils, BaseActionTask, - BaseTask, BaseTaskImpl, - }, + tasks::{utils::TransactionUtils, BaseActionTask, BaseTask, BaseTaskImpl}, transactions::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, }; From e1adceb5082ab8e5700d7b3094afb438074a0f21 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 10 Jun 2026 21:48:53 +0700 Subject: [PATCH 42/97] wip --- .../intent_execution_client.rs | 1 + .../src/intent_executor/patcher.rs | 78 +++++++++---------- .../src/intent_executor/two_stage_executor.rs | 3 +- .../src/intent_executor/utils.rs | 10 ++- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index ccf29419c..fa1dad53e 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -125,6 +125,7 @@ impl IntentExecutionClient { signatures: &[Signature], ) -> MagicBlockRpcClientResult>>> { + // TODO(edwin): add metrics/log for this one let response = self .rpc_client .get_inner() diff --git a/magicblock-committor-service/src/intent_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/patcher.rs index cde0d9ed1..28bf53abf 100644 --- a/magicblock-committor-service/src/intent_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/patcher.rs @@ -202,6 +202,45 @@ pub(in crate::intent_executor) struct CommitStagePatcher<'a, F, A, T> { pub transaction_preparator: &'a T, } + +impl CommitStagePatcher<'_, F, A, T> +where + T: TransactionPreparator, +{ + /// Handles unfinalized account error for commit stage execution. + /// Sends a separate tx to finalize account and then continues execution. + async fn handle_unfinalized_account_error( + &self, + failed_signature: &Option, + delegated_account: Pubkey, + ) -> IntentExecutorResult> { + let finalize_task: BaseTaskImpl = + FinalizeTask { delegated_account }.into(); + prepare_and_execute_strategy( + self.intent_client, + self.authority, + self.transaction_preparator, + &mut TransactionStrategy { + optimized_tasks: vec![finalize_task], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + }, + ) + .await + .map_err(IntentExecutorError::FailedCommitPreparationError)? + .map_err(|err| IntentExecutorError::FailedToCommitError { + err, + signature: *failed_signature, + })?; + + Ok(ControlFlow::Continue(TransactionStrategy { + optimized_tasks: vec![], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + })) + } +} + #[async_trait] impl Patcher for CommitStagePatcher<'_, F, A, T> where @@ -297,45 +336,6 @@ where } } -impl CommitStagePatcher<'_, F, A, T> -where - T: TransactionPreparator, -{ - /// Handles unfinalized account error for commit stage execution. - /// Sends a separate tx to finalize account and then continues execution. - async fn handle_unfinalized_account_error( - &self, - failed_signature: &Option, - delegated_account: Pubkey, - ) -> IntentExecutorResult> { - let finalize_task: BaseTaskImpl = - FinalizeTask { delegated_account }.into(); - prepare_and_execute_strategy( - self.intent_client, - self.authority, - self.transaction_preparator, - &mut TransactionStrategy { - optimized_tasks: vec![finalize_task], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - }, - ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)? - .map_err(|err| IntentExecutorError::FailedToCommitError { - err, - signature: *failed_signature, - })?; - - Ok(ControlFlow::Continue(TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - })) - } -} - -// --- FinalizeStagePatcher --- pub(in crate::intent_executor) struct FinalizeStagePatcher<'a, A> { pub authority: &'a Keypair, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs index 553decf1f..0e5b630c2 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs @@ -156,7 +156,7 @@ where transaction_preparator, }; - // Execute stage + // Execute commit stage let commit_result = stage_execution_loop( &self.authority, &self.intent_client, @@ -294,6 +294,7 @@ where authority: &self.authority, callback_scheduler: &self.callback_scheduler, }; + // Execute finalize stage let finalize_result = stage_execution_loop( &self.authority, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 65b34bd35..c7fd7b995 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -55,7 +55,7 @@ pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( intent_id: u64, make_outbox_stage: impl Fn(Signature) -> ExecutionStage, map_preparation_err: fn(TransactionPreparatorError) -> IntentExecutorError, - mut state: ExecutionState<'a>, + state: ExecutionState<'a>, ) -> IntentExecutorResult> where T: TransactionPreparator, @@ -68,7 +68,7 @@ where let flow = check_pending_signature(intent_client, sig).await?; // Pending signature corresponds to succeeded transaction - break if let ControlFlow::Break(()) = flow { - return Ok(Ok(*sig)); + break Ok(Ok(*sig)); } *state.pending_signature = None; @@ -86,7 +86,7 @@ where .await .map_err(map_preparation_err)?; - // Record in outbox + // Record in outbox before sending let signature = prepared_transaction.get_signature(); outbox_client .set_intent_execution_stage( @@ -115,7 +115,6 @@ where Ok(()) => return Ok(Ok(*signature)), Err(err) => err, }; - let flow = patcher .patch( &execution_err, @@ -425,6 +424,7 @@ where junk } +// TODO(edwin): docs pub(in crate::intent_executor) async fn execute_with_timeout( time_left: Option, mut executor: impl StageExecutor, @@ -450,6 +450,8 @@ pub(in crate::intent_executor) async fn execute_with_timeout( } } + // Timeout concerns only callbacks + // We still need to drive intent to completion executor.execute().await } From f05d9f0d5cd3f906181d9dfb03dcadd78e1d6abc Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 12 Jun 2026 12:47:31 +0700 Subject: [PATCH 43/97] wip --- .../accepted_intent_executor.rs | 349 +++++++++++ .../src/intent_executor/cleanup_handle.rs | 49 ++ .../intent_executor_factory.rs | 27 +- .../src/intent_executor/mod.rs | 434 ++----------- .../single_stage_intent_executor.rs | 88 +++ .../intent_executor/strategy_executor/mod.rs | 4 + .../{ => strategy_executor}/patcher.rs | 18 +- .../single_stage.rs} | 12 +- .../two_stage.rs} | 20 +- .../strategy_executor/utils.rs | 576 ++++++++++++++++++ .../two_stage_intent_executor.rs | 127 ++++ .../src/intent_executor/utils.rs | 573 ----------------- .../test-committor-service/tests/common.rs | 6 +- .../tests/test_intent_executor.rs | 50 +- 14 files changed, 1325 insertions(+), 1008 deletions(-) create mode 100644 magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs create mode 100644 magicblock-committor-service/src/intent_executor/cleanup_handle.rs create mode 100644 magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs create mode 100644 magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs rename magicblock-committor-service/src/intent_executor/{ => strategy_executor}/patcher.rs (98%) rename magicblock-committor-service/src/intent_executor/{single_stage_executor.rs => strategy_executor/single_stage.rs} (93%) rename magicblock-committor-service/src/intent_executor/{two_stage_executor.rs => strategy_executor/two_stage.rs} (95%) create mode 100644 magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs create mode 100644 magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs new file mode 100644 index 000000000..dbb336101 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -0,0 +1,349 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use futures_util::future::{join, try_join_all}; +use magicblock_core::traits::ActionsCallbackScheduler; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, validator::validator_authority, +}; +use magicblock_rpc_client::MagicblockRpcClient; +use solana_keypair::{Address as Pubkey, Keypair}; +use solana_signer::Signer; +use tracing::trace; + +use crate::{ + intent_executor::{ + cleanup_handle::CleanupHandle, + error::{IntentExecutorError, IntentExecutorResult}, + intent_execution_client::IntentExecutionClient, + strategy_executor::{ + single_stage::SingleStageStrategyExecutor, + two_stage::{Initialized, TwoStageStrategyExecutor}, + utils::{ + execute_with_timeout, handle_cpi_limit_error, CommitStage, + FinalizeStage, SingleStage, + }, + }, + task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, + ExecutionOutput, IntentExecutionReport, IntentExecutionResult, + IntentExecutor, IntentExecutorCtx, + }, + outbox_client::OutboxClient, + tasks::{ + task_builder::TasksBuilder, + task_strategist::{ + StrategyExecutionMode, TaskStrategist, TransactionStrategy, + }, + TaskBuilderImpl, + }, + transaction_preparator::{ + delivery_preparator::BufferExecutionError, TransactionPreparator, + }, +}; + +pub struct AcceptedIntentExecutor { + ctx: IntentExecutorCtx, + authority: Keypair, + /// Intent execution started at + pub started_at: Instant, + /// Junk that needs to be cleaned up + junk: Vec, + /// Set to false on execution failure so cleanup only releases ALT + /// reservations without closing buffer PDAs (see race condition note in + /// intent_execution_engine) + close_buffers: bool, +} + +impl AcceptedIntentExecutor +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + pub fn new(ctx: IntentExecutorCtx) -> Self { + let authority = validator_authority(); + Self { + ctx, + authority, + started_at: Instant::now(), + junk: vec![], + close_buffers: true, + } + } + + async fn execute_inner( + &mut self, + intent_bundle: ScheduledIntentBundle, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + if intent_bundle.is_empty() { + return Err(IntentExecutorError::EmptyIntentError); + } + let all_committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); + + if all_committed_pubkeys.is_empty() { + // Build tasks for commit stage + // TODO (snawaz): it's actually MagicBaseIntent::BaseActions scenario, not Commit + // scenario, so the related code needs little bit of refactoring and proper renaming. + let commit_tasks = TaskBuilderImpl::commit_tasks( + &self.ctx.task_info_fetcher, + &intent_bundle, + ) + .await?; + + // Standalone actions executed in single stage + let mut strategy = TaskStrategist::build_strategy( + commit_tasks, + &self.authority.pubkey(), + )?; + strategy.standalone_action_nonce = Some(intent_bundle.id); + return self + .single_stage_execution_flow( + intent_bundle, + strategy, + execution_report, + ) + .await; + }; + + // Build tasks for commit & finalize stages + let (commit_tasks, finalize_tasks) = { + let commit_tasks_fut = TaskBuilderImpl::commit_tasks( + &self.ctx.task_info_fetcher, + &intent_bundle, + ); + let finalize_tasks_fut = TaskBuilderImpl::finalize_tasks( + &self.ctx.task_info_fetcher, + &intent_bundle, + ); + let (commit_tasks, finalize_tasks) = + join(commit_tasks_fut, finalize_tasks_fut).await; + + (commit_tasks?, finalize_tasks?) + }; + + // Build execution strategy + match TaskStrategist::build_execution_strategy( + commit_tasks, + finalize_tasks, + &self.authority.pubkey(), + )? { + StrategyExecutionMode::SingleStage(strategy) => { + trace!("Single stage execution"); + self.single_stage_execution_flow( + intent_bundle, + strategy, + execution_report, + ) + .await + } + StrategyExecutionMode::TwoStage { + commit_stage, + finalize_stage, + } => { + trace!("Two stage execution"); + self.two_stage_execution_flow( + intent_bundle.id, + &all_committed_pubkeys, + commit_stage, + finalize_stage, + execution_report, + ) + .await + } + } + } + + fn time_left(&self) -> Option { + self.ctx + .actions_timeout + .checked_sub(self.started_at.elapsed()) + } + + /// Starting execution from single stage + pub async fn single_stage_execution_flow( + &mut self, + base_intent: ScheduledIntentBundle, + transaction_strategy: TransactionStrategy, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + let committed_pubkeys = base_intent.get_all_committed_pubkeys(); + + let mut single_stage_executor = SingleStageStrategyExecutor::new( + self.authority.insecure_clone(), + base_intent.id, + None, + self.ctx.intent_client.clone(), + self.ctx.task_info_fetcher.clone(), + self.ctx.outbox_client.clone(), + transaction_strategy, + self.ctx.actions_callback_executor.clone(), + execution_report, + ); + let res = execute_with_timeout( + self.time_left(), + SingleStage { + inner: &mut single_stage_executor, + transaction_preparator: &self.ctx.transaction_preparator, + committed_pubkeys: &committed_pubkeys, + }, + ) + .await; + + // Here we continue only IF the error is a limit-type execution error + // We can recover that Error by splitting execution + // in 2 stages - commit & finalize + // Otherwise we return error + let execution_err = match res { + Err(IntentExecutorError::FailedToFinalizeError { + err, + commit_signature: _, + finalize_signature: _, + }) if !committed_pubkeys.is_empty() + && err.is_single_stage_split_limit_error() => + { + err + } + res => { + let signature = res.as_ref().ok().copied(); + single_stage_executor + .execute_callbacks(signature, res.as_ref().map(|_| ())); + let transaction_strategy = + single_stage_executor.consume_strategy(); + execution_report.dispose(transaction_strategy); + return res.map(ExecutionOutput::SingleStage); + } + }; + + // With actions, we can't predict num of CPIs + // If we get here we will try to switch from Single stage to Two Stage commit + // Note that this not necessarily will pass at the end due to the same reason + let strategy = single_stage_executor.consume_strategy(); + let (commit_strategy, finalize_strategy, cleanup) = + handle_cpi_limit_error(&self.authority.pubkey(), strategy); + execution_report.dispose(cleanup); + execution_report.add_patched_error(execution_err); + + self.two_stage_execution_flow( + base_intent.id, + &committed_pubkeys, + commit_strategy, + finalize_strategy, + execution_report, + ) + .await + } + + pub async fn two_stage_execution_flow( + &mut self, + intent_id: u64, + committed_pubkeys: &[Pubkey], + commit_strategy: TransactionStrategy, + finalize_strategy: TransactionStrategy, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + let state = Initialized::new(commit_strategy, finalize_strategy, None); + let mut executor = TwoStageStrategyExecutor::new( + state, + self.authority.insecure_clone(), + intent_id, + self.ctx.intent_client.clone(), + self.ctx.outbox_client.clone(), + self.ctx.actions_callback_executor.clone(), + execution_report, + ); + + let commit_signature = execute_with_timeout( + self.time_left(), + CommitStage { + inner: &mut executor, + transaction_preparator: &self.ctx.transaction_preparator, + task_info_fetcher: &self.ctx.task_info_fetcher, + committed_pubkeys, + }, + ) + .await?; + + let mut finalize_executor = executor.done(commit_signature); + let finalize_signature = execute_with_timeout( + self.time_left(), + FinalizeStage { + inner: &mut finalize_executor, + transaction_preparator: &self.ctx.transaction_preparator, + }, + ) + .await?; + + let finalized_stage = finalize_executor.done(finalize_signature); + Ok(ExecutionOutput::TwoStage { + commit_signature: finalized_stage.commit_signature, + finalize_signature: finalized_stage.finalize_signature, + }) + } +} + +#[async_trait] +impl IntentExecutor for AcceptedIntentExecutor +where + T: TransactionPreparator, + C: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + /// Executes Message on Base layer + /// Returns `ExecutionOutput` or an `Error` + async fn execute( + mut self: Box, + base_intent: ScheduledIntentBundle, + ) -> (IntentExecutionResult, CleanupHandle) { + self.started_at = Instant::now(); + let is_undelegate = base_intent.has_undelegate_intent(); + let pubkeys = base_intent.get_all_committed_pubkeys(); + + let mut execution_report = IntentExecutionReport::default(); + let result = + self.execute_inner(base_intent, &mut execution_report).await; + if !pubkeys.is_empty() { + // Reset TaskInfoFetcher, as cache could become invalid + // NOTE: if undelegation was removed - we still reset + // We assume its safe since all consecutive commits will fail + if result.is_err() || is_undelegate { + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&pubkeys)); + } + } + + // Gather metrics in separate task + let intent_client = self.ctx.intent_client.clone(); + let result = result.inspect(|output| { + let output_copy = *output; + tokio::spawn(async move { + intent_client.intent_metrics(output_copy).await + }); + }); + + self.close_buffers = result.is_ok(); + self.junk = execution_report.junk; + let result = IntentExecutionResult { + inner: result, + patched_errors: execution_report.patched_errors, + callbacks_report: execution_report.callbacks_report, + }; + let cleanup_handle = CleanupHandle::new( + self.authority, + self.junk, + self.close_buffers, + self.ctx.transaction_preparator, + ); + + (result, cleanup_handle) + } +} diff --git a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs new file mode 100644 index 000000000..a15ae10b2 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs @@ -0,0 +1,49 @@ +use futures_util::future::try_join_all; +use solana_keypair::Keypair; + +use crate::{ + tasks::task_strategist::TransactionStrategy, + transaction_preparator::{ + delivery_preparator::BufferExecutionError, TransactionPreparator, + }, +}; + +pub(crate) struct CleanupHandle { + authority: Keypair, + junk: Vec, + /// When false (execution failed), only releases ALT reservations without + /// closing buffer PDAs — avoids a race condition where a concurrent + /// executor may still be using them. + close_buffers: bool, + transaction_preparator: T, +} + +impl CleanupHandle { + pub(crate) fn new( + authority: Keypair, + junk: Vec, + close_buffers: bool, + transaction_preparator: T, + ) -> Self { + Self { + junk, + close_buffers, + authority, + transaction_preparator, + } + } + + pub(crate) async fn clean(self) -> Result<(), BufferExecutionError> { + let close_buffers = self.close_buffers; + let cleanup_futs = self.junk.iter().map(|to_cleanup| { + self.transaction_preparator.cleanup_for_strategy( + &self.authority, + &to_cleanup.optimized_tasks, + &to_cleanup.lookup_tables_keys, + close_buffers, + ) + }); + + try_join_all(cleanup_futs).await.map(|_| ()) + } +} diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index 57e6ada61..2b8693054 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -6,9 +6,11 @@ use magicblock_table_mania::TableMania; use crate::{ intent_executor::{ + accepted_intent_executor::AcceptedIntentExecutor, error::IntentExecutorError, + intent_execution_client::IntentExecutionClient, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, - IntentExecutor, IntentExecutorImpl, + IntentExecutor, IntentExecutorCtx, }, outbox_client::OutboxClient, transaction_preparator::TransactionPreparatorImpl, @@ -42,8 +44,12 @@ where O: OutboxClient, O::Error: Into, { - type Executor = - IntentExecutorImpl; + type Executor = AcceptedIntentExecutor< + TransactionPreparatorImpl, + RpcTaskInfoFetcher, + A, + O, + >; fn create_instance(&self) -> Self::Executor { let transaction_preparator = TransactionPreparatorImpl::new( @@ -51,13 +57,14 @@ where self.table_mania.clone(), self.executor_config.compute_budget_config.clone(), ); - Self::Executor::new( - self.rpc_client.clone(), + let ctx = IntentExecutorCtx { + intent_client: IntentExecutionClient::new(self.rpc_client.clone()), transaction_preparator, - self.task_info_fetcher.clone(), - self.outbox_client.clone(), - self.actions_callback_executor.clone(), - self.executor_config.actions_timeout, - ) + task_info_fetcher: self.task_info_fetcher.clone(), + outbox_client: self.outbox_client.clone(), + actions_callback_executor: self.actions_callback_executor.clone(), + actions_timeout: self.executor_config.actions_timeout, + }; + Self::Executor::new(ctx) } } diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 76f583f63..33ece5c85 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -1,62 +1,96 @@ +pub mod accepted_intent_executor; +pub mod cleanup_handle; pub mod error; pub mod intent_execution_client; pub(crate) mod intent_executor_factory; pub mod intent_executor_gateway; -pub mod patcher; -pub mod single_stage_executor; +pub mod single_stage_intent_executor; +pub mod strategy_executor; pub mod task_info_fetcher; -pub mod two_stage_executor; +pub mod two_stage_intent_executor; pub mod utils; -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use futures_util::future::{join, try_join_all}; use magicblock_core::traits::{ ActionsCallbackScheduler, CallbackScheduleError, }; use magicblock_metrics::metrics; use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, validator::validator_authority, + magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, + outbox_intent_bundles::OutboxIntentBundleStatus, }; -use magicblock_rpc_client::MagicblockRpcClient; -use solana_keypair::Keypair; -use solana_pubkey::Pubkey; use solana_signature::Signature; use solana_signer::Signer; -use tracing::trace; use crate::{ intent_executor::{ + accepted_intent_executor::AcceptedIntentExecutor, + cleanup_handle::CleanupHandle, error::{ IntentExecutorError, IntentExecutorResult, TransactionStrategyExecutionError, }, intent_execution_client::IntentExecutionClient, - single_stage_executor::SingleStageExecutor, - task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, - two_stage_executor::{Initialized, TwoStageExecutor}, - utils::{ - execute_with_timeout, handle_cpi_limit_error, CommitStage, - FinalizeStage, SingleStage, - }, + single_stage_intent_executor::SingleStageIntentExecutor, + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + two_stage_intent_executor::TwoStageIntentExecutor, }, outbox_client::OutboxClient, - tasks::{ - task_builder::{TaskBuilderImpl, TasksBuilder}, - task_strategist::{ - StrategyExecutionMode, TaskStrategist, TransactionStrategy, - }, - }, - transaction_preparator::{ - delivery_preparator::BufferExecutionError, TransactionPreparator, - }, + tasks::{task_builder::TasksBuilder, task_strategist::TransactionStrategy}, + transaction_preparator::TransactionPreparator, }; +#[async_trait] +pub trait IntentExecutor: Send + Sync + 'static { + /// Executes Message on Base layer + /// Returns result of intent execution `IntentExecutionResult` + /// and `CleanupHandle` for cleanup after intent + async fn execute( + self: Box, + base_intent: ScheduledIntentBundle, + ) -> (IntentExecutionResult, CleanupHandle); +} + +fn build_stage_intent_executor( + ctx: IntentExecutorCtx, + status: OutboxIntentBundleStatus, +) -> Box> +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + match status { + OutboxIntentBundleStatus::Accepted => { + Box::new(AcceptedIntentExecutor::new(ctx)) + as Box<(dyn IntentExecutor + 'static)> + } + OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( + sig, + )) => Box::new(SingleStageIntentExecutor::new(ctx, sig)) + as Box<(dyn IntentExecutor + 'static)>, + OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( + value, + )) => Box::new(TwoStageIntentExecutor::new(ctx, value)) + as Box<(dyn IntentExecutor + 'static)>, + } +} + +pub struct IntentExecutorCtx { + pub intent_client: IntentExecutionClient, + pub transaction_preparator: T, + pub task_info_fetcher: Arc>, + pub outbox_client: Arc, + pub actions_callback_executor: A, + // TODO(edwin): more like config field. exclude? + /// Timeout for Intent's actions + pub actions_timeout: Duration, +} + #[derive(Clone, Copy, Debug)] pub enum ExecutionOutput { // TODO: with arrival of challenge window remove SingleStage @@ -93,19 +127,6 @@ pub struct IntentExecutionResult { pub callbacks_report: Vec>, } -#[async_trait] -pub trait IntentExecutor: Send + Sync + 'static { - /// Executes Message on Base layer - /// Returns `ExecutionOutput` or an `Error` - async fn execute( - &mut self, - base_intent: OutboxIntentBundle, - ) -> IntentExecutionResult; - - /// Cleans up after intent - async fn cleanup(self) -> Result<(), BufferExecutionError>; -} - #[derive(Default)] pub struct IntentExecutionReport { /// Junk that needs to be cleaned up @@ -143,330 +164,3 @@ impl IntentExecutionReport { &self.junk } } - -pub struct IntentExecutorImpl { - authority: Keypair, - intent_client: IntentExecutionClient, - transaction_preparator: T, - task_info_fetcher: Arc>, - actions_callback_executor: A, - outbox_client: Arc, - /// Timeout for Intent's actions - actions_timeout: Duration, - - /// Intent execution started at - pub started_at: Instant, - /// Junk that needs to be cleaned up - junk: Vec, - /// Set to false on execution failure so cleanup only releases ALT - /// reservations without closing buffer PDAs (see race condition note in - /// intent_execution_engine) - close_buffers: bool, -} - -impl IntentExecutorImpl -where - T: TransactionPreparator, - F: TaskInfoFetcher, - A: ActionsCallbackScheduler, - O: OutboxClient, - O::Error: Into, -{ - pub fn new( - rpc_client: MagicblockRpcClient, - transaction_preparator: T, - task_info_fetcher: Arc>, - outbox_client: Arc, - actions_callback_executor: A, - actions_timeout: Duration, - ) -> Self { - let authority = validator_authority(); - let intent_client = IntentExecutionClient::new(rpc_client); - Self { - authority, - intent_client, - transaction_preparator, - task_info_fetcher, - outbox_client, - actions_callback_executor, - actions_timeout, - - started_at: Instant::now(), - junk: vec![], - close_buffers: true, - } - } - - async fn execute_inner( - &mut self, - intent_bundle: ScheduledIntentBundle, - execution_report: &mut IntentExecutionReport, - ) -> IntentExecutorResult { - if intent_bundle.is_empty() { - return Err(IntentExecutorError::EmptyIntentError); - } - let all_committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); - - if all_committed_pubkeys.is_empty() { - // Build tasks for commit stage - // TODO (snawaz): it's actually MagicBaseIntent::BaseActions scenario, not Commit - // scenario, so the related code needs little bit of refactoring and proper renaming. - let commit_tasks = TaskBuilderImpl::commit_tasks( - &self.task_info_fetcher, - &intent_bundle, - ) - .await?; - - // Standalone actions executed in single stage - let mut strategy = TaskStrategist::build_strategy( - commit_tasks, - &self.authority.pubkey(), - )?; - strategy.standalone_action_nonce = Some(intent_bundle.id); - return self - .single_stage_execution_flow( - intent_bundle, - strategy, - execution_report, - ) - .await; - }; - - // Build tasks for commit & finalize stages - let (commit_tasks, finalize_tasks) = { - let commit_tasks_fut = TaskBuilderImpl::commit_tasks( - &self.task_info_fetcher, - &intent_bundle, - ); - let finalize_tasks_fut = TaskBuilderImpl::finalize_tasks( - &self.task_info_fetcher, - &intent_bundle, - ); - let (commit_tasks, finalize_tasks) = - join(commit_tasks_fut, finalize_tasks_fut).await; - - (commit_tasks?, finalize_tasks?) - }; - - // Build execution strategy - match TaskStrategist::build_execution_strategy( - commit_tasks, - finalize_tasks, - &self.authority.pubkey(), - )? { - StrategyExecutionMode::SingleStage(strategy) => { - trace!("Single stage execution"); - self.single_stage_execution_flow( - intent_bundle, - strategy, - execution_report, - ) - .await - } - StrategyExecutionMode::TwoStage { - commit_stage, - finalize_stage, - } => { - trace!("Two stage execution"); - self.two_stage_execution_flow( - intent_bundle.id, - &all_committed_pubkeys, - commit_stage, - finalize_stage, - execution_report, - ) - .await - } - } - } - - fn time_left(&self) -> Option { - self.actions_timeout.checked_sub(self.started_at.elapsed()) - } - - /// Starting execution from single stage - pub async fn single_stage_execution_flow( - &mut self, - base_intent: ScheduledIntentBundle, - transaction_strategy: TransactionStrategy, - execution_report: &mut IntentExecutionReport, - ) -> IntentExecutorResult { - let committed_pubkeys = base_intent.get_all_committed_pubkeys(); - - let mut single_stage_executor = SingleStageExecutor::new( - self.authority.insecure_clone(), - base_intent.id, - None, - self.intent_client.clone(), - self.task_info_fetcher.clone(), - self.outbox_client.clone(), - transaction_strategy, - self.actions_callback_executor.clone(), - execution_report, - ); - let res = execute_with_timeout( - self.time_left(), - SingleStage { - inner: &mut single_stage_executor, - transaction_preparator: &self.transaction_preparator, - committed_pubkeys: &committed_pubkeys, - }, - ) - .await; - - // Here we continue only IF the error is a limit-type execution error - // We can recover that Error by splitting execution - // in 2 stages - commit & finalize - // Otherwise we return error - let execution_err = match res { - Err(IntentExecutorError::FailedToFinalizeError { - err, - commit_signature: _, - finalize_signature: _, - }) if !committed_pubkeys.is_empty() - && err.is_single_stage_split_limit_error() => - { - err - } - res => { - let signature = res.as_ref().ok().copied(); - single_stage_executor - .execute_callbacks(signature, res.as_ref().map(|_| ())); - let transaction_strategy = - single_stage_executor.consume_strategy(); - execution_report.dispose(transaction_strategy); - return res.map(ExecutionOutput::SingleStage); - } - }; - - // With actions, we can't predict num of CPIs - // If we get here we will try to switch from Single stage to Two Stage commit - // Note that this not necessarily will pass at the end due to the same reason - let strategy = single_stage_executor.consume_strategy(); - let (commit_strategy, finalize_strategy, cleanup) = - handle_cpi_limit_error(&self.authority.pubkey(), strategy); - execution_report.dispose(cleanup); - execution_report.add_patched_error(execution_err); - - self.two_stage_execution_flow( - base_intent.id, - &committed_pubkeys, - commit_strategy, - finalize_strategy, - execution_report, - ) - .await - } - - pub async fn two_stage_execution_flow( - &mut self, - intent_id: u64, - committed_pubkeys: &[Pubkey], - commit_strategy: TransactionStrategy, - finalize_strategy: TransactionStrategy, - execution_report: &mut IntentExecutionReport, - ) -> IntentExecutorResult { - let state = Initialized::new(commit_strategy, finalize_strategy, None); - let mut executor = TwoStageExecutor::new( - state, - self.authority.insecure_clone(), - intent_id, - self.intent_client.clone(), - self.outbox_client.clone(), - self.actions_callback_executor.clone(), - execution_report, - ); - - let commit_signature = execute_with_timeout( - self.time_left(), - CommitStage { - inner: &mut executor, - transaction_preparator: &self.transaction_preparator, - task_info_fetcher: &self.task_info_fetcher, - committed_pubkeys, - }, - ) - .await?; - - let mut finalize_executor = executor.done(commit_signature); - let finalize_signature = execute_with_timeout( - self.time_left(), - FinalizeStage { - inner: &mut finalize_executor, - transaction_preparator: &self.transaction_preparator, - }, - ) - .await?; - - let finalized_stage = finalize_executor.done(finalize_signature); - Ok(ExecutionOutput::TwoStage { - commit_signature: finalized_stage.commit_signature, - finalize_signature: finalized_stage.finalize_signature, - }) - } -} - -#[async_trait] -impl IntentExecutor for IntentExecutorImpl -where - T: TransactionPreparator, - C: TaskInfoFetcher, - A: ActionsCallbackScheduler, - O: OutboxClient, - O::Error: Into, -{ - /// Executes Message on Base layer - /// Returns `ExecutionOutput` or an `Error` - async fn execute( - &mut self, - base_intent: OutboxIntentBundle, - ) -> IntentExecutionResult { - self.started_at = Instant::now(); - let is_undelegate = base_intent.has_undelegate_intent(); - let pubkeys = base_intent.get_all_committed_pubkeys(); - - let mut execution_report = IntentExecutionReport::default(); - let result = self - .execute_inner(base_intent.inner, &mut execution_report) - .await; - if !pubkeys.is_empty() { - // Reset TaskInfoFetcher, as cache could become invalid - // NOTE: if undelegation was removed - we still reset - // We assume its safe since all consecutive commits will fail - if result.is_err() || is_undelegate { - self.task_info_fetcher.reset(ResetType::Specific(&pubkeys)); - } - } - - // Gather metrics in separate task - let intent_client = self.intent_client.clone(); - let result = result.inspect(|output| { - let output_copy = *output; - tokio::spawn(async move { - intent_client.intent_metrics(output_copy).await - }); - }); - - self.close_buffers = result.is_ok(); - self.junk = execution_report.junk; - IntentExecutionResult { - inner: result, - patched_errors: execution_report.patched_errors, - callbacks_report: execution_report.callbacks_report, - } - } - - async fn cleanup(mut self) -> Result<(), BufferExecutionError> { - let close_buffers = self.close_buffers; - let cleanup_futs = self.junk.iter().map(|to_cleanup| { - self.transaction_preparator.cleanup_for_strategy( - &self.authority, - &to_cleanup.optimized_tasks, - &to_cleanup.lookup_tables_keys, - close_buffers, - ) - }); - - try_join_all(cleanup_futs).await.map(|_| ()) - } -} diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs new file mode 100644 index 000000000..5c0b7c6b6 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -0,0 +1,88 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use magicblock_core::traits::ActionsCallbackScheduler; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + validator::validator_authority, +}; +use magicblock_rpc_client::MagicblockRpcClient; +use solana_keypair::Keypair; +use solana_signature::Signature; +use solana_signer::Signer; + +use crate::{ + intent_executor::{ + accepted_intent_executor::AcceptedIntentExecutor, + cleanup_handle::CleanupHandle, + error::IntentExecutorError, + intent_execution_client::IntentExecutionClient, + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + IntentExecutionResult, IntentExecutor, IntentExecutorCtx, + }, + outbox_client::OutboxClient, + tasks::task_strategist::TransactionStrategy, + transaction_preparator::TransactionPreparator, +}; + +pub struct SingleStageIntentExecutor { + authority: Keypair, + /// Signature of committing + pending_signature: Signature, + /// Intent Executor context + ctx: IntentExecutorCtx, + + /// Intent execution started at + pub started_at: Instant, + /// Junk that needs to be cleaned up + junk: Vec, + /// Set to false on execution failure so cleanup only releases ALT + /// reservations without closing buffer PDAs (see race condition note in + /// intent_execution_engine) + close_buffers: bool, +} + +impl SingleStageIntentExecutor +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + pub fn new( + ctx: IntentExecutorCtx, + pending_signature: Signature, + ) -> Self { + let authority = validator_authority(); + Self { + authority, + pending_signature, + ctx, + + started_at: Instant::now(), + junk: vec![], + close_buffers: true, + } + } +} + +#[async_trait] +impl IntentExecutor for SingleStageIntentExecutor +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + async fn execute( + mut self: Box, + base_intent: ScheduledIntentBundle, + ) -> (IntentExecutionResult, CleanupHandle) { + todo!() + } +} diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs new file mode 100644 index 000000000..fe27b25cd --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs @@ -0,0 +1,4 @@ +pub mod patcher; +pub mod single_stage; +pub mod two_stage; +pub mod utils; diff --git a/magicblock-committor-service/src/intent_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs similarity index 98% rename from magicblock-committor-service/src/intent_executor/patcher.rs rename to magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs index 28bf53abf..e3277f583 100644 --- a/magicblock-committor-service/src/intent_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs @@ -15,11 +15,11 @@ use crate::{ TransactionStrategyExecutionError, }, intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - utils::{ + strategy_executor::utils::{ handle_actions_result, handle_commit_id_error, handle_undelegation_error, prepare_and_execute_strategy, }, + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, IntentExecutionReport, }, tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, @@ -202,7 +202,6 @@ pub(in crate::intent_executor) struct CommitStagePatcher<'a, F, A, T> { pub transaction_preparator: &'a T, } - impl CommitStagePatcher<'_, F, A, T> where T: TransactionPreparator, @@ -226,12 +225,12 @@ where standalone_action_nonce: None, }, ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)? - .map_err(|err| IntentExecutorError::FailedToCommitError { - err, - signature: *failed_signature, - })?; + .await + .map_err(IntentExecutorError::FailedCommitPreparationError)? + .map_err(|err| IntentExecutorError::FailedToCommitError { + err, + signature: *failed_signature, + })?; Ok(ControlFlow::Continue(TransactionStrategy { optimized_tasks: vec![], @@ -336,7 +335,6 @@ where } } - pub(in crate::intent_executor) struct FinalizeStagePatcher<'a, A> { pub authority: &'a Keypair, pub callback_scheduler: &'a A, diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs similarity index 93% rename from magicblock-committor-service/src/intent_executor/single_stage_executor.rs rename to magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 9e5c973d4..3858b5f01 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -12,9 +12,13 @@ use crate::{ intent_executor::{ error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, - patcher::SingleStagePatcher, + strategy_executor::{ + patcher::SingleStagePatcher, + utils::{ + handle_actions_result, stage_execution_loop, ExecutionState, + }, + }, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - utils::{handle_actions_result, stage_execution_loop, ExecutionState}, IntentExecutionReport, }, outbox_client::OutboxClient, @@ -22,7 +26,7 @@ use crate::{ transaction_preparator::TransactionPreparator, }; -pub struct SingleStageExecutor<'a, F, A, O> { +pub struct SingleStageStrategyExecutor<'a, F, A, O> { current_attempt: u8, pending_signature: Option, execution_report: &'a mut IntentExecutionReport, @@ -36,7 +40,7 @@ pub struct SingleStageExecutor<'a, F, A, O> { transaction_strategy: TransactionStrategy, } -impl<'a, F, A, O> SingleStageExecutor<'a, F, A, O> +impl<'a, F, A, O> SingleStageStrategyExecutor<'a, F, A, O> where F: TaskInfoFetcher, A: ActionsCallbackScheduler, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs similarity index 95% rename from magicblock-committor-service/src/intent_executor/two_stage_executor.rs rename to magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index 0e5b630c2..3d583e834 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -12,10 +12,14 @@ use crate::{ intent_executor::{ error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, - patcher::{CommitStagePatcher, FinalizeStagePatcher}, + strategy_executor::{ + patcher::{CommitStagePatcher, FinalizeStagePatcher}, + two_stage::sealed::Sealed, + utils::{ + handle_actions_result, stage_execution_loop, ExecutionState, + }, + }, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - two_stage_executor::sealed::Sealed, - utils::{handle_actions_result, stage_execution_loop, ExecutionState}, IntentExecutionReport, }, outbox_client::OutboxClient, @@ -82,7 +86,7 @@ pub struct Finalized { pub finalize_signature: Signature, } -pub struct TwoStageExecutor<'a, A, O, S: Sealed> { +pub struct TwoStageStrategyExecutor<'a, A, O, S: Sealed> { /// State per stage state: S, /// Common state across the stages @@ -94,7 +98,7 @@ pub struct TwoStageExecutor<'a, A, O, S: Sealed> { execution_report: &'a mut IntentExecutionReport, } -impl<'a, A, O> TwoStageExecutor<'a, A, O, Initialized> +impl<'a, A, O> TwoStageStrategyExecutor<'a, A, O, Initialized> where A: ActionsCallbackScheduler, O: OutboxClient, @@ -226,8 +230,8 @@ where pub fn done( self, commit_signature: Signature, - ) -> TwoStageExecutor<'a, A, O, Committed> { - TwoStageExecutor { + ) -> TwoStageStrategyExecutor<'a, A, O, Committed> { + TwoStageStrategyExecutor { authority: self.authority, intent_id: self.intent_id, intent_client: self.intent_client, @@ -244,7 +248,7 @@ where } } -impl<'a, A, O> TwoStageExecutor<'a, A, O, Committed> +impl<'a, A, O> TwoStageStrategyExecutor<'a, A, O, Committed> where A: ActionsCallbackScheduler, O: OutboxClient, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs new file mode 100644 index 000000000..720f370f2 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -0,0 +1,576 @@ +use std::{ops::ControlFlow, time::Duration}; + +use async_trait::async_trait; +use magicblock_core::traits::{ + ActionError, ActionResult, ActionsCallbackScheduler, +}; +use magicblock_program::outbox::ExecutionStage; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_signature::Signature; +use solana_transaction::versioned::VersionedTransaction; +use tokio::time::timeout; +use tracing::{error, info, warn}; + +use crate::{ + intent_executor::{ + error::{ + IntentExecutorError, IntentExecutorResult, + TransactionStrategyExecutionError, + }, + intent_execution_client::IntentExecutionClient, + strategy_executor::{ + patcher::Patcher, + single_stage::SingleStageStrategyExecutor, + two_stage::{Committed, Initialized, TwoStageStrategyExecutor}, + }, + task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, + IntentExecutionReport, + }, + outbox_client::OutboxClient, + tasks::{ + task_builder::TaskBuilderError, + task_strategist::{TaskStrategist, TransactionStrategy}, + BaseTaskImpl, + }, + transaction_preparator::{ + error::TransactionPreparatorError, TransactionPreparator, + }, +}; + +const STAGE_LOOP_CEILING: u8 = 10; + +pub(in crate::intent_executor) struct ExecutionState<'a> { + pub current_attempt: &'a mut u8, + pub transaction_strategy: &'a mut TransactionStrategy, + pub pending_signature: &'a mut Option, + pub execution_report: &'a mut IntentExecutionReport, +} + +pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( + authority: &Keypair, + intent_client: &IntentExecutionClient, + outbox_client: &O, + transaction_preparator: &T, + mut patcher: P, + intent_id: u64, + make_outbox_stage: impl Fn(Signature) -> ExecutionStage, + map_preparation_err: fn(TransactionPreparatorError) -> IntentExecutorError, + state: ExecutionState<'a>, +) -> IntentExecutorResult> +where + T: TransactionPreparator, + O: OutboxClient, + O::Error: Into, + P: Patcher, +{ + loop { + if let Some(ref sig) = state.pending_signature { + let flow = check_pending_signature(intent_client, sig).await?; + // Pending signature corresponds to succeeded transaction - break + if let ControlFlow::Break(()) = flow { + break Ok(Ok(*sig)); + } + + *state.pending_signature = None; + } + + *state.current_attempt += 1; + + // Prepare message + let prepared_transaction = prepare_transaction( + intent_client, + authority, + transaction_preparator, + state.transaction_strategy, + ) + .await + .map_err(map_preparation_err)?; + + // Record in outbox before sending + let signature = prepared_transaction.get_signature(); + outbox_client + .set_intent_execution_stage( + intent_id, + make_outbox_stage(*signature), + ) + .await + .map_err(Into::into)?; + + // Now record locally signature of tx we about to send + // Precaution for timeout in between + *state.pending_signature = Some(*signature); + + // Send signed tx + let execution_result = intent_client + .send_signed_tx_with_retries( + &prepared_transaction, + &state.transaction_strategy.optimized_tasks, + ) + .await; + + // Result returned, cleanup pending signature + *state.pending_signature = None; + + let execution_err = match execution_result { + Ok(()) => return Ok(Ok(*signature)), + Err(err) => err, + }; + let flow = patcher + .patch( + &execution_err, + state.transaction_strategy, + state.execution_report, + ) + .await?; + let cleanup = match flow { + ControlFlow::Continue(value) => value, + ControlFlow::Break(()) => return Ok(Err(execution_err)), + }; + intent_client.invalidate_cached_blockhash().await; + state.execution_report.dispose(cleanup); + + if *state.current_attempt >= STAGE_LOOP_CEILING { + error!("CRITICAL! Recursion ceiling reached"); + return Ok(Err(execution_err)); + } else { + state.execution_report.add_patched_error(execution_err); + } + } +} + +pub async fn prepare_transaction( + client: &IntentExecutionClient, + authority: &Keypair, + transaction_preparator: &T, + transaction_strategy: &mut TransactionStrategy, +) -> Result { + let mut prepared_message = transaction_preparator + .prepare_for_strategy(&authority, transaction_strategy) + .await?; + + // Get latest blockhash(Part of preparation I guess) + let blockhash = client + .get_latest_blockhash() + .await + .map_err(TransactionPreparatorError::GetLatestBlockhashError)?; + prepared_message.set_recent_blockhash(blockhash); + + // Create and sign transaction(Part of transaction preparation i guess, hence the errors) + let transaction = + VersionedTransaction::try_new(prepared_message, &[&authority]) + .map_err(TransactionPreparatorError::SignerError)?; + Ok(transaction) +} + +/// Checks a `pending_signature` loaded from the outbox against the full +/// transaction history. Returns `Break(sig)` if the tx succeeded — the caller +/// can short-circuit and skip re-execution. Returns `Continue` for every other +/// outcome (tx failed, never sent, or RPC returned no entry), letting the +/// normal execution loop proceed. +pub(in crate::intent_executor) async fn check_pending_signature( + client: &IntentExecutionClient, + sig: &Signature, +) -> IntentExecutorResult> { + let statuses = client + .get_signature_statuses_with_history(std::slice::from_ref(sig)) + .await + .map_err(IntentExecutorError::GetPendingSignatureStatusError)?; + + match statuses.get(0) { + Some(Some(Ok(()))) => Ok(ControlFlow::Break(())), + // TODO(edwin): well, that is bizarre one + None => { + warn!(pending_signature = ?sig, "RPC did not return status for signature"); + Ok(ControlFlow::Continue(())) + } + // Any case below means tx failed in execution we continue attempts + Some(None) => { + info!(pending_signature = ?sig, "Transaction corresponding to pending signature was never sent"); + Ok(ControlFlow::Continue(())) + } + Some(Some(Err(err))) => { + info!(pending_signature = ?sig, error = ?err, "Transaction corresponding to pending signature failed"); + Ok(ControlFlow::Continue(())) + } + } +} + +pub async fn prepare_and_execute_strategy( + client: &IntentExecutionClient, + authority: &Keypair, + transaction_preparator: &T, + transaction_strategy: &mut TransactionStrategy, +) -> Result< + Result, + TransactionPreparatorError, +> +where + T: TransactionPreparator, +{ + let prepared_message = transaction_preparator + .prepare_for_strategy(authority, transaction_strategy) + .await?; + + let execution_result = client + .execute_message_with_retries( + authority, + prepared_message, + &transaction_strategy.optimized_tasks, + ) + .await; + + Ok(execution_result) +} + +/// Handles out of sync commit id error, fixes current strategy +/// Returns strategy to be cleaned up +/// TODO(edwin): TransactionStrategy -> CleanupStrategy or something, naming is confusing for something that is cleaned up +pub(in crate::intent_executor) async fn handle_commit_id_error< + T: TaskInfoFetcher, +>( + authority: &Pubkey, + task_info_fetcher: &CacheTaskInfoFetcher, + committed_pubkeys: &[Pubkey], + strategy: &mut TransactionStrategy, +) -> Result { + let min_context_slot = strategy + .optimized_tasks + .iter() + .filter_map(|task| match task { + BaseTaskImpl::Commit(task) => { + Some(task.committed_account.remote_slot) + } + BaseTaskImpl::CommitFinalize(task) => { + Some(task.committed_account.remote_slot) + } + _ => None, + }) + .max() + .unwrap_or_default(); + + // We reset TaskInfoFetcher for all committed accounts + // We re-fetch them to fix out of sync tasks + task_info_fetcher.reset(ResetType::Specific(committed_pubkeys)); + let commit_ids = task_info_fetcher + .fetch_next_commit_nonces(committed_pubkeys, min_context_slot) + .await + .map_err(TaskBuilderError::CommitTasksBuildError)?; + + // Here we find the broken tasks and reset them + // Broken tasks are prepared incorrectly so they have to be cleaned up + let mut to_cleanup = Vec::new(); + for task in &mut strategy.optimized_tasks { + match task { + BaseTaskImpl::Commit(task) => { + let Some(commit_id) = + commit_ids.get(&task.committed_account.pubkey) + else { + continue; + }; + if commit_id == &task.commit_id { + continue; + } + + // Handle invalid tasks + to_cleanup.push(BaseTaskImpl::Commit(task.clone())); + task.reset_commit_id(*commit_id); + } + BaseTaskImpl::CommitFinalize(task) => { + let Some(commit_id) = + commit_ids.get(&task.committed_account.pubkey) + else { + continue; + }; + if commit_id == &task.commit_id { + continue; + } + + // Handle invalid tasks + to_cleanup.push(BaseTaskImpl::CommitFinalize(task.clone())); + task.reset_commit_id(*commit_id); + } + _ => {} + } + } + + let old_alts = strategy.dummy_revaluate_alts(authority); + Ok(TransactionStrategy { + optimized_tasks: to_cleanup, + lookup_tables_keys: old_alts, + standalone_action_nonce: None, + }) +} + +/// Handle CPI limit error, splits single strategy flow into 2 +/// Returns Commit stage strategy, Finalize stage strategy and strategy to clean up +pub(in crate::intent_executor) fn handle_cpi_limit_error( + authority: &Pubkey, + strategy: TransactionStrategy, +) -> ( + TransactionStrategy, + TransactionStrategy, + TransactionStrategy, +) { + // We encountered error "Max instruction trace length exceeded" + // All the tasks a prepared to be executed at this point + // We attempt Two stages commit flow, need to split tasks up + let last_commit_ind = strategy.optimized_tasks.iter().rposition(|el| { + matches!( + el, + BaseTaskImpl::Commit(_) | BaseTaskImpl::CommitFinalize(_) + ) + }); + let (mut commit_stage_tasks, mut finalize_stage_tasks) = (vec![], vec![]); + for (i, el) in strategy.optimized_tasks.into_iter().enumerate() { + if Some(i) <= last_commit_ind { + commit_stage_tasks.push(el); + } else { + finalize_stage_tasks.push(el); + } + } + + let commit_alt_pubkeys = if strategy.lookup_tables_keys.is_empty() { + vec![] + } else { + TaskStrategist::collect_lookup_table_keys( + authority, + &commit_stage_tasks, + ) + }; + let commit_strategy = TransactionStrategy { + optimized_tasks: commit_stage_tasks, + lookup_tables_keys: commit_alt_pubkeys, + standalone_action_nonce: None, + }; + + let finalize_alt_pubkeys = if strategy.lookup_tables_keys.is_empty() { + vec![] + } else { + TaskStrategist::collect_lookup_table_keys( + authority, + &finalize_stage_tasks, + ) + }; + let finalize_strategy = TransactionStrategy { + optimized_tasks: finalize_stage_tasks, + lookup_tables_keys: finalize_alt_pubkeys, + standalone_action_nonce: None, + }; + + // We clean up only ALTs + let to_cleanup = TransactionStrategy { + optimized_tasks: vec![], + lookup_tables_keys: strategy.lookup_tables_keys, + standalone_action_nonce: None, + }; + + (commit_strategy, finalize_strategy, to_cleanup) +} + +/// Handles undelegation error, stripping away actions +/// Returns [`TransactionStrategy`] to be cleaned up +pub(in crate::intent_executor) fn handle_undelegation_error( + authority: &Pubkey, + strategy: &mut TransactionStrategy, +) -> TransactionStrategy { + let position = strategy + .optimized_tasks + .iter() + .position(|el| matches!(el, BaseTaskImpl::Undelegate(_))); + + if let Some(position) = position { + // Remove everything after undelegation including post undelegation actions + let removed_task = strategy.optimized_tasks.drain(position..).collect(); + let old_alts = strategy.dummy_revaluate_alts(authority); + TransactionStrategy { + optimized_tasks: removed_task, + lookup_tables_keys: old_alts, + standalone_action_nonce: None, + } + } else { + TransactionStrategy { + optimized_tasks: vec![], + lookup_tables_keys: vec![], + standalone_action_nonce: None, + } + } +} + +pub(in crate::intent_executor) fn handle_actions_result( + authority: &Pubkey, + callback_scheduler: &A, + execution_report: &mut IntentExecutionReport, + transaction_strategy: &mut TransactionStrategy, + signature: Option, + result: ActionResult, +) -> TransactionStrategy +where + A: ActionsCallbackScheduler, +{ + let (callbacks, junk) = if result.is_ok() { + let callbacks = transaction_strategy.extract_action_callbacks(); + (callbacks, TransactionStrategy::default()) + } else { + let mut removed_actions = + transaction_strategy.remove_actions(authority); + let callbacks = removed_actions.extract_action_callbacks(); + (callbacks, removed_actions) + }; + if !callbacks.is_empty() { + let result = callback_scheduler.schedule(callbacks, signature, result); + execution_report.add_callback_report(result); + } + + junk +} + +// TODO(edwin): docs +pub(in crate::intent_executor) async fn execute_with_timeout( + time_left: Option, + mut executor: impl StageExecutor, +) -> IntentExecutorResult { + if executor.has_callbacks() { + if let Some(time_left) = time_left { + match timeout(time_left, executor.execute()).await { + Ok(res) => return res, + Err(_) => { + // The race between callback and intent txn is handled + // on the user smart contract side via TimeoutError. + // We must respect the timeout contract. + info!("Intent execution timed out, cleaning up actions"); + executor.execute_callbacks( + None, + Err(ActionError::TimeoutError), + ); + } + } + } else { + // Already timed out; see comment above. + executor.execute_callbacks(None, Err(ActionError::TimeoutError)); + } + } + + // Timeout concerns only callbacks + // We still need to drive intent to completion + executor.execute().await +} + +#[async_trait] +pub(in crate::intent_executor) trait StageExecutor { + fn has_callbacks(&self) -> bool; + async fn execute(&mut self) -> IntentExecutorResult; + fn execute_callbacks( + &mut self, + signature: Option, + result: ActionResult, + ); +} + +pub(in crate::intent_executor) struct SingleStage<'a, 'e, A, T, F, O> { + pub(in crate::intent_executor) inner: + &'a mut SingleStageStrategyExecutor<'e, F, A, O>, + pub(in crate::intent_executor) transaction_preparator: &'a T, + pub(in crate::intent_executor) committed_pubkeys: &'a [Pubkey], +} + +#[async_trait] +impl<'a, 'e, A, T, F, O> StageExecutor for SingleStage<'a, 'e, A, T, F, O> +where + A: ActionsCallbackScheduler, + T: TransactionPreparator, + F: TaskInfoFetcher, + O: OutboxClient, + O::Error: Into, +{ + fn has_callbacks(&self) -> bool { + self.inner.has_callbacks() + } + + async fn execute(&mut self) -> IntentExecutorResult { + self.inner + .execute(self.committed_pubkeys, self.transaction_preparator) + .await + } + + fn execute_callbacks( + &mut self, + signature: Option, + result: ActionResult, + ) { + self.inner.execute_callbacks(signature, result) + } +} + +pub(in crate::intent_executor) struct CommitStage<'a, 'e, A, T, F, O> { + pub(in crate::intent_executor) inner: + &'a mut TwoStageStrategyExecutor<'e, A, O, Initialized>, + pub(in crate::intent_executor) transaction_preparator: &'a T, + pub(in crate::intent_executor) task_info_fetcher: + &'a CacheTaskInfoFetcher, + pub(in crate::intent_executor) committed_pubkeys: &'a [Pubkey], +} + +#[async_trait] +impl<'a, 'e, A, T, F, O> StageExecutor for CommitStage<'a, 'e, A, T, F, O> +where + A: ActionsCallbackScheduler, + T: TransactionPreparator, + F: TaskInfoFetcher, + O: OutboxClient, + O::Error: Into, +{ + fn has_callbacks(&self) -> bool { + self.inner.has_callbacks() + } + + async fn execute(&mut self) -> IntentExecutorResult { + self.inner + .commit( + self.committed_pubkeys, + self.transaction_preparator, + self.task_info_fetcher, + ) + .await + } + + fn execute_callbacks( + &mut self, + signature: Option, + result: ActionResult, + ) { + self.inner.execute_callbacks(signature, result) + } +} + +pub(in crate::intent_executor) struct FinalizeStage<'a, 'e, A, T, O> { + pub(in crate::intent_executor) inner: + &'a mut TwoStageStrategyExecutor<'e, A, O, Committed>, + pub(in crate::intent_executor) transaction_preparator: &'a T, +} + +#[async_trait] +impl<'a, 'e, A, T, O> StageExecutor for FinalizeStage<'a, 'e, A, T, O> +where + A: ActionsCallbackScheduler, + T: TransactionPreparator, + O: OutboxClient, + O::Error: Into, +{ + fn has_callbacks(&self) -> bool { + self.inner.has_callbacks() + } + + async fn execute(&mut self) -> IntentExecutorResult { + self.inner.finalize(self.transaction_preparator).await + } + + fn execute_callbacks( + &mut self, + signature: Option, + result: ActionResult, + ) { + self.inner.execute_callbacks(signature, result) + } +} diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs new file mode 100644 index 000000000..13e9ea8a9 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -0,0 +1,127 @@ +use std::time::Instant; + +use async_trait::async_trait; +use magicblock_core::traits::ActionsCallbackScheduler; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox::TwoStageProgress, validator::validator_authority, +}; +use solana_keypair::Keypair; +use solana_signature::Signature; + +use crate::{ + intent_executor::{ + cleanup_handle::CleanupHandle, + error::{IntentExecutorError, IntentExecutorResult}, + task_info_fetcher::TaskInfoFetcher, + ExecutionOutput, IntentExecutionReport, IntentExecutionResult, + IntentExecutor, IntentExecutorCtx, + }, + outbox_client::OutboxClient, + tasks::task_strategist::TransactionStrategy, + transaction_preparator::TransactionPreparator, +}; +use crate::intent_executor::task_info_fetcher::ResetType; + +pub struct TwoStageIntentExecutor { + authority: Keypair, + /// Current stage of TwoStage execution flow + stage: TwoStageProgress, + /// Intent Executor context + ctx: IntentExecutorCtx, + + /// Intent execution started at + pub started_at: Instant, + /// Junk that needs to be cleaned up + junk: Vec, + /// Set to false on execution failure so cleanup only releases ALT + /// reservations without closing buffer PDAs (see race condition note in + /// intent_execution_engine) + close_buffers: bool, +} + +impl TwoStageIntentExecutor +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + pub fn new( + ctx: IntentExecutorCtx, + stage: TwoStageProgress, + ) -> Self { + let authority = validator_authority(); + Self { + authority, + stage, + ctx, + + started_at: Instant::now(), + junk: vec![], + close_buffers: true, + } + } + + async fn execute_committing_intent( + &mut self, + intent: ScheduledIntentBundle, + pending_signature: Signature, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + todo!() + } + + async fn execute_finalizing_intent( + &mut self, + intent: ScheduledIntentBundle, + commit_signature: Signature, + pending_finalize_signature: Signature, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + + todo!() + } +} + +#[async_trait] +impl IntentExecutor for TwoStageIntentExecutor +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + async fn execute( + mut self: Box, + intent: ScheduledIntentBundle, + ) -> (IntentExecutionResult, CleanupHandle) { + let is_undelegate = intent.has_undelegate_intent(); + // TODO(edwin): should validate non emptiness of pubkeys? Shouldn't be possible tho + let pubkeys = intent.get_all_committed_pubkeys(); + + let mut execution_report = IntentExecutionReport::default(); + let result = match &self.stage { + TwoStageProgress::Committing(signature) => { + self.execute_committing_intent(intent, *signature, &mut execution_report).await + } + TwoStageProgress::Finalizing { + commit, + finalize + } => { + self.execute_finalizing_intent(intent, *commit, *finalize, &mut execution_report).await + } + }; + + + if is_undelegate { + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&pubkeys)); + } + + todo!() + } +} diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index c7fd7b995..8b1378917 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -1,574 +1 @@ -use std::{ops::ControlFlow, time::Duration}; -use async_trait::async_trait; -use magicblock_core::traits::{ - ActionError, ActionResult, ActionsCallbackScheduler, -}; -use magicblock_program::outbox::ExecutionStage; -use solana_keypair::Keypair; -use solana_pubkey::Pubkey; -use solana_rpc_client::rpc_client::SerializableTransaction; -use solana_signature::Signature; -use solana_transaction::versioned::VersionedTransaction; -use tokio::time::timeout; -use tracing::{error, info, warn}; - -use crate::{ - intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, - }, - intent_execution_client::IntentExecutionClient, - patcher::Patcher, - single_stage_executor::SingleStageExecutor, - task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, - two_stage_executor::{Committed, Initialized, TwoStageExecutor}, - IntentExecutionReport, - }, - outbox_client::OutboxClient, - tasks::{ - task_builder::TaskBuilderError, - task_strategist::{TaskStrategist, TransactionStrategy}, - BaseTaskImpl, - }, - transaction_preparator::{ - error::TransactionPreparatorError, TransactionPreparator, - }, -}; - -const STAGE_LOOP_CEILING: u8 = 10; - -pub(in crate::intent_executor) struct ExecutionState<'a> { - pub current_attempt: &'a mut u8, - pub transaction_strategy: &'a mut TransactionStrategy, - pub pending_signature: &'a mut Option, - pub execution_report: &'a mut IntentExecutionReport, -} - -pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( - authority: &Keypair, - intent_client: &IntentExecutionClient, - outbox_client: &O, - transaction_preparator: &T, - mut patcher: P, - intent_id: u64, - make_outbox_stage: impl Fn(Signature) -> ExecutionStage, - map_preparation_err: fn(TransactionPreparatorError) -> IntentExecutorError, - state: ExecutionState<'a>, -) -> IntentExecutorResult> -where - T: TransactionPreparator, - O: OutboxClient, - O::Error: Into, - P: Patcher, -{ - loop { - if let Some(ref sig) = state.pending_signature { - let flow = check_pending_signature(intent_client, sig).await?; - // Pending signature corresponds to succeeded transaction - break - if let ControlFlow::Break(()) = flow { - break Ok(Ok(*sig)); - } - - *state.pending_signature = None; - } - - *state.current_attempt += 1; - - // Prepare message - let prepared_transaction = prepare_transaction( - intent_client, - authority, - transaction_preparator, - state.transaction_strategy, - ) - .await - .map_err(map_preparation_err)?; - - // Record in outbox before sending - let signature = prepared_transaction.get_signature(); - outbox_client - .set_intent_execution_stage( - intent_id, - make_outbox_stage(*signature), - ) - .await - .map_err(Into::into)?; - - // Now record locally signature of tx we about to send - // Precaution for timeout in between - *state.pending_signature = Some(*signature); - - // Send signed tx - let execution_result = intent_client - .send_signed_tx_with_retries( - &prepared_transaction, - &state.transaction_strategy.optimized_tasks, - ) - .await; - - // Result returned, cleanup pending signature - *state.pending_signature = None; - - let execution_err = match execution_result { - Ok(()) => return Ok(Ok(*signature)), - Err(err) => err, - }; - let flow = patcher - .patch( - &execution_err, - state.transaction_strategy, - state.execution_report, - ) - .await?; - let cleanup = match flow { - ControlFlow::Continue(value) => value, - ControlFlow::Break(()) => return Ok(Err(execution_err)), - }; - intent_client.invalidate_cached_blockhash().await; - state.execution_report.dispose(cleanup); - - if *state.current_attempt >= STAGE_LOOP_CEILING { - error!("CRITICAL! Recursion ceiling reached"); - return Ok(Err(execution_err)); - } else { - state.execution_report.add_patched_error(execution_err); - } - } -} - -pub async fn prepare_transaction( - client: &IntentExecutionClient, - authority: &Keypair, - transaction_preparator: &T, - transaction_strategy: &mut TransactionStrategy, -) -> Result { - let mut prepared_message = transaction_preparator - .prepare_for_strategy(&authority, transaction_strategy) - .await?; - - // Get latest blockhash(Part of preparation I guess) - let blockhash = client - .get_latest_blockhash() - .await - .map_err(TransactionPreparatorError::GetLatestBlockhashError)?; - prepared_message.set_recent_blockhash(blockhash); - - // Create and sign transaction(Part of transaction preparation i guess, hence the errors) - let transaction = - VersionedTransaction::try_new(prepared_message, &[&authority]) - .map_err(TransactionPreparatorError::SignerError)?; - Ok(transaction) -} - -/// Checks a `pending_signature` loaded from the outbox against the full -/// transaction history. Returns `Break(sig)` if the tx succeeded — the caller -/// can short-circuit and skip re-execution. Returns `Continue` for every other -/// outcome (tx failed, never sent, or RPC returned no entry), letting the -/// normal execution loop proceed. -pub(in crate::intent_executor) async fn check_pending_signature( - client: &IntentExecutionClient, - sig: &Signature, -) -> IntentExecutorResult> { - let statuses = client - .get_signature_statuses_with_history(std::slice::from_ref(sig)) - .await - .map_err(IntentExecutorError::GetPendingSignatureStatusError)?; - - match statuses.get(0) { - Some(Some(Ok(()))) => Ok(ControlFlow::Break(())), - // TODO(edwin): well, that is bizarre one - None => { - warn!(pending_signature = ?sig, "RPC did not return status for signature"); - Ok(ControlFlow::Continue(())) - } - // Any case below means tx failed in execution we continue attempts - Some(None) => { - info!(pending_signature = ?sig, "Transaction corresponding to pending signature was never sent"); - Ok(ControlFlow::Continue(())) - } - Some(Some(Err(err))) => { - info!(pending_signature = ?sig, error = ?err, "Transaction corresponding to pending signature failed"); - Ok(ControlFlow::Continue(())) - } - } -} - -pub async fn prepare_and_execute_strategy( - client: &IntentExecutionClient, - authority: &Keypair, - transaction_preparator: &T, - transaction_strategy: &mut TransactionStrategy, -) -> Result< - Result, - TransactionPreparatorError, -> -where - T: TransactionPreparator, -{ - let prepared_message = transaction_preparator - .prepare_for_strategy(authority, transaction_strategy) - .await?; - - let execution_result = client - .execute_message_with_retries( - authority, - prepared_message, - &transaction_strategy.optimized_tasks, - ) - .await; - - Ok(execution_result) -} - -/// Handles out of sync commit id error, fixes current strategy -/// Returns strategy to be cleaned up -/// TODO(edwin): TransactionStrategy -> CleanupStrategy or something, naming is confusing for something that is cleaned up -pub(in crate::intent_executor) async fn handle_commit_id_error< - T: TaskInfoFetcher, ->( - authority: &Pubkey, - task_info_fetcher: &CacheTaskInfoFetcher, - committed_pubkeys: &[Pubkey], - strategy: &mut TransactionStrategy, -) -> Result { - let min_context_slot = strategy - .optimized_tasks - .iter() - .filter_map(|task| match task { - BaseTaskImpl::Commit(task) => { - Some(task.committed_account.remote_slot) - } - BaseTaskImpl::CommitFinalize(task) => { - Some(task.committed_account.remote_slot) - } - _ => None, - }) - .max() - .unwrap_or_default(); - - // We reset TaskInfoFetcher for all committed accounts - // We re-fetch them to fix out of sync tasks - task_info_fetcher.reset(ResetType::Specific(committed_pubkeys)); - let commit_ids = task_info_fetcher - .fetch_next_commit_nonces(committed_pubkeys, min_context_slot) - .await - .map_err(TaskBuilderError::CommitTasksBuildError)?; - - // Here we find the broken tasks and reset them - // Broken tasks are prepared incorrectly so they have to be cleaned up - let mut to_cleanup = Vec::new(); - for task in &mut strategy.optimized_tasks { - match task { - BaseTaskImpl::Commit(task) => { - let Some(commit_id) = - commit_ids.get(&task.committed_account.pubkey) - else { - continue; - }; - if commit_id == &task.commit_id { - continue; - } - - // Handle invalid tasks - to_cleanup.push(BaseTaskImpl::Commit(task.clone())); - task.reset_commit_id(*commit_id); - } - BaseTaskImpl::CommitFinalize(task) => { - let Some(commit_id) = - commit_ids.get(&task.committed_account.pubkey) - else { - continue; - }; - if commit_id == &task.commit_id { - continue; - } - - // Handle invalid tasks - to_cleanup.push(BaseTaskImpl::CommitFinalize(task.clone())); - task.reset_commit_id(*commit_id); - } - _ => {} - } - } - - let old_alts = strategy.dummy_revaluate_alts(authority); - Ok(TransactionStrategy { - optimized_tasks: to_cleanup, - lookup_tables_keys: old_alts, - standalone_action_nonce: None, - }) -} - -/// Handle CPI limit error, splits single strategy flow into 2 -/// Returns Commit stage strategy, Finalize stage strategy and strategy to clean up -pub(in crate::intent_executor) fn handle_cpi_limit_error( - authority: &Pubkey, - strategy: TransactionStrategy, -) -> ( - TransactionStrategy, - TransactionStrategy, - TransactionStrategy, -) { - // We encountered error "Max instruction trace length exceeded" - // All the tasks a prepared to be executed at this point - // We attempt Two stages commit flow, need to split tasks up - let last_commit_ind = strategy.optimized_tasks.iter().rposition(|el| { - matches!( - el, - BaseTaskImpl::Commit(_) | BaseTaskImpl::CommitFinalize(_) - ) - }); - let (mut commit_stage_tasks, mut finalize_stage_tasks) = (vec![], vec![]); - for (i, el) in strategy.optimized_tasks.into_iter().enumerate() { - if Some(i) <= last_commit_ind { - commit_stage_tasks.push(el); - } else { - finalize_stage_tasks.push(el); - } - } - - let commit_alt_pubkeys = if strategy.lookup_tables_keys.is_empty() { - vec![] - } else { - TaskStrategist::collect_lookup_table_keys( - authority, - &commit_stage_tasks, - ) - }; - let commit_strategy = TransactionStrategy { - optimized_tasks: commit_stage_tasks, - lookup_tables_keys: commit_alt_pubkeys, - standalone_action_nonce: None, - }; - - let finalize_alt_pubkeys = if strategy.lookup_tables_keys.is_empty() { - vec![] - } else { - TaskStrategist::collect_lookup_table_keys( - authority, - &finalize_stage_tasks, - ) - }; - let finalize_strategy = TransactionStrategy { - optimized_tasks: finalize_stage_tasks, - lookup_tables_keys: finalize_alt_pubkeys, - standalone_action_nonce: None, - }; - - // We clean up only ALTs - let to_cleanup = TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: strategy.lookup_tables_keys, - standalone_action_nonce: None, - }; - - (commit_strategy, finalize_strategy, to_cleanup) -} - -/// Handles undelegation error, stripping away actions -/// Returns [`TransactionStrategy`] to be cleaned up -pub(in crate::intent_executor) fn handle_undelegation_error( - authority: &Pubkey, - strategy: &mut TransactionStrategy, -) -> TransactionStrategy { - let position = strategy - .optimized_tasks - .iter() - .position(|el| matches!(el, BaseTaskImpl::Undelegate(_))); - - if let Some(position) = position { - // Remove everything after undelegation including post undelegation actions - let removed_task = strategy.optimized_tasks.drain(position..).collect(); - let old_alts = strategy.dummy_revaluate_alts(authority); - TransactionStrategy { - optimized_tasks: removed_task, - lookup_tables_keys: old_alts, - standalone_action_nonce: None, - } - } else { - TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - } - } -} - -pub(in crate::intent_executor) fn handle_actions_result( - authority: &Pubkey, - callback_scheduler: &A, - execution_report: &mut IntentExecutionReport, - transaction_strategy: &mut TransactionStrategy, - signature: Option, - result: ActionResult, -) -> TransactionStrategy -where - A: ActionsCallbackScheduler, -{ - let (callbacks, junk) = if result.is_ok() { - let callbacks = transaction_strategy.extract_action_callbacks(); - (callbacks, TransactionStrategy::default()) - } else { - let mut removed_actions = - transaction_strategy.remove_actions(authority); - let callbacks = removed_actions.extract_action_callbacks(); - (callbacks, removed_actions) - }; - if !callbacks.is_empty() { - let result = callback_scheduler.schedule(callbacks, signature, result); - execution_report.add_callback_report(result); - } - - junk -} - -// TODO(edwin): docs -pub(in crate::intent_executor) async fn execute_with_timeout( - time_left: Option, - mut executor: impl StageExecutor, -) -> IntentExecutorResult { - if executor.has_callbacks() { - if let Some(time_left) = time_left { - match timeout(time_left, executor.execute()).await { - Ok(res) => return res, - Err(_) => { - // The race between callback and intent txn is handled - // on the user smart contract side via TimeoutError. - // We must respect the timeout contract. - info!("Intent execution timed out, cleaning up actions"); - executor.execute_callbacks( - None, - Err(ActionError::TimeoutError), - ); - } - } - } else { - // Already timed out; see comment above. - executor.execute_callbacks(None, Err(ActionError::TimeoutError)); - } - } - - // Timeout concerns only callbacks - // We still need to drive intent to completion - executor.execute().await -} - -#[async_trait] -pub(in crate::intent_executor) trait StageExecutor { - fn has_callbacks(&self) -> bool; - async fn execute(&mut self) -> IntentExecutorResult; - fn execute_callbacks( - &mut self, - signature: Option, - result: ActionResult, - ); -} - -pub(in crate::intent_executor) struct SingleStage<'a, 'e, A, T, F, O> { - pub(in crate::intent_executor) inner: - &'a mut SingleStageExecutor<'e, F, A, O>, - pub(in crate::intent_executor) transaction_preparator: &'a T, - pub(in crate::intent_executor) committed_pubkeys: &'a [Pubkey], -} - -#[async_trait] -impl<'a, 'e, A, T, F, O> StageExecutor for SingleStage<'a, 'e, A, T, F, O> -where - A: ActionsCallbackScheduler, - T: TransactionPreparator, - F: TaskInfoFetcher, - O: OutboxClient, - O::Error: Into, -{ - fn has_callbacks(&self) -> bool { - self.inner.has_callbacks() - } - - async fn execute(&mut self) -> IntentExecutorResult { - self.inner - .execute(self.committed_pubkeys, self.transaction_preparator) - .await - } - - fn execute_callbacks( - &mut self, - signature: Option, - result: ActionResult, - ) { - self.inner.execute_callbacks(signature, result) - } -} - -pub(in crate::intent_executor) struct CommitStage<'a, 'e, A, T, F, O> { - pub(in crate::intent_executor) inner: - &'a mut TwoStageExecutor<'e, A, O, Initialized>, - pub(in crate::intent_executor) transaction_preparator: &'a T, - pub(in crate::intent_executor) task_info_fetcher: - &'a CacheTaskInfoFetcher, - pub(in crate::intent_executor) committed_pubkeys: &'a [Pubkey], -} - -#[async_trait] -impl<'a, 'e, A, T, F, O> StageExecutor for CommitStage<'a, 'e, A, T, F, O> -where - A: ActionsCallbackScheduler, - T: TransactionPreparator, - F: TaskInfoFetcher, - O: OutboxClient, - O::Error: Into, -{ - fn has_callbacks(&self) -> bool { - self.inner.has_callbacks() - } - - async fn execute(&mut self) -> IntentExecutorResult { - self.inner - .commit( - self.committed_pubkeys, - self.transaction_preparator, - self.task_info_fetcher, - ) - .await - } - - fn execute_callbacks( - &mut self, - signature: Option, - result: ActionResult, - ) { - self.inner.execute_callbacks(signature, result) - } -} - -pub(in crate::intent_executor) struct FinalizeStage<'a, 'e, A, T, O> { - pub(in crate::intent_executor) inner: - &'a mut TwoStageExecutor<'e, A, O, Committed>, - pub(in crate::intent_executor) transaction_preparator: &'a T, -} - -#[async_trait] -impl<'a, 'e, A, T, O> StageExecutor for FinalizeStage<'a, 'e, A, T, O> -where - A: ActionsCallbackScheduler, - T: TransactionPreparator, - O: OutboxClient, - O::Error: Into, -{ - fn has_callbacks(&self) -> bool { - self.inner.has_callbacks() - } - - async fn execute(&mut self) -> IntentExecutorResult { - self.inner.finalize(self.transaction_preparator).await - } - - fn execute_callbacks( - &mut self, - signature: Option, - result: ActionResult, - ) { - self.inner.execute_callbacks(signature, result) - } -} diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 66f896211..25807cc93 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -9,11 +9,11 @@ use std::{ use async_trait::async_trait; use magicblock_committor_service::{ intent_executor::{ + accepted_intent_executor::AcceptedIntentExecutor, task_info_fetcher::{ CacheTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, TaskInfoFetcherResult, }, - IntentExecutorImpl, }, tasks::commit_task::{CommitDelivery, CommitTask}, transaction_preparator::{ @@ -108,14 +108,14 @@ impl TestFixture { #[allow(dead_code)] pub fn create_intent_executor( &self, - ) -> IntentExecutorImpl< + ) -> AcceptedIntentExecutor< TransactionPreparatorImpl, MockTaskInfoFetcher, MockActionsCallbackExecutor, > { let transaction_preparator = self.create_transaction_preparator(); - IntentExecutorImpl::new( + AcceptedIntentExecutor::new( self.rpc_client.clone(), transaction_preparator, self.create_task_info_fetcher(), diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 72cfbcc9f..09fb47592 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -14,16 +14,19 @@ use futures::future::{join_all, try_join_all}; use magicblock_committor_program::pdas; use magicblock_committor_service::{ intent_executor::{ + accepted_intent_executor::AcceptedIntentExecutor, error::{IntentExecutorError, TransactionStrategyExecutionError}, intent_execution_client::IntentExecutionClient, + strategy_executor::{ + two_stage::{Initialized, TwoStageStrategyExecutor}, + utils::prepare_and_execute_strategy, + }, task_info_fetcher::{ CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, }, - two_stage_executor::{Initialized, TwoStageExecutor}, - utils::prepare_and_execute_strategy, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, - IntentExecutor, IntentExecutorImpl, + IntentExecutor, }, persist::IntentPersisterImpl, tasks::{ @@ -86,7 +89,7 @@ const ACTOR_ESCROW_INDEX: u8 = 1; struct TestEnv { fixture: TestFixture, task_info_fetcher: Arc>, - intent_executor: IntentExecutorImpl< + intent_executor: AcceptedIntentExecutor< TransactionPreparatorImpl, RpcTaskInfoFetcher, MockActionsCallbackExecutor, @@ -115,7 +118,7 @@ impl TestEnv { } let callback_executor = MockActionsCallbackExecutor::default(); - let intent_executor = IntentExecutorImpl::new( + let intent_executor = AcceptedIntentExecutor::new( fixture.rpc_client.clone(), transaction_preparator, task_info_fetcher.clone(), @@ -1278,7 +1281,7 @@ async fn test_action_callback_fired_on_timeout() { let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( RpcTaskInfoFetcher::new(fixture.rpc_client.clone()), )); - let mut intent_executor = IntentExecutorImpl::new( + let mut intent_executor = AcceptedIntentExecutor::new( fixture.rpc_client.clone(), fixture.create_transaction_preparator(), task_info_fetcher, @@ -1413,7 +1416,7 @@ async fn test_callbacks_fired_in_two_stage() { assert!(calls[1].1.is_ok()); } -/// Builds a [`TwoStageExecutor`] directly from an intent by constructing the +/// Builds a [`TwoStageStrategyExecutor`] directly from an intent by constructing the /// commit and finalize strategies independently, without going through /// `execute_inner` or any CPI-limit recovery path. async fn create_two_stage_executor<'a>( @@ -1422,7 +1425,7 @@ async fn create_two_stage_executor<'a>( intent: &ScheduledIntentBundle, task_info_fetcher: &Arc>, execution_report: &'a mut IntentExecutionReport, -) -> TwoStageExecutor<'a, MockActionsCallbackExecutor, Initialized> { +) -> TwoStageStrategyExecutor<'a, MockActionsCallbackExecutor, Initialized> { let authority = &fixture.authority.pubkey(); let commit_tasks = TaskBuilderImpl::commit_tasks( task_info_fetcher, @@ -1435,17 +1438,11 @@ async fn create_two_stage_executor<'a>( TaskBuilderImpl::finalize_tasks(task_info_fetcher, intent) .await .unwrap(); - let commit_strategy = TaskStrategist::build_strategy( - commit_tasks, - authority, - ) - .unwrap(); - let finalize_strategy = TaskStrategist::build_strategy( - finalize_tasks, - authority, - ) - .unwrap(); - TwoStageExecutor::new( + let commit_strategy = + TaskStrategist::build_strategy(commit_tasks, authority).unwrap(); + let finalize_strategy = + TaskStrategist::build_strategy(finalize_tasks, authority).unwrap(); + TwoStageStrategyExecutor::new( fixture.authority.insecure_clone(), commit_strategy, finalize_strategy, @@ -1691,23 +1688,16 @@ async fn single_flow_transaction_strategy( task_info_fetcher: &Arc>, intent: &ScheduledIntentBundle, ) -> TransactionStrategy { - let mut tasks = TaskBuilderImpl::commit_tasks( - task_info_fetcher, - intent, - ) - .await - .unwrap(); + let mut tasks = TaskBuilderImpl::commit_tasks(task_info_fetcher, intent) + .await + .unwrap(); let finalize_tasks = TaskBuilderImpl::finalize_tasks(task_info_fetcher, intent) .await .unwrap(); tasks.extend(finalize_tasks); - TaskStrategist::build_strategy( - tasks, - authority, - ) - .unwrap() + TaskStrategist::build_strategy(tasks, authority).unwrap() } async fn verify_committed_accounts_state( From 4dfd1d4f562798849e26d28797714cd49bc6b65b Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 15 Jun 2026 11:31:04 +0700 Subject: [PATCH 44/97] wip --- .../accepted_intent_executor.rs | 68 ++------ .../single_stage_intent_executor.rs | 79 +++++++++- .../two_stage_intent_executor.rs | 145 ++++++++++++++++-- .../src/intent_executor/utils.rs | 96 ++++++++++++ .../src/tasks/task_strategist.rs | 39 +++-- 5 files changed, 343 insertions(+), 84 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index dbb336101..0ae838373 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -22,13 +22,13 @@ use crate::{ intent_execution_client::IntentExecutionClient, strategy_executor::{ single_stage::SingleStageStrategyExecutor, - two_stage::{Initialized, TwoStageStrategyExecutor}, + two_stage::Initialized, utils::{ - execute_with_timeout, handle_cpi_limit_error, CommitStage, - FinalizeStage, SingleStage, + execute_with_timeout, handle_cpi_limit_error, SingleStage, }, }, task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, + utils::{build_commit_finalize_tasks, execute_two_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, @@ -37,6 +37,7 @@ use crate::{ task_builder::TasksBuilder, task_strategist::{ StrategyExecutionMode, TaskStrategist, TransactionStrategy, + TwoStageExecutionMode, }, TaskBuilderImpl, }, @@ -113,20 +114,11 @@ where }; // Build tasks for commit & finalize stages - let (commit_tasks, finalize_tasks) = { - let commit_tasks_fut = TaskBuilderImpl::commit_tasks( - &self.ctx.task_info_fetcher, - &intent_bundle, - ); - let finalize_tasks_fut = TaskBuilderImpl::finalize_tasks( - &self.ctx.task_info_fetcher, - &intent_bundle, - ); - let (commit_tasks, finalize_tasks) = - join(commit_tasks_fut, finalize_tasks_fut).await; - - (commit_tasks?, finalize_tasks?) - }; + let (commit_tasks, finalize_tasks) = build_commit_finalize_tasks( + &intent_bundle, + &self.ctx.task_info_fetcher, + ) + .await?; // Build execution strategy match TaskStrategist::build_execution_strategy( @@ -143,10 +135,10 @@ where ) .await } - StrategyExecutionMode::TwoStage { + StrategyExecutionMode::TwoStage(TwoStageExecutionMode { commit_stage, finalize_stage, - } => { + }) => { trace!("Two stage execution"); self.two_stage_execution_flow( intent_bundle.id, @@ -249,42 +241,16 @@ where execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { let state = Initialized::new(commit_strategy, finalize_strategy, None); - let mut executor = TwoStageStrategyExecutor::new( + execute_two_stage_flow( + &self.ctx, state, - self.authority.insecure_clone(), + &self.authority, intent_id, - self.ctx.intent_client.clone(), - self.ctx.outbox_client.clone(), - self.ctx.actions_callback_executor.clone(), + committed_pubkeys, execution_report, - ); - - let commit_signature = execute_with_timeout( - self.time_left(), - CommitStage { - inner: &mut executor, - transaction_preparator: &self.ctx.transaction_preparator, - task_info_fetcher: &self.ctx.task_info_fetcher, - committed_pubkeys, - }, + || self.time_left(), ) - .await?; - - let mut finalize_executor = executor.done(commit_signature); - let finalize_signature = execute_with_timeout( - self.time_left(), - FinalizeStage { - inner: &mut finalize_executor, - transaction_preparator: &self.ctx.transaction_preparator, - }, - ) - .await?; - - let finalized_stage = finalize_executor.done(finalize_signature); - Ok(ExecutionOutput::TwoStage { - commit_signature: finalized_stage.commit_signature, - finalize_signature: finalized_stage.finalize_signature, - }) + .await } } diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 5c0b7c6b6..857fe7854 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -18,13 +18,18 @@ use crate::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, cleanup_handle::CleanupHandle, - error::IntentExecutorError, + error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - IntentExecutionResult, IntentExecutor, IntentExecutorCtx, + task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, + utils::build_commit_finalize_tasks, + ExecutionOutput, IntentExecutionReport, IntentExecutionResult, + IntentExecutor, IntentExecutorCtx, }, outbox_client::OutboxClient, - tasks::task_strategist::TransactionStrategy, + tasks::{ + task_strategist::{TaskStrategist, TransactionStrategy}, + TaskBuilderImpl, + }, transaction_preparator::TransactionPreparator, }; @@ -68,6 +73,29 @@ where close_buffers: true, } } + + pub async fn execute_inner( + &mut self, + intent_bundle: ScheduledIntentBundle, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + // It we're here so previous run determined this should + let (commit_tasks, finalize_tasks) = build_commit_finalize_tasks( + &intent_bundle, + &self.ctx.task_info_fetcher, + ) + .await?; + + let single_stage_tasks = [commit_tasks, finalize_tasks].concat(); + let transaction_strategy = TaskStrategist::build_strategy( + single_stage_tasks, + &self.authority.pubkey(), + )?; + + + + todo!() + } } #[async_trait] @@ -83,6 +111,47 @@ where mut self: Box, base_intent: ScheduledIntentBundle, ) -> (IntentExecutionResult, CleanupHandle) { - todo!() + self.started_at = Instant::now(); + let is_undelegate = base_intent.has_undelegate_intent(); + let pubkeys = base_intent.get_all_committed_pubkeys(); + + let mut execution_report = IntentExecutionReport::default(); + let result = + self.execute_inner(base_intent, &mut execution_report).await; + if !pubkeys.is_empty() { + // Reset TaskInfoFetcher, as cache could become invalid + // NOTE: if undelegation was removed - we still reset + // We assume its safe since all consecutive commits will fail + if result.is_err() || is_undelegate { + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&pubkeys)); + } + } + + // Gather metrics in separate task + let intent_client = self.ctx.intent_client.clone(); + let result = result.inspect(|output| { + let output_copy = *output; + tokio::spawn(async move { + intent_client.intent_metrics(output_copy).await + }); + }); + + self.close_buffers = result.is_ok(); + self.junk = execution_report.junk; + let result = IntentExecutionResult { + inner: result, + patched_errors: execution_report.patched_errors, + callbacks_report: execution_report.callbacks_report, + }; + let cleanup_handle = CleanupHandle::new( + self.authority, + self.junk, + self.close_buffers, + self.ctx.transaction_preparator, + ); + + (result, cleanup_handle) } } diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 13e9ea8a9..228e446f6 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -1,4 +1,4 @@ -use std::time::Instant; +use std::time::{Duration, Instant}; use async_trait::async_trait; use magicblock_core::traits::ActionsCallbackScheduler; @@ -8,20 +8,30 @@ use magicblock_program::{ }; use solana_keypair::Keypair; use solana_signature::Signature; +use solana_signer::Signer; use crate::{ intent_executor::{ cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, - task_info_fetcher::TaskInfoFetcher, + strategy_executor::{ + two_stage::{Committed, Initialized, TwoStageStrategyExecutor}, + utils::{execute_with_timeout, FinalizeStage}, + }, + task_info_fetcher::{ResetType, TaskInfoFetcher}, + utils::{build_commit_finalize_tasks, execute_two_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, outbox_client::OutboxClient, - tasks::task_strategist::TransactionStrategy, + tasks::{ + task_builder::{TaskBuilderImpl, TasksBuilder}, + task_strategist::{ + TaskStrategist, TransactionStrategy, TwoStageExecutionMode, + }, + }, transaction_preparator::TransactionPreparator, }; -use crate::intent_executor::task_info_fetcher::ResetType; pub struct TwoStageIntentExecutor { authority: Keypair, @@ -58,19 +68,59 @@ where stage, ctx, + // TODO(edwin): deduce started_at properly started_at: Instant::now(), junk: vec![], close_buffers: true, } } + fn time_left(&self) -> Option { + self.ctx + .actions_timeout + .checked_sub(self.started_at.elapsed()) + } + async fn execute_committing_intent( &mut self, - intent: ScheduledIntentBundle, + intent_bundle: ScheduledIntentBundle, pending_signature: Signature, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - todo!() + // This stage was chosen prior so we build tasks for it + // Build tasks for commit & finalize stages + let (commit_tasks, finalize_tasks) = build_commit_finalize_tasks( + &intent_bundle, + &self.ctx.task_info_fetcher, + ) + .await?; + + // As strategy was chosen build two stage + let TwoStageExecutionMode { + commit_stage, + finalize_stage, + } = TaskStrategist::build_two_stage( + commit_tasks, + finalize_tasks, + &self.authority.pubkey(), + )?; + + let committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); + let state = Initialized::new( + commit_stage, + finalize_stage, + Some(pending_signature), + ); + execute_two_stage_flow( + &self.ctx, + state, + &self.authority, + intent_bundle.id, + &committed_pubkeys, + execution_report, + || self.time_left(), + ) + .await } async fn execute_finalizing_intent( @@ -80,8 +130,50 @@ where pending_finalize_signature: Signature, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { + // Commit succeeded so we skip those tasks all together + let finalize_tasks = TaskBuilderImpl::finalize_tasks( + &self.ctx.task_info_fetcher, + &intent, + ) + .await?; - todo!() + // Build strategy for finalize tasks + let finalize_strategy = TaskStrategist::build_strategy( + finalize_tasks, + &self.authority.pubkey(), + )?; + + let committed_state = Committed::new( + commit_signature, + finalize_strategy, + Some(pending_finalize_signature), + ); + let mut finalize_strategy_executor = + TwoStageStrategyExecutor::committed( + committed_state, + self.authority.insecure_clone(), + intent.id, + self.ctx.intent_client.clone(), + self.ctx.outbox_client.clone(), + self.ctx.actions_callback_executor.clone(), + execution_report, + ); + + let finalize_signature = execute_with_timeout( + self.time_left(), + FinalizeStage { + inner: &mut finalize_strategy_executor, + transaction_preparator: &self.ctx.transaction_preparator, + }, + ) + .await?; + + let finalized_stage = + finalize_strategy_executor.done(finalize_signature); + Ok(ExecutionOutput::TwoStage { + commit_signature: finalized_stage.commit_signature, + finalize_signature: finalized_stage.finalize_signature, + }) } } @@ -98,6 +190,8 @@ where mut self: Box, intent: ScheduledIntentBundle, ) -> (IntentExecutionResult, CleanupHandle) { + // TODO(edwin): see if can be extracted into single utils + // Duplicates AcceptedIntentExecutor::execute let is_undelegate = intent.has_undelegate_intent(); // TODO(edwin): should validate non emptiness of pubkeys? Shouldn't be possible tho let pubkeys = intent.get_all_committed_pubkeys(); @@ -105,23 +199,44 @@ where let mut execution_report = IntentExecutionReport::default(); let result = match &self.stage { TwoStageProgress::Committing(signature) => { - self.execute_committing_intent(intent, *signature, &mut execution_report).await + self.execute_committing_intent( + intent, + *signature, + &mut execution_report, + ) + .await } - TwoStageProgress::Finalizing { - commit, - finalize - } => { - self.execute_finalizing_intent(intent, *commit, *finalize, &mut execution_report).await + TwoStageProgress::Finalizing { commit, finalize } => { + self.execute_finalizing_intent( + intent, + *commit, + *finalize, + &mut execution_report, + ) + .await } }; - if is_undelegate { self.ctx .task_info_fetcher .reset(ResetType::Specific(&pubkeys)); } - todo!() + self.close_buffers = result.is_ok(); + self.junk = execution_report.junk; + let result = IntentExecutionResult { + inner: result, + patched_errors: execution_report.patched_errors, + callbacks_report: execution_report.callbacks_report, + }; + let cleanup_handle = CleanupHandle::new( + self.authority, + self.junk, + self.close_buffers, + self.ctx.transaction_preparator, + ); + + (result, cleanup_handle) } } diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 8b1378917..2910acfbe 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -1 +1,97 @@ +use std::{sync::Arc, time::Duration}; +use futures_util::future::join; +use magicblock_core::traits::ActionsCallbackScheduler; +use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; + +use crate::{ + intent_executor::{ + error::{IntentExecutorError, IntentExecutorResult}, + strategy_executor::{ + two_stage, + two_stage::TwoStageStrategyExecutor, + utils::{execute_with_timeout, CommitStage, FinalizeStage}, + }, + task_info_fetcher::TaskInfoFetcher, + ExecutionOutput, IntentExecutionReport, IntentExecutorCtx, + }, + outbox_client::OutboxClient, + tasks::{ + task_builder::{TaskBuilderImpl, TasksBuilder}, + task_strategist::TransactionStrategy, + BaseTaskImpl, + }, + transaction_preparator::TransactionPreparator, +}; + +pub(in crate::intent_executor) async fn build_commit_finalize_tasks< + F: TaskInfoFetcher, +>( + intent_bundle: &ScheduledIntentBundle, + task_info_fetcher: &Arc, +) -> IntentExecutorResult<(Vec, Vec)> { + let commit_tasks_fut = + TaskBuilderImpl::commit_tasks(&task_info_fetcher, &intent_bundle); + let finalize_tasks_fut = + TaskBuilderImpl::finalize_tasks(&task_info_fetcher, &intent_bundle); + let (commit_tasks, finalize_tasks) = + join(commit_tasks_fut, finalize_tasks_fut).await; + + Ok((commit_tasks?, finalize_tasks?)) +} + +pub(in crate::intent_executor) async fn execute_two_stage_flow( + ctx: &IntentExecutorCtx, + state: two_stage::Initialized, + authority: &Keypair, + intent_id: u64, + committed_pubkeys: &[Pubkey], + execution_report: &mut IntentExecutionReport, + time_left: impl Fn() -> Option, +) -> IntentExecutorResult +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + let mut executor = TwoStageStrategyExecutor::new( + state, + authority.insecure_clone(), + intent_id, + ctx.intent_client.clone(), + ctx.outbox_client.clone(), + ctx.actions_callback_executor.clone(), + execution_report, + ); + + let commit_signature = execute_with_timeout( + time_left(), + CommitStage { + inner: &mut executor, + transaction_preparator: &ctx.transaction_preparator, + task_info_fetcher: &ctx.task_info_fetcher, + committed_pubkeys, + }, + ) + .await?; + + let mut finalize_executor = executor.done(commit_signature); + let finalize_signature = execute_with_timeout( + time_left(), + FinalizeStage { + inner: &mut finalize_executor, + transaction_preparator: &ctx.transaction_preparator, + }, + ) + .await?; + + let finalized_stage = finalize_executor.done(finalize_signature); + Ok(ExecutionOutput::TwoStage { + commit_signature: finalized_stage.commit_signature, + finalize_signature: finalized_stage.finalize_signature, + }) +} diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 42b9beb4b..d549426dc 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -93,24 +93,35 @@ impl TransactionStrategy { pub enum StrategyExecutionMode { SingleStage(TransactionStrategy), - TwoStage { - commit_stage: TransactionStrategy, - finalize_stage: TransactionStrategy, - }, + TwoStage(TwoStageExecutionMode), } impl StrategyExecutionMode { pub fn uses_alts(&self) -> bool { match self { Self::SingleStage(value) => value.uses_alts(), - Self::TwoStage { - commit_stage, - finalize_stage, - } => commit_stage.uses_alts() || finalize_stage.uses_alts(), + Self::TwoStage(value) => value.uses_alts(), } } } +pub struct TwoStageExecutionMode { + commit_stage: TransactionStrategy, + finalize_stage: TransactionStrategy, +} + +impl TwoStageExecutionMode { + pub fn uses_alts(&self) -> bool { + self.commit_stage.uses_alts() || self.finalize_stage.uses_alts() + } +} + +impl From for StrategyExecutionMode { + fn from(value: TwoStageExecutionMode) -> Self { + Self::TwoStage(value) + } +} + /// Takes [`BaseTask`]s and chooses the best way to fit them in TX /// It may change Task execution strategy so all task would fit in tx pub struct TaskStrategist; @@ -136,7 +147,8 @@ impl TaskStrategist { commit_tasks, finalize_tasks, authority, - ); + ) + .map(Into::into); } // Clone tasks since strategies applied to united case maybe suboptimal for regular one @@ -153,7 +165,8 @@ impl TaskStrategist { commit_tasks, finalize_tasks, authority, - ); + ) + .map(Into::into); } Err(TaskStrategistError::SignerError(err)) => { return Err(err.into()) @@ -173,7 +186,7 @@ impl TaskStrategist { if two_stage.uses_alts() { Ok(single_stage_strategy) } else { - Ok(two_stage) + Ok(two_stage.into()) } } @@ -181,7 +194,7 @@ impl TaskStrategist { commit_tasks: Vec, finalize_tasks: Vec, authority: &Pubkey, - ) -> TaskStrategistResult { + ) -> TaskStrategistResult { // Build strategy for Commit stage let commit_strategy = TaskStrategist::build_strategy(commit_tasks, authority)?; @@ -190,7 +203,7 @@ impl TaskStrategist { let finalize_strategy = TaskStrategist::build_strategy(finalize_tasks, authority)?; - Ok(StrategyExecutionMode::TwoStage { + Ok(TwoStageExecutionMode { commit_stage: commit_strategy, finalize_stage: finalize_strategy, }) From 8f7a4f2f5a8487391494a1fb38f000ea466b31ae Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 15 Jun 2026 12:32:02 +0700 Subject: [PATCH 45/97] wip single stage utils --- .../accepted_intent_executor.rs | 2 +- .../src/intent_executor/error.rs | 21 +- .../intent_execution_client.rs | 28 +- .../intent_executor/single_stage_executor.rs | 305 ---------- .../single_stage_intent_executor.rs | 2 - .../src/intent_executor/two_stage_executor.rs | 527 ------------------ .../two_stage_intent_executor.rs | 2 + .../src/intent_executor/utils.rs | 104 +++- magicblock-rpc-client/src/lib.rs | 21 + 9 files changed, 142 insertions(+), 870 deletions(-) delete mode 100644 magicblock-committor-service/src/intent_executor/single_stage_executor.rs delete mode 100644 magicblock-committor-service/src/intent_executor/two_stage_executor.rs diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 0ae838373..31f86efe4 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -198,7 +198,7 @@ where commit_signature: _, finalize_signature: _, }) if !committed_pubkeys.is_empty() - && err.is_single_stage_split_limit_error() => + && err.is_recoverable_by_two_stage() => { err } diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index a9576ae93..0c3b03e4b 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -45,25 +45,10 @@ impl InternalError { } pub fn is_transaction_too_large(&self) -> bool { - fn is_send_transaction_too_large(err: &RpcClientError) -> bool { - matches!(err.request, Some(RpcRequest::SendTransaction)) - && matches!( - &*err.kind, - RpcClientErrorKind::RpcError( - RpcError::RpcResponseError { code: -32602, message, .. } - ) if message.contains("VersionedTransaction too large") - || message.contains("base64 encoded too large") - ) - } - match self { - Self::MagicBlockRpcClientError(err) => match err.as_ref() { - MagicBlockRpcClientError::RpcClientError(err) - | MagicBlockRpcClientError::SendTransaction(err) => { - is_send_transaction_too_large(err) - } - _ => false, - }, + Self::MagicBlockRpcClientError(err) => { + err.is_transaction_too_large() + } Self::SignerError(_) => false, } } diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 27beca82f..fae53cadb 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -255,16 +255,17 @@ where type ExecutionError = TransactionStrategyExecutionError; fn map(&self, error: InternalError) -> Self::ExecutionError { if error.is_transaction_too_large() { - return TransactionStrategyExecutionError::TransactionTooLargeError(error); - } - match error { - InternalError::MagicBlockRpcClientError(err) => { - map_magicblock_client_error( - &self.transaction_error_mapper, - *err, - ) + TransactionStrategyExecutionError::TransactionTooLargeError(error) + } else { + match error { + InternalError::MagicBlockRpcClientError(err) => { + map_magicblock_client_error( + &self.transaction_error_mapper, + *err, + ) + } + err => TransactionStrategyExecutionError::InternalError(err), } - err => TransactionStrategyExecutionError::InternalError(err), } } @@ -287,10 +288,13 @@ where { type ExecutionError = TransactionStrategyExecutionError; fn map(&self, error: MagicBlockRpcClientError) -> Self::ExecutionError { - if error.is_transaction_too_large() { - return TransactionStrategyExecutionError::TransactionTooLargeError(error); + if error.is_transaction_too_large() { + TransactionStrategyExecutionError::TransactionTooLargeError( + error.into(), + ) + } else { + map_magicblock_client_error(&self.transaction_error_mapper, error) } - map_magicblock_client_error(&self.transaction_error_mapper, error) } fn decide_flow(err: &Self::ExecutionError) -> ControlFlow<(), Duration> { diff --git a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_executor.rs deleted file mode 100644 index 3b9dddfce..000000000 --- a/magicblock-committor-service/src/intent_executor/single_stage_executor.rs +++ /dev/null @@ -1,305 +0,0 @@ -use std::{ops::ControlFlow, sync::Arc}; - -use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; -use solana_keypair::Keypair; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_signer::Signer; -use tracing::{error, instrument}; - -use crate::{ - intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, - }, - intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - utils::{ - handle_actions_result, handle_commit_id_error, - handle_undelegation_error, prepare_and_execute_strategy, - }, - IntentExecutionReport, - }, - persist::{IntentPersister, IntentPersisterImpl}, - tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, - transaction_preparator::TransactionPreparator, -}; - -pub struct SingleStageExecutor<'a, F, A> { - current_attempt: u8, - execution_report: &'a mut IntentExecutionReport, - - authority: Keypair, - intent_client: IntentExecutionClient, - task_info_fetcher: Arc>, - callback_scheduler: A, - transaction_strategy: TransactionStrategy, -} - -impl<'a, F, A> SingleStageExecutor<'a, F, A> -where - F: TaskInfoFetcher, - A: ActionsCallbackScheduler, -{ - pub fn new( - authority: Keypair, - intent_client: IntentExecutionClient, - task_info_fetcher: Arc>, - transaction_strategy: TransactionStrategy, - callback_scheduler: A, - execution_report: &'a mut IntentExecutionReport, - ) -> Self { - Self { - current_attempt: 0, - authority, - intent_client, - task_info_fetcher, - transaction_strategy, - callback_scheduler, - execution_report, - } - } - - #[instrument( - skip(self, committed_pubkeys, transaction_preparator, persister), - fields(stage = "single_stage") - )] - pub async fn execute( - &mut self, - committed_pubkeys: &[Pubkey], - transaction_preparator: &T, - persister: &Option

, - ) -> IntentExecutorResult - where - T: TransactionPreparator, - P: IntentPersister, - { - const RECURSION_CEILING: u8 = 10; - - let result = loop { - self.current_attempt += 1; - - // Prepare & execute message - let execution_result = prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.transaction_strategy, - persister, - ) - .await - .map_err(IntentExecutorError::FailedFinalizePreparationError)?; - - // Process error: Ok - return, Err - handle further - let execution_err = match execution_result { - // break with result, strategy that was executed at this point has to be returned for cleanup - Ok(value) => { - break Ok(value); - } - Err(err) => err, - }; - - // Attempt patching - let flow = self - .patch_strategy( - &execution_err, - committed_pubkeys, - transaction_preparator, - ) - .await?; - let cleanup = match flow { - ControlFlow::Continue(cleanup) => cleanup, - ControlFlow::Break(()) => { - break Err(execution_err); - } - }; - self.intent_client.invalidate_cached_blockhash().await; - self.execution_report.dispose(cleanup); - - if self.current_attempt >= RECURSION_CEILING { - error!( - attempt = self.current_attempt, - ceiling = RECURSION_CEILING, - error = ?execution_err, - "Recursion ceiling exceeded" - ); - break Err(execution_err); - } else { - self.execution_report.add_patched_error(execution_err); - } - }; - - result.map_err(|err| { - IntentExecutorError::from_finalize_execution_error( - err, - // TODO(edwin): shall one stage have same signature for commit & finalize - None, - ) - }) - } - - pub fn has_callbacks(&self) -> bool { - self.transaction_strategy.has_actions_callbacks() - } - - pub fn execute_callbacks( - &mut self, - signature: Option, - result: Result<(), impl Into>, - ) { - let junk_strategy = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.transaction_strategy, - signature, - result.map_err(|err| err.into()), - ); - self.execution_report.dispose(junk_strategy); - } - - pub fn consume_strategy(self) -> TransactionStrategy { - self.transaction_strategy - } - - /// Patch the current `transaction_strategy` in response to a recoverable - /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data - /// to be applied after a retry. - /// - /// [`TransactionStrategyExecutionError`], returning either: - /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or - /// - `Break(())` when this stage cannot be recovered here. - pub async fn patch_strategy( - &mut self, - err: &TransactionStrategyExecutionError, - committed_pubkeys: &[Pubkey], - transaction_preparator: &T, - ) -> IntentExecutorResult> { - if committed_pubkeys.is_empty() { - // No patching is applicable if intent doesn't commit accounts - return Ok(ControlFlow::Break(())); - } - - match err { - TransactionStrategyExecutionError::ActionsError(err, signature) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); - let to_cleanup = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.transaction_strategy, - *signature, - action_error, - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::CommitIDError(_, _) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let to_cleanup = handle_commit_id_error( - &self.authority.pubkey(), - &self.task_info_fetcher, - committed_pubkeys, - &mut self.transaction_strategy, - ) - .await?; - Ok(ControlFlow::Continue(to_cleanup)) - } - err - @ TransactionStrategyExecutionError::UnfinalizedAccountError( - _, - signature, - ) => { - let optimized_tasks = - self.transaction_strategy.optimized_tasks.as_slice(); - if let Some(delegated_account) = err - .task_index() - .and_then(|index| optimized_tasks.get(index as usize)) - .and_then(|task| match task { - BaseTaskImpl::Commit(task) => { - Some(task.committed_account.pubkey) - } - BaseTaskImpl::CommitFinalize(task) => { - Some(task.committed_account.pubkey) - } - _ => None, - }) - { - self.handle_unfinalized_account_error( - signature, - delegated_account, - transaction_preparator, - ) - .await - } else { - error!( - task_index = err.task_index(), - optimized_tasks_len = optimized_tasks.len(), - error = ?err, - "RPC returned unexpected task index" - ); - Ok(ControlFlow::Break(())) - } - } - TransactionStrategyExecutionError::UndelegationError(_, _) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let to_cleanup = handle_undelegation_error( - &self.authority.pubkey(), - &mut self.transaction_strategy - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::CpiLimitError(_, _) - | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded(_, _) - | TransactionStrategyExecutionError::TransactionTooLargeError(_) => { - // Can't be handled in scope of single stage execution - // We signal flow break - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::InternalError(_) => { - // Error that we can't handle - break with cleanup data - Ok(ControlFlow::Break(())) - } - } - } - - /// Handles unfinalized account error - /// Sends a separate tx to finalize account and then continues execution - async fn handle_unfinalized_account_error( - &self, - failed_signature: &Option, - delegated_account: Pubkey, - transaction_preparator: &T, - ) -> IntentExecutorResult> { - let finalize_task: BaseTaskImpl = - FinalizeTask { delegated_account }.into(); - prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut TransactionStrategy { - optimized_tasks: vec![finalize_task], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - }, - &None::, - ) - .await - .map_err(IntentExecutorError::FailedFinalizePreparationError)? - .map_err(|err| IntentExecutorError::FailedToFinalizeError { - err, - commit_signature: None, - finalize_signature: *failed_signature, - })?; - - Ok(ControlFlow::Continue(TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - })) - } -} diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 857fe7854..3aab343bf 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -92,8 +92,6 @@ where &self.authority.pubkey(), )?; - - todo!() } } diff --git a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_executor.rs deleted file mode 100644 index 196ae72be..000000000 --- a/magicblock-committor-service/src/intent_executor/two_stage_executor.rs +++ /dev/null @@ -1,527 +0,0 @@ -use std::{mem, ops::ControlFlow}; - -use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; -use solana_keypair::Keypair; -use solana_pubkey::Pubkey; -use solana_signature::Signature; -use solana_signer::Signer; -use tracing::{error, instrument, warn}; - -use crate::{ - intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, - }, - intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - two_stage_executor::sealed::Sealed, - utils::{ - handle_actions_result, handle_commit_id_error, - handle_undelegation_error, prepare_and_execute_strategy, - }, - IntentExecutionReport, - }, - persist::{IntentPersister, IntentPersisterImpl}, - tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, - transaction_preparator::TransactionPreparator, -}; - -pub struct Initialized { - /// Commit stage strategy - commit_strategy: TransactionStrategy, - /// Finalize stage strategy - finalize_strategy: TransactionStrategy, - - current_attempt: u8, -} - -pub struct Committed { - /// Signature of commit stage - commit_signature: Signature, - /// Finalize stage strategy - finalize_strategy: TransactionStrategy, - - current_attempt: u8, -} - -pub struct Finalized { - /// Signature of commit stage - pub commit_signature: Signature, - /// Signature of finalize stage - pub finalize_signature: Signature, -} - -pub struct TwoStageExecutor<'a, A, S: Sealed> { - state: S, - authority: Keypair, - intent_client: IntentExecutionClient, - callback_scheduler: A, - execution_report: &'a mut IntentExecutionReport, -} - -impl<'a, A> TwoStageExecutor<'a, A, Initialized> -where - A: ActionsCallbackScheduler, -{ - const RECURSION_CEILING: u8 = 10; - - pub fn new( - authority: Keypair, - commit_strategy: TransactionStrategy, - finalize_strategy: TransactionStrategy, - intent_client: IntentExecutionClient, - callback_scheduler: A, - execution_report: &'a mut IntentExecutionReport, - ) -> Self { - Self { - authority, - intent_client, - execution_report, - callback_scheduler, - state: Initialized { - commit_strategy, - finalize_strategy, - current_attempt: 0, - }, - } - } - - #[instrument( - skip( - self, - committed_pubkeys, - transaction_preparator, - task_info_fetcher, - persister - ), - fields(stage = "commit") - )] - pub async fn commit( - &mut self, - committed_pubkeys: &[Pubkey], - transaction_preparator: &T, - task_info_fetcher: &CacheTaskInfoFetcher, - persister: &Option

, - ) -> IntentExecutorResult - where - T: TransactionPreparator, - F: TaskInfoFetcher, - P: IntentPersister, - { - let commit_result = loop { - self.state.current_attempt += 1; - - // Prepare & execute message - let execution_result = prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.state.commit_strategy, - persister, - ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)?; - let execution_err = match execution_result { - Ok(value) => break Ok(value), - Err(err) => err, - }; - - let flow = self - .patch_commit_strategy( - &execution_err, - task_info_fetcher, - transaction_preparator, - committed_pubkeys, - ) - .await?; - let cleanup = match flow { - ControlFlow::Continue(value) => value, - ControlFlow::Break(()) => { - break Err(execution_err); - } - }; - self.intent_client.invalidate_cached_blockhash().await; - self.execution_report.dispose(cleanup); - - if self.state.current_attempt >= Self::RECURSION_CEILING { - error!("CRITICAL! Recursion ceiling reached"); - break Err(execution_err); - } else { - self.execution_report.add_patched_error(execution_err); - } - }; - - self.execute_callbacks( - commit_result.as_ref().ok().copied(), - commit_result.as_ref().map(|_| ()), - ); - self.execution_report - .dispose(mem::take(&mut self.state.commit_strategy)); - if commit_result.is_err() { - self.execution_report - .dispose(mem::take(&mut self.state.finalize_strategy)); - } - commit_result.map_err(|err| { - IntentExecutorError::from_commit_execution_error(err) - }) - } - - /// Patches Commit stage `transaction_strategy` in response to a recoverable - /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data - /// to be applied after a retry. - /// - /// [`TransactionStrategyExecutionError`], returning either: - /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or - /// - `Break(())` when this stage cannot be recovered. - pub async fn patch_commit_strategy( - &mut self, - err: &TransactionStrategyExecutionError, - task_info_fetcher: &CacheTaskInfoFetcher, - transaction_preparator: &T, - committed_pubkeys: &[Pubkey], - ) -> IntentExecutorResult> - where - T: TransactionPreparator, - F: TaskInfoFetcher, - { - match err { - TransactionStrategyExecutionError::CommitIDError(_, _) => { - let to_cleanup = handle_commit_id_error( - &self.authority.pubkey(), - task_info_fetcher, - committed_pubkeys, - &mut self.state.commit_strategy - ) - .await?; - Ok(ControlFlow::Continue(to_cleanup)) - } - err - @ TransactionStrategyExecutionError::UnfinalizedAccountError( - _, - signature, - ) => { - let optimized_tasks = - self.state.commit_strategy.optimized_tasks.as_slice(); - let task_index = err.task_index(); - if let Some(delegated_account) = task_index - .and_then(|index| optimized_tasks.get(index as usize)) - .and_then(|task| match task { - BaseTaskImpl::Commit(task) => { - Some(task.committed_account.pubkey) - } - BaseTaskImpl::CommitFinalize(task) => { - Some(task.committed_account.pubkey) - } - _ => None, - }) - { - self.handle_unfinalized_account_error( - signature, - delegated_account, - transaction_preparator, - ) - .await - } else { - error!( - task_index = ?task_index, - optimized_tasks_len = optimized_tasks.len(), - error = ?err, - "RPC returned unexpected task index" - ); - Ok(ControlFlow::Break(())) - } - } - TransactionStrategyExecutionError::ActionsError(err, signature) => { - // Intent bundles allow for actions to be in commit stage - let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); - let to_cleanup = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.commit_strategy, - *signature, - action_error - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::UndelegationError(_, _) => { - // Unexpected in Two Stage commit - // That would mean that Two Stage executes undelegation in commit phase - error!(error = ?err, "Unexpected error in two stage commit flow"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::CpiLimitError(_, _) - | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( - _, - _, - ) - | TransactionStrategyExecutionError::TransactionTooLargeError(_) => { - // Can't be handled - error!(error = ?err, "Commit tasks exceeded execution limit"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::InternalError(_) => { - // Can't be handled - Ok(ControlFlow::Break(())) - } - } - } - - /// Handles unfinalized account error - /// Sends a separate tx to finalize account and then continues execution - async fn handle_unfinalized_account_error( - &self, - failed_signature: &Option, - delegated_account: Pubkey, - transaction_preparator: &T, - ) -> IntentExecutorResult> { - let finalize_task: BaseTaskImpl = - FinalizeTask { delegated_account }.into(); - prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut TransactionStrategy { - optimized_tasks: vec![finalize_task], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - }, - &None::, - ) - .await - .map_err(IntentExecutorError::FailedCommitPreparationError)? - .map_err(|err| IntentExecutorError::FailedToCommitError { - err, - signature: *failed_signature, - })?; - - Ok(ControlFlow::Continue(TransactionStrategy { - optimized_tasks: vec![], - lookup_tables_keys: vec![], - standalone_action_nonce: None, - })) - } - - pub fn has_callbacks(&self) -> bool { - self.state.commit_strategy.has_actions_callbacks() - || self.state.finalize_strategy.has_actions_callbacks() - } - - /// On `Err`: removes actions from both commit and finalize strategies and - /// executes all their callbacks with the error. - /// On `Ok`: removes actions only from commit strategy and executes their - /// callbacks, preserving finalize-stage actions for the finalize phase. - pub fn execute_callbacks( - &mut self, - signature: Option, - result: Result<(), impl Into>, - ) { - let result = result.map(|_| ()).map_err(|err| err.into()); - let junk_strategy = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.commit_strategy, - signature, - result.clone(), - ); - self.execution_report.dispose(junk_strategy); - - if result.is_err() { - let junk_strategy = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.finalize_strategy, - signature, - result, - ); - self.execution_report.dispose(junk_strategy); - } - } - - /// Transitions to next executor state - pub fn done( - self, - commit_signature: Signature, - ) -> TwoStageExecutor<'a, A, Committed> { - TwoStageExecutor { - authority: self.authority, - intent_client: self.intent_client, - callback_scheduler: self.callback_scheduler, - execution_report: self.execution_report, - state: Committed { - commit_signature, - finalize_strategy: self.state.finalize_strategy, - current_attempt: 0, - }, - } - } -} - -impl<'a, A> TwoStageExecutor<'a, A, Committed> -where - A: ActionsCallbackScheduler, -{ - const RECURSION_CEILING: u8 = 10; - - #[instrument( - skip(self, transaction_preparator, persister), - fields(stage = "finalize") - )] - pub async fn finalize( - &mut self, - transaction_preparator: &T, - persister: &Option

, - ) -> IntentExecutorResult - where - T: TransactionPreparator, - P: IntentPersister, - { - let finalize_result = loop { - self.state.current_attempt += 1; - - // Prepare & execute message - let execution_result = prepare_and_execute_strategy( - &self.intent_client, - &self.authority, - transaction_preparator, - &mut self.state.finalize_strategy, - persister, - ) - .await - .map_err(IntentExecutorError::FailedFinalizePreparationError)?; - let execution_err = match execution_result { - Ok(value) => break Ok(value), - Err(err) => err, - }; - - let flow = self.patch_finalize_strategy(&execution_err).await?; - - let cleanup = match flow { - ControlFlow::Continue(cleanup) => cleanup, - ControlFlow::Break(()) => { - break Err(execution_err); - } - }; - self.intent_client.invalidate_cached_blockhash().await; - self.execution_report.dispose(cleanup); - - if self.state.current_attempt >= Self::RECURSION_CEILING { - error!("CRITICAL! Recursion ceiling reached"); - break Err(execution_err); - } else { - self.execution_report.add_patched_error(execution_err); - } - }; - - // Even if failed - dump finalize into junk - self.execute_callbacks( - finalize_result.as_ref().ok().copied(), - finalize_result.as_ref().map(|_| ()), - ); - self.execution_report - .dispose(mem::take(&mut self.state.finalize_strategy)); - finalize_result.map_err(|err| { - IntentExecutorError::from_finalize_execution_error( - err, - Some(self.state.commit_signature), - ) - }) - } - - pub fn has_callbacks(&self) -> bool { - self.state.finalize_strategy.has_actions_callbacks() - } - - /// Removes actions from finalize strategy - /// Executes callbacks - pub fn execute_callbacks( - &mut self, - signature: Option, - result: Result<(), impl Into>, - ) { - let junk_strategy = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.finalize_strategy, - signature, - result.map_err(|err| err.into()), - ); - self.execution_report.dispose(junk_strategy); - } - - /// Patches Finalize stage `transaction_strategy` in response to a recoverable - /// [`TransactionStrategyExecutionError`], optionally preparing cleanup data - /// to be applied after a retry. - /// - /// [`TransactionStrategyExecutionError`], returning either: - /// - `Continue(to_cleanup)` when a retry should be attempted with cleanup metadata, or - /// - `Break(())` when this stage cannot be recovered. - pub async fn patch_finalize_strategy( - &mut self, - err: &TransactionStrategyExecutionError, - ) -> IntentExecutorResult> { - match err { - TransactionStrategyExecutionError::CommitIDError(_, _) - | TransactionStrategyExecutionError::UnfinalizedAccountError( - _, - _, - ) => { - // Unexpected error in Two Stage commit - error!(error = ?err, "Unexpected error in two stage finalize flow"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::ActionsError(err, signature) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let action_error = Err(ActionError::ActionsError(err.clone(), *signature)); - let to_cleanup = handle_actions_result( - &self.authority.pubkey(), - &self.callback_scheduler, - self.execution_report, - &mut self.state.finalize_strategy, - *signature, - action_error - ); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::UndelegationError(_, _) => { - // Here we patch strategy for it to be retried in next iteration - // & we also record data that has to be cleaned up after patch - let to_cleanup = - handle_undelegation_error(&self.authority.pubkey(), &mut self.state.finalize_strategy); - Ok(ControlFlow::Continue(to_cleanup)) - } - TransactionStrategyExecutionError::CpiLimitError(_, _) - | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded(_, _) - | TransactionStrategyExecutionError::TransactionTooLargeError(_) => { - // Can't be handled - warn!(error = ?err, "Finalization tasks exceeded execution limit"); - Ok(ControlFlow::Break(())) - } - TransactionStrategyExecutionError::InternalError(_) => { - // Can't be handled - Ok(ControlFlow::Break(())) - } - } - } - - /// Transitions to next executor state - pub fn done(self, finalize_signature: Signature) -> Finalized { - Finalized { - commit_signature: self.state.commit_signature, - finalize_signature, - } - } -} - -mod sealed { - pub trait Sealed {} - - impl Sealed for super::Initialized {} - impl Sealed for super::Committed {} - impl Sealed for super::Finalized {} -} diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 228e446f6..fcf751c8a 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -81,6 +81,7 @@ where .checked_sub(self.started_at.elapsed()) } + /// Picks up execution from commit stage signature async fn execute_committing_intent( &mut self, intent_bundle: ScheduledIntentBundle, @@ -123,6 +124,7 @@ where .await } + /// Picks up execution from pending finalize signature async fn execute_finalizing_intent( &mut self, intent: ScheduledIntentBundle, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 2910acfbe..6a8570bbf 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -5,14 +5,20 @@ use magicblock_core::traits::ActionsCallbackScheduler; use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use solana_keypair::Keypair; use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_signer::Signer; use crate::{ intent_executor::{ error::{IntentExecutorError, IntentExecutorResult}, strategy_executor::{ + single_stage::SingleStageStrategyExecutor, two_stage, two_stage::TwoStageStrategyExecutor, - utils::{execute_with_timeout, CommitStage, FinalizeStage}, + utils::{ + execute_with_timeout, handle_cpi_limit_error, CommitStage, + FinalizeStage, SingleStage, + }, }, task_info_fetcher::TaskInfoFetcher, ExecutionOutput, IntentExecutionReport, IntentExecutorCtx, @@ -42,12 +48,99 @@ pub(in crate::intent_executor) async fn build_commit_finalize_tasks< Ok((commit_tasks?, finalize_tasks?)) } +pub(in crate::intent_executor) async fn execute_single_stage_flow( + ctx: &IntentExecutorCtx, + authority: &Keypair, + intent_bundle: ScheduledIntentBundle, + pending_signature: Option, + transaction_strategy: TransactionStrategy, + execution_report: &mut IntentExecutionReport, + time_left: impl Fn() -> Option, +) -> IntentExecutorResult +where + T: TransactionPreparator, + F: TaskInfoFetcher, + A: ActionsCallbackScheduler, + O: OutboxClient, + O::Error: Into, +{ + let committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); + + let mut single_stage_executor = SingleStageStrategyExecutor::new( + authority.insecure_clone(), + intent_bundle.id, + pending_signature, + ctx.intent_client.clone(), + ctx.task_info_fetcher.clone(), + ctx.outbox_client.clone(), + transaction_strategy, + ctx.actions_callback_executor.clone(), + execution_report, + ); + let res = execute_with_timeout( + time_left(), + SingleStage { + inner: &mut single_stage_executor, + transaction_preparator: &ctx.transaction_preparator, + committed_pubkeys: &committed_pubkeys, + }, + ) + .await; + + // Here we continue only IF the error is a limit-type execution error + // We can recover that Error by splitting execution + // in 2 stages - commit & finalize + // Otherwise we return error + let execution_err = match res { + Err(IntentExecutorError::FailedToFinalizeError { + err, + commit_signature: _, + finalize_signature: _, + }) if !committed_pubkeys.is_empty() + && err.is_recoverable_by_two_stage() => + { + err + } + res => { + let signature = res.as_ref().ok().copied(); + single_stage_executor + .execute_callbacks(signature, res.as_ref().map(|_| ())); + let transaction_strategy = single_stage_executor.consume_strategy(); + execution_report.dispose(transaction_strategy); + return res.map(ExecutionOutput::SingleStage); + } + }; + + // With actions, we can't predict num of CPIs + // If we get here we will try to switch from Single stage to Two Stage commit + // Note that this not necessarily will pass at the end due to the same reason + let strategy = single_stage_executor.consume_strategy(); + let (commit_strategy, finalize_strategy, cleanup) = + handle_cpi_limit_error(&authority.pubkey(), strategy); + execution_report.dispose(cleanup); + execution_report.add_patched_error(execution_err); + + // Create state for two stage flow + // Pending signature set to None as if we're here tx failed + let state = + two_stage::Initialized::new(commit_strategy, finalize_strategy, None); + + execute_two_stage_flow( + ctx, + state, + authority, + intent_bundle, + execution_report, + time_left, + ) + .await +} + pub(in crate::intent_executor) async fn execute_two_stage_flow( ctx: &IntentExecutorCtx, state: two_stage::Initialized, authority: &Keypair, - intent_id: u64, - committed_pubkeys: &[Pubkey], + intent_bundle: ScheduledIntentBundle, execution_report: &mut IntentExecutionReport, time_left: impl Fn() -> Option, ) -> IntentExecutorResult @@ -58,10 +151,11 @@ where O: OutboxClient, O::Error: Into, { + let committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); let mut executor = TwoStageStrategyExecutor::new( state, authority.insecure_clone(), - intent_id, + intent_bundle.id, ctx.intent_client.clone(), ctx.outbox_client.clone(), ctx.actions_callback_executor.clone(), @@ -74,7 +168,7 @@ where inner: &mut executor, transaction_preparator: &ctx.transaction_preparator, task_info_fetcher: &ctx.task_info_fetcher, - committed_pubkeys, + committed_pubkeys: &committed_pubkeys, }, ) .await?; diff --git a/magicblock-rpc-client/src/lib.rs b/magicblock-rpc-client/src/lib.rs index 900b4fb56..867320e6f 100644 --- a/magicblock-rpc-client/src/lib.rs +++ b/magicblock-rpc-client/src/lib.rs @@ -110,6 +110,27 @@ impl MagicBlockRpcClientError { _ => None, } } + + pub fn is_transaction_too_large(&self) -> bool { + fn is_send_transaction_too_large(err: &RpcClientError) -> bool { + matches!(err.request, Some(RpcRequest::SendTransaction)) + && matches!( + &*err.kind, + RpcClientErrorKind::RpcError( + RpcError::RpcResponseError { code: -32602, message, .. } + ) if message.contains("VersionedTransaction too large") + || message.contains("base64 encoded too large") + ) + } + + match self { + MagicBlockRpcClientError::RpcClientError(err) + | MagicBlockRpcClientError::SendTransaction(err) => { + is_send_transaction_too_large(err) + } + _ => false, + } + } } pub type MagicBlockRpcClientResult = From 7e1608b449cbbbef0a27fa17622cc79053b4ef6c Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 15 Jun 2026 15:04:16 +0700 Subject: [PATCH 46/97] feat: integration of executors within committor service fix: committor service compilation errors --- magicblock-api/src/magic_validator.rs | 3 +- .../src/committor_processor.rs | 20 +--- .../src/intent_execution_manager.rs | 6 +- .../intent_execution_engine.rs | 36 +++--- .../accepted_intent_executor.rs | 105 +++------------- .../src/intent_executor/cleanup_handle.rs | 2 +- .../src/intent_executor/error.rs | 11 +- .../intent_executor_factory.rs | 32 ++--- .../intent_executor_gateway.rs | 35 ------ .../src/intent_executor/mod.rs | 10 +- .../single_stage_intent_executor.rs | 36 +++--- .../strategy_executor/patcher.rs | 11 +- .../two_stage_intent_executor.rs | 4 +- .../src/intent_executor/utils.rs | 1 - .../src/outbox_client.rs | 2 +- magicblock-committor-service/src/service.rs | 113 ++++-------------- .../src/tasks/task_strategist.rs | 4 +- 17 files changed, 133 insertions(+), 298 deletions(-) delete mode 100644 magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index abe134319..b874bf790 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -478,7 +478,6 @@ impl MagicValidator { ); Ok(CommittorProcessor::try_new( authority, - committor_persist_path, base_chain_config, shared_chain_slot.clone(), actions_callback_executor, @@ -502,7 +501,7 @@ impl MagicValidator { IntentExecutionServiceImpl::new( chainlink.clone(), - intent_client, + Arc::new(intent_client), committor_processor.clone(), slot_interval, cancellation_token.clone(), diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index b2978aedf..1264ef32d 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -18,10 +18,10 @@ use crate::{ config::ChainConfig, error::{CommittorServiceError, CommittorServiceResult}, intent_execution_manager::{ - db::{DumberDB, DummyDB}, - BroadcastedIntentExecutionResult, IntentExecutionManager, + db::DummyDB, BroadcastedIntentExecutionResult, IntentExecutionManager, }, intent_executor::{ + error::IntentExecutorError, intent_executor_factory::ExecutorConfig, task_info_fetcher::{ CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, @@ -57,6 +57,7 @@ impl CommittorProcessor { where A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { let rpc_client = RpcClient::new_with_commitment( chain_config.rpc_uri.to_string(), @@ -196,21 +197,6 @@ impl CommittorProcessor { Ok(results) } - #[instrument(skip(self, intent_bundles))] - pub async fn schedule_recovered_intent_bundles( - &self, - intent_bundles: Vec, - ) -> CommittorServiceResult<()> { - self.commits_scheduler - .schedule(intent_bundles) - .await - .inspect_err(|err| { - error!(error = ?err, "Failed to schedule recovered intent"); - })?; - - Ok(()) - } - /// Creates a subscription for results of BaseIntent execution pub fn subscribe_for_results( &self, diff --git a/magicblock-committor-service/src/intent_execution_manager.rs b/magicblock-committor-service/src/intent_execution_manager.rs index 72001505b..2d519526a 100644 --- a/magicblock-committor-service/src/intent_execution_manager.rs +++ b/magicblock-committor-service/src/intent_execution_manager.rs @@ -19,7 +19,8 @@ use crate::{ intent_execution_engine::{IntentExecutionEngine, ResultSubscriber}, }, intent_executor::{ - intent_executor_factory::{ExecutorConfig, IntentExecutorFactoryImpl}, + error::IntentExecutorError, + intent_executor_factory::{ExecutorConfig, IntentExecutorBuilderImpl}, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }, outbox_client::OutboxClient, @@ -45,10 +46,11 @@ impl IntentExecutionManager { where A: ActionsCallbackScheduler, O: OutboxClient, + O::Error: Into, { let db = Arc::new(Mutex::new(db)); - let executor_factory = IntentExecutorFactoryImpl { + let executor_factory = IntentExecutorBuilderImpl { rpc_client, table_mania, executor_config, diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index a3d5d3008..24f550151 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -1,4 +1,5 @@ use std::{ + marker::PhantomData, ops::Deref, sync::{Arc, Mutex}, time::{Duration, Instant}, @@ -27,9 +28,10 @@ use crate::{ IntentExecutorError, IntentExecutorResult, TransactionStrategyExecutionError, }, - intent_executor_factory::IntentExecutorFactory, + intent_executor_factory::IntentExecutorBuilder, ExecutionOutput, IntentExecutionResult, IntentExecutor, }, + transaction_preparator::TransactionPreparator, }; const SEMAPHORE_CLOSED_MSG: &str = "Executors semaphore closed!"; @@ -85,30 +87,32 @@ impl ResultSubscriber { } } -pub(crate) struct IntentExecutionEngine { +pub(crate) struct IntentExecutionEngine { intent_stream: IntentStream, - executor_factory: F, + executor_builder: F, inner: Arc>, running_executors: FuturesUnordered>, executors_semaphore: Arc, + _phantom_data: PhantomData, } -impl IntentExecutionEngine +impl IntentExecutionEngine where D: DB, - F: IntentExecutorFactory + Send + Sync + 'static, - E: IntentExecutor, + T: TransactionPreparator, + F: IntentExecutorBuilder + Send + Sync + 'static, { pub fn new(intent_stream: IntentStream, executor_factory: F) -> Self { Self { intent_stream, - executor_factory, + executor_builder: executor_factory, running_executors: FuturesUnordered::new(), executors_semaphore: Arc::new(Semaphore::new( MAX_EXECUTORS as usize, )), inner: Arc::new(Mutex::new(IntentScheduler::new())), + _phantom_data: PhantomData::default(), } } @@ -159,7 +163,8 @@ where .expect(SEMAPHORE_CLOSED_MSG); // Spawn executor - let executor = self.executor_factory.create_instance(); + let executor = + self.executor_builder.create_instance(intent.status.clone()); let inner = self.inner.clone(); let handle = tokio::spawn(Self::execute( @@ -242,7 +247,7 @@ where /// Wrapper on [`IntentExecutor`] that handles its results and drops execution permit #[instrument(skip(executor, intent, inner_scheduler, execution_permit, result_sender), fields(intent_id = intent.id))] async fn execute( - mut executor: E, + executor: Box>, intent: OutboxIntentBundle, inner_scheduler: Arc>, execution_permit: OwnedSemaphorePermit, @@ -251,7 +256,8 @@ where let instant = Instant::now(); // Execute an Intent - let result = executor.execute(intent.clone()).await; + let (result, cleanup_handle) = + executor.execute(intent.inner.clone()).await; let _ = result.inner.as_ref().inspect_err(|err| { error!(intent_id = intent.id, error = ?err, "Failed to execute intent bundle"); }); @@ -279,7 +285,7 @@ where .expect("Valid completion of previously scheduled message"); tokio::spawn(async move { - if let Err(err) = executor.cleanup().await { + if let Err(err) = cleanup_handle.clean().await { error!(error = ?err, "Failed to cleanup after intent"); } }); @@ -509,7 +515,7 @@ mod tests { let active_tasks = Arc::new(AtomicUsize::new(0)); let max_concurrent = Arc::new(AtomicUsize::new(0)); worker - .executor_factory + .executor_builder .with_concurrency_tracking(&active_tasks, &max_concurrent); let result_subscriber = worker.spawn(); @@ -577,7 +583,7 @@ mod tests { let active_tasks = Arc::new(AtomicUsize::new(0)); let max_concurrent = Arc::new(AtomicUsize::new(0)); worker - .executor_factory + .executor_builder .with_concurrency_tracking(&active_tasks, &max_concurrent); let result_subscriber = worker.spawn(); @@ -635,7 +641,7 @@ mod tests { let active_tasks = Arc::new(AtomicUsize::new(0)); let max_concurrent = Arc::new(AtomicUsize::new(0)); worker - .executor_factory + .executor_builder .with_concurrency_tracking(&active_tasks, &max_concurrent); let result_subscriber = worker.spawn(); @@ -708,7 +714,7 @@ mod tests { } } - impl IntentExecutorFactory for MockIntentExecutorFactory { + impl IntentExecutorBuilder for MockIntentExecutorFactory { type Executor = MockIntentExecutor; fn create_instance(&self) -> Self::Executor { diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 31f86efe4..e66fdf878 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -1,17 +1,12 @@ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use async_trait::async_trait; -use futures_util::future::{join, try_join_all}; use magicblock_core::traits::ActionsCallbackScheduler; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, validator::validator_authority, + validator::validator_authority, }; -use magicblock_rpc_client::MagicblockRpcClient; -use solana_keypair::{Address as Pubkey, Keypair}; +use solana_keypair::Keypair; use solana_signer::Signer; use tracing::trace; @@ -19,16 +14,12 @@ use crate::{ intent_executor::{ cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, - intent_execution_client::IntentExecutionClient, - strategy_executor::{ - single_stage::SingleStageStrategyExecutor, - two_stage::Initialized, - utils::{ - execute_with_timeout, handle_cpi_limit_error, SingleStage, - }, + strategy_executor::two_stage::Initialized, + task_info_fetcher::{ResetType, TaskInfoFetcher}, + utils::{ + build_commit_finalize_tasks, execute_single_stage_flow, + execute_two_stage_flow, }, - task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, - utils::{build_commit_finalize_tasks, execute_two_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, @@ -41,9 +32,7 @@ use crate::{ }, TaskBuilderImpl, }, - transaction_preparator::{ - delivery_preparator::BufferExecutionError, TransactionPreparator, - }, + transaction_preparator::TransactionPreparator, }; pub struct AcceptedIntentExecutor { @@ -141,8 +130,7 @@ where }) => { trace!("Two stage execution"); self.two_stage_execution_flow( - intent_bundle.id, - &all_committed_pubkeys, + intent_bundle, commit_stage, finalize_stage, execution_report, @@ -161,81 +149,25 @@ where /// Starting execution from single stage pub async fn single_stage_execution_flow( &mut self, - base_intent: ScheduledIntentBundle, + intent_bundle: ScheduledIntentBundle, transaction_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - let committed_pubkeys = base_intent.get_all_committed_pubkeys(); - - let mut single_stage_executor = SingleStageStrategyExecutor::new( - self.authority.insecure_clone(), - base_intent.id, + execute_single_stage_flow( + &self.ctx, + &self.authority, + intent_bundle, None, - self.ctx.intent_client.clone(), - self.ctx.task_info_fetcher.clone(), - self.ctx.outbox_client.clone(), transaction_strategy, - self.ctx.actions_callback_executor.clone(), - execution_report, - ); - let res = execute_with_timeout( - self.time_left(), - SingleStage { - inner: &mut single_stage_executor, - transaction_preparator: &self.ctx.transaction_preparator, - committed_pubkeys: &committed_pubkeys, - }, - ) - .await; - - // Here we continue only IF the error is a limit-type execution error - // We can recover that Error by splitting execution - // in 2 stages - commit & finalize - // Otherwise we return error - let execution_err = match res { - Err(IntentExecutorError::FailedToFinalizeError { - err, - commit_signature: _, - finalize_signature: _, - }) if !committed_pubkeys.is_empty() - && err.is_recoverable_by_two_stage() => - { - err - } - res => { - let signature = res.as_ref().ok().copied(); - single_stage_executor - .execute_callbacks(signature, res.as_ref().map(|_| ())); - let transaction_strategy = - single_stage_executor.consume_strategy(); - execution_report.dispose(transaction_strategy); - return res.map(ExecutionOutput::SingleStage); - } - }; - - // With actions, we can't predict num of CPIs - // If we get here we will try to switch from Single stage to Two Stage commit - // Note that this not necessarily will pass at the end due to the same reason - let strategy = single_stage_executor.consume_strategy(); - let (commit_strategy, finalize_strategy, cleanup) = - handle_cpi_limit_error(&self.authority.pubkey(), strategy); - execution_report.dispose(cleanup); - execution_report.add_patched_error(execution_err); - - self.two_stage_execution_flow( - base_intent.id, - &committed_pubkeys, - commit_strategy, - finalize_strategy, execution_report, + || self.time_left(), ) .await } pub async fn two_stage_execution_flow( &mut self, - intent_id: u64, - committed_pubkeys: &[Pubkey], + intent_bundle: ScheduledIntentBundle, commit_strategy: TransactionStrategy, finalize_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, @@ -245,8 +177,7 @@ where &self.ctx, state, &self.authority, - intent_id, - committed_pubkeys, + intent_bundle, execution_report, || self.time_left(), ) diff --git a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs index a15ae10b2..f058eaf23 100644 --- a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs +++ b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs @@ -8,7 +8,7 @@ use crate::{ }, }; -pub(crate) struct CleanupHandle { +pub struct CleanupHandle { authority: Keypair, junk: Vec, /// When false (execution failed), only releases ALT reservations without diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index 0c3b03e4b..b7006ba64 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -4,10 +4,6 @@ use magicblock_rpc_client::{ utils::TransactionErrorMapper, MagicBlockRpcClientError, }; use solana_instruction::error::InstructionError; -use solana_rpc_client_api::{ - client_error::{Error as RpcClientError, ErrorKind as RpcClientErrorKind}, - request::{RpcError, RpcRequest}, -}; use solana_signature::Signature; use solana_signer::SignerError; use solana_transaction_error::TransactionError; @@ -111,7 +107,8 @@ impl IntentExecutorError { } } - pub fn signatures(&self) -> Option<(Signature, Option)> { + /// Returns signatures of transaction sent to Base layer + pub fn base_signatures(&self) -> Option<(Signature, Option)> { match self { IntentExecutorError::FailedToCommitError { signature, err: _ } => { signature.map(|el| (el, None)) @@ -130,7 +127,9 @@ impl IntentExecutorError { } => commit_signature.map(|el| (el, *finalize_signature)), IntentExecutorError::EmptyIntentError | IntentExecutorError::FailedToFitError - | IntentExecutorError::SignerError(_) => None, + | IntentExecutorError::SignerError(_) + | IntentExecutorError::OutboxClientError(_) + | IntentExecutorError::GetPendingSignatureStatusError(_) => None, } } } diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index 2b8693054..1731214a7 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -1,12 +1,13 @@ use std::{sync::Arc, time::Duration}; use magicblock_core::traits::ActionsCallbackScheduler; +use magicblock_program::outbox_intent_bundles::OutboxIntentBundleStatus; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::TableMania; use crate::{ intent_executor::{ - accepted_intent_executor::AcceptedIntentExecutor, + build_stage_intent_executor, error::IntentExecutorError, intent_execution_client::IntentExecutionClient, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, @@ -17,10 +18,12 @@ use crate::{ ComputeBudgetConfig, }; -pub trait IntentExecutorFactory { - type Executor: IntentExecutor; - - fn create_instance(&self) -> Self::Executor; +// TODO(edwin): T could be removed if cleanuphandle is a trait +pub trait IntentExecutorBuilder { + fn create_instance( + &self, + status: OutboxIntentBundleStatus, + ) -> Box>; } pub struct ExecutorConfig { @@ -29,7 +32,7 @@ pub struct ExecutorConfig { } /// Dummy struct to simplify signature of CommitSchedulerWorker -pub struct IntentExecutorFactoryImpl { +pub struct IntentExecutorBuilderImpl { pub rpc_client: MagicblockRpcClient, pub table_mania: TableMania, pub executor_config: ExecutorConfig, @@ -38,20 +41,17 @@ pub struct IntentExecutorFactoryImpl { pub actions_callback_executor: A, } -impl IntentExecutorFactory for IntentExecutorFactoryImpl +impl IntentExecutorBuilder + for IntentExecutorBuilderImpl where A: ActionsCallbackScheduler, O: OutboxClient, O::Error: Into, { - type Executor = AcceptedIntentExecutor< - TransactionPreparatorImpl, - RpcTaskInfoFetcher, - A, - O, - >; - - fn create_instance(&self) -> Self::Executor { + fn create_instance( + &self, + status: OutboxIntentBundleStatus, + ) -> Box> { let transaction_preparator = TransactionPreparatorImpl::new( self.rpc_client.clone(), self.table_mania.clone(), @@ -65,6 +65,6 @@ where actions_callback_executor: self.actions_callback_executor.clone(), actions_timeout: self.executor_config.actions_timeout, }; - Self::Executor::new(ctx) + build_stage_intent_executor(ctx, status) } } diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs b/magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs deleted file mode 100644 index 0945ac37e..000000000 --- a/magicblock-committor-service/src/intent_executor/intent_executor_gateway.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use magicblock_core::traits::ActionsCallbackScheduler; -use solana_keypair::Keypair; - -use crate::{ - intent_executor::{ - intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, - }, - transaction_preparator::TransactionPreparator, -}; - -pub struct IntentExecutorGateway { - authority: Keypair, - intent_client: IntentExecutionClient, - transaction_preparator: T, - task_info_fetcher: Arc>, - actions_callback_executor: A, - /// Timeout for Intent's actions - actions_timeout: Duration, -} - -impl IntentExecutorGateway -where - T: TransactionPreparator, - F: TaskInfoFetcher, - A: ActionsCallbackScheduler, -{ - pub fn new() -> Self { - Self {} - } -} - -async fn intent_executor_gateway() {} diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 33ece5c85..822e3d8bf 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -3,7 +3,6 @@ pub mod cleanup_handle; pub mod error; pub mod intent_execution_client; pub(crate) mod intent_executor_factory; -pub mod intent_executor_gateway; pub mod single_stage_intent_executor; pub mod strategy_executor; pub mod task_info_fetcher; @@ -22,7 +21,6 @@ use magicblock_program::{ outbox_intent_bundles::OutboxIntentBundleStatus, }; use solana_signature::Signature; -use solana_signer::Signer; use crate::{ intent_executor::{ @@ -38,7 +36,7 @@ use crate::{ two_stage_intent_executor::TwoStageIntentExecutor, }, outbox_client::OutboxClient, - tasks::{task_builder::TasksBuilder, task_strategist::TransactionStrategy}, + tasks::task_strategist::TransactionStrategy, transaction_preparator::TransactionPreparator, }; @@ -67,16 +65,16 @@ where match status { OutboxIntentBundleStatus::Accepted => { Box::new(AcceptedIntentExecutor::new(ctx)) - as Box<(dyn IntentExecutor + 'static)> + as Box + 'static> } OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( sig, )) => Box::new(SingleStageIntentExecutor::new(ctx, sig)) - as Box<(dyn IntentExecutor + 'static)>, + as Box + 'static>, OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( value, )) => Box::new(TwoStageIntentExecutor::new(ctx, value)) - as Box<(dyn IntentExecutor + 'static)>, + as Box + 'static>, } } diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 3aab343bf..f6663b7f0 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -1,7 +1,4 @@ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use async_trait::async_trait; use magicblock_core::traits::ActionsCallbackScheduler; @@ -9,27 +6,21 @@ use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, validator::validator_authority, }; -use magicblock_rpc_client::MagicblockRpcClient; use solana_keypair::Keypair; use solana_signature::Signature; use solana_signer::Signer; use crate::{ intent_executor::{ - accepted_intent_executor::AcceptedIntentExecutor, cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, - intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, - utils::build_commit_finalize_tasks, + task_info_fetcher::{ResetType, TaskInfoFetcher}, + utils::{build_commit_finalize_tasks, execute_single_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, outbox_client::OutboxClient, - tasks::{ - task_strategist::{TaskStrategist, TransactionStrategy}, - TaskBuilderImpl, - }, + tasks::task_strategist::{TaskStrategist, TransactionStrategy}, transaction_preparator::TransactionPreparator, }; @@ -74,12 +65,18 @@ where } } + fn time_left(&self) -> Option { + self.ctx + .actions_timeout + .checked_sub(self.started_at.elapsed()) + } + pub async fn execute_inner( &mut self, intent_bundle: ScheduledIntentBundle, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - // It we're here so previous run determined this should + // It we're here so previous run determined this should be single stage let (commit_tasks, finalize_tasks) = build_commit_finalize_tasks( &intent_bundle, &self.ctx.task_info_fetcher, @@ -92,7 +89,16 @@ where &self.authority.pubkey(), )?; - todo!() + execute_single_stage_flow( + &self.ctx, + &self.authority, + intent_bundle, + Some(self.pending_signature), + transaction_strategy, + execution_report, + || self.time_left(), + ) + .await } } diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs index e3277f583..222ff708c 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs @@ -178,7 +178,8 @@ where | TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( _, _, - ) => { + ) + | TransactionStrategyExecutionError::TransactionTooLargeError(_) => { // Can't be handled in scope of single stage execution // We signal flow break Ok(ControlFlow::Break(())) @@ -327,6 +328,10 @@ where error!(error = ?err, "Commit tasks exceeded execution limit"); Ok(ControlFlow::Break(())) } + TransactionStrategyExecutionError::TransactionTooLargeError(err) => { + error!(error = ?err, "Failed to execute intent: transaction too large!"); + Ok(ControlFlow::Break(())) + } TransactionStrategyExecutionError::InternalError(_) => { // Can't be handled Ok(ControlFlow::Break(())) @@ -395,6 +400,10 @@ where warn!(error = ?err, "Finalization tasks exceeded execution limit"); Ok(ControlFlow::Break(())) } + TransactionStrategyExecutionError::TransactionTooLargeError(err) => { + error!(error = ?err, "Failed to execute intent: transaction too large!"); + Ok(ControlFlow::Break(())) + } TransactionStrategyExecutionError::InternalError(_) => { // Can't be handled Ok(ControlFlow::Break(())) diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index fcf751c8a..7be89f74f 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -106,7 +106,6 @@ where &self.authority.pubkey(), )?; - let committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); let state = Initialized::new( commit_stage, finalize_stage, @@ -116,8 +115,7 @@ where &self.ctx, state, &self.authority, - intent_bundle.id, - &committed_pubkeys, + intent_bundle, execution_report, || self.time_left(), ) diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 6a8570bbf..20d38675b 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -4,7 +4,6 @@ use futures_util::future::join; use magicblock_core::traits::ActionsCallbackScheduler; use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use solana_keypair::Keypair; -use solana_pubkey::Pubkey; use solana_signature::Signature; use solana_signer::Signer; diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index fb700c569..e638d7979 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -120,7 +120,7 @@ impl OutboxClient for InternalOutboxClient { Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) } - fn set_intent_execution_stage( + async fn set_intent_execution_stage( &self, intent_id: u64, stage: ExecutionStage, diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 5ffe03bdd..d5de95cea 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -15,7 +15,8 @@ use magicblock_account_cloner::ChainlinkCloner; use magicblock_chainlink::{ProdChainlink, ProdInnerChainlink}; use magicblock_metrics::metrics::{self, AccountFetchOrigin}; use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, Pubkey, SentCommit, + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, Pubkey, SentCommit, }; use solana_hash::Hash; use solana_transaction::Transaction; @@ -31,7 +32,7 @@ use crate::{ committor_processor::CommittorProcessor, error::CommittorServiceResult, intent_execution_manager::BroadcastedIntentExecutionResult, - intent_executor::ExecutionOutput, + intent_executor::{error::IntentExecutorError, ExecutionOutput}, outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::{ InternalOutboxIntentBundlesReaderError, OutboxIntentBundlesReader, @@ -43,25 +44,27 @@ const POISONED_MUTEX_MSG: &str = "ServiceInner intents_meta_map mutex poisoned"; pub type InnerChainlinkImpl = ProdInnerChainlink; pub type ChainlinkImpl = ProdChainlink; -pub enum IntentExecutionService { - Created(ServiceInner), +pub enum IntentExecutionService { + Created(ServiceInner), Started(JoinHandle<()>), Stopped, Error, } -impl IntentExecutionService +impl IntentExecutionService where - R: OutboxClient, - // ERIntentClient errors should be convertible to Service errors - R::Error: Into, + O: OutboxClient, + // OutboxClient errors should be convertible to Service errors + O::Error: Into, + // OutboxClient errors should be convertible to IntentExecutor error + O::Error: Into, // OutboxReader errors should be convertible to Service errors - ::Error: + ::Error: Into, { pub fn new( chainlink: Arc, - intent_client: R, + intent_client: Arc, processor: Arc, slot_interval: Duration, cancellation_token: CancellationToken, @@ -122,8 +125,10 @@ pub struct ServiceInner { impl ServiceInner where O: OutboxClient, - // ERIntentClient errors should be convertible to Service errors + // OutboxClient errors should be convertible to Service errors O::Error: Into, + // OutboxClient errors should be convertible into IntentExecutor errors + O::Error: Into, // OutboxReader errors should be convertible to Service errors ::Error: Into, @@ -195,6 +200,7 @@ where } }; + let intent_bundles= intent_bundles.into_iter().map(OutboxIntentBundle::accepted).collect(); if let Err(err) = self.schedule_intent_execution(intent_bundles).await { error!("Failed to schedule intent execution: {}", err); } @@ -223,15 +229,10 @@ where // TODO(edwin): use status let read_len = intent_bundles_chunk.len(); - let intent_bundles = intent_bundles_chunk - .into_iter() - .map(|el| el.inner) - .collect(); - // Schedule without initial persistence as bundle already exists in db let result = self - .process_intent_bundles(intent_bundles, |bundles| { - self.processor.schedule_recovered_intent_bundles(bundles) + .process_intent_bundles(intent_bundles_chunk, |bundles| { + self.processor.schedule_intent_bundles(bundles) }) .await; if let Err(err) = result { @@ -245,29 +246,9 @@ where } } - async fn reschedule_pending_bundles(&self) -> CommittorServiceResult<()> { - // Fetch pending bundles from DB - let mut bundles = - self.processor.pending_intent_bundles().await.inspect_err(|err| { - error!(error = ?err, "Failed to load pending intent bundles for recovery"); - })?; - if bundles.is_empty() { - return Ok(()); - } - - // Retain only recoverable bundles - self.retain_recoverable_intent_bundles(&mut bundles).await; - - // Schedule without initial persisitance as bundle already exists in db - self.process_intent_bundles(bundles, |bundles| { - self.processor.schedule_recovered_intent_bundles(bundles) - }) - .await - } - async fn schedule_intent_execution( &self, - intent_bundles: Vec, + intent_bundles: Vec, ) -> CommittorServiceResult<()> { if intent_bundles.is_empty() { return Ok(()); @@ -283,11 +264,11 @@ where async fn process_intent_bundles( &self, - intent_bundles: Vec, + intent_bundles: Vec, schedule: F, ) -> CommittorServiceResult<()> where - F: FnOnce(Vec) -> Fut, + F: FnOnce(Vec) -> Fut, Fut: Future>, { if intent_bundles.is_empty() { @@ -466,7 +447,7 @@ where "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", intent_id, intent_meta.slot, intent_meta.blockhash, err ); - err.signatures() + err.base_signatures() .map(|(commit, finalize)| { finalize .map(|finalize| vec![commit, finalize]) @@ -515,52 +496,6 @@ where callbacks_scheduling_results: callbacks_report, } } - - /// Retains bundles whose accounts are still delegated - async fn retain_recoverable_intent_bundles( - &self, - bundles: &mut Vec, - ) { - let results = join_all( - bundles.iter().map(|b| b.get_all_committed_pubkeys()).map( - |pubkeys| async move { - self.chainlink - .accounts_delegated_on_base_and_er( - &pubkeys, - AccountFetchOrigin::GetAccount, - ) - .await - }, - ), - ) - .await; - - let mut results_iter = results.into_iter(); - bundles.retain(|bundle| { - let Some(result) = results_iter.next() else { - error!("Results and bundles must have equal length"); - return false; - }; - match result { - Ok(delegated) if delegated.iter().all(|d| *d) => true, - Ok(_) => { - warn!( - intent_id = bundle.id, - "Skipping recovered commit intent because not all accounts are delegated on base and ER" - ); - false - } - Err(err) => { - error!( - intent_id = bundle.id, - error = ?err, - "Failed to verify recovered commit intent accounts" - ); - false - } - } - }); - } } struct ScheduledBaseIntentMeta { @@ -595,4 +530,6 @@ pub enum IntentExecutionServiceError { IntentRpcClientError(#[from] InternalOutboxClientError), #[error("OutboxReaderError")] OutboxReaderError(#[from] InternalOutboxIntentBundlesReaderError), + #[error("asd")] + ASdError(#[from] IntentExecutorError), } diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index d549426dc..81776ff04 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -106,8 +106,8 @@ impl StrategyExecutionMode { } pub struct TwoStageExecutionMode { - commit_stage: TransactionStrategy, - finalize_stage: TransactionStrategy, + pub commit_stage: TransactionStrategy, + pub finalize_stage: TransactionStrategy, } impl TwoStageExecutionMode { From 8c8d92604ebb53df0b2e0e3a5a12be1294964f32 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 15 Jun 2026 15:05:48 +0700 Subject: [PATCH 47/97] fix: fmt --- .../src/committor_processor.rs | 1 - .../src/intent_executor/intent_execution_client.rs | 14 -------------- .../intent_executor/strategy_executor/two_stage.rs | 4 ---- magicblock-committor-service/src/service.rs | 3 +-- 4 files changed, 1 insertion(+), 21 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 1264ef32d..e9f61b3c2 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -33,7 +33,6 @@ use crate::{ const POISONED_MUTEX_MSG: &str = "CommittorProcessor pending messages mutex poisoned!"; -pub(crate) const RECOVERY_MAX_AGE_SECS: u64 = 14 * 24 * 60 * 60; type BundleResultListener = oneshot::Sender; diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index fae53cadb..7c1e9b820 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -102,20 +102,6 @@ impl IntentExecutionClient { Ok(()) } - /// Checks whether a previously-sent transaction confirmed. - /// `Err` means the signature was not confirmed within the timeout window. - pub(in crate::intent_executor) async fn wait_for_confirmed_status( - &self, - signature: &Signature, - ) -> MagicBlockRpcClientResult> { - const TIMEOUT: Duration = Duration::from_secs(5); - const INTERVAL: Duration = Duration::from_millis(200); - - self.rpc_client - .wait_for_confirmed_status(signature, &TIMEOUT, &Some(INTERVAL)) - .await - } - /// Queries the full transaction history for the given signatures. /// Each entry is `None` if the signature was never included in a block. /// Use this for restart recovery where txs may be older than the RPC diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index 3d583e834..1d978f0a0 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -104,8 +104,6 @@ where O: OutboxClient, O::Error: Into, { - const RECURSION_CEILING: u8 = 10; - pub fn new( state: Initialized, authority: Keypair, @@ -254,8 +252,6 @@ where O: OutboxClient, O::Error: Into, { - const RECURSION_CEILING: u8 = 10; - pub fn committed( state: Committed, authority: Keypair, diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index d5de95cea..b577cc622 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -10,10 +10,9 @@ use std::{ time::Duration, }; -use futures_util::future::join_all; use magicblock_account_cloner::ChainlinkCloner; use magicblock_chainlink::{ProdChainlink, ProdInnerChainlink}; -use magicblock_metrics::metrics::{self, AccountFetchOrigin}; +use magicblock_metrics::metrics::{self}; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, outbox_intent_bundles::OutboxIntentBundle, Pubkey, SentCommit, From 9513a1a008df9a9c73064f623a1a85349d450123 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 16 Jun 2026 16:22:56 +0700 Subject: [PATCH 48/97] feat: InternalOutboxIntentBundlesReader --- magicblock-committor-service/src/service.rs | 5 +- .../src/service/acceptor.rs | 1 - .../service/outbox_intent_bundles_reader.rs | 167 ++++++++++++++---- 3 files changed, 138 insertions(+), 35 deletions(-) delete mode 100644 magicblock-committor-service/src/service/acceptor.rs diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index b577cc622..f832f4408 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,4 +1,3 @@ -pub mod acceptor; pub mod outbox_intent_bundles_reader; use std::{ @@ -34,7 +33,7 @@ use crate::{ intent_executor::{error::IntentExecutorError, ExecutionOutput}, outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::{ - InternalOutboxIntentBundlesReaderError, OutboxIntentBundlesReader, + OutboxIntentBundlesReader, OutboxIntentBundlesReaderError, }, }; @@ -528,7 +527,7 @@ pub enum IntentExecutionServiceError { #[error("IntentRpcClientError: {0}")] IntentRpcClientError(#[from] InternalOutboxClientError), #[error("OutboxReaderError")] - OutboxReaderError(#[from] InternalOutboxIntentBundlesReaderError), + OutboxReaderError(#[from] OutboxIntentBundlesReaderError), #[error("asd")] ASdError(#[from] IntentExecutorError), } diff --git a/magicblock-committor-service/src/service/acceptor.rs b/magicblock-committor-service/src/service/acceptor.rs deleted file mode 100644 index 8b1378917..000000000 --- a/magicblock-committor-service/src/service/acceptor.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index f7cd5baf6..95fc3e4e0 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -1,29 +1,37 @@ -use std::{num::NonZeroUsize, sync::Arc}; +use std::{ + cmp::Ordering, + collections::{BinaryHeap, VecDeque}, + num::NonZeroUsize, + sync::Arc, +}; use async_trait::async_trait; -use magicblock_accounts_db::AccountsDb; +use magicblock_accounts_db::{error::AccountsDbError, AccountsDb}; +use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; -use solana_account::ReadableAccount; +use solana_account::{AccountSharedData, ReadableAccount}; use solana_pubkey::Pubkey; -// TODO(edwin): name - OutboxIntentAccount/PendingIntentAccount +use tracing::warn; -// TODO(edwin): name - PendingIntentBundlesReader? #[async_trait] pub trait OutboxIntentBundlesReader: Send + 'static { type Error: Send; - /// Reads `n` outbox intents - /// Returns `OutboxIntentBundle` in ascending order by `ScheduledIntentBundle::id` - /// If Vec::len != n, that means that there's no more Outbox intents + /// Returns up to `n` outbox intents sorted ascending by `ScheduledIntentBundle::id`. + /// If `Vec::len() < n`, no more intents are available. async fn read( &mut self, - n: NonZeroUsize, + n: usize, ) -> Result, Self::Error>; } pub struct InternalOutboxIntentBundlesReader { - /// Current intent id pos - intent_id_pos: Option, + /// Capacity of the buffer + capacity: NonZeroUsize, + /// Sorted ascending by intent id, drained from front on read() + buffer: VecDeque, + /// Max id returned so far; refill skips ids <= this + last_consumed_id: Option, /// Accounts DB where we read intents from /// Could be an RpcClient in the future accounts_db: Arc, @@ -31,44 +39,141 @@ pub struct InternalOutboxIntentBundlesReader { impl InternalOutboxIntentBundlesReader { const TARGET_PROGRAM_ID: Pubkey = magicblock_program::ID; - const ACCOUNT_DISCRIMINATOR: [u8; 8] = todo!(); - pub fn new(accounts_db: Arc) -> Self { + pub fn new(accounts_db: Arc, capacity: NonZeroUsize) -> Self { Self { - // Uninitialized state - intent_id_pos: None, + capacity, + buffer: VecDeque::new(), + last_consumed_id: None, accounts_db, } } - pub fn initialize_intent_pos( - &mut self, - ) -> Result<(), InternalOutboxIntentBundlesReader> { - let asd = self - .accounts_db - .get_program_accounts(&Self::TARGET_PROGRAM_ID, |account| { - if !account.data().starts_with(&Self::ACCOUNT_DISCRIMINATOR) { - return false; - } + // Refills buffer with OutboxIntents up to capacity + fn refill(&mut self) -> Result<(), AccountsDbError> { + #[repr(transparent)] + struct OrderedIntent { + inner: OutboxIntentBundle, + } + impl Eq for OrderedIntent {} + impl PartialEq for OrderedIntent { + fn eq(&self, other: &Self) -> bool { + self.inner.id.eq(&other.inner.id) + } + } + impl PartialOrd for OrderedIntent { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + impl Ord for OrderedIntent { + fn cmp(&self, other: &Self) -> Ordering { + self.inner.id.cmp(&other.inner.id) + } + } - true + let outbox_candidates_iter = self.accounts_db.get_program_accounts( + &magicblock_program::ID, + Self::outbox_accounts_filter, + )?; + + // Create iterator that yields valid, unconsumed intents + let outbox_iter = outbox_candidates_iter + .map(|(address, account)| { + (address, OutboxIntentBundle::try_from_bytes(account.data())) + }) + // Filter out failed deserializations + warn + .filter_map(|(pubkey, account_result)| match account_result { + Ok(value) => Some(value), + Err(err) => { + warn!(error = ?err, "Account with OutboxIntentBundle failed to deserialize, pubkey; {}", pubkey); + None + } }) - .unwrap(); - todo!() + // Filter out already consumed intents + .filter(|outbox_intent| if let Some(last_consumed_id) = self.last_consumed_id { + outbox_intent.id > last_consumed_id + } else { + true + }); + + // Retain only `capacity` of smallest id's intents + let capacity = self.capacity.get(); + let mut heap: BinaryHeap = + BinaryHeap::with_capacity(capacity); + for outbox_intent_bundle in outbox_iter { + heap.push(OrderedIntent { + inner: outbox_intent_bundle, + }); + if heap.len() > capacity { + heap.pop(); + } + } + + // SAFETY: OrderedIntent is #[repr(transparent)] over OutboxIntentBundle — + // identical memory layout, zero-copy cast + let mut items: Vec = unsafe { + let mut v = std::mem::ManuallyDrop::new(heap.into_vec()); + Vec::from_raw_parts( + v.as_mut_ptr() as *mut OutboxIntentBundle, + v.len(), + v.capacity(), + ) + }; + items.sort_unstable_by_key(|b| b.id); + self.last_consumed_id = + items.last().map(|b| b.id).or(self.last_consumed_id); + self.buffer.extend(items); + Ok(()) + } + + /// Filter for OutboxIntentBundle accounts, those start with OUTBOX_INTENT_DISCRIMINATOR + fn outbox_accounts_filter(account: &AccountSharedData) -> bool { + account.data().starts_with(&OUTBOX_INTENT_DISCRIMINATOR) } } #[async_trait] impl OutboxIntentBundlesReader for InternalOutboxIntentBundlesReader { - type Error = InternalOutboxIntentBundlesReaderError; + type Error = OutboxIntentBundlesReaderError; + /// Returns up to `n` outbox intents sorted ascending by `ScheduledIntentBundle::id`. + /// If `Vec::len() < n`, no more intents are available. + /// + /// When the internal buffer runs low, triggers a full `getProgramAccounts` scan + /// to refill — O(N log C) where N is open intent count and C is capacity. + /// Callers recommended to drive each read batch to completion and close the accounts + /// before reading again; closed accounts are removed from the DB and won't + /// be scanned on the next refill. async fn read( &mut self, - n: NonZeroUsize, + n: usize, ) -> Result, Self::Error> { - todo!() + if n == 0 { + return Ok(vec![]); + } + if n > self.capacity.get() { + return Err(Self::Error::ReadExceedsCapacityError(n, capacity)); + } + + // Refill if buffer runs low + if self.buffer.len() < n { + self.refill()?; + } + + // Take available amount + let take = n.min(self.buffer.len()); + Ok(self.buffer.drain(..take).collect()) } } #[derive(thiserror::Error, Debug)] -pub enum InternalOutboxIntentBundlesReaderError {} +pub enum OutboxIntentBundlesReaderError { + #[error("Requested read exceeded capacity, n: {0}, capacity: {1}")] + ReadExceedsCapacityError(usize, usize), + #[error("AccountsDbError: {0}")] + AccountsDbError(#[from] AccountsDbError), +} + +pub type OutboxIntentBundlesReaderResult = + Result; From c9225f6431dd5d266838363e4366e29578a0da7e Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 16 Jun 2026 16:31:04 +0700 Subject: [PATCH 49/97] feat: added tests for OutboxReader --- .../service/outbox_intent_bundles_reader.rs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index 95fc3e4e0..e99ab2d48 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -177,3 +177,131 @@ pub enum OutboxIntentBundlesReaderError { pub type OutboxIntentBundlesReaderResult = Result; + +#[cfg(test)] +mod tests { + use std::{num::NonZeroUsize, sync::Arc}; + + use magicblock_accounts_db::{AccountsDb, AccountsDbConfig}; + use magicblock_core::intent::outbox::outbox_intent_pda; + use magicblock_program::{ + intent_bundles::{ + magic_scheduled_base_intent::{ + MagicIntentBundle, ScheduledIntentBundle, + }, + outbox_intent_bundles::OutboxIntentBundle, + }, + }; + use solana_account::{AccountSharedData, WritableAccount}; + use solana_hash::Hash; + use solana_pubkey::Pubkey; + use solana_transaction::Transaction; + + use super::{InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader}; + + fn make_db() -> (Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("temp dir"); + let db = AccountsDb::new(&AccountsDbConfig::default(), dir.path(), 0) + .expect("db init") + .into(); + (db, dir) + } + + fn make_bundle(id: u64) -> OutboxIntentBundle { + let inner = ScheduledIntentBundle { + id, + slot: 0, + blockhash: Hash::default(), + sent_transaction: Transaction::default(), + payer: Pubkey::default(), + intent_bundle: MagicIntentBundle::default(), + }; + OutboxIntentBundle::accepted(inner) + } + + fn insert_bundle(db: &AccountsDb, bundle: &OutboxIntentBundle) { + let bytes = bundle.try_to_bytes().expect("serialize"); + let pubkey = outbox_intent_pda(bundle.inner.id); + let mut account = + AccountSharedData::new(1, bytes.len(), &magicblock_program::ID); + account.data_as_mut_slice().copy_from_slice(&bytes); + db.insert_account(&pubkey, &account).expect("insert"); + } + + #[tokio::test] + async fn read_returns_ascending_order() { + let (db, _dir) = make_db(); + insert_bundle(&db, &make_bundle(9)); + insert_bundle(&db, &make_bundle(1)); + insert_bundle(&db, &make_bundle(4)); + + let mut reader = InternalOutboxIntentBundlesReader::new( + db, + NonZeroUsize::new(10).unwrap(), + ); + let result = reader.read(3).await.unwrap(); + assert_eq!(result.len(), 3); + assert_eq!(result[0].inner.id, 1); + assert_eq!(result[1].inner.id, 4); + assert_eq!(result[2].inner.id, 9); + } + + #[tokio::test] + async fn read_fewer_than_n_when_db_has_less() { + let (db, _dir) = make_db(); + insert_bundle(&db, &make_bundle(3)); + + let mut reader = InternalOutboxIntentBundlesReader::new( + db, + NonZeroUsize::new(10).unwrap(), + ); + let result = reader.read(5).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].inner.id, 3); + } + + #[tokio::test] + async fn read_respects_capacity_smallest() { + let (db, _dir) = make_db(); + for id in [10, 5, 1, 8, 3] { + insert_bundle(&db, &make_bundle(id)); + } + + let mut reader = InternalOutboxIntentBundlesReader::new( + db, + NonZeroUsize::new(3).unwrap(), + ); + let result = reader.read(3).await.unwrap(); + assert_eq!(result.iter().map(|b| b.inner.id).collect::>(), vec![1, 3, 5]); + } + + #[tokio::test] + async fn read_exceeds_capacity_errors() { + let (db, _dir) = make_db(); + let mut reader = InternalOutboxIntentBundlesReader::new( + db, + NonZeroUsize::new(3).unwrap(), + ); + assert!(reader.read(4).await.is_err()); + } + + #[tokio::test] + async fn sequential_reads_dont_repeat() { + let (db, _dir) = make_db(); + for id in 1..=6 { + insert_bundle(&db, &make_bundle(id)); + } + + let mut reader = InternalOutboxIntentBundlesReader::new( + db, + NonZeroUsize::new(5).unwrap(), + ); + let first = reader.read(3).await.unwrap(); + let second = reader.read(3).await.unwrap(); + + let first_ids: Vec<_> = first.iter().map(|b| b.inner.id).collect(); + let second_ids: Vec<_> = second.iter().map(|b| b.inner.id).collect(); + assert_eq!(first_ids, vec![1, 2, 3]); + assert_eq!(second_ids, vec![4, 5, 6]); + } +} From 898fab2f4e31ec0b53f6a9ea59c94cb42466355b Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 16 Jun 2026 18:55:49 +0700 Subject: [PATCH 50/97] fix: engine tests --- .../intent_execution_engine.rs | 117 +++++++++++++----- .../src/outbox_client.rs | 8 +- magicblock-committor-service/src/service.rs | 2 +- .../service/outbox_intent_bundles_reader.rs | 22 ++-- .../src/tasks/task_strategist.rs | 1 - 5 files changed, 101 insertions(+), 49 deletions(-) diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 24f550151..6ba342644 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -343,6 +343,8 @@ mod tests { }; use async_trait::async_trait; + use solana_keypair::Keypair; + use solana_message::VersionedMessage; use solana_pubkey::{pubkey, Pubkey}; use solana_signature::Signature; use solana_signer::SignerError; @@ -357,18 +359,57 @@ mod tests { intent_scheduler::{create_test_intent, create_test_intent_bundle}, }, intent_executor::{ + cleanup_handle::CleanupHandle, error::{IntentExecutorError as ExecutorError, InternalError}, + intent_executor_factory::IntentExecutorBuilder, IntentExecutionResult, }, + tasks::task_strategist::TransactionStrategy, test_utils, - transaction_preparator::delivery_preparator::BufferExecutionError, + transaction_preparator::{ + delivery_preparator::{ + BufferExecutionError, DeliveryPreparatorResult, + }, + error::PreparatorResult, + TransactionPreparator, + }, }; - type MockIntentExecutionEngine = - IntentExecutionEngine; + struct MockTransactionPreparator; + + #[async_trait] + impl TransactionPreparator for MockTransactionPreparator { + async fn prepare_for_strategy( + &self, + _authority: &Keypair, + _transaction_strategy: &mut TransactionStrategy, + ) -> PreparatorResult { + unimplemented!() + } + + async fn cleanup_for_strategy( + &self, + _authority: &Keypair, + _tasks: &[crate::tasks::BaseTaskImpl], + _lookup_table_keys: &[Pubkey], + _close_buffers: bool, + ) -> DeliveryPreparatorResult<(), BufferExecutionError> { + Ok(()) + } + } + + type MockIntentExecutionEngine = IntentExecutionEngine< + DummyDB, + MockIntentExecutorFactory, + MockTransactionPreparator, + >; fn setup_engine( should_fail: bool, - ) -> (IntentScheduleHandle, MockIntentExecutionEngine) { + ) -> ( + IntentScheduleHandle, + MockIntentExecutionEngine, + Arc>, + ) { test_utils::init_test_logger(); let db = Arc::new(Mutex::new(DummyDB::new())); @@ -381,12 +422,12 @@ mod tests { let worker = IntentExecutionEngine::new(intent_stream, executor_factory); - (handle, worker) + (handle, worker, db) } #[tokio::test] async fn test_worker_processes_messages() { - let (sender, worker) = setup_engine(false); + let (sender, worker, _db) = setup_engine(false); let result_subscriber = worker.spawn(); let mut result_receiver = result_subscriber.subscribe(); @@ -406,7 +447,7 @@ mod tests { #[tokio::test] async fn test_worker_handles_conflicting_messages() { - let (sender, worker) = setup_engine(false); + let (sender, worker, _db) = setup_engine(false); let result_subscriber = worker.spawn(); let mut result_receiver = result_subscriber.subscribe(); @@ -415,8 +456,8 @@ mod tests { let msg1 = create_test_intent(1, &[pubkey], false); let msg2 = create_test_intent(2, &[pubkey], false); - sender.send(msg1.clone()).await.unwrap(); - sender.send(msg2.clone()).await.unwrap(); + sender.schedule(vec![msg1.clone()]).unwrap(); + sender.schedule(vec![msg2.clone()]).unwrap(); // First message should be processed immediately let result1 = result_receiver.recv().await.unwrap(); @@ -431,7 +472,7 @@ mod tests { #[tokio::test] async fn test_worker_handles_conflicting_bundles() { - let (sender, worker) = setup_engine(false); + let (sender, worker, _db) = setup_engine(false); let result_subscriber = worker.spawn(); let mut result_receiver = result_subscriber.subscribe(); @@ -441,8 +482,8 @@ mod tests { let msg1 = create_test_intent_bundle(1, &[a], &[b]); let msg2 = create_test_intent(2, &[a], false); - sender.send(msg1.clone()).await.unwrap(); - sender.send(msg2.clone()).await.unwrap(); + sender.schedule(vec![msg1.clone()]).unwrap(); + sender.schedule(vec![msg2.clone()]).unwrap(); // First message should be processed immediately let result1 = result_receiver.recv().await.unwrap(); @@ -457,7 +498,7 @@ mod tests { #[tokio::test] async fn test_worker_handles_executor_failure() { - let (sender, worker) = setup_engine(true); + let (sender, worker, _db) = setup_engine(true); let result_subscriber = worker.spawn(); let mut result_receiver = result_subscriber.subscribe(); @@ -485,7 +526,7 @@ mod tests { #[tokio::test] async fn test_worker_falls_back_to_db_when_channel_empty() { - let (_sender, worker) = setup_engine(false); + let (_sender, worker, db) = setup_engine(false); // Add a message to the DB let msg = create_test_intent( @@ -493,7 +534,7 @@ mod tests { &[pubkey!("1111111111111111111111111111111111111111111")], false, ); - worker.db.store_intent_bundle(msg.clone()).await.unwrap(); + db.lock().unwrap().store_intent_bundle(msg.clone()).unwrap(); // Start worker let result_subscriber = worker.spawn(); @@ -510,7 +551,7 @@ mod tests { async fn test_high_throughput_message_processing() { const NUM_MESSAGES: usize = 20; - let (sender, mut worker) = setup_engine(false); + let (sender, mut worker, _db) = setup_engine(false); let active_tasks = Arc::new(AtomicUsize::new(0)); let max_concurrent = Arc::new(AtomicUsize::new(0)); @@ -552,7 +593,7 @@ mod tests { /// Tests that errors from executor propagated gracefully #[tokio::test] async fn test_multiple_failures() { - let (sender, worker) = setup_engine(true); // Worker that always fails + let (sender, worker, _db) = setup_engine(true); // Worker that always fails let result_subscriber = worker.spawn(); let mut result_receiver = result_subscriber.subscribe(); @@ -578,7 +619,7 @@ mod tests { async fn test_non_blocking_messages() { const NUM_MESSAGES: u64 = 200; - let (sender, mut worker) = setup_engine(false); + let (sender, mut worker, _db) = setup_engine(false); let active_tasks = Arc::new(AtomicUsize::new(0)); let max_concurrent = Arc::new(AtomicUsize::new(0)); @@ -636,7 +677,7 @@ mod tests { // 30% blocking messages const BLOCKING_RATIO: f32 = 0.3; - let (sender, mut worker) = setup_engine(false); + let (sender, mut worker, _db) = setup_engine(false); let active_tasks = Arc::new(AtomicUsize::new(0)); let max_concurrent = Arc::new(AtomicUsize::new(0)); @@ -714,15 +755,18 @@ mod tests { } } - impl IntentExecutorBuilder for MockIntentExecutorFactory { - type Executor = MockIntentExecutor; - - fn create_instance(&self) -> Self::Executor { - MockIntentExecutor { + impl IntentExecutorBuilder + for MockIntentExecutorFactory + { + fn create_instance( + &self, + _status: magicblock_program::outbox_intent_bundles::OutboxIntentBundleStatus, + ) -> Box> { + Box::new(MockIntentExecutor { should_fail: self.should_fail, active_tasks: self.active_tasks.clone(), max_concurrent: self.max_concurrent.clone(), - } + }) } } @@ -764,11 +808,14 @@ mod tests { } #[async_trait] - impl IntentExecutor for MockIntentExecutor { + impl IntentExecutor for MockIntentExecutor { async fn execute( - &mut self, - _base_intent: OutboxIntentBundle, - ) -> IntentExecutionResult { + self: Box, + _base_intent: magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle, + ) -> ( + IntentExecutionResult, + CleanupHandle, + ) { self.on_task_started(); // Simulate some work @@ -805,11 +852,13 @@ mod tests { self.on_task_finished(); - result - } - - async fn cleanup(self) -> Result<(), BufferExecutionError> { - Ok(()) + let cleanup = CleanupHandle::new( + Keypair::new(), + vec![], + false, + MockTransactionPreparator, + ); + (result, cleanup) } } } diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index e638d7979..7c136b5e5 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{num::NonZeroUsize, sync::Arc}; use async_trait::async_trait; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; @@ -152,7 +152,11 @@ impl OutboxClient for InternalOutboxClient { } fn outbox_reader(&self) -> Self::OutboxReader { - InternalOutboxIntentBundlesReader::new(self.accounts_db.clone()) + const CAPACITY: NonZeroUsize = NonZeroUsize::new(1000).unwrap(); + InternalOutboxIntentBundlesReader::new( + self.accounts_db.clone(), + CAPACITY, + ) } } diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index f832f4408..5b7ffda7b 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -218,7 +218,7 @@ where loop { // Read by chunks in order not to overload `IntentExecutionEngine` let intent_bundles_chunk = outbox_bundles_reader - .read(RESCHEDULE_CHUNK_SIZE) + .read(RESCHEDULE_CHUNK_SIZE.get()) .await .map_err(Into::into)?; if intent_bundles_chunk.is_empty() { diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index e99ab2d48..865290334 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -152,7 +152,8 @@ impl OutboxIntentBundlesReader for InternalOutboxIntentBundlesReader { if n == 0 { return Ok(vec![]); } - if n > self.capacity.get() { + let capacity = self.capacity.get(); + if n > capacity { return Err(Self::Error::ReadExceedsCapacityError(n, capacity)); } @@ -182,15 +183,13 @@ pub type OutboxIntentBundlesReaderResult = mod tests { use std::{num::NonZeroUsize, sync::Arc}; - use magicblock_accounts_db::{AccountsDb, AccountsDbConfig}; + use magicblock_accounts_db::AccountsDb; use magicblock_core::intent::outbox::outbox_intent_pda; use magicblock_program::{ - intent_bundles::{ - magic_scheduled_base_intent::{ - MagicIntentBundle, ScheduledIntentBundle, - }, - outbox_intent_bundles::OutboxIntentBundle, + magic_scheduled_base_intent::{ + MagicIntentBundle, ScheduledIntentBundle, }, + outbox_intent_bundles::OutboxIntentBundle, }; use solana_account::{AccountSharedData, WritableAccount}; use solana_hash::Hash; @@ -201,9 +200,7 @@ mod tests { fn make_db() -> (Arc, tempfile::TempDir) { let dir = tempfile::tempdir().expect("temp dir"); - let db = AccountsDb::new(&AccountsDbConfig::default(), dir.path(), 0) - .expect("db init") - .into(); + let db = AccountsDb::open(dir.path()).expect("db init").into(); (db, dir) } @@ -272,7 +269,10 @@ mod tests { NonZeroUsize::new(3).unwrap(), ); let result = reader.read(3).await.unwrap(); - assert_eq!(result.iter().map(|b| b.inner.id).collect::>(), vec![1, 3, 5]); + assert_eq!( + result.iter().map(|b| b.inner.id).collect::>(), + vec![1, 3, 5] + ); } #[tokio::test] diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 81776ff04..8cb262f12 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -399,7 +399,6 @@ mod tests { intent_executor::task_info_fetcher::{ TaskInfoFetcher, TaskInfoFetcherResult, }, - persist::IntentPersisterImpl, tasks::{ commit_task::CommitTask, task_builder::{ From 5ca4419156cdca7039365d1f918e788d77f162b2 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 16 Jun 2026 18:58:27 +0700 Subject: [PATCH 51/97] fix: task strategist tests --- .../src/tasks/task_strategist.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 8cb262f12..c8a5111ec 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -744,7 +744,6 @@ mod tests { let commit_task = TaskBuilderImpl::commit_tasks( &info_fetcher, &intent, - &None::, ) .await .unwrap(); @@ -757,7 +756,6 @@ mod tests { commit_task, finalize_task, &Pubkey::new_unique(), - &None::, ) .expect("Execution mode created"); @@ -776,7 +774,6 @@ mod tests { let commit_task = TaskBuilderImpl::commit_tasks( &info_fetcher, &intent, - &None::, ) .await .unwrap(); @@ -789,19 +786,14 @@ mod tests { commit_task, finalize_task, &Pubkey::new_unique(), - &None::, ) .expect("Execution mode created"); - let StrategyExecutionMode::TwoStage { - commit_stage, - finalize_stage, - } = execution_mode - else { + let StrategyExecutionMode::TwoStage(two_stage) = execution_mode else { panic!("Unexpected execution mode"); }; - assert!(!commit_stage.uses_alts()); - assert!(!finalize_stage.uses_alts()); + assert!(!two_stage.commit_stage.uses_alts()); + assert!(!two_stage.finalize_stage.uses_alts()); } #[tokio::test] @@ -813,7 +805,6 @@ mod tests { let commit_task = TaskBuilderImpl::commit_tasks( &info_fetcher, &intent, - &None::, ) .await .unwrap(); @@ -826,7 +817,6 @@ mod tests { commit_task, finalize_task, &Pubkey::new_unique(), - &None::, ) .expect("Execution mode created"); From 363c96d63026ce82bc84717eac7200799bef0610 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 19 Jun 2026 15:01:11 +0700 Subject: [PATCH 52/97] wip --- Cargo.lock | 52 +++++-- Cargo.toml | 1 + magicblock-committor-service/Cargo.toml | 1 + .../src/outbox_client.rs | 140 ++++++++++++++---- .../src/tasks/task_strategist.rs | 27 ++-- programs/magicblock/src/magic_context.rs | 37 +++-- .../magicblock/src/utils/instruction_utils.rs | 22 ++- test-integration/Cargo.lock | 56 +++++-- 8 files changed, 249 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9137735f8..57654fc83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,7 +264,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -275,7 +275,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -760,6 +760,20 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.17", + "instant", + "pin-project-lite", + "rand 0.8.6", + "tokio", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -808,7 +822,7 @@ dependencies = [ "bitflags 2.11.1", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "proc-macro2", @@ -828,7 +842,7 @@ dependencies = [ "bitflags 2.11.1", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -1949,7 +1963,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3088,6 +3102,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -3890,6 +3913,7 @@ name = "magicblock-committor-service" version = "0.12.3" dependencies = [ "async-trait", + "backoff", "bincode", "borsh", "futures-util", @@ -4600,7 +4624,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5196,7 +5220,7 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", "heck", - "itertools 0.10.5", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -5216,7 +5240,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "petgraph 0.8.3", @@ -5237,7 +5261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.117", @@ -5250,7 +5274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5943,7 +5967,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6536,7 +6560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9844,7 +9868,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10895,7 +10919,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 26e005f89..4bd95a648 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ agave-geyser-plugin-interface = { version = "3.1" } agave-precompiles = { version = "=3.1.12" } agave-syscalls = { version = "=3.1.12" } anyhow = "1.0.86" +backoff = { version = "0.4", features = ["tokio"] } arc-swap = { version = "1.7" } assert_matches = "1.5.0" async-nats = "0.46" diff --git a/magicblock-committor-service/Cargo.toml b/magicblock-committor-service/Cargo.toml index ea8dce441..1a32777cb 100644 --- a/magicblock-committor-service/Cargo.toml +++ b/magicblock-committor-service/Cargo.toml @@ -12,6 +12,7 @@ doctest = false [dependencies] async-trait = { workspace = true } +backoff = { workspace = true } bincode = { workspace = true } borsh = { workspace = true } futures-util = { workspace = true } diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index 7c136b5e5..eb5255891 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -1,18 +1,35 @@ -use std::{num::NonZeroUsize, sync::Arc}; +use std::{ + num::{NonZeroU64, NonZeroUsize}, + sync::Arc, + time::Duration, +}; use async_trait::async_trait; +use backoff::{future::retry, ExponentialBackoff}; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_core::{ - link::transactions::{with_encoded, TransactionSchedulerHandle}, + intent::outbox::outbox_intent_pda, + link::{ + accounts::LockedAccount, + transactions::{with_encoded, TransactionSchedulerHandle}, + }, traits::LatestBlockProvider, }; use magicblock_program::{ instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, - register_scheduled_commit_sent, MagicContext, SentCommit, - TransactionScheduler, MAGIC_CONTEXT_PUBKEY, + outbox_intent_bundles::OutboxIntentBundle, register_scheduled_commit_sent, + MagicContext, Pubkey, SentCommit, TransactionScheduler, + MAGIC_CONTEXT_PUBKEY, +}; +use solana_account::{Account, ReadableAccount}; +use solana_rpc_client::{ + nonblocking::rpc_client::RpcClient, rpc_client::SerializableTransaction, +}; +use solana_rpc_client_api::{ + client_error, + client_error::{Error as RpcClientError, ErrorKind as RpcClientErrorKind}, }; -use solana_account::ReadableAccount; use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tracing::{debug, error}; @@ -58,6 +75,8 @@ pub trait OutboxClient: Send + Sync + 'static { pub struct InternalOutboxClient { /// Provides access to MagicContext accounts_db: Arc, + /// RPC client for sending accept transactions to the ER + rpc_client: Arc, /// Internal endpoint for scheduling ER TXs transaction_scheduler: TransactionSchedulerHandle, /// Provides access to ER latest block for TX creation @@ -67,34 +86,98 @@ pub struct InternalOutboxClient { impl InternalOutboxClient { pub fn new( accounts_db: Arc, + rpc_client: Arc, transaction_scheduler: TransactionSchedulerHandle, latest_block_provider: L, ) -> Self { Self { accounts_db, + rpc_client, transaction_scheduler, latest_block_provider, } } - /// Sends transaction to move the scheduled commits from the `MagicContext` - /// to the global ScheduledCommit store - async fn send_accept_tx(&self) -> Result<(), InternalOutboxClientError> { - let tx = InstructionUtils::accept_scheduled_commits( - self.latest_block_provider.blockhash(), - ); - let encoded_tx = with_encoded(tx).inspect_err(|err| { - error!(error = ?err, "Failed to bincode intent transaction"); - })?; - self.transaction_scheduler - .execute(encoded_tx) - .await - .inspect_err(|err| { - error!(error = ?err, "Failed to accept scheduled commits"); - })?; + async fn send_with_backoff( + &self, + backoff_config: ExponentialBackoff, + tx: &impl SerializableTransaction, + ) -> Result<(), client_error::Error> { + let signature = tx.get_signature(); + retry(backoff_config, || async { + self.rpc_client + .send_and_confirm_transaction(tx) + .await + .map_err(|err| { + match err.kind() { + RpcClientErrorKind::TransactionError(_) => { + backoff::Error::Permanent(err) + } + _ => { + error!(signature = ?signature, error = ?err, "Transient error accepting intents, retrying"); + backoff::Error::transient(err) + } + } + }) + }).await?; Ok(()) } + + /// Sends `AcceptScheduledCommits` transactions to the ER, moving scheduled + /// commits from `MagicContext` into outbox PDA accounts, up to CHUNK_SIZE intents per transaction. + /// On first error returns the successfully accepted intents so far alongside the error. + async fn send_accept_tx( + &self, + scheduled_intents: Vec, + ) -> Result, (Vec, InternalOutboxClientError)> { + const CHUNK_SIZE: usize = 50; + + let mut remaining = scheduled_intents; + let mut accepted = Vec::with_capacity(remaining.len()); + while !remaining.is_empty() { + let chunk_size = CHUNK_SIZE.min(remaining.len()); + let tx = InstructionUtils::accept_scheduled_commits( + self.latest_block_provider.blockhash(), + remaining[..chunk_size].iter().map(|i| i.id), + ); + let backoff_config = ExponentialBackoff { + max_elapsed_time: Some(Duration::from_secs(25)), + max_interval: Duration::from_secs(5), + ..ExponentialBackoff::default() + }; + match self.send_with_backoff(backoff_config, &tx).await { + Ok(_) => accepted.extend(remaining.drain(..chunk_size)), + Err(err) => return Err((accepted, err.into())), + } + } + + Ok(accepted) + } + + /// Returns account corresponding to `pubkey` if it exists + /// Safely handles race-conditions and any concurrent changes to an account + fn safe_get_account(&self, pubkey: &Pubkey) -> Option { + let shared_account = self.accounts_db.get_account(pubkey)?; + let locked_account = LockedAccount::new(*pubkey, shared_account); + let output = locked_account + .read_locked(|pubkey, account| Account::from(account.clone())); + + Some(output) + } + + /// Returns just accepted OutboxIntents + fn read_outbox_intent( + &self, + intent_id: u64, + ) -> InternalOutboxClientResult> { + let pda = outbox_intent_pda(intent_id); + let Some(account) = self.safe_get_account(&pda) else { + return Ok(None); + }; + + todo!() + } } #[async_trait] @@ -104,17 +187,16 @@ impl OutboxClient for InternalOutboxClient { async fn accept_scheduled_intents( &self, - ) -> Result, Self::Error> { + ) -> Result, (Vec, Self::Error)> { // If accounts were scheduled to be committed, we accept them here // and processs the commits let magic_context_acc = - self.accounts_db.get_account(&MAGIC_CONTEXT_PUBKEY).expect( + self.safe_get_account(&MAGIC_CONTEXT_PUBKEY).expect( "Validator found to be running without MagicContext account!", ); - if !MagicContext::has_scheduled_commits(magic_context_acc.data()) { - return Ok(vec![]); - } - self.send_accept_tx().await?; + + let magic_context = MagicContext::deserialize(magic_context_acc.data())?; + self.send_accept_tx(magic_context.scheduled_base_intents).await?; // Return intents from global store Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) @@ -164,4 +246,10 @@ impl OutboxClient for InternalOutboxClient { pub enum InternalOutboxClientError { #[error("TransactionError: {0}")] TransactionError(#[from] TransactionError), + #[error("RpcClientError: {0}")] + RpcClientError(#[from] client_error::Error), + #[error("BincodeError: {0}")] + BincodeError(#[from] bincode::Error), } + +pub type InternalOutboxClientResult = Result; diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index c8a5111ec..b388b86dc 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -741,12 +741,9 @@ mod tests { let intent = create_test_intent(0, &pubkey, false); let info_fetcher = Arc::new(MockInfoFetcher); - let commit_task = TaskBuilderImpl::commit_tasks( - &info_fetcher, - &intent, - ) - .await - .unwrap(); + let commit_task = TaskBuilderImpl::commit_tasks(&info_fetcher, &intent) + .await + .unwrap(); let finalize_task = TaskBuilderImpl::finalize_tasks(&info_fetcher, &intent) .await @@ -771,12 +768,9 @@ mod tests { let intent = create_test_intent(0, &pubkeys, true); let info_fetcher = Arc::new(MockInfoFetcher); - let commit_task = TaskBuilderImpl::commit_tasks( - &info_fetcher, - &intent, - ) - .await - .unwrap(); + let commit_task = TaskBuilderImpl::commit_tasks(&info_fetcher, &intent) + .await + .unwrap(); let finalize_task = TaskBuilderImpl::finalize_tasks(&info_fetcher, &intent) .await @@ -802,12 +796,9 @@ mod tests { let intent = create_test_intent(0, &pubkeys, false); let info_fetcher = Arc::new(MockInfoFetcher); - let commit_task = TaskBuilderImpl::commit_tasks( - &info_fetcher, - &intent, - ) - .await - .unwrap(); + let commit_task = TaskBuilderImpl::commit_tasks(&info_fetcher, &intent) + .await + .unwrap(); let finalize_task = TaskBuilderImpl::finalize_tasks(&info_fetcher, &intent) .await diff --git a/programs/magicblock/src/magic_context.rs b/programs/magicblock/src/magic_context.rs index bf9a54a20..7daad1ad5 100644 --- a/programs/magicblock/src/magic_context.rs +++ b/programs/magicblock/src/magic_context.rs @@ -16,7 +16,7 @@ impl MagicContext { pub const SIZE: usize = MAGIC_CONTEXT_SIZE; pub const ZERO: [u8; Self::SIZE] = [0; Self::SIZE]; - pub(crate) fn deserialize(data: &[u8]) -> Result { + pub fn deserialize(data: &[u8]) -> Result { if data.is_empty() || is_zeroed(data) { Ok(Self::default()) } else { @@ -67,20 +67,39 @@ impl MagicContext { self.scheduled_base_intents.drain(..n).collect() } - pub fn has_scheduled_commits(data: &[u8]) -> bool { - const LEN_OFF: usize = mem::size_of::(); - const LEN_END: usize = LEN_OFF + mem::size_of::(); + /// Returns `intent_id` store in `MagicContext` without deserializing whole account + pub fn intent_id(data: &[u8]) -> u64 { + const ID_OFFSET: usize = 0; + const ID_END: usize = mem::size_of::(); + + // TODO(edwin): should exist, return Option and cast to error + let Some(raw_id) = data.get(ID_OFFSET..ID_END) else { + return 0; + }; + + let mut buf = [0; mem::size_of::()]; + buf.copy_from_slice(raw_id); + u64::from_le_bytes(buf) + } + + pub fn scheduled_intents_len(data: &[u8]) -> u64 { + const LEN_OFFSET: usize = mem::size_of::(); + const LEN_END: usize = LEN_OFFSET + mem::size_of::(); if is_zeroed(data) { - return false; + return 0; } - - let Some(raw_len) = data.get(LEN_OFF..LEN_END) else { - return false; + let Some(raw_len) = data.get(LEN_OFFSET..LEN_END) else { + return 0; }; + let mut len = [0; mem::size_of::()]; len.copy_from_slice(raw_len); - u64::from_le_bytes(len) != 0 + u64::from_le_bytes(len) + } + + pub fn has_scheduled_intents(data: &[u8]) -> bool { + Self::scheduled_intents_len(data) != 0 } } diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index bd289fe99..547d5b74f 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -3,6 +3,7 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, }; +use magicblock_core::intent::outbox::outbox_intent_pda; use magicblock_magic_program_api::{ args::ScheduleTaskArgs, instruction::{ @@ -150,16 +151,29 @@ impl InstructionUtils { // ----------------- // Accept Scheduled Commits // ----------------- - pub fn accept_scheduled_commits(recent_blockhash: Hash) -> Transaction { - let ix = Self::accept_scheduled_commits_instruction(); + pub fn accept_scheduled_commits( + recent_blockhash: Hash, + intent_ids: impl IntoIterator, + ) -> Transaction { + let ix = Self::accept_scheduled_commits_instruction(intent_ids); Self::into_transaction(&validator_authority(), ix, recent_blockhash) } - pub(crate) fn accept_scheduled_commits_instruction() -> Instruction { - let account_metas = vec![ + pub(crate) fn accept_scheduled_commits_instruction( + intent_ids: impl IntoIterator, + ) -> Instruction { + let mut account_metas = vec![ AccountMeta::new_readonly(validator_authority_id(), true), AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), ]; + + // Add outbox intent accounts + let outbox_intent_metas = intent_ids + .into_iter() + .map(|intent_id| outbox_intent_pda(intent_id)) + .map(|intent_pda| AccountMeta::new(intent_pda, false)); + account_metas.extend(outbox_intent_metas); + Instruction::new_with_bincode( crate::id(), &MagicBlockInstruction::AcceptScheduleCommits, diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 771b01131..c190ab71f 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -396,7 +396,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -407,7 +407,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -997,6 +997,20 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.16", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -1066,7 +1080,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "proc-macro2", @@ -1086,7 +1100,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -2375,7 +2389,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3579,6 +3593,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "integration-test-tools" version = "0.0.0" @@ -4436,6 +4459,7 @@ name = "magicblock-committor-service" version = "0.12.3" dependencies = [ "async-trait", + "backoff", "bincode", "borsh", "futures-util", @@ -5190,7 +5214,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5958,8 +5982,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -5978,8 +6002,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.13.0", "log", "multimap", "petgraph 0.8.3", @@ -6000,7 +6024,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.117", @@ -6013,7 +6037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -6737,7 +6761,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6838,7 +6862,7 @@ dependencies = [ "security-framework 3.5.1", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11101,7 +11125,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -12378,7 +12402,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] From f96a9b467d880a3d4fea742dd56a9db6b82ad049 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 19 Jun 2026 18:10:16 +0700 Subject: [PATCH 53/97] refactor: move TransactionExecutioError into separate fild --- .../intent_execution_engine.rs | 2 +- .../src/intent_executor/error.rs | 267 +----------------- .../intent_execution_client.rs | 4 +- .../src/intent_executor/mod.rs | 3 +- .../strategy_executor/error.rs | 251 ++++++++++++++++ .../intent_executor/strategy_executor/mod.rs | 1 + .../strategy_executor/patcher.rs | 2 +- .../strategy_executor/utils.rs | 2 +- .../src/outbox_client.rs | 24 +- magicblock-committor-service/src/service.rs | 15 +- .../tests/test_intent_executor.rs | 4 +- 11 files changed, 292 insertions(+), 283 deletions(-) create mode 100644 magicblock-committor-service/src/intent_executor/strategy_executor/error.rs diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 6ba342644..5cd49d597 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -26,13 +26,13 @@ use crate::{ intent_executor::{ error::{ IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, }, intent_executor_factory::IntentExecutorBuilder, ExecutionOutput, IntentExecutionResult, IntentExecutor, }, transaction_preparator::TransactionPreparator, }; +use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; const SEMAPHORE_CLOSED_MSG: &str = "Executors semaphore closed!"; /// Max number of executors that can send messages in parallel to Base layer diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index b7006ba64..deabf4dab 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -3,20 +3,17 @@ use magicblock_metrics::metrics; use magicblock_rpc_client::{ utils::TransactionErrorMapper, MagicBlockRpcClientError, }; -use solana_instruction::error::InstructionError; use solana_signature::Signature; use solana_signer::SignerError; -use solana_transaction_error::TransactionError; use tracing::error; use crate::{ outbox_client::InternalOutboxClientError, - tasks::{ - task_builder::TaskBuilderError, task_strategist::TaskStrategistError, - BaseTaskImpl, - }, + tasks::task_builder::TaskBuilderError, transaction_preparator::error::TransactionPreparatorError, }; +use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; +use crate::tasks::task_strategist::TaskStrategistError; #[derive(thiserror::Error, Debug)] pub enum InternalError { @@ -150,233 +147,18 @@ impl metrics::LabelValue for IntentExecutorError { } } -/// Those are the errors that may occur during Commit/Finalize stages on Base layer -#[derive(thiserror::Error, Debug)] -pub enum TransactionStrategyExecutionError { - #[error("User supplied actions are ill-formed: {0}. {1:?}")] - ActionsError(#[source] TransactionError, Option), - #[error("Invalid undelegation: {0}. {1:?}")] - UndelegationError(#[source] TransactionError, Option), - #[error("Accounts committed with an invalid Commit id: {0}. {1:?}")] - CommitIDError(#[source] TransactionError, Option), - #[error("Max instruction trace length exceeded: {0}. {1:?}")] - CpiLimitError(#[source] TransactionError, Option), - #[error("Loaded accounts data size exceeded: {0}. {1:?}")] - LoadedAccountsDataSizeExceeded( - #[source] TransactionError, - Option, - ), - #[error("Unfinalized account error: {0}, {1:?}")] - UnfinalizedAccountError(#[source] TransactionError, Option), - #[error("Transaction too large to send over the wire: {0}")] - TransactionTooLargeError(#[source] InternalError), - #[error("InternalError: {0}")] - InternalError(#[from] InternalError), -} - -impl From for TransactionStrategyExecutionError { - fn from(value: MagicBlockRpcClientError) -> Self { - Self::InternalError(InternalError::from(value)) - } -} - -impl TransactionStrategyExecutionError { - /// Number of compute budget instructions prepended to every transaction. - /// Used to map instruction indices back to task indices. - const TASK_OFFSET: u8 = 2; - - pub fn is_cpi_limit_error(&self) -> bool { - matches!( - self, - Self::CpiLimitError(_, _) - | Self::LoadedAccountsDataSizeExceeded(_, _) - ) - } - - pub fn is_recoverable_by_two_stage(&self) -> bool { - self.is_cpi_limit_error() - || matches!(self, Self::TransactionTooLargeError(_)) - } - - pub fn task_index(&self) -> Option { - match self { - Self::CommitIDError( - TransactionError::InstructionError(index, _), - _, - ) - | Self::ActionsError( - TransactionError::InstructionError(index, _), - _, - ) - | Self::UndelegationError( - TransactionError::InstructionError(index, _), - _, - ) - | Self::UnfinalizedAccountError( - TransactionError::InstructionError(index, _), - _, - ) - | Self::CpiLimitError( - TransactionError::InstructionError(index, _), - _, - ) => index.checked_sub(Self::TASK_OFFSET), - _ => None, - } - } - - pub fn signature(&self) -> Option { - match self { - Self::InternalError(err) => err.signature(), - Self::TransactionTooLargeError(err) => err.signature(), - Self::CommitIDError(_, signature) - | Self::ActionsError(_, signature) - | Self::UndelegationError(_, signature) - | Self::UnfinalizedAccountError(_, signature) - | Self::CpiLimitError(_, signature) - | Self::LoadedAccountsDataSizeExceeded(_, signature) => *signature, - } - } - - /// Convert [`TransactionError`] into known errors that can be handled - /// Otherwise return original [`TransactionError`] - /// [`TransactionStrategyExecutionError`] - pub fn try_from_transaction_error( - err: TransactionError, - signature: Option, - tasks: &[BaseTaskImpl], - ) -> Result { - // Commit Nonce order error - const NONCE_OUT_OF_ORDER: u32 = - dlp_api::error::DlpError::NonceOutOfOrder as u32; - // Errors when commit state already exists - const COMMIT_STATE_INVALID_ACCOUNT_OWNER: u32 = - dlp_api::error::DlpError::CommitStateInvalidAccountOwner as u32; - const COMMIT_STATE_ALREADY_INITIALIZED: u32 = - dlp_api::error::DlpError::CommitStateAlreadyInitialized as u32; - const COMMIT_RECORD_INVALID_ACCOUNT_OWNER: u32 = - dlp_api::error::DlpError::CommitRecordInvalidAccountOwner as u32; - const COMMIT_RECORD_ALREADY_INITIALIZED: u32 = - dlp_api::error::DlpError::CommitRecordAlreadyInitialized as u32; - - match err { - // Some tx may use too much CPIs and we can handle it in certain cases - transaction_err @ TransactionError::InstructionError( - _, - InstructionError::MaxInstructionTraceLengthExceeded, - ) => Ok(TransactionStrategyExecutionError::CpiLimitError( - transaction_err, - signature, - )), - err @ TransactionError::MaxLoadedAccountsDataSizeExceeded => { - Ok(TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( - err, - signature, - )) - } - // Map per-task InstructionError into CommitID / Actions / Undelegation errors when possible - TransactionError::InstructionError(index, instruction_err) => { - let tx_err_helper = |instruction_err| -> TransactionError { - TransactionError::InstructionError(index, instruction_err) - }; - let Some(action_index) = index.checked_sub(Self::TASK_OFFSET) - else { - return Err(tx_err_helper(instruction_err)); - }; - - let Some(task) = tasks.get(action_index as usize) else { - return Err(tx_err_helper(instruction_err)); - }; - - match (task, instruction_err) { - ( - BaseTaskImpl::Commit(_) - | BaseTaskImpl::CommitFinalize(_), - instruction_err, - ) => match instruction_err { - InstructionError::Custom(NONCE_OUT_OF_ORDER) => Ok( - TransactionStrategyExecutionError::CommitIDError( - tx_err_helper(InstructionError::Custom( - NONCE_OUT_OF_ORDER, - )), - signature, - ), - ), - instruction_err @ (InstructionError::Custom( - COMMIT_STATE_INVALID_ACCOUNT_OWNER, - ) - | InstructionError::Custom( - COMMIT_STATE_ALREADY_INITIALIZED, - ) - | InstructionError::Custom( - COMMIT_RECORD_INVALID_ACCOUNT_OWNER, - ) - | InstructionError::Custom( - COMMIT_RECORD_ALREADY_INITIALIZED, - )) => { - Ok(TransactionStrategyExecutionError::UnfinalizedAccountError( - tx_err_helper(instruction_err), - signature - )) - } - err => Err(tx_err_helper(err)), - }, - (BaseTaskImpl::BaseAction(_), instruction_err) => { - Ok(TransactionStrategyExecutionError::ActionsError( - tx_err_helper(instruction_err), - signature, - )) - } - (BaseTaskImpl::Undelegate(_), instruction_err) => Ok( - TransactionStrategyExecutionError::UndelegationError( - tx_err_helper(instruction_err), - signature, - ), - ), - (_, instruction_err) => Err(tx_err_helper(instruction_err)), - } - } - // This means transaction failed to other reasons that we don't handle - propagate - err => { - error!(error = ?err, "Message execution failed"); - Err(err) - } - } - } -} - -impl metrics::LabelValue for TransactionStrategyExecutionError { - fn value(&self) -> &str { - match self { - Self::ActionsError(_, _) => "actions_failed", - Self::CpiLimitError(_, _) => "cpi_limit_failed", - Self::LoadedAccountsDataSizeExceeded(_, _) => { - "loaded_accounts_data_limit_exceeded" +impl From<&IntentExecutorError> for ActionError { + fn from(value: &IntentExecutorError) -> Self { + match value { + IntentExecutorError::FailedToCommitError { err, .. } + | IntentExecutorError::FailedToFinalizeError { err, .. } => { + err.into() } - Self::CommitIDError(_, _) => "commit_nonce_failed", - Self::UndelegationError(_, _) => "undelegation_failed", - Self::UnfinalizedAccountError(_, _) => "unfinalized_account_failed", - Self::TransactionTooLargeError(_) => "transaction_too_large", - _ => "failed", + err => ActionError::IntentFailedError(err.to_string()), } } } -pub(crate) struct IntentTransactionErrorMapper<'a> { - pub tasks: &'a [BaseTaskImpl], -} -impl TransactionErrorMapper for IntentTransactionErrorMapper<'_> { - type ExecutionError = TransactionStrategyExecutionError; - fn try_map( - &self, - error: TransactionError, - signature: Option, - ) -> Result { - TransactionStrategyExecutionError::try_from_transaction_error( - error, signature, self.tasks, - ) - } -} - impl From for IntentExecutorError { fn from(value: TaskStrategistError) -> Self { match value { @@ -385,31 +167,6 @@ impl From for IntentExecutorError { } } } - -impl From<&TransactionStrategyExecutionError> for ActionError { - fn from(value: &TransactionStrategyExecutionError) -> Self { - if let TransactionStrategyExecutionError::ActionsError(err, signature) = - value - { - Self::ActionsError(err.clone(), *signature) - } else { - Self::IntentFailedError(value.to_string()) - } - } -} - -impl From<&IntentExecutorError> for ActionError { - fn from(value: &IntentExecutorError) -> Self { - match value { - IntentExecutorError::FailedToCommitError { err, .. } - | IntentExecutorError::FailedToFinalizeError { err, .. } => { - err.into() - } - err => ActionError::IntentFailedError(err.to_string()), - } - } -} - pub type IntentExecutorResult = Result; #[cfg(test)] @@ -421,8 +178,8 @@ mod tests { }, request::{RpcError, RpcRequest, RpcResponseErrorData}, }; - - use super::{InternalError, TransactionStrategyExecutionError}; + use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; + use super::InternalError; const TX_TOO_LARGE_SOLANA: &str = "base64 encoded too large"; const TX_TOO_LARGE_MAGICBLOCK: &str = diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 7c1e9b820..9ff17c489 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -22,13 +22,13 @@ use tracing::warn; use crate::{ intent_executor::{ error::{ - IntentExecutorResult, IntentTransactionErrorMapper, InternalError, - TransactionStrategyExecutionError, + IntentExecutorResult, InternalError, }, ExecutionOutput, }, tasks::BaseTaskImpl, }; +use crate::intent_executor::strategy_executor::error::{IntentTransactionErrorMapper, TransactionStrategyExecutionError}; #[derive(Clone)] pub struct IntentExecutionClient { diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 822e3d8bf..4bd67e46a 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -21,14 +21,13 @@ use magicblock_program::{ outbox_intent_bundles::OutboxIntentBundleStatus, }; use solana_signature::Signature; - +use strategy_executor::error::TransactionStrategyExecutionError; use crate::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, cleanup_handle::CleanupHandle, error::{ IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, }, intent_execution_client::IntentExecutionClient, single_stage_intent_executor::SingleStageIntentExecutor, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs new file mode 100644 index 000000000..1d0185bd7 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs @@ -0,0 +1,251 @@ +use solana_instruction::error::InstructionError; +use magicblock_rpc_client::utils::TransactionErrorMapper; +use solana_transaction_error::TransactionError; +use solana_signature::Signature; +use tracing::error; +use magicblock_core::traits::ActionError; +use magicblock_metrics::metrics; +use magicblock_rpc_client::MagicBlockRpcClientError; +use crate::intent_executor::error::{InternalError}; +use crate::tasks::BaseTaskImpl; +use crate::tasks::task_strategist::TaskStrategistError; + +/// Those are the errors that may occur during Commit/Finalize stages on Base layer +#[derive(thiserror::Error, Debug)] +pub enum TransactionStrategyExecutionError { + #[error("User supplied actions are ill-formed: {0}. {1:?}")] + ActionsError(#[source] TransactionError, Option), + #[error("Invalid undelegation: {0}. {1:?}")] + UndelegationError(#[source] TransactionError, Option), + #[error("Accounts committed with an invalid Commit id: {0}. {1:?}")] + CommitIDError(#[source] TransactionError, Option), + #[error("Max instruction trace length exceeded: {0}. {1:?}")] + CpiLimitError(#[source] TransactionError, Option), + #[error("Loaded accounts data size exceeded: {0}. {1:?}")] + LoadedAccountsDataSizeExceeded( + #[source] TransactionError, + Option, + ), + #[error("Unfinalized account error: {0}, {1:?}")] + UnfinalizedAccountError(#[source] TransactionError, Option), + #[error("Transaction too large to send over the wire: {0}")] + TransactionTooLargeError(#[source] InternalError), + #[error("InternalError: {0}")] + InternalError(#[from] InternalError), +} + +impl From for TransactionStrategyExecutionError { + fn from(value: MagicBlockRpcClientError) -> Self { + Self::InternalError(InternalError::from(value)) + } +} + +impl TransactionStrategyExecutionError { + /// Number of compute budget instructions prepended to every transaction. + /// Used to map instruction indices back to task indices. + const TASK_OFFSET: u8 = 2; + + pub fn is_cpi_limit_error(&self) -> bool { + matches!( + self, + Self::CpiLimitError(_, _) + | Self::LoadedAccountsDataSizeExceeded(_, _) + ) + } + + pub fn is_recoverable_by_two_stage(&self) -> bool { + self.is_cpi_limit_error() + || matches!(self, Self::TransactionTooLargeError(_)) + } + + pub fn task_index(&self) -> Option { + match self { + Self::CommitIDError( + TransactionError::InstructionError(index, _), + _, + ) + | Self::ActionsError( + TransactionError::InstructionError(index, _), + _, + ) + | Self::UndelegationError( + TransactionError::InstructionError(index, _), + _, + ) + | Self::UnfinalizedAccountError( + TransactionError::InstructionError(index, _), + _, + ) + | Self::CpiLimitError( + TransactionError::InstructionError(index, _), + _, + ) => index.checked_sub(Self::TASK_OFFSET), + _ => None, + } + } + + pub fn signature(&self) -> Option { + match self { + Self::InternalError(err) => err.signature(), + Self::TransactionTooLargeError(err) => err.signature(), + Self::CommitIDError(_, signature) + | Self::ActionsError(_, signature) + | Self::UndelegationError(_, signature) + | Self::UnfinalizedAccountError(_, signature) + | Self::CpiLimitError(_, signature) + | Self::LoadedAccountsDataSizeExceeded(_, signature) => *signature, + } + } + + /// Convert [`TransactionError`] into known errors that can be handled + /// Otherwise return original [`TransactionError`] + /// [`TransactionStrategyExecutionError`] + pub fn try_from_transaction_error( + err: TransactionError, + signature: Option, + tasks: &[BaseTaskImpl], + ) -> Result { + // Commit Nonce order error + const NONCE_OUT_OF_ORDER: u32 = + dlp_api::error::DlpError::NonceOutOfOrder as u32; + // Errors when commit state already exists + const COMMIT_STATE_INVALID_ACCOUNT_OWNER: u32 = + dlp_api::error::DlpError::CommitStateInvalidAccountOwner as u32; + const COMMIT_STATE_ALREADY_INITIALIZED: u32 = + dlp_api::error::DlpError::CommitStateAlreadyInitialized as u32; + const COMMIT_RECORD_INVALID_ACCOUNT_OWNER: u32 = + dlp_api::error::DlpError::CommitRecordInvalidAccountOwner as u32; + const COMMIT_RECORD_ALREADY_INITIALIZED: u32 = + dlp_api::error::DlpError::CommitRecordAlreadyInitialized as u32; + + match err { + // Some tx may use too much CPIs and we can handle it in certain cases + transaction_err @ TransactionError::InstructionError( + _, + InstructionError::MaxInstructionTraceLengthExceeded, + ) => Ok(TransactionStrategyExecutionError::CpiLimitError( + transaction_err, + signature, + )), + err @ TransactionError::MaxLoadedAccountsDataSizeExceeded => { + Ok(TransactionStrategyExecutionError::LoadedAccountsDataSizeExceeded( + err, + signature, + )) + } + // Map per-task InstructionError into CommitID / Actions / Undelegation errors when possible + TransactionError::InstructionError(index, instruction_err) => { + let tx_err_helper = |instruction_err| -> TransactionError { + TransactionError::InstructionError(index, instruction_err) + }; + let Some(action_index) = index.checked_sub(Self::TASK_OFFSET) + else { + return Err(tx_err_helper(instruction_err)); + }; + + let Some(task) = tasks.get(action_index as usize) else { + return Err(tx_err_helper(instruction_err)); + }; + + match (task, instruction_err) { + ( + BaseTaskImpl::Commit(_) + | BaseTaskImpl::CommitFinalize(_), + instruction_err, + ) => match instruction_err { + InstructionError::Custom(NONCE_OUT_OF_ORDER) => Ok( + TransactionStrategyExecutionError::CommitIDError( + tx_err_helper(InstructionError::Custom( + NONCE_OUT_OF_ORDER, + )), + signature, + ), + ), + instruction_err @ (InstructionError::Custom( + COMMIT_STATE_INVALID_ACCOUNT_OWNER, + ) + | InstructionError::Custom( + COMMIT_STATE_ALREADY_INITIALIZED, + ) + | InstructionError::Custom( + COMMIT_RECORD_INVALID_ACCOUNT_OWNER, + ) + | InstructionError::Custom( + COMMIT_RECORD_ALREADY_INITIALIZED, + )) => { + Ok(TransactionStrategyExecutionError::UnfinalizedAccountError( + tx_err_helper(instruction_err), + signature + )) + } + err => Err(tx_err_helper(err)), + }, + (BaseTaskImpl::BaseAction(_), instruction_err) => { + Ok(TransactionStrategyExecutionError::ActionsError( + tx_err_helper(instruction_err), + signature, + )) + } + (BaseTaskImpl::Undelegate(_), instruction_err) => Ok( + TransactionStrategyExecutionError::UndelegationError( + tx_err_helper(instruction_err), + signature, + ), + ), + (_, instruction_err) => Err(tx_err_helper(instruction_err)), + } + } + // This means transaction failed to other reasons that we don't handle - propagate + err => { + error!(error = ?err, "Message execution failed"); + Err(err) + } + } + } +} + +impl metrics::LabelValue for TransactionStrategyExecutionError { + fn value(&self) -> &str { + match self { + Self::ActionsError(_, _) => "actions_failed", + Self::CpiLimitError(_, _) => "cpi_limit_failed", + Self::LoadedAccountsDataSizeExceeded(_, _) => { + "loaded_accounts_data_limit_exceeded" + } + Self::CommitIDError(_, _) => "commit_nonce_failed", + Self::UndelegationError(_, _) => "undelegation_failed", + Self::UnfinalizedAccountError(_, _) => "unfinalized_account_failed", + Self::TransactionTooLargeError(_) => "transaction_too_large", + _ => "failed", + } + } +} + +pub(crate) struct IntentTransactionErrorMapper<'a> { + pub tasks: &'a [BaseTaskImpl], +} + +impl TransactionErrorMapper for IntentTransactionErrorMapper<'_> { + type ExecutionError = TransactionStrategyExecutionError; + fn try_map( + &self, + error: TransactionError, + signature: Option, + ) -> Result { + TransactionStrategyExecutionError::try_from_transaction_error( + error, signature, self.tasks, + ) + } +} + +impl From<&TransactionStrategyExecutionError> for ActionError { + fn from(value: &TransactionStrategyExecutionError) -> Self { + if let TransactionStrategyExecutionError::ActionsError(err, signature) = + value + { + Self::ActionsError(err.clone(), *signature) + } else { + Self::IntentFailedError(value.to_string()) + } + } +} \ No newline at end of file diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs index fe27b25cd..8c8fa3f0f 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs @@ -2,3 +2,4 @@ pub mod patcher; pub mod single_stage; pub mod two_stage; pub mod utils; +pub mod error; diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs index 222ff708c..cc7048368 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs @@ -12,7 +12,6 @@ use crate::{ intent_executor::{ error::{ IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, }, intent_execution_client::IntentExecutionClient, strategy_executor::utils::{ @@ -25,6 +24,7 @@ use crate::{ tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, transaction_preparator::TransactionPreparator, }; +use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; #[async_trait] pub(in crate::intent_executor) trait Patcher { diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index 720f370f2..a633b4dbe 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -17,7 +17,6 @@ use crate::{ intent_executor::{ error::{ IntentExecutorError, IntentExecutorResult, - TransactionStrategyExecutionError, }, intent_execution_client::IntentExecutionClient, strategy_executor::{ @@ -38,6 +37,7 @@ use crate::{ error::TransactionPreparatorError, TransactionPreparator, }, }; +use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; const STAGE_LOOP_CEILING: u8 = 10; diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index eb5255891..01e509432 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -48,7 +48,10 @@ pub trait OutboxClient: Send + Sync + 'static { /// Executes `Accept` tx and returns accepted intents async fn accept_scheduled_intents( &self, - ) -> Result, Self::Error>; + ) -> Result< + Vec, + (Vec, Self::Error), + >; /// Sets execution stage for outbox intent /// Note: intent has to be accepted prior @@ -130,7 +133,10 @@ impl InternalOutboxClient { async fn send_accept_tx( &self, scheduled_intents: Vec, - ) -> Result, (Vec, InternalOutboxClientError)> { + ) -> Result< + Vec, + (Vec, InternalOutboxClientError), + > { const CHUNK_SIZE: usize = 50; let mut remaining = scheduled_intents; @@ -187,7 +193,10 @@ impl OutboxClient for InternalOutboxClient { async fn accept_scheduled_intents( &self, - ) -> Result, (Vec, Self::Error)> { + ) -> Result< + Vec, + (Vec, Self::Error), + > { // If accounts were scheduled to be committed, we accept them here // and processs the commits let magic_context_acc = @@ -195,11 +204,10 @@ impl OutboxClient for InternalOutboxClient { "Validator found to be running without MagicContext account!", ); - let magic_context = MagicContext::deserialize(magic_context_acc.data())?; - self.send_accept_tx(magic_context.scheduled_base_intents).await?; - - // Return intents from global store - Ok(TransactionScheduler::default().take_scheduled_intent_bundles()) + let magic_context = + MagicContext::deserialize(magic_context_acc.data()).map_err(|err| (vec![], err.into()))?; + self.send_accept_tx(magic_context.scheduled_base_intents) + .await } async fn set_intent_execution_stage( diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 5b7ffda7b..6ac1bcb16 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -174,10 +174,6 @@ where // TODO(edwin): early shutdown or cleanup errors to avoid this .expect("Failed to reschedule intents"); - // if let Err(err) = self.reschedule_pending_bundles().await { - // error!(error = ?err, "Failed to reschedule pending bundles") - // } - let mut interval = tokio::time::interval(self.slot_interval); loop { tokio::select! { @@ -190,13 +186,10 @@ where .outbox_client .accept_scheduled_intents() .await; - let intent_bundles = match accept_result { - Ok(value) => value, - Err(err) => { - error!("Failed to accept intents: {}", err); - continue; - } - }; + let intent_bundles = accept_result.unwrap_or_else(|(accepted_intents, err)| { + error!("Failed to accept intents: {}", err); + accepted_intents + }); let intent_bundles= intent_bundles.into_iter().map(OutboxIntentBundle::accepted).collect(); if let Err(err) = self.schedule_intent_execution(intent_bundles).await { diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 09fb47592..047e761c3 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -15,7 +15,7 @@ use magicblock_committor_program::pdas; use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, - error::{IntentExecutorError, TransactionStrategyExecutionError}, + error::IntentExecutorError, intent_execution_client::IntentExecutionClient, strategy_executor::{ two_stage::{Initialized, TwoStageStrategyExecutor}, @@ -69,7 +69,7 @@ use solana_sdk::{ transaction::{Transaction, TransactionError}, }; use solana_sdk_ids::system_program; - +use magicblock_committor_service::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; use crate::{ common::{MockActionsCallbackExecutor, TestFixture}, utils::{ From 61ea35741fae9aa738540f4fc170dfb771d699c0 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 22 Jun 2026 14:39:06 +0700 Subject: [PATCH 54/97] fix: compilation --- magicblock-api/src/magic_validator.rs | 71 +++++++++-------- .../src/committor_processor.rs | 8 +- .../intent_execution_engine.rs | 6 +- .../src/intent_executor/error.rs | 10 ++- .../intent_execution_client.rs | 6 +- .../src/intent_executor/mod.rs | 5 +- .../strategy_executor/error.rs | 21 ++--- .../intent_executor/strategy_executor/mod.rs | 2 +- .../strategy_executor/patcher.rs | 14 ++-- .../strategy_executor/utils.rs | 6 +- .../src/outbox_client.rs | 23 +++++- .../src/instruction.rs | 12 +-- .../outbox/process_scheduled_commit_sent.rs | 77 +++++++++++++++---- .../schedule/process_schedule_commit_tests.rs | 2 +- .../magicblock/src/magicblock_processor.rs | 9 +-- .../magicblock/src/utils/instruction_utils.rs | 60 ++++++++++----- .../tests/test_ix_commit_local.rs | 8 +- 17 files changed, 209 insertions(+), 131 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index b874bf790..4f65f6fe4 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -231,22 +231,30 @@ impl MagicValidator { log_timing("startup", "chainlink_init", step_start); let step_start = Instant::now(); + let outbox_client = { + let val = Self::init_outbox_client( + &config, + &accountsdb, + &dispatch.transaction_scheduler, + &ledger.latest_block(), + ); + Arc::new(val) + }; let committor_processor = { let processor = Self::init_committor_processor( &config, ledger.latest_block(), + &outbox_client, &shared_chain_slot, - )?; + ); Arc::new(processor) }; - let intent_execution_service = Self::init_intent_execution_service( - &accountsdb, - &chainlink, - &dispatch.transaction_scheduler, - ledger.latest_block(), - &committor_processor, + let intent_execution_service = IntentExecutionServiceImpl::new( + chainlink.clone(), + outbox_client.clone(), + committor_processor.clone(), config.ledger.block_time, - &token, + token.clone(), ); log_timing("startup", "committor_service_init", step_start); init_magic_sys(Arc::new(MagicSysAdapter::new( @@ -449,14 +457,28 @@ impl MagicValidator { }) } + fn init_outbox_client( + config: &ValidatorParams, + accounts_db: &Arc, + transaction_scheduler: &TransactionSchedulerHandle, + latest_block: &LatestBlock, + ) -> InternalOutboxClient { + let rpc_client = RpcClient::new(config.aperture.listen.http()); + InternalOutboxClient::new( + accounts_db.clone(), + Arc::new(rpc_client), + transaction_scheduler.clone(), + latest_block.clone(), + ) + } + pub fn init_committor_processor( config: &ValidatorParams, latest_block: &LatestBlock, + outbox_client: &Arc>, shared_chain_slot: &Option>, - ) -> ApiResult { + ) -> CommittorProcessor { let authority = config.validator.keypair.insecure_clone(); - let committor_persist_path = - config.storage.join("committor_service.sqlite"); let base_chain_config = ChainConfig { rpc_uri: config.rpc_url().to_owned(), commitment: CommitmentConfig::confirmed(), @@ -476,35 +498,12 @@ impl MagicValidator { config.validator.keypair.insecure_clone(), latest_block.clone(), ); - Ok(CommittorProcessor::try_new( + CommittorProcessor::new( authority, base_chain_config, shared_chain_slot.clone(), + outbox_client.clone(), actions_callback_executor, - )?) - } - - fn init_intent_execution_service( - accounts_db: &Arc, - chainlink: &Arc, - transaction_scheduler: &TransactionSchedulerHandle, - latest_block: &LatestBlock, - committor_processor: &Arc, - slot_interval: Duration, - cancellation_token: &CancellationToken, - ) -> IntentExecutionServiceImpl { - let intent_client = InternalOutboxClient::new( - accounts_db.clone(), - transaction_scheduler.clone(), - latest_block.clone(), - ); - - IntentExecutionServiceImpl::new( - chainlink.clone(), - Arc::new(intent_client), - committor_processor.clone(), - slot_interval, - cancellation_token.clone(), ) } diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index e9f61b3c2..1c11e4908 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -46,13 +46,13 @@ pub struct CommittorProcessor { } impl CommittorProcessor { - pub fn try_new( + pub fn new( authority: Keypair, chain_config: ChainConfig, chain_slot: Option>, outbox_client: Arc, actions_callback_executor: A, - ) -> CommittorServiceResult + ) -> Self where A: ActionsCallbackScheduler, O: OutboxClient, @@ -115,14 +115,14 @@ impl CommittorProcessor { pending_result_listeners.clone(), )); - Ok(Self { + Self { authority, _table_mania: table_mania, magic_rpc_client: magic_block_rpc_client, commits_scheduler, task_info_fetcher, pending_result_listeners, - }) + } } #[instrument(skip(self, intent_bundles))] diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 5cd49d597..9f90435d0 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -24,15 +24,13 @@ use crate::{ intent_scheduler::{IntentScheduler, POISONED_INNER_MSG}, }, intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - }, + error::{IntentExecutorError, IntentExecutorResult}, intent_executor_factory::IntentExecutorBuilder, + strategy_executor::error::TransactionStrategyExecutionError, ExecutionOutput, IntentExecutionResult, IntentExecutor, }, transaction_preparator::TransactionPreparator, }; -use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; const SEMAPHORE_CLOSED_MSG: &str = "Executors semaphore closed!"; /// Max number of executors that can send messages in parallel to Base layer diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index deabf4dab..5f2bcfcb7 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -8,12 +8,13 @@ use solana_signer::SignerError; use tracing::error; use crate::{ + intent_executor::strategy_executor::error::TransactionStrategyExecutionError, outbox_client::InternalOutboxClientError, - tasks::task_builder::TaskBuilderError, + tasks::{ + task_builder::TaskBuilderError, task_strategist::TaskStrategistError, + }, transaction_preparator::error::TransactionPreparatorError, }; -use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; -use crate::tasks::task_strategist::TaskStrategistError; #[derive(thiserror::Error, Debug)] pub enum InternalError { @@ -178,8 +179,9 @@ mod tests { }, request::{RpcError, RpcRequest, RpcResponseErrorData}, }; - use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; + use super::InternalError; + use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; const TX_TOO_LARGE_SOLANA: &str = "base64 encoded too large"; const TX_TOO_LARGE_MAGICBLOCK: &str = diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 9ff17c489..7a5c7b697 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -21,14 +21,14 @@ use tracing::warn; use crate::{ intent_executor::{ - error::{ - IntentExecutorResult, InternalError, + error::{IntentExecutorResult, InternalError}, + strategy_executor::error::{ + IntentTransactionErrorMapper, TransactionStrategyExecutionError, }, ExecutionOutput, }, tasks::BaseTaskImpl, }; -use crate::intent_executor::strategy_executor::error::{IntentTransactionErrorMapper, TransactionStrategyExecutionError}; #[derive(Clone)] pub struct IntentExecutionClient { diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 4bd67e46a..7289ef597 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -22,13 +22,12 @@ use magicblock_program::{ }; use solana_signature::Signature; use strategy_executor::error::TransactionStrategyExecutionError; + use crate::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, cleanup_handle::CleanupHandle, - error::{ - IntentExecutorError, IntentExecutorResult, - }, + error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, single_stage_intent_executor::SingleStageIntentExecutor, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs index 1d0185bd7..afa3ca278 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs @@ -1,14 +1,17 @@ +use magicblock_core::traits::ActionError; +use magicblock_metrics::metrics; +use magicblock_rpc_client::{ + utils::TransactionErrorMapper, MagicBlockRpcClientError, +}; use solana_instruction::error::InstructionError; -use magicblock_rpc_client::utils::TransactionErrorMapper; -use solana_transaction_error::TransactionError; use solana_signature::Signature; +use solana_transaction_error::TransactionError; use tracing::error; -use magicblock_core::traits::ActionError; -use magicblock_metrics::metrics; -use magicblock_rpc_client::MagicBlockRpcClientError; -use crate::intent_executor::error::{InternalError}; -use crate::tasks::BaseTaskImpl; -use crate::tasks::task_strategist::TaskStrategistError; + +use crate::{ + intent_executor::error::InternalError, + tasks::{task_strategist::TaskStrategistError, BaseTaskImpl}, +}; /// Those are the errors that may occur during Commit/Finalize stages on Base layer #[derive(thiserror::Error, Debug)] @@ -248,4 +251,4 @@ impl From<&TransactionStrategyExecutionError> for ActionError { Self::IntentFailedError(value.to_string()) } } -} \ No newline at end of file +} diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs index 8c8fa3f0f..46187a866 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/mod.rs @@ -1,5 +1,5 @@ +pub mod error; pub mod patcher; pub mod single_stage; pub mod two_stage; pub mod utils; -pub mod error; diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs index cc7048368..3b11eb3bf 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs @@ -10,13 +10,14 @@ use tracing::{error, warn}; use crate::{ intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - }, + error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, - strategy_executor::utils::{ - handle_actions_result, handle_commit_id_error, - handle_undelegation_error, prepare_and_execute_strategy, + strategy_executor::{ + error::TransactionStrategyExecutionError, + utils::{ + handle_actions_result, handle_commit_id_error, + handle_undelegation_error, prepare_and_execute_strategy, + }, }, task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, IntentExecutionReport, @@ -24,7 +25,6 @@ use crate::{ tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, transaction_preparator::TransactionPreparator, }; -use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; #[async_trait] pub(in crate::intent_executor) trait Patcher { diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index a633b4dbe..bdd0d63e1 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -15,11 +15,10 @@ use tracing::{error, info, warn}; use crate::{ intent_executor::{ - error::{ - IntentExecutorError, IntentExecutorResult, - }, + error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, strategy_executor::{ + error::TransactionStrategyExecutionError, patcher::Patcher, single_stage::SingleStageStrategyExecutor, two_stage::{Committed, Initialized, TwoStageStrategyExecutor}, @@ -37,7 +36,6 @@ use crate::{ error::TransactionPreparatorError, TransactionPreparator, }, }; -use crate::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; const STAGE_LOOP_CEILING: u8 = 10; diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index 01e509432..6eb23f090 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -79,6 +79,7 @@ pub struct InternalOutboxClient { /// Provides access to MagicContext accounts_db: Arc, /// RPC client for sending accept transactions to the ER + // TODO(edwin): check if needs to be Arc rpc_client: Arc, /// Internal endpoint for scheduling ER TXs transaction_scheduler: TransactionSchedulerHandle, @@ -204,8 +205,8 @@ impl OutboxClient for InternalOutboxClient { "Validator found to be running without MagicContext account!", ); - let magic_context = - MagicContext::deserialize(magic_context_acc.data()).map_err(|err| (vec![], err.into()))?; + let magic_context = MagicContext::deserialize(magic_context_acc.data()) + .map_err(|err| (vec![], err.into()))?; self.send_accept_tx(magic_context.scheduled_base_intents) .await } @@ -215,8 +216,22 @@ impl OutboxClient for InternalOutboxClient { intent_id: u64, stage: ExecutionStage, ) -> Result<(), Self::Error> { - // TODO(edwin): use rpc of scheduler - todo!() + let tx = InstructionUtils::set_intent_execution_stage( + self.latest_block_provider.blockhash(), + intent_id, + stage, + ); + + self.send_with_backoff( + ExponentialBackoff { + max_elapsed_time: Some(Duration::from_secs(25)), + max_interval: Duration::from_secs(5), + ..ExponentialBackoff::default() + }, + &tx, + ) + .await + .map_err(Into::into) } async fn notify_commit_sent( diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index 8f84136c2..9c29bd7c9 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -84,15 +84,17 @@ pub enum MagicBlockInstruction { }, /// Records the attempt to realize a scheduled commit on chain. + /// Closes the associated outbox intent PDA account. /// /// The signature of this transaction can be pre-calculated since we pass the /// ID of the scheduled commit and retrieve the signature from a globally - /// stored hashmap. + /// stored hashmap. Transaction uniqueness is guaranteed by the per-intent PDA. /// - /// We implement it this way so we can log the signature of this transaction - /// as part of the [MagicBlockInstruction::ScheduleCommit] instruction. - /// Args: (intent_id, bump) - bump is needed in order to guarantee unique transactions - ScheduledCommitSent((u64, u64)), + /// # Account references + /// - **0.** `[WRITE, SIGNER]` Validator Authority (receives rent refund from close) + /// - **1.** `[WRITE]` Outbox intent PDA to close, seeds: `["outbox-intent", intent_id.to_le_bytes()]` + /// - **2.** `[WRITE]` Ephemeral vault + ScheduledCommitSent(u64), /// Schedules execution of a single *base intent*. /// diff --git a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs index e42338c07..6bad10e8a 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs @@ -4,15 +4,17 @@ use std::{ }; use lazy_static::lazy_static; -use magicblock_core::coordination_mode; +use magicblock_core::{coordination_mode, intent::outbox::outbox_intent_pda}; +use magicblock_magic_program_api::{ + instruction::MagicBlockInstruction, EPHEMERAL_VAULT_PUBKEY, +}; use solana_clock::Slot; use solana_hash::Hash; -use solana_instruction::error::InstructionError; +use solana_instruction::{error::InstructionError, AccountMeta, Instruction}; use solana_log_collector::ic_msg; use solana_program_runtime::invoke_context::InvokeContext; use solana_pubkey::Pubkey; use solana_signature::Signature; -use solana_transaction_context::TransactionContext; use crate::{ errors::custom_error_codes, @@ -114,8 +116,7 @@ fn get_scheduled_commit(id: u64) -> Option { pub fn process_scheduled_commit_sent( signers: HashSet, - invoke_context: &InvokeContext, - transaction_context: &TransactionContext, + invoke_context: &mut InvokeContext, commit_id: u64, ) -> Result<(), InstructionError> { let mode = coordination_mode::CoordinationMode::current(); @@ -128,19 +129,19 @@ pub fn process_scheduled_commit_sent( return Ok(()); } - const PROGRAM_IDX: u16 = 0; - const VALIDATOR_IDX: u16 = 1; + const VALIDATOR_IDX: u16 = 0; + const CLOSING_PDA_IDX: u16 = 1; + + let transaction_context = &invoke_context.transaction_context; + let ix_ctx = transaction_context.get_current_instruction_context()?; // Assert MagicBlock program - let program_id = - get_instruction_pubkey_with_idx(transaction_context, PROGRAM_IDX)?; - if program_id.ne(&crate::id()) { + if ix_ctx.get_program_key()? != &crate::id() { ic_msg!( invoke_context, - "ScheduleCommitSent ERR: Invalid program id '{}'", - program_id + "ScheduleCommitSent ERR: Magic program account not found" ); - return Err(InstructionError::IncorrectProgramId); + return Err(InstructionError::UnsupportedProgramId); } // Assert validator identity matches @@ -165,6 +166,22 @@ pub fn process_scheduled_commit_sent( return Err(InstructionError::MissingRequiredSignature); } + // Validate outbox intent PDA + let provided_pda = + get_instruction_pubkey_with_idx(transaction_context, CLOSING_PDA_IDX)?; + let expected_pda = outbox_intent_pda(commit_id); + if *provided_pda != expected_pda { + ic_msg!( + invoke_context, + "ScheduleCommitSent ERR: account at idx {} is {}, expected PDA {} for intent {}", + CLOSING_PDA_IDX, + provided_pda, + expected_pda, + commit_id + ); + return Err(InstructionError::InvalidArgument); + } + // Only after we passed all checks do we remove the commit from the global hashmap // Otherwise a malicious actor could remove a commit from the hashmap without // signing as the validator @@ -250,17 +267,45 @@ pub fn process_scheduled_commit_sent( ); } - if let Some(error_message) = commit.error_message { + let result = if let Some(error_message) = commit.error_message { ic_msg!( invoke_context, "ScheduledCommitSent error message: {}", error_message ); - Err(InstructionError::Custom(INTENT_FAILED_CODE)) } else { Ok(()) - } + }; + + close_outbox_account_cpi( + invoke_context, + validator_authority_id, + expected_pda, + )?; + + result +} + +fn close_outbox_account_cpi( + invoke_context: &mut InvokeContext, + sponsor: Pubkey, + pda: Pubkey, +) -> Result<(), InstructionError> { + invoke_context.native_invoke( + Instruction { + program_id: crate::id(), + accounts: vec![ + AccountMeta::new(sponsor, true), + AccountMeta::new(pda, false), + AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), + ], + data: MagicBlockInstruction::CloseEphemeralAccount + .try_to_vec() + .map_err(|_| InstructionError::InvalidInstructionData)?, + }, + &[], + ) } #[cfg(test)] diff --git a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs index eeb724135..4dd93ddd6 100644 --- a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs +++ b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs @@ -260,7 +260,7 @@ fn assert_first_commit( assert!(intent_bundle.commit_finalize.is_some()); assert!(intent_bundle.commit_finalize_and_undelegate.is_none()); } - let _instruction = MagicBlockInstruction::ScheduledCommitSent((*id, 0)); + let _instruction = MagicBlockInstruction::ScheduledCommitSent(*id); // TODO(edwin) @@@ this fails in CI only with the similar to the below // left: [4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] // right: [4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] diff --git a/programs/magicblock/src/magicblock_processor.rs b/programs/magicblock/src/magicblock_processor.rs index 5e8a5d3da..d438a788b 100644 --- a/programs/magicblock/src/magicblock_processor.rs +++ b/programs/magicblock/src/magicblock_processor.rs @@ -104,12 +104,9 @@ declare_process_instruction!( stage, ) } - ScheduledCommitSent((id, _bump)) => process_scheduled_commit_sent( - signers, - invoke_context, - transaction_context, - id, - ), + ScheduledCommitSent(id) => { + process_scheduled_commit_sent(signers, invoke_context, id) + } ScheduleBaseIntent(args) => process_schedule_intent_bundle( signers, invoke_context, diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index 547d5b74f..24bdc52ef 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -1,7 +1,4 @@ -use std::{ - collections::HashMap, - sync::atomic::{AtomicU64, Ordering}, -}; +use std::collections::HashMap; use magicblock_core::intent::outbox::outbox_intent_pda; use magicblock_magic_program_api::{ @@ -10,8 +7,9 @@ use magicblock_magic_program_api::{ AccountModification, AccountModificationForInstruction, MagicBlockInstruction, PostDelegationActionExecutorInstruction, }, + outbox, pda::crank_signer_pda, - CRANK_PROGRAM_ID, MAGIC_CONTEXT_PUBKEY, + CRANK_PROGRAM_ID, EPHEMERAL_VAULT_PUBKEY, MAGIC_CONTEXT_PUBKEY, POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, }; use solana_hash::Hash; @@ -120,30 +118,21 @@ impl InstructionUtils { scheduled_commit_id: u64, recent_blockhash: Hash, ) -> Transaction { - let ix = Self::scheduled_commit_sent_instruction( - &crate::id(), - &validator_authority_id(), - scheduled_commit_id, - ); + let ix = Self::scheduled_commit_sent_instruction(scheduled_commit_id); Self::into_transaction(&validator_authority(), ix, recent_blockhash) } pub(crate) fn scheduled_commit_sent_instruction( - magic_block_program: &Pubkey, - validator_authority: &Pubkey, scheduled_commit_id: u64, ) -> Instruction { - static COMMIT_SENT_BUMP: AtomicU64 = AtomicU64::new(0); let account_metas = vec![ - AccountMeta::new_readonly(*magic_block_program, false), - AccountMeta::new_readonly(*validator_authority, true), + AccountMeta::new(validator_authority_id(), true), + AccountMeta::new(outbox_intent_pda(scheduled_commit_id), false), + AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), ]; Instruction::new_with_bincode( - *magic_block_program, - &MagicBlockInstruction::ScheduledCommitSent(( - scheduled_commit_id, - COMMIT_SENT_BUMP.fetch_add(1, Ordering::SeqCst), - )), + crate::id(), + &MagicBlockInstruction::ScheduledCommitSent(scheduled_commit_id), account_metas, ) } @@ -181,6 +170,37 @@ impl InstructionUtils { ) } + // ----------------- + // SetIntentExecutionStage + // ----------------- + + pub fn set_intent_execution_stage( + recent_blockhash: Hash, + intent_id: u64, + stage: outbox::ExecutionStage, + ) -> Transaction { + let ix = Self::set_intent_execution_stage_instruction(intent_id, stage); + Self::into_transaction(&validator_authority(), ix, recent_blockhash) + } + + pub(crate) fn set_intent_execution_stage_instruction( + intent_id: u64, + stage: outbox::ExecutionStage, + ) -> Instruction { + let account_metas = vec![ + AccountMeta::new_readonly(validator_authority_id(), true), + AccountMeta::new(outbox_intent_pda(intent_id), false), + ]; + Instruction::new_with_bincode( + crate::id(), + &MagicBlockInstruction::SetIntentExecutionStage { + intent_id, + stage, + }, + account_metas, + ) + } + // ----------------- // ModifyAccounts // ----------------- diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 72ae07121..348a9a103 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -240,7 +240,7 @@ async fn commit_single_account( // Run each test with and without finalizing let processor = Arc::new( - CommittorProcessor::try_new( + CommittorProcessor::new( validator_auth.insecure_clone(), ":memory:", ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), @@ -324,7 +324,7 @@ async fn commit_book_order_account( // Run each test with and without finalizing let processor = Arc::new( - CommittorProcessor::try_new( + CommittorProcessor::new( validator_auth.insecure_clone(), ":memory:", ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), @@ -799,7 +799,7 @@ async fn commit_multiple_accounts( fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; let processor = Arc::new( - CommittorProcessor::try_new( + CommittorProcessor::new( validator_auth.insecure_clone(), ":memory:", ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), @@ -870,7 +870,7 @@ async fn execute_intent_bundle( fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; let processor = Arc::new( - CommittorProcessor::try_new( + CommittorProcessor::new( validator_auth.insecure_clone(), ":memory:", ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), From 7d62d1ca3ad8c4543d5885f23cbadad1b64faa10 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 22 Jun 2026 14:44:38 +0700 Subject: [PATCH 55/97] fix: warnings cleanup --- .../src/committor_processor.rs | 1 + .../src/intent_executor/error.rs | 5 +--- .../strategy_executor/error.rs | 5 +--- .../src/outbox_client.rs | 28 +++---------------- magicblock-committor-service/src/service.rs | 2 +- .../service/outbox_intent_bundles_reader.rs | 3 -- 6 files changed, 8 insertions(+), 36 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 1c11e4908..a08da1afc 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -95,6 +95,7 @@ impl CommittorProcessor { )); let commits_scheduler = IntentExecutionManager::new( magic_block_rpc_client.clone(), + // TODO(edwin): use DumberDb DummyDB::new(), task_info_fetcher.clone(), outbox_client, diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index 5f2bcfcb7..d906825b2 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -1,11 +1,8 @@ use magicblock_core::traits::ActionError; use magicblock_metrics::metrics; -use magicblock_rpc_client::{ - utils::TransactionErrorMapper, MagicBlockRpcClientError, -}; +use magicblock_rpc_client::MagicBlockRpcClientError; use solana_signature::Signature; use solana_signer::SignerError; -use tracing::error; use crate::{ intent_executor::strategy_executor::error::TransactionStrategyExecutionError, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs index afa3ca278..c12d5e1c1 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/error.rs @@ -8,10 +8,7 @@ use solana_signature::Signature; use solana_transaction_error::TransactionError; use tracing::error; -use crate::{ - intent_executor::error::InternalError, - tasks::{task_strategist::TaskStrategistError, BaseTaskImpl}, -}; +use crate::{intent_executor::error::InternalError, tasks::BaseTaskImpl}; /// Those are the errors that may occur during Commit/Finalize stages on Base layer #[derive(thiserror::Error, Debug)] diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index 6eb23f090..6211ff289 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -1,14 +1,9 @@ -use std::{ - num::{NonZeroU64, NonZeroUsize}, - sync::Arc, - time::Duration, -}; +use std::{num::NonZeroUsize, sync::Arc, time::Duration}; use async_trait::async_trait; use backoff::{future::retry, ExponentialBackoff}; use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_core::{ - intent::outbox::outbox_intent_pda, link::{ accounts::LockedAccount, transactions::{with_encoded, TransactionSchedulerHandle}, @@ -18,8 +13,7 @@ use magicblock_core::{ use magicblock_program::{ instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, - outbox_intent_bundles::OutboxIntentBundle, register_scheduled_commit_sent, - MagicContext, Pubkey, SentCommit, TransactionScheduler, + register_scheduled_commit_sent, MagicContext, Pubkey, SentCommit, MAGIC_CONTEXT_PUBKEY, }; use solana_account::{Account, ReadableAccount}; @@ -27,8 +21,7 @@ use solana_rpc_client::{ nonblocking::rpc_client::RpcClient, rpc_client::SerializableTransaction, }; use solana_rpc_client_api::{ - client_error, - client_error::{Error as RpcClientError, ErrorKind as RpcClientErrorKind}, + client_error, client_error::ErrorKind as RpcClientErrorKind, }; use solana_transaction::Transaction; use solana_transaction_error::TransactionError; @@ -168,23 +161,10 @@ impl InternalOutboxClient { let shared_account = self.accounts_db.get_account(pubkey)?; let locked_account = LockedAccount::new(*pubkey, shared_account); let output = locked_account - .read_locked(|pubkey, account| Account::from(account.clone())); + .read_locked(|_, account| Account::from(account.clone())); Some(output) } - - /// Returns just accepted OutboxIntents - fn read_outbox_intent( - &self, - intent_id: u64, - ) -> InternalOutboxClientResult> { - let pda = outbox_intent_pda(intent_id); - let Some(account) = self.safe_get_account(&pda) else { - return Ok(None); - }; - - todo!() - } } #[async_trait] diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 6ac1bcb16..d7db56ec2 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -24,7 +24,7 @@ use tokio::{ task::{JoinError, JoinHandle}, }; use tokio_util::sync::CancellationToken; -use tracing::{error, info, instrument, warn}; +use tracing::{error, info, instrument}; use crate::{ committor_processor::CommittorProcessor, diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index 865290334..6ee2f9d04 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -10,7 +10,6 @@ use magicblock_accounts_db::{error::AccountsDbError, AccountsDb}; use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::{AccountSharedData, ReadableAccount}; -use solana_pubkey::Pubkey; use tracing::warn; #[async_trait] @@ -38,8 +37,6 @@ pub struct InternalOutboxIntentBundlesReader { } impl InternalOutboxIntentBundlesReader { - const TARGET_PROGRAM_ID: Pubkey = magicblock_program::ID; - pub fn new(accounts_db: Arc, capacity: NonZeroUsize) -> Self { Self { capacity, From efbef309464a1d4b12f9427cd20fdc2f9855ab9f Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 23 Jun 2026 15:15:02 +0700 Subject: [PATCH 56/97] fix: test-integration compilation --- magicblock-committor-service/Cargo.toml | 1 + .../intent_execution_engine.rs | 6 + .../accepted_intent_executor.rs | 3 + .../src/intent_executor/cleanup_handle.rs | 2 +- .../src/intent_executor/mod.rs | 15 +- .../single_stage_intent_executor.rs | 3 + .../strategy_executor/single_stage.rs | 7 +- .../strategy_executor/two_stage.rs | 10 + .../two_stage_intent_executor.rs | 3 + .../src/intent_executor/utils.rs | 4 +- .../src/tasks/task_strategist.rs | 1 + .../test-committor-service/Cargo.toml | 2 +- .../test-committor-service/tests/common.rs | 80 ++++++- .../tests/test_delivery_preparator.rs | 53 +---- .../tests/test_intent_executor.rs | 135 +++++------ .../tests/test_ix_commit_local.rs | 212 +++++++++++++----- .../tests/test_transaction_preparator.rs | 25 +-- 17 files changed, 353 insertions(+), 209 deletions(-) diff --git a/magicblock-committor-service/Cargo.toml b/magicblock-committor-service/Cargo.toml index 1a32777cb..ba2692959 100644 --- a/magicblock-committor-service/Cargo.toml +++ b/magicblock-committor-service/Cargo.toml @@ -66,3 +66,4 @@ test-kit = { workspace = true } [features] default = [] +dev-context-only-utils = [] \ No newline at end of file diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 9f90435d0..6930e6450 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -44,6 +44,9 @@ pub struct BroadcastedIntentExecutionResult { pub id: u64, pub patched_errors: Arc, pub callbacks_report: Vec>>, + #[cfg(feature = "dev-context-only-utils")] + pub successful_transaction_strategies: + Vec, } impl BroadcastedIntentExecutionResult { @@ -61,6 +64,9 @@ impl BroadcastedIntentExecutionResult { patched_errors, inner, callbacks_report, + #[cfg(feature = "dev-context-only-utils")] + successful_transaction_strategies: execution_result + .successful_transaction_strategies, } } } diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index e66fdf878..207b1f2c9 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -233,6 +233,9 @@ where inner: result, patched_errors: execution_report.patched_errors, callbacks_report: execution_report.callbacks_report, + #[cfg(feature = "dev-context-only-utils")] + successful_transaction_strategies: execution_report + .successful_transaction_strategies, }; let cleanup_handle = CleanupHandle::new( self.authority, diff --git a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs index f058eaf23..43d7806fa 100644 --- a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs +++ b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs @@ -33,7 +33,7 @@ impl CleanupHandle { } } - pub(crate) async fn clean(self) -> Result<(), BufferExecutionError> { + pub async fn clean(self) -> Result<(), BufferExecutionError> { let close_buffers = self.close_buffers; let cleanup_futs = self.junk.iter().map(|to_cleanup| { self.transaction_preparator.cleanup_for_strategy( diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 7289ef597..22c370356 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -113,7 +113,6 @@ impl metrics::LabelValue for ExecutionOutput { } } -#[derive(Debug)] pub struct IntentExecutionResult { /// Final result of Intent Execution pub inner: IntentExecutorResult, @@ -121,6 +120,9 @@ pub struct IntentExecutionResult { pub patched_errors: Vec, /// Callbacks result pub callbacks_report: Vec>, + #[cfg(feature = "dev-context-only-utils")] + /// Strategies that were successfully executed (test only) + pub successful_transaction_strategies: Vec, } #[derive(Default)] @@ -131,6 +133,9 @@ pub struct IntentExecutionReport { patched_errors: Vec, /// Report of scheduled callbacks callbacks_report: Vec>, + #[cfg(feature = "dev-context-only-utils")] + /// Succeeded transaction strategies report (test only) + successful_transaction_strategies: Vec, } impl IntentExecutionReport { @@ -159,4 +164,12 @@ impl IntentExecutionReport { pub fn junk(&self) -> &Vec { &self.junk } + + #[cfg(feature = "dev-context-only-utils")] + pub fn add_succeeded_transaction_strategy( + &mut self, + value: TransactionStrategy, + ) { + self.successful_transaction_strategies.push(value); + } } diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index f6663b7f0..1f52968e6 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -148,6 +148,9 @@ where inner: result, patched_errors: execution_report.patched_errors, callbacks_report: execution_report.callbacks_report, + #[cfg(feature = "dev-context-only-utils")] + successful_transaction_strategies: execution_report + .successful_transaction_strategies, }; let cleanup_handle = CleanupHandle::new( self.authority, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 3858b5f01..3c61826ce 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -140,7 +140,12 @@ where self.execution_report.dispose(junk_strategy); } - pub fn consume_strategy(self) -> TransactionStrategy { + pub fn done(self) -> TransactionStrategy { + #[cfg(feature = "dev-context-only-utils")] + self.execution_report.add_succeeded_transaction_strategy( + self.transaction_strategy.clone(), + ); + self.transaction_strategy } } diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index 1d978f0a0..daaaeac03 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -229,6 +229,11 @@ where self, commit_signature: Signature, ) -> TwoStageStrategyExecutor<'a, A, O, Committed> { + #[cfg(feature = "dev-context-only-utils")] + self.execution_report.add_succeeded_transaction_strategy( + self.state.commit_strategy.clone(), + ); + TwoStageStrategyExecutor { authority: self.authority, intent_id: self.intent_id, @@ -352,6 +357,11 @@ where /// Transitions to next executor state pub fn done(self, finalize_signature: Signature) -> Finalized { + #[cfg(feature = "dev-context-only-utils")] + self.execution_report.add_succeeded_transaction_strategy( + self.state.finalize_strategy.clone(), + ); + Finalized { commit_signature: self.state.commit_signature, finalize_signature, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 7be89f74f..f60deec8a 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -229,6 +229,9 @@ where inner: result, patched_errors: execution_report.patched_errors, callbacks_report: execution_report.callbacks_report, + #[cfg(feature = "dev-context-only-utils")] + successful_transaction_strategies: execution_report + .successful_transaction_strategies, }; let cleanup_handle = CleanupHandle::new( self.authority, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 20d38675b..b5236e967 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -104,7 +104,7 @@ where let signature = res.as_ref().ok().copied(); single_stage_executor .execute_callbacks(signature, res.as_ref().map(|_| ())); - let transaction_strategy = single_stage_executor.consume_strategy(); + let transaction_strategy = single_stage_executor.done(); execution_report.dispose(transaction_strategy); return res.map(ExecutionOutput::SingleStage); } @@ -113,7 +113,7 @@ where // With actions, we can't predict num of CPIs // If we get here we will try to switch from Single stage to Two Stage commit // Note that this not necessarily will pass at the end due to the same reason - let strategy = single_stage_executor.consume_strategy(); + let strategy = single_stage_executor.done(); let (commit_strategy, finalize_strategy, cleanup) = handle_cpi_limit_error(&authority.pubkey(), strategy); execution_report.dispose(cleanup); diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index b388b86dc..8ad488450 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -11,6 +11,7 @@ use crate::{ }; #[derive(Default)] +#[cfg_attr(feature = "dev-context-only-utils", derive(Clone, Debug))] pub struct TransactionStrategy { pub optimized_tasks: Vec, pub lookup_tables_keys: Vec, diff --git a/test-integration/test-committor-service/Cargo.toml b/test-integration/test-committor-service/Cargo.toml index 6035b5ec8..0fe490d69 100644 --- a/test-integration/test-committor-service/Cargo.toml +++ b/test-integration/test-committor-service/Cargo.toml @@ -12,7 +12,7 @@ magicblock-core = { workspace = true } magicblock-committor-program = { workspace = true, features = [ "no-entrypoint", ] } -magicblock-committor-service = { workspace = true } +magicblock-committor-service = { workspace = true, features = ["dev-context-only-utils"] } magicblock-delegation-program-api = { workspace = true } magicblock-program = { workspace = true } magicblock-rpc-client = { workspace = true } diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 25807cc93..13107c321 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -10,11 +10,16 @@ use async_trait::async_trait; use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, + error::IntentExecutorError, + intent_execution_client::IntentExecutionClient, task_info_fetcher::{ CacheTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, TaskInfoFetcherResult, }, + IntentExecutorCtx, }, + outbox_client::{InternalOutboxClientError, OutboxClient}, + service::outbox_intent_bundles_reader::OutboxIntentBundlesReader, tasks::commit_task::{CommitDelivery, CommitTask}, transaction_preparator::{ delivery_preparator::DeliveryPreparator, TransactionPreparatorImpl, @@ -25,6 +30,10 @@ use magicblock_core::{ intent::{BaseActionCallback, CommittedAccount}, traits::{ActionResult, ActionsCallbackScheduler, CallbackScheduleError}, }; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, + outbox_intent_bundles::OutboxIntentBundle, SentCommit, +}; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use solana_account::Account; @@ -34,6 +43,7 @@ use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ signature::{Keypair, Signature}, signer::Signer, + transaction::Transaction, }; // Helper function to create a test RPC client @@ -112,16 +122,16 @@ impl TestFixture { TransactionPreparatorImpl, MockTaskInfoFetcher, MockActionsCallbackExecutor, + MockOutboxClient, > { - let transaction_preparator = self.create_transaction_preparator(); - - AcceptedIntentExecutor::new( - self.rpc_client.clone(), - transaction_preparator, - self.create_task_info_fetcher(), - MockActionsCallbackExecutor::default(), - DEFAULT_ACTIONS_TIMEOUT, - ) + AcceptedIntentExecutor::new(IntentExecutorCtx { + intent_client: IntentExecutionClient::new(self.rpc_client.clone()), + transaction_preparator: self.create_transaction_preparator(), + task_info_fetcher: self.create_task_info_fetcher(), + outbox_client: Arc::new(MockOutboxClient), + actions_callback_executor: MockActionsCallbackExecutor::default(), + actions_timeout: DEFAULT_ACTIONS_TIMEOUT, + }) } #[allow(dead_code)] @@ -134,6 +144,58 @@ impl TestFixture { } } +pub struct MockOutboxReader; + +#[async_trait] +impl OutboxIntentBundlesReader for MockOutboxReader { + type Error = std::convert::Infallible; + + async fn read( + &mut self, + _n: usize, + ) -> Result, Self::Error> { + Ok(vec![]) + } +} + +#[derive(Clone)] +pub struct MockOutboxClient; + +#[async_trait] +impl OutboxClient for MockOutboxClient { + type Error = InternalOutboxClientError; + type OutboxReader = MockOutboxReader; + + async fn accept_scheduled_intents( + &self, + ) -> Result< + Vec, + (Vec, Self::Error), + > { + Ok(vec![]) + } + + async fn set_intent_execution_stage( + &self, + _intent_id: u64, + _stage: ExecutionStage, + ) -> Result<(), Self::Error> { + Ok(()) + } + + async fn notify_commit_sent( + &self, + _sent_tx: Transaction, + _sent_commit: SentCommit, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn outbox_reader(&self) -> Self::OutboxReader { + MockOutboxReader + } +} + type CallbackCalls = Vec<(Vec, ActionResult)>; #[derive(Clone, Default)] diff --git a/test-integration/test-committor-service/tests/test_delivery_preparator.rs b/test-integration/test-committor-service/tests/test_delivery_preparator.rs index 403cd6f7e..db4441d61 100644 --- a/test-integration/test-committor-service/tests/test_delivery_preparator.rs +++ b/test-integration/test-committor-service/tests/test_delivery_preparator.rs @@ -1,12 +1,9 @@ use borsh::BorshDeserialize; use magicblock_committor_program::Chunks; -use magicblock_committor_service::{ - persist::IntentPersisterImpl, - tasks::{ - commit_task::{CommitBufferStage, CommitDelivery}, - task_strategist::{TaskStrategist, TransactionStrategy}, - BaseTaskImpl, - }, +use magicblock_committor_service::tasks::{ + commit_task::{CommitBufferStage, CommitDelivery}, + task_strategist::{TaskStrategist, TransactionStrategy}, + BaseTaskImpl, }; use solana_sdk::signer::Signer; @@ -31,11 +28,7 @@ async fn test_prepare_10kb_buffer() { // Test preparation let result = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy) .await; assert!(result.is_ok(), "Preparation failed: {:?}", result.err()); @@ -101,11 +94,7 @@ async fn test_prepare_multiple_buffers() { // Test preparation let result = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy) .await; assert!(result.is_ok(), "Preparation failed: {:?}", result.err()); @@ -184,11 +173,7 @@ async fn test_lookup_tables() { }; let result = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy) .await; assert!(result.is_ok(), "Failed to prepare lookup tables"); @@ -221,11 +206,7 @@ async fn test_already_initialized_error_handled() { // Test preparation let result = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy) .await; assert!(result.is_ok(), "Preparation failed: {:?}", result.err()); @@ -263,11 +244,7 @@ async fn test_already_initialized_error_handled() { // Test preparation let result = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy) .await; assert!(result.is_ok(), "Preparation failed: {:?}", result.err()); @@ -321,11 +298,7 @@ async fn test_prepare_cleanup_and_reprepare_mixed_tasks() { // --- Step 1: initial prepare --- let res = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy) .await; assert!(res.is_ok(), "Initial prepare failed: {:?}", res.err()); @@ -424,11 +397,7 @@ async fn test_prepare_cleanup_and_reprepare_mixed_tasks() { }; let res2 = preparator - .prepare_for_delivery( - &fixture.authority, - &mut strategy2, - &None::, - ) + .prepare_for_delivery(&fixture.authority, &mut strategy2) .await; assert!( res2.is_ok(), diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 047e761c3..fd8045053 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -18,6 +18,7 @@ use magicblock_committor_service::{ error::IntentExecutorError, intent_execution_client::IntentExecutionClient, strategy_executor::{ + error::TransactionStrategyExecutionError, two_stage::{Initialized, TwoStageStrategyExecutor}, utils::prepare_and_execute_strategy, }, @@ -26,9 +27,8 @@ use magicblock_committor_service::{ TaskInfoFetcherError, }, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, - IntentExecutor, + IntentExecutor, IntentExecutorCtx, }, - persist::IntentPersisterImpl, tasks::{ task_builder::{TaskBuilderError, TaskBuilderImpl, TasksBuilder}, task_strategist::{TaskStrategist, TransactionStrategy}, @@ -69,9 +69,9 @@ use solana_sdk::{ transaction::{Transaction, TransactionError}, }; use solana_sdk_ids::system_program; -use magicblock_committor_service::intent_executor::strategy_executor::error::TransactionStrategyExecutionError; + use crate::{ - common::{MockActionsCallbackExecutor, TestFixture}, + common::{MockActionsCallbackExecutor, MockOutboxClient, TestFixture}, utils::{ ensure_validator_authority, transactions::{ @@ -93,6 +93,7 @@ struct TestEnv { TransactionPreparatorImpl, RpcTaskInfoFetcher, MockActionsCallbackExecutor, + MockOutboxClient, >, callback_executor: MockActionsCallbackExecutor, pre_test_tablemania_state: HashMap, @@ -118,13 +119,16 @@ impl TestEnv { } let callback_executor = MockActionsCallbackExecutor::default(); - let intent_executor = AcceptedIntentExecutor::new( - fixture.rpc_client.clone(), + let intent_executor = AcceptedIntentExecutor::new(IntentExecutorCtx { + intent_client: IntentExecutionClient::new( + fixture.rpc_client.clone(), + ), transaction_preparator, - task_info_fetcher.clone(), - callback_executor.clone(), - DEFAULT_ACTIONS_TIMEOUT, - ); + task_info_fetcher: task_info_fetcher.clone(), + outbox_client: Arc::new(MockOutboxClient), + actions_callback_executor: callback_executor.clone(), + actions_timeout: DEFAULT_ACTIONS_TIMEOUT, + }); Self { fixture, @@ -181,7 +185,6 @@ async fn test_commit_id_error_parsing() { &fixture.authority, &transaction_preparator, &mut transaction_strategy, - &None::, ) .await; assert!(execution_result.is_ok(), "Preparation is expected to pass!"); @@ -245,7 +248,6 @@ async fn test_undelegation_error_parsing() { &fixture.authority, &transaction_preparator, &mut transaction_strategy, - &None::, ) .await; assert!(execution_result.is_ok(), "Preparation is expected to pass!"); @@ -317,7 +319,6 @@ async fn test_action_error_parsing() { &fixture.authority, &transaction_preparator, &mut transaction_strategy, - &None::, ) .await; assert!(execution_result.is_ok(), "Preparation is expected to pass!"); @@ -386,7 +387,6 @@ async fn test_cpi_limits_error_parsing() { &fixture.authority, &transaction_preparator, &mut transaction_strategy, - &None::, ) .await; assert!(execution_result.is_ok(), "Preparation is expected to pass!"); @@ -420,7 +420,7 @@ async fn test_min_context_slot_not_reached_error_parsing() { let TestEnv { fixture: _, - mut intent_executor, + intent_executor, task_info_fetcher: _, callback_executor: _, pre_test_tablemania_state: _, @@ -436,9 +436,7 @@ async fn test_min_context_slot_not_reached_error_parsing() { true, ); - let execution_result = intent_executor - .execute(intent, None::) - .await; + let (execution_result, _) = Box::new(intent_executor).execute(intent).await; // Verify that we got MinContextSlotNotReachedError assert!(execution_result.inner.is_err()); @@ -467,7 +465,7 @@ async fn test_commit_id_error_recovery() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher, callback_executor: _, pre_test_tablemania_state, @@ -495,13 +493,12 @@ async fn test_commit_id_error_recovery() { assert!(res.unwrap().contains_key(&committed_account.pubkey)); // Now execute intent - let res = intent_executor - .execute(intent, None::) - .await; + let (res, cleanup_handle) = Box::new(intent_executor).execute(intent).await; let IntentExecutionResult { inner: res, patched_errors, callbacks_report, + .. } = res; assert!( @@ -521,7 +518,7 @@ async fn test_commit_id_error_recovery() { )); // Cleanup succeeds - assert!(intent_executor.cleanup().await.is_ok()); + assert!(cleanup_handle.clean().await.is_ok()); let mut commit_ids_by_pk = HashMap::new(); for el in [&committed_account].iter() { let nonce = task_info_fetcher @@ -547,7 +544,7 @@ async fn test_undelegation_error_recovery() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher: _, callback_executor: _, pre_test_tablemania_state, @@ -570,13 +567,12 @@ async fn test_undelegation_error_recovery() { let intent = create_intent(vec![committed_account.clone()], true); // Execute intent - let res = intent_executor - .execute(intent, None::) - .await; + let (res, cleanup_handle) = Box::new(intent_executor).execute(intent).await; let IntentExecutionResult { inner: res, patched_errors, callbacks_report, + .. } = res; assert!(res.is_ok()); @@ -592,7 +588,7 @@ async fn test_undelegation_error_recovery() { )); // Cleanup succeeds - assert!(intent_executor.cleanup().await.is_ok()); + assert!(cleanup_handle.clean().await.is_ok()); verify( &fixture.table_mania, fixture.rpc_client.get_inner(), @@ -609,7 +605,7 @@ async fn test_action_error_recovery() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher: _, callback_executor: _, pre_test_tablemania_state, @@ -637,13 +633,12 @@ async fn test_action_error_recovery() { }); let scheduled_intent = create_scheduled_intent(base_intent); - let res = intent_executor - .execute(scheduled_intent, None::) - .await; + let (res, _) = Box::new(intent_executor).execute(scheduled_intent).await; let IntentExecutionResult { inner: res, patched_errors, callbacks_report, + .. } = res; assert!(res.is_ok()); @@ -679,7 +674,7 @@ async fn test_commit_id_and_action_errors_recovery() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher, callback_executor: _, pre_test_tablemania_state, @@ -716,13 +711,13 @@ async fn test_commit_id_and_action_errors_recovery() { let scheduled_intent = create_scheduled_intent(base_intent); // Execute intent - let res = intent_executor - .execute(scheduled_intent, None::) - .await; + let (res, cleanup_handle) = + Box::new(intent_executor).execute(scheduled_intent).await; let IntentExecutionResult { inner: res, patched_errors, callbacks_report, + .. } = res; assert!(res.is_ok()); @@ -745,7 +740,7 @@ async fn test_commit_id_and_action_errors_recovery() { )); // Cleanup succeeds - assert!(intent_executor.cleanup().await.is_ok()); + assert!(cleanup_handle.clean().await.is_ok()); verify_committed_accounts_state( fixture.rpc_client.get_inner(), @@ -818,7 +813,6 @@ async fn test_cpi_limits_error_recovery() { scheduled_intent, strategy, &mut execution_report, - &None::, ) .await; assert!(execution_result.is_ok(), "Intent expected to recover"); @@ -941,7 +935,6 @@ async fn test_commit_id_actions_cpi_limit_errors_recovery() { scheduled_intent, strategy, &mut execution_report, - &None::, ) .await; @@ -1020,7 +1013,7 @@ async fn test_commit_id_actions_cpi_limit_errors_recovery() { async fn test_commit_unfinalized_account_recovery() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher: _, callback_executor: _, pre_test_tablemania_state: _, @@ -1075,9 +1068,7 @@ async fn test_commit_unfinalized_account_recovery() { remote_slot: Default::default(), }; let intent = create_intent(vec![committed_account], false); - let result = intent_executor - .execute(intent, None::) - .await; + let (result, _) = Box::new(intent_executor).execute(intent).await; assert!(result.inner.is_ok()); assert!(matches!( result.inner.unwrap(), @@ -1099,7 +1090,7 @@ async fn test_commit_unfinalized_account_recovery() { async fn test_commit_unfinalized_account_recovery_two_stage() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher: _, callback_executor: _, pre_test_tablemania_state: _, @@ -1162,9 +1153,7 @@ async fn test_commit_unfinalized_account_recovery_two_stage() { .collect(); let intent = create_intent(committed_accounts, true); - let result = intent_executor - .execute(intent, None::) - .await; + let (result, _) = Box::new(intent_executor).execute(intent).await; assert!(result.inner.is_ok()); assert!(matches!( result.inner.unwrap(), @@ -1191,7 +1180,7 @@ async fn test_action_callback_fired_on_failure() { let TestEnv { fixture, - mut intent_executor, + intent_executor, task_info_fetcher: _, callback_executor, pre_test_tablemania_state: _, @@ -1223,9 +1212,7 @@ async fn test_action_callback_fired_on_failure() { }); let scheduled_intent = create_scheduled_intent(base_intent); - let res = intent_executor - .execute(scheduled_intent, None::) - .await; + let (res, _) = Box::new(intent_executor).execute(scheduled_intent).await; assert!(res.inner.is_ok()); assert_eq!(res.callbacks_report.len(), 1, "1 callback scheduled"); @@ -1281,18 +1268,18 @@ async fn test_action_callback_fired_on_timeout() { let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( RpcTaskInfoFetcher::new(fixture.rpc_client.clone()), )); - let mut intent_executor = AcceptedIntentExecutor::new( - fixture.rpc_client.clone(), - fixture.create_transaction_preparator(), + let intent_executor = AcceptedIntentExecutor::new(IntentExecutorCtx { + intent_client: IntentExecutionClient::new(fixture.rpc_client.clone()), + transaction_preparator: fixture.create_transaction_preparator(), task_info_fetcher, - callback_executor.clone(), - Duration::ZERO, - ); + outbox_client: Arc::new(MockOutboxClient), + actions_callback_executor: callback_executor.clone(), + actions_timeout: Duration::ZERO, + }); let scheduled_intent = create_scheduled_intent(base_intent); - let res = intent_executor - .execute(scheduled_intent, None::) - .await; + let (res, cleanup_handle) = + Box::new(intent_executor).execute(scheduled_intent).await; assert!(res.inner.is_ok()); assert!(res.patched_errors.is_empty()); @@ -1306,7 +1293,7 @@ async fn test_action_callback_fired_on_timeout() { assert_eq!(callbacks[0], expected_callback); assert!(matches!(result, Err(ActionError::TimeoutError))); - assert!(intent_executor.cleanup().await.is_ok()); + assert!(cleanup_handle.clean().await.is_ok()); verify_committed_accounts_state( fixture.rpc_client.get_inner(), &[committed_account], @@ -1383,7 +1370,6 @@ async fn test_callbacks_fired_in_two_stage() { &committed_pubkeys, &transaction_preparator, &task_info_fetcher, - &None::, ) .await .expect("commit must succeed"); @@ -1401,7 +1387,7 @@ async fn test_callbacks_fired_in_two_stage() { // Execute finalize stage let mut finalize_executor = executor.done(commit_sig); finalize_executor - .finalize(&transaction_preparator, &None::) + .finalize(&transaction_preparator) .await .expect("finalize must succeed"); @@ -1425,15 +1411,16 @@ async fn create_two_stage_executor<'a>( intent: &ScheduledIntentBundle, task_info_fetcher: &Arc>, execution_report: &'a mut IntentExecutionReport, -) -> TwoStageStrategyExecutor<'a, MockActionsCallbackExecutor, Initialized> { +) -> TwoStageStrategyExecutor< + 'a, + MockActionsCallbackExecutor, + MockOutboxClient, + Initialized, +> { let authority = &fixture.authority.pubkey(); - let commit_tasks = TaskBuilderImpl::commit_tasks( - task_info_fetcher, - intent, - &None::, - ) - .await - .unwrap(); + let commit_tasks = TaskBuilderImpl::commit_tasks(task_info_fetcher, intent) + .await + .unwrap(); let finalize_tasks = TaskBuilderImpl::finalize_tasks(task_info_fetcher, intent) .await @@ -1442,11 +1429,13 @@ async fn create_two_stage_executor<'a>( TaskStrategist::build_strategy(commit_tasks, authority).unwrap(); let finalize_strategy = TaskStrategist::build_strategy(finalize_tasks, authority).unwrap(); + let state = Initialized::new(commit_strategy, finalize_strategy, None); TwoStageStrategyExecutor::new( + state, fixture.authority.insecure_clone(), - commit_strategy, - finalize_strategy, + intent.id, IntentExecutionClient::new(fixture.rpc_client.clone()), + Arc::new(MockOutboxClient), callback_executor.clone(), execution_report, ) diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 348a9a103..34ef494fe 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -5,13 +5,19 @@ use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, intent_executor::{error::IntentExecutorError, ExecutionOutput}, - persist::CommitStrategy, + tasks::{ + commit_task::CommitDelivery, task_strategist::TransactionStrategy, + BaseTaskImpl, + }, ComputeBudgetConfig, }; use magicblock_core::intent::CommittedAccount; -use magicblock_program::magic_scheduled_base_intent::{ - CommitAndUndelegate, CommitType, MagicBaseIntent, MagicIntentBundle, - ScheduledIntentBundle, UndelegateType, +use magicblock_program::{ + magic_scheduled_base_intent::{ + CommitAndUndelegate, CommitType, MagicBaseIntent, MagicIntentBundle, + ScheduledIntentBundle, UndelegateType, + }, + outbox_intent_bundles::OutboxIntentBundle, }; use program_flexi_counter::state::FlexiCounter; use solana_account::{Account, ReadableAccount}; @@ -43,6 +49,107 @@ mod utils; // ----------------- type ExpectedStrategies = HashMap; +pub struct AccountCommitInfo { + data: Vec, + commit_nonce: u64, + allow_undelegation: bool, + strategy: CommitStrategy, +} + +impl AccountCommitInfo { + fn from_strategy( + strategy: TransactionStrategy, + ) -> Vec<(Pubkey, AccountCommitInfo)> { + let uses_alts = strategy.uses_alts(); + strategy + .optimized_tasks + .iter() + .fold(vec![], |mut infos, task| { + let commit_info = match task { + BaseTaskImpl::Commit(val) => { + let info = AccountCommitInfo { + commit_nonce: val.commit_id, + data: val.committed_account.account.data.clone(), + allow_undelegation: val.allow_undelegation, + strategy: CommitStrategy::new( + uses_alts, + val.delivery_details.clone(), + ), + }; + + Some((val.committed_account.pubkey, info)) + } + BaseTaskImpl::CommitFinalize(val) => { + let info = AccountCommitInfo { + commit_nonce: val.commit_id, + data: val.committed_account.account.data.clone(), + allow_undelegation: val.allow_undelegation, + strategy: CommitStrategy::new( + uses_alts, + val.delivery.clone(), + ), + }; + + Some((val.committed_account.pubkey, info)) + } + _ => None, + }; + + if let Some(commit_info) = commit_info { + infos.push(commit_info); + infos + } else { + infos + } + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum CommitStrategy { + /// Args without the use of a lookup table + #[default] + StateArgs, + /// Args with the use of a lookup table + StateArgsWithLookupTable, + /// Buffer and chunks which has the most overhead + StateBuffer, + /// Buffer and chunks with the use of a lookup table + StateBufferWithLookupTable, + + /// Args without the use of a lookup table + DiffArgs, + /// Args with the use of a lookup table + DiffArgsWithLookupTable, + /// Buffer and chunks which has the most overhead + DiffBuffer, + /// Buffer and chunks with the use of a lookup table + DiffBufferWithLookupTable, +} + +impl CommitStrategy { + fn new(uses_alts: bool, commit_delivery: CommitDelivery) -> Self { + match (uses_alts, commit_delivery) { + (false, CommitDelivery::StateInArgs) => Self::StateArgs, + (false, CommitDelivery::StateInBuffer { .. }) => Self::StateBuffer, + (false, CommitDelivery::DiffInArgs { .. }) => Self::DiffArgs, + (false, CommitDelivery::DiffInBuffer { .. }) => Self::DiffBuffer, + (true, CommitDelivery::StateInArgs) => { + Self::StateArgsWithLookupTable + } + (true, CommitDelivery::StateInBuffer { .. }) => { + Self::StateBufferWithLookupTable + } + (true, CommitDelivery::DiffInArgs { .. }) => { + Self::DiffArgsWithLookupTable + } + (true, CommitDelivery::DiffInBuffer { .. }) => { + Self::DiffBufferWithLookupTable + } + } + } +} + /// /// Unlike ScheduleCommitType which always implies Finalize (because that /// simulates "user-facing" schedule commit intent), CommitIntentKind simulates @@ -239,16 +346,13 @@ async fn commit_single_account( fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; // Run each test with and without finalizing - let processor = Arc::new( - CommittorProcessor::new( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - None, - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(), - ); + let processor = Arc::new(CommittorProcessor::new( + validator_auth.insecure_clone(), + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + None, + Arc::new(common::MockOutboxClient), + common::MockActionsCallbackExecutor::default(), + )); let counter_auth = Keypair::new(); let (pubkey, mut account) = @@ -323,16 +427,13 @@ async fn commit_book_order_account( fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; // Run each test with and without finalizing - let processor = Arc::new( - CommittorProcessor::new( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - None, - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(), - ); + let processor = Arc::new(CommittorProcessor::new( + validator_auth.insecure_clone(), + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + None, + Arc::new(common::MockOutboxClient), + common::MockActionsCallbackExecutor::default(), + )); let payer = Keypair::new(); let (order_book_pk, mut order_book_ac) = @@ -798,16 +899,13 @@ async fn commit_multiple_accounts( let validator_auth = ensure_validator_authority(); fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; - let processor = Arc::new( - CommittorProcessor::new( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - None, - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(), - ); + let processor = Arc::new(CommittorProcessor::new( + validator_auth.insecure_clone(), + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + None, + Arc::new(common::MockOutboxClient), + common::MockActionsCallbackExecutor::default(), + )); // Create bundles of committed accounts let bundles_of_committees = create_bundles(bundle_size, bytess).await; @@ -869,16 +967,13 @@ async fn execute_intent_bundle( let validator_auth = ensure_validator_authority(); fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; - let processor = Arc::new( - CommittorProcessor::new( - validator_auth.insecure_clone(), - ":memory:", - ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), - None, - common::MockActionsCallbackExecutor::default(), - ) - .unwrap(), - ); + let processor = Arc::new(CommittorProcessor::new( + validator_auth.insecure_clone(), + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + None, + Arc::new(common::MockOutboxClient), + common::MockActionsCallbackExecutor::default(), + )); // Create bundles of committed accounts let to_commit = create_and_delegate_accounts(bytess_to_commit); @@ -954,8 +1049,12 @@ async fn ix_commit_local( expected_strategies: ExpectedStrategies, program_id: Pubkey, ) { + let outbox_bundles = intent_bundles + .iter() + .map(|b| OutboxIntentBundle::accepted(b.clone())) + .collect::>(); let execution_outputs = processor - .execute_intent_bundles(intent_bundles.clone()) + .execute_intent_bundles(outbox_bundles) .await .unwrap() .into_iter() @@ -1062,23 +1161,20 @@ async fn ix_commit_local( }) .collect(); - let statuses = processor.get_commit_statuses(base_intent.id).unwrap(); - debug!( - "{}", - statuses + let account_commit_infos: Vec<(Pubkey, AccountCommitInfo)> = + execution_result + .successful_transaction_strategies .iter() - .map(|x| x.to_string()) - .collect::>() - .join("\n") - ); + .flat_map(|s| AccountCommitInfo::from_strategy(s.clone())) + .collect(); - assert_eq!(statuses.len(), committed_accounts.len()); + assert_eq!(account_commit_infos.len(), committed_accounts.len()); - for commit_status in statuses { + for (pubkey, commit_info) in account_commit_infos { let (is_undelegate, account) = committed_accounts - .remove(&commit_status.pubkey) + .remove(&pubkey) .expect("Account should be persisted"); - println!("account: {}", account.pubkey); + println!("account: {}", pubkey); // When we finalize it is possible to also undelegate the account let expected_owner = if is_undelegate { @@ -1106,7 +1202,7 @@ async fn ix_commit_local( ); // Track the strategy used - let strategy = commit_status.commit_strategy; + let strategy = commit_info.strategy; let strategy_count = strategies.entry(strategy).or_insert(0); *strategy_count += 1; } diff --git a/test-integration/test-committor-service/tests/test_transaction_preparator.rs b/test-integration/test-committor-service/tests/test_transaction_preparator.rs index 207f3fc48..8ac829f6e 100644 --- a/test-integration/test-committor-service/tests/test_transaction_preparator.rs +++ b/test-integration/test-committor-service/tests/test_transaction_preparator.rs @@ -1,7 +1,6 @@ use borsh::BorshDeserialize; use magicblock_committor_program::Chunks; use magicblock_committor_service::{ - persist::IntentPersisterImpl, tasks::{ commit_task::CommitBufferStage, task_strategist::{TaskStrategist, TransactionStrategy}, @@ -56,11 +55,7 @@ async fn test_prepare_commit_tx_with_single_account() { // Test preparation let result = preparator - .prepare_for_strategy( - &fixture.authority, - &mut tx_strategy, - &None::, - ) + .prepare_for_strategy(&fixture.authority, &mut tx_strategy) .await; assert!(result.is_ok(), "Preparation failed: {:?}", result.err()); @@ -127,11 +122,7 @@ async fn test_prepare_commit_tx_with_multiple_accounts() { // Test preparation let mut actual_message = preparator - .prepare_for_strategy( - &fixture.authority, - &mut tx_strategy, - &None::, - ) + .prepare_for_strategy(&fixture.authority, &mut tx_strategy) .await .unwrap(); @@ -222,11 +213,7 @@ async fn test_prepare_commit_tx_with_base_actions() { // Test preparation let mut actual_message = preparator - .prepare_for_strategy( - &fixture.authority, - &mut tx_strategy, - &None::, - ) + .prepare_for_strategy(&fixture.authority, &mut tx_strategy) .await .unwrap(); @@ -303,11 +290,7 @@ async fn test_prepare_finalize_tx_with_undelegate_with_atls() { // Test preparation let result = preparator - .prepare_for_strategy( - &fixture.authority, - &mut tx_strategy, - &None::, - ) + .prepare_for_strategy(&fixture.authority, &mut tx_strategy) .await; assert!(result.is_ok()); From c264990e519767fb83719bc97a31ff4d00e98872 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 23 Jun 2026 15:54:01 +0700 Subject: [PATCH 57/97] fix: tests --- .../intent_execution_engine.rs | 4 ++-- .../strategy_executor/two_stage.rs | 22 ++++++++++--------- .../src/tasks/task_strategist.rs | 4 ++-- .../tests/test_ix_commit_local.rs | 5 +++++ 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 6930e6450..361e55fb3 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -29,6 +29,7 @@ use crate::{ strategy_executor::error::TransactionStrategyExecutionError, ExecutionOutput, IntentExecutionResult, IntentExecutor, }, + tasks::task_strategist::TransactionStrategy, transaction_preparator::TransactionPreparator, }; @@ -45,8 +46,7 @@ pub struct BroadcastedIntentExecutionResult { pub patched_errors: Arc, pub callbacks_report: Vec>>, #[cfg(feature = "dev-context-only-utils")] - pub successful_transaction_strategies: - Vec, + pub successful_transaction_strategies: Vec, } impl BroadcastedIntentExecutionResult { diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index daaaeac03..94c6949ea 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -176,7 +176,7 @@ where commit_result.as_ref().map(|_| ()), ); self.execution_report - .dispose(mem::take(&mut self.state.commit_strategy)); + .dispose(self.state.commit_strategy.clone()); if commit_result.is_err() { self.execution_report .dispose(mem::take(&mut self.state.finalize_strategy)); @@ -226,13 +226,14 @@ where /// Transitions to next executor state pub fn done( - self, + mut self, commit_signature: Signature, ) -> TwoStageStrategyExecutor<'a, A, O, Committed> { #[cfg(feature = "dev-context-only-utils")] - self.execution_report.add_succeeded_transaction_strategy( - self.state.commit_strategy.clone(), - ); + self.execution_report + .add_succeeded_transaction_strategy(mem::take( + &mut self.state.commit_strategy, + )); TwoStageStrategyExecutor { authority: self.authority, @@ -324,7 +325,7 @@ where finalize_result.as_ref().map(|_| ()), ); self.execution_report - .dispose(mem::take(&mut self.state.finalize_strategy)); + .dispose(self.state.finalize_strategy.clone()); finalize_result.map_err(|err| { IntentExecutorError::from_finalize_execution_error( err, @@ -356,11 +357,12 @@ where } /// Transitions to next executor state - pub fn done(self, finalize_signature: Signature) -> Finalized { + pub fn done(mut self, finalize_signature: Signature) -> Finalized { #[cfg(feature = "dev-context-only-utils")] - self.execution_report.add_succeeded_transaction_strategy( - self.state.finalize_strategy.clone(), - ); + self.execution_report + .add_succeeded_transaction_strategy(mem::take( + &mut self.state.finalize_strategy, + )); Finalized { commit_signature: self.state.commit_signature, diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 8ad488450..2b5803bd9 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -10,8 +10,8 @@ use crate::{ transactions::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, }; -#[derive(Default)] -#[cfg_attr(feature = "dev-context-only-utils", derive(Clone, Debug))] +#[derive(Default, Clone)] +#[cfg_attr(feature = "dev-context-only-utils", derive(Debug))] pub struct TransactionStrategy { pub optimized_tasks: Vec, pub lookup_tables_keys: Vec, diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 34ef494fe..4fc48d949 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -1060,6 +1060,9 @@ async fn ix_commit_local( .into_iter() .collect::>(); + execution_outputs.iter().for_each(|asd| { + println!("kek: {:?}", asd.successful_transaction_strategies); + }); // Assert that all completed assert_eq!(execution_outputs.len(), intent_bundles.len()); @@ -1161,6 +1164,8 @@ async fn ix_commit_local( }) .collect(); + println!("ffff {:?}", execution_result + .successful_transaction_strategies); let account_commit_infos: Vec<(Pubkey, AccountCommitInfo)> = execution_result .successful_transaction_strategies From cf25f6a1fee5fb916fabf6c9fe67d3f2d0ad47bd Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 23 Jun 2026 16:07:09 +0700 Subject: [PATCH 58/97] fix: tests --- .../src/intent_executor/strategy_executor/single_stage.rs | 7 +------ magicblock-committor-service/src/intent_executor/utils.rs | 8 ++++++-- .../test-committor-service/tests/test_ix_commit_local.rs | 6 ------ 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 3c61826ce..3858b5f01 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -140,12 +140,7 @@ where self.execution_report.dispose(junk_strategy); } - pub fn done(self) -> TransactionStrategy { - #[cfg(feature = "dev-context-only-utils")] - self.execution_report.add_succeeded_transaction_strategy( - self.transaction_strategy.clone(), - ); - + pub fn consume_strategy(self) -> TransactionStrategy { self.transaction_strategy } } diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index b5236e967..55e2411f1 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -104,7 +104,11 @@ where let signature = res.as_ref().ok().copied(); single_stage_executor .execute_callbacks(signature, res.as_ref().map(|_| ())); - let transaction_strategy = single_stage_executor.done(); + let transaction_strategy = single_stage_executor.consume_strategy(); + #[cfg(feature = "dev-context-only-utils")] + execution_report.add_succeeded_transaction_strategy( + transaction_strategy.clone(), + ); execution_report.dispose(transaction_strategy); return res.map(ExecutionOutput::SingleStage); } @@ -113,7 +117,7 @@ where // With actions, we can't predict num of CPIs // If we get here we will try to switch from Single stage to Two Stage commit // Note that this not necessarily will pass at the end due to the same reason - let strategy = single_stage_executor.done(); + let strategy = single_stage_executor.consume_strategy(); let (commit_strategy, finalize_strategy, cleanup) = handle_cpi_limit_error(&authority.pubkey(), strategy); execution_report.dispose(cleanup); diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 4fc48d949..0c11bbf08 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -1060,9 +1060,6 @@ async fn ix_commit_local( .into_iter() .collect::>(); - execution_outputs.iter().for_each(|asd| { - println!("kek: {:?}", asd.successful_transaction_strategies); - }); // Assert that all completed assert_eq!(execution_outputs.len(), intent_bundles.len()); @@ -1164,8 +1161,6 @@ async fn ix_commit_local( }) .collect(); - println!("ffff {:?}", execution_result - .successful_transaction_strategies); let account_commit_infos: Vec<(Pubkey, AccountCommitInfo)> = execution_result .successful_transaction_strategies @@ -1179,7 +1174,6 @@ async fn ix_commit_local( let (is_undelegate, account) = committed_accounts .remove(&pubkey) .expect("Account should be persisted"); - println!("account: {}", pubkey); // When we finalize it is possible to also undelegate the account let expected_owner = if is_undelegate { From 9fc7110e4c3f3c3e1cc320ab889ade09ce3a1bfe Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 23 Jun 2026 19:45:09 +0700 Subject: [PATCH 59/97] feat: init outbox tests --- test-integration/Cargo.lock | 9 + .../test-schedule-intent/Cargo.toml | 9 + .../test-schedule-intent/tests/common/mod.rs | 162 ++++++++++ .../tests/test_outbox_flow.rs | 306 ++++++++++++++++++ .../tests/test_schedule_intents.rs | 166 +--------- .../src/integration_test_context.rs | 9 + 6 files changed, 499 insertions(+), 162 deletions(-) create mode 100644 test-integration/test-schedule-intent/tests/common/mod.rs create mode 100644 test-integration/test-schedule-intent/tests/test_outbox_flow.rs diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index c190ab71f..6f5612cd2 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -11312,15 +11312,24 @@ dependencies = [ name = "test-schedule-intent" version = "0.0.0" dependencies = [ + "async-trait", "ephemeral-rollups-sdk", "integration-test-tools", "log", + "magicblock-committor-service", + "magicblock-core", "magicblock-delegation-program-api 0.3.0 (git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b)", "magicblock-magic-program-api 0.12.3", + "magicblock-program", + "magicblock-rpc-client", + "magicblock-table-mania", "program-flexi-counter", + "solana-commitment-config", + "solana-rpc-client", "solana-rpc-client-api", "solana-sdk", "test-kit", + "tokio", "tracing", ] diff --git a/test-integration/test-schedule-intent/Cargo.toml b/test-integration/test-schedule-intent/Cargo.toml index 6f1140712..9f31b4d95 100644 --- a/test-integration/test-schedule-intent/Cargo.toml +++ b/test-integration/test-schedule-intent/Cargo.toml @@ -7,12 +7,21 @@ version.workspace = true log = "0.4.29" [dev-dependencies] +async-trait = { workspace = true } ephemeral-rollups-sdk = { workspace = true } integration-test-tools = { workspace = true } +magicblock-committor-service = { workspace = true } +magicblock-core = { workspace = true } magicblock-delegation-program-api = { workspace = true } magicblock-magic-program-api = { workspace = true } +magicblock-program = { workspace = true } +magicblock-rpc-client = { workspace = true } +magicblock-table-mania = { workspace = true } program-flexi-counter = { workspace = true, features = ["no-entrypoint"] } +solana-commitment-config = { workspace = true } +solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-sdk = { workspace = true } test-kit = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } diff --git a/test-integration/test-schedule-intent/tests/common/mod.rs b/test-integration/test-schedule-intent/tests/common/mod.rs new file mode 100644 index 000000000..fc0be184d --- /dev/null +++ b/test-integration/test-schedule-intent/tests/common/mod.rs @@ -0,0 +1,162 @@ +use dlp_api::pda::ephemeral_balance_pda_from_payer; +use integration_test_tools::IntegrationTestContext; +use program_flexi_counter::{ + delegation_program_id, + instruction::{create_add_ix, create_delegate_ix, create_init_ix}, + state::FlexiCounter, +}; +use solana_sdk::{ + native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, rent::Rent, + signature::Keypair, signer::Signer, +}; + +pub const LABEL: &str = "I am a label"; + +pub fn setup_payer(ctx: &IntegrationTestContext) -> Keypair { + let payer = Keypair::new(); + ctx.airdrop_chain(&payer.pubkey(), LAMPORTS_PER_SOL).unwrap(); + + let ix = dlp_api::instruction_builder::top_up_ephemeral_balance( + payer.pubkey(), + payer.pubkey(), + Some(LAMPORTS_PER_SOL / 2), + Some(1), + ); + ctx.send_and_confirm_instructions_with_payer_chain(&[ix], &payer) + .unwrap(); + + let escrow_pda = ephemeral_balance_pda_from_payer(&payer.pubkey(), 1); + let rent = Rent::default().minimum_balance(0); + assert_eq!( + ctx.fetch_chain_account(escrow_pda).unwrap().lamports, + LAMPORTS_PER_SOL / 2 + rent + ); + + payer +} + +pub fn init_counter(ctx: &IntegrationTestContext, payer: &Keypair) { + let ix = create_init_ix(payer.pubkey(), LABEL.to_string()); + let (_, confirmed) = ctx + .send_and_confirm_instructions_with_payer_chain(&[ix], payer) + .unwrap(); + assert!(confirmed, "Should confirm transaction"); + + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + let counter = ctx + .fetch_chain_account_struct::(counter_pda) + .unwrap(); + assert_eq!( + counter, + FlexiCounter { + count: 0, + updates: 0, + label: LABEL.to_string() + }, + ) +} + +pub fn delegate_counter(ctx: &IntegrationTestContext, payer: &Keypair) { + ctx.wait_for_next_slot_ephem().unwrap(); + + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + let ix = create_delegate_ix(payer.pubkey()); + ctx.send_and_confirm_instructions_with_payer_chain(&[ix], payer) + .unwrap(); + + let owner = ctx.fetch_chain_account_owner(counter_pda).unwrap(); + assert_eq!(owner, delegation_program_id()); +} + +pub fn add_to_counter( + ctx: &IntegrationTestContext, + payer: &Keypair, + value: u8, +) { + ctx.wait_for_next_slot_ephem().unwrap(); + + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + let counter_before = + ctx.fetch_ephem_account_struct::(counter_pda) + .unwrap_or(FlexiCounter { + count: 0, + updates: 0, + label: LABEL.to_string(), + }); + + let ix = create_add_ix(payer.pubkey(), value); + ctx.send_and_confirm_instructions_with_payer_ephem(&[ix], payer) + .unwrap(); + + let counter = ctx + .fetch_ephem_account_struct::(counter_pda) + .unwrap(); + assert_eq!( + counter, + FlexiCounter { + count: counter_before.count + value as u64, + updates: counter_before.updates + 1, + label: LABEL.to_string() + }, + ) +} + +pub struct ExpectedCounter { + pub pda: Pubkey, + pub expected: u64, +} + +pub fn assert_counters( + ctx: &IntegrationTestContext, + expected_counters: &[ExpectedCounter], + is_base: bool, +) { + let actual = expected_counters + .iter() + .map(|c| { + if is_base { + ctx.fetch_chain_account_struct::(c.pda).unwrap() + } else { + ctx.fetch_ephem_account_struct::(c.pda).unwrap() + } + }) + .collect::>(); + + for (i, actual_counter) in actual.iter().enumerate() { + assert_eq!(actual_counter.count, expected_counters[i].expected); + } +} + +pub fn verify_undelegation_in_ephem_via_owner( + pubkeys: &[Pubkey], + ctx: &IntegrationTestContext, +) { + const RETRY_LIMIT: usize = 20; + let mut retries = 0; + + loop { + ctx.wait_for_next_slot_ephem().unwrap(); + let mut not_verified = vec![]; + for pk in pubkeys.iter() { + let counter_pda = FlexiCounter::pda(pk).0; + let owner = ctx.fetch_ephem_account_owner(counter_pda).unwrap(); + if owner == delegation_program_id() { + not_verified.push(*pk); + } + } + if not_verified.is_empty() { + break; + } + retries += 1; + if retries >= RETRY_LIMIT { + panic!( + "Failed to verify undelegation for pubkeys: {}", + not_verified + .iter() + .map(|k| k.to_string()) + .collect::>() + .join(", ") + ); + } + } +} diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs new file mode 100644 index 000000000..cd3c46e09 --- /dev/null +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -0,0 +1,306 @@ +mod common; +use common::*; + +use std::sync::{Arc, Mutex, Once}; + +use async_trait::async_trait; +use integration_test_tools::{ + loaded_accounts::DLP_TEST_AUTHORITY_BYTES, IntegrationTestContext, +}; +use magicblock_committor_service::{ + intent_executor::{ + accepted_intent_executor::AcceptedIntentExecutor, + intent_execution_client::IntentExecutionClient, + task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, + IntentExecutorCtx, + }, + outbox_client::{InternalOutboxClientError, OutboxClient}, + service::outbox_intent_bundles_reader::OutboxIntentBundlesReader, + transaction_preparator::TransactionPreparatorImpl, + ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, +}; +use magicblock_core::{ + intent::{outbox::outbox_intent_pda, BaseActionCallback}, + traits::{ActionResult, ActionsCallbackScheduler, CallbackScheduleError}, +}; +use magicblock_magic_program_api::{ + instruction::MagicBlockInstruction, outbox::ExecutionStage, + MAGIC_CONTEXT_PUBKEY, +}; +use magicblock_committor_service::intent_executor::IntentExecutor; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, + outbox_intent_bundles::OutboxIntentBundle, + validator::init_validator_authority, + MagicContext, SentCommit, +}; +use magicblock_rpc_client::MagicblockRpcClient; +use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; +use program_flexi_counter::{instruction::create_intent_bundle_ix, state::FlexiCounter}; +use solana_rpc_client::nonblocking::rpc_client::RpcClient as AsyncRpcClient; +use solana_sdk::{ + instruction::{AccountMeta, Instruction}, + signature::{Keypair, Signature}, + signer::Signer, + transaction::Transaction, +}; + + +// --------------------------------------------------------------------------- +// NoopCallbackScheduler +// --------------------------------------------------------------------------- + +#[derive(Clone, Default)] +struct NoopCallbackScheduler; + +impl ActionsCallbackScheduler for NoopCallbackScheduler { + fn schedule( + &self, + callbacks: Vec, + _signature: Option, + _result: ActionResult, + ) -> Vec> { + callbacks.iter().map(|_| Ok(Signature::new_unique())).collect() + } +} + +// --------------------------------------------------------------------------- +// TestOutboxClient +// --------------------------------------------------------------------------- + +struct TestOutboxReader; + +#[async_trait] +impl OutboxIntentBundlesReader for TestOutboxReader { + type Error = std::convert::Infallible; + + async fn read( + &mut self, + _n: usize, + ) -> Result, Self::Error> { + Ok(vec![]) + } +} + +struct TestOutboxClient { + ephem_rpc: Arc, + validator_keypair: Keypair, + pub stage_calls: Arc>>, +} + +impl TestOutboxClient { + fn new(ephem_rpc: Arc, validator_keypair: Keypair) -> Self { + Self { + ephem_rpc, + validator_keypair, + stage_calls: Default::default(), + } + } +} + +#[async_trait] +impl OutboxClient for TestOutboxClient { + type Error = InternalOutboxClientError; + type OutboxReader = TestOutboxReader; + + async fn accept_scheduled_intents( + &self, + ) -> Result< + Vec, + (Vec, Self::Error), + > { + Ok(vec![]) + } + + async fn set_intent_execution_stage( + &self, + intent_id: u64, + stage: ExecutionStage, + ) -> Result<(), Self::Error> { + self.stage_calls.lock().unwrap().push((intent_id, stage.clone())); + let blockhash = self + .ephem_rpc + .get_latest_blockhash() + .await + .map_err(InternalOutboxClientError::RpcClientError)?; + let ix = Instruction::new_with_bincode( + magicblock_magic_program_api::id(), + &MagicBlockInstruction::SetIntentExecutionStage { intent_id, stage }, + vec![ + AccountMeta::new_readonly(self.validator_keypair.pubkey(), true), + AccountMeta::new(outbox_intent_pda(intent_id), false), + ], + ); + let tx = Transaction::new_signed_with_payer( + &[ix], + Some(&self.validator_keypair.pubkey()), + &[&self.validator_keypair], + blockhash, + ); + self.ephem_rpc + .send_and_confirm_transaction(&tx) + .await + .map_err(InternalOutboxClientError::RpcClientError)?; + Ok(()) + } + + async fn notify_commit_sent( + &self, + _sent_tx: Transaction, + _sent_commit: SentCommit, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn outbox_reader(&self) -> Self::OutboxReader { + TestOutboxReader + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn ensure_validator_authority() -> Keypair { + static ONCE: Once = Once::new(); + let kp = Keypair::try_from(&DLP_TEST_AUTHORITY_BYTES[..]).unwrap(); + ONCE.call_once(|| { + init_validator_authority(kp.insecure_clone()); + }); + kp +} + +fn read_next_intent_id(ctx: &IntegrationTestContext) -> u64 { + let data = ctx + .fetch_ephem_account_data(MAGIC_CONTEXT_PUBKEY) + .unwrap(); + MagicContext::intent_id(&data) +} + +/// Sends a single ER transaction: [schedule_commit_ix, accept_ix]. +/// Both instructions land atomically — MagicContext is drained within the +/// same tx so CommittorService never sees the intent. +/// Returns the OutboxIntentBundle in Accepted stage. +fn schedule_and_accept( + ctx: &IntegrationTestContext, + payer: &Keypair, +) -> OutboxIntentBundle { + ctx.wait_for_next_slot_ephem().unwrap(); + + let validator_keypair = ensure_validator_authority(); + let intent_id = read_next_intent_id(ctx); + + let destination = Keypair::new(); + let schedule_ix = create_intent_bundle_ix( + vec![payer.pubkey()], + vec![], + destination.pubkey(), + vec![], + 100_000, + ); + let accept_ix = Instruction::new_with_bincode( + magicblock_magic_program_api::id(), + &MagicBlockInstruction::AcceptScheduleCommits, + vec![ + AccountMeta::new_readonly(validator_keypair.pubkey(), true), + AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), + AccountMeta::new(outbox_intent_pda(intent_id), false), + ], + ); + + let mut tx = Transaction::new_with_payer( + &[schedule_ix, accept_ix], + Some(&payer.pubkey()), + ); + let (_, confirmed) = ctx + .send_and_confirm_transaction_ephem(&mut tx, &[payer, &validator_keypair]) + .unwrap(); + assert!(confirmed, "schedule_and_accept tx not confirmed"); + + let pda = outbox_intent_pda(intent_id); + let data = ctx.fetch_ephem_account_data(pda).unwrap(); + OutboxIntentBundle::try_from_bytes(&data).unwrap() +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +/// AcceptedIntentExecutor drives the full commit flow using TestOutboxClient. +/// Verifies: executor succeeds, set_intent_execution_stage is called, +/// and the committed value lands on the base chain. +#[tokio::test] +#[ignore] +async fn test_accepted_executor_outbox_flow() { + let ctx = IntegrationTestContext::try_new().unwrap(); + let payer = setup_payer(&ctx); + + init_counter(&ctx, &payer); + delegate_counter(&ctx, &payer); + add_to_counter(&ctx, &payer, 42); + + let outbox_bundle = schedule_and_accept(&ctx, &payer); + let intent_id = outbox_bundle.inner.id; + + let validator_keypair = ensure_validator_authority(); + let chain_rpc = Arc::new(ctx.try_chain_client_async().unwrap()); + let ephem_rpc = Arc::new(ctx.try_ephem_client_async().unwrap()); + let chain_mb_rpc = MagicblockRpcClient::new(chain_rpc); + let ephem_mb_rpc = MagicblockRpcClient::new(ephem_rpc.clone()); + + let outbox_client = Arc::new(TestOutboxClient::new( + ephem_rpc, + validator_keypair.insecure_clone(), + )); + let stage_calls = outbox_client.stage_calls.clone(); + + let gc_config = GarbageCollectorConfig::default(); + let table_mania = TableMania::new( + chain_mb_rpc.clone(), + &validator_keypair, + Some(gc_config), + ); + let compute_budget = ComputeBudgetConfig::new(1_000_000); + let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( + RpcTaskInfoFetcher::new(ephem_mb_rpc), + )); + let transaction_preparator = TransactionPreparatorImpl::new( + chain_mb_rpc.clone(), + table_mania, + compute_budget, + ); + + let executor = AcceptedIntentExecutor::new(IntentExecutorCtx { + intent_client: IntentExecutionClient::new(chain_mb_rpc), + transaction_preparator, + task_info_fetcher, + outbox_client, + actions_callback_executor: NoopCallbackScheduler, + actions_timeout: DEFAULT_ACTIONS_TIMEOUT, + }); + + let (result, cleanup_handle) = + Box::new(executor).execute(outbox_bundle.inner).await; + let _ = cleanup_handle.clean().await; + + assert!( + result.inner.is_ok(), + "Executor failed: {:?}", + result.inner + ); + let calls = stage_calls.lock().unwrap(); + assert!( + !calls.is_empty(), + "Expected at least one set_intent_execution_stage call" + ); + assert!( + calls.iter().all(|(id, _)| *id == intent_id), + "Stage calls contain unexpected intent ids" + ); + + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + let counter = ctx + .fetch_chain_account_struct::(counter_pda) + .unwrap(); + assert_eq!(counter.count, 42); +} diff --git a/test-integration/test-schedule-intent/tests/test_schedule_intents.rs b/test-integration/test-schedule-intent/tests/test_schedule_intents.rs index 2a16d1b90..e309ef483 100644 --- a/test-integration/test-schedule-intent/tests/test_schedule_intents.rs +++ b/test-integration/test-schedule-intent/tests/test_schedule_intents.rs @@ -1,4 +1,6 @@ -use dlp_api::pda::ephemeral_balance_pda_from_payer; +mod common; +use common::*; + use integration_test_tools::{ conversions::stringify_simulation_result, loaded_accounts::DLP_TEST_AUTHORITY_BYTES, IntegrationTestContext, @@ -6,7 +8,6 @@ use integration_test_tools::{ use program_flexi_counter::{ delegation_program_id, instruction::{ - create_add_ix, create_delegate_ix, create_init_ix, create_intent_bundle_commit_and_finalize_ix, create_intent_bundle_ix, create_intent_ix, create_transfer_intent_ix, }, @@ -14,14 +15,11 @@ use program_flexi_counter::{ }; use solana_rpc_client_api::config::RpcSimulateTransactionConfig; use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, rent::Rent, - signature::Keypair, signer::Signer, transaction::Transaction, + pubkey::Pubkey, signature::Keypair, signer::Signer, transaction::Transaction, }; use test_kit::init_logger; use tracing::*; -const LABEL: &str = "I am a label"; - #[test] fn test_schedule_intent_basic() { // Init context @@ -645,129 +643,6 @@ fn test_intent_bundle_commit_and_commit_finalize() { assert_eq!(owner_finalize, delegation_program_id()); } -fn setup_payer(ctx: &IntegrationTestContext) -> Keypair { - // Airdrop to payer on chain - let payer = Keypair::new(); - ctx.airdrop_chain(&payer.pubkey(), LAMPORTS_PER_SOL) - .unwrap(); - - // Create actor escrow - let ix = dlp_api::instruction_builder::top_up_ephemeral_balance( - payer.pubkey(), - payer.pubkey(), - Some(LAMPORTS_PER_SOL / 2), - Some(1), - ); - ctx.send_and_confirm_instructions_with_payer_chain(&[ix], &payer) - .unwrap(); - - // Confirm actor escrow - let escrow_pda = ephemeral_balance_pda_from_payer(&payer.pubkey(), 1); - let rent = Rent::default().minimum_balance(0); - assert_eq!( - ctx.fetch_chain_account(escrow_pda).unwrap().lamports, - LAMPORTS_PER_SOL / 2 + rent - ); - - payer -} - -fn init_counter(ctx: &IntegrationTestContext, payer: &Keypair) { - let ix = create_init_ix(payer.pubkey(), LABEL.to_string()); - let (_, confirmed) = ctx - .send_and_confirm_instructions_with_payer_chain(&[ix], payer) - .unwrap(); - assert!(confirmed, "Should confirm transaction"); - - let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; - let counter = ctx - .fetch_chain_account_struct::(counter_pda) - .unwrap(); - assert_eq!( - counter, - FlexiCounter { - count: 0, - updates: 0, - label: LABEL.to_string() - }, - ) -} - -// ER action -fn delegate_counter(ctx: &IntegrationTestContext, payer: &Keypair) { - ctx.wait_for_next_slot_ephem().unwrap(); - - let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; - let ix = create_delegate_ix(payer.pubkey()); - ctx.send_and_confirm_instructions_with_payer_chain(&[ix], payer) - .unwrap(); - - // Confirm delegated - let owner = ctx.fetch_chain_account_owner(counter_pda).unwrap(); - assert_eq!(owner, delegation_program_id()); -} - -// ER action -fn add_to_counter(ctx: &IntegrationTestContext, payer: &Keypair, value: u8) { - ctx.wait_for_next_slot_ephem().unwrap(); - - let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; - let counter_before = ctx - .fetch_ephem_account_struct::(counter_pda) - .unwrap_or(FlexiCounter { - count: 0, - updates: 0, - label: LABEL.to_string(), - }); - - // Add value to counter - let ix = create_add_ix(payer.pubkey(), value); - ctx.send_and_confirm_instructions_with_payer_ephem(&[ix], payer) - .unwrap(); - - let counter = ctx - .fetch_ephem_account_struct::(counter_pda) - .unwrap(); - assert_eq!( - counter, - FlexiCounter { - count: counter_before.count + value as u64, - updates: counter_before.updates + 1, - label: LABEL.to_string() - }, - ) -} - -struct ExpectedCounter { - pda: Pubkey, - expected: u64, -} - -fn assert_counters( - ctx: &IntegrationTestContext, - expected_counters: &[ExpectedCounter], - is_base: bool, -) { - // Confirm results on base lauer - let actual_counter = expected_counters - .iter() - .map(|counter| { - if is_base { - ctx.fetch_chain_account_struct::(counter.pda) - .unwrap() - } else { - ctx.fetch_ephem_account_struct::(counter.pda) - .unwrap() - } - }) - .collect::>(); - - for i in 0..actual_counter.len() { - let actual_counter = &actual_counter[i]; - let expected_counter = &expected_counters[i]; - assert_eq!(actual_counter.count, expected_counter.expected); - } -} fn schedule_intent( ctx: &IntegrationTestContext, @@ -990,36 +865,3 @@ fn dump_ephem_simulation( } } -fn verify_undelegation_in_ephem_via_owner( - pubkeys: &[Pubkey], - ctx: &IntegrationTestContext, -) { - const RETRY_LIMIT: usize = 20; - let mut retries = 0; - - loop { - ctx.wait_for_next_slot_ephem().unwrap(); - let mut not_verified = vec![]; - for pk in pubkeys.iter() { - let counter_pda = FlexiCounter::pda(pk).0; - let owner = ctx.fetch_ephem_account_owner(counter_pda).unwrap(); - if owner == delegation_program_id() { - not_verified.push(*pk); - } - } - if not_verified.is_empty() { - break; - } - retries += 1; - if retries >= RETRY_LIMIT { - panic!( - "Failed to verify undelegation for pubkeys: {}", - not_verified - .iter() - .map(|k| k.to_string()) - .collect::>() - .join(", ") - ); - } - } -} diff --git a/test-integration/test-tools/src/integration_test_context.rs b/test-integration/test-tools/src/integration_test_context.rs index f1246007d..822bf05a5 100644 --- a/test-integration/test-tools/src/integration_test_context.rs +++ b/test-integration/test-tools/src/integration_test_context.rs @@ -287,6 +287,15 @@ impl IntegrationTestContext { Ok(ephem_client) } + pub fn try_ephem_client_async( + &self, + ) -> anyhow::Result { + let Some(ephem_client) = self.ephem_client.as_ref() else { + return Err(anyhow::anyhow!("Ephem client not available")); + }; + Ok(async_rpc_client(ephem_client)) + } + pub fn fetch_ephem_account_data( &self, pubkey: Pubkey, From bec92c019a30f4a2d1bafa5a2c64edb73136068c Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 24 Jun 2026 19:14:46 +0700 Subject: [PATCH 60/97] fix: tests --- Cargo.lock | 8 +-- Cargo.toml | 17 ++++--- .../src/instruction.rs | 22 ++++----- .../process_accept_scheduled_commits.rs | 14 +++--- .../outbox/process_scheduled_commit_sent.rs | 6 ++- .../intent_bundles/outbox_intent_bundles.rs | 39 ++++++++++++--- .../magicblock/src/utils/instruction_utils.rs | 7 ++- test-integration/Cargo.lock | 14 +++--- test-integration/Cargo.toml | 10 ++-- .../test-schedule-intent/tests/common/mod.rs | 23 +++++---- .../tests/test_outbox_flow.rs | 49 +++++++++++-------- .../tests/test_schedule_intents.rs | 6 +-- .../test-tools/src/transactions.rs | 2 +- 13 files changed, 130 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcca3fb9a..6dc3abdc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "solana-account" version = "3.4.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "bincode", "qualifier_attr", @@ -7872,7 +7872,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "base64 0.22.1", "bincode", @@ -8568,7 +8568,7 @@ dependencies = [ [[package]] name = "solana-svm" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "ahash 0.8.12", "log", @@ -8884,7 +8884,7 @@ dependencies = [ [[package]] name = "solana-transaction-context" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "bincode", "qualifier_attr", diff --git a/Cargo.toml b/Cargo.toml index 17cbcf57a..13f2cdc73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -141,7 +141,7 @@ prometheus = "0.13.4" # Keep in sync with `solana-storage-proto` codegen. -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82", features = [ +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00", features = [ "dev-context-only-utils" ] } solana-transaction-error = { version = "3.0" } @@ -186,7 +186,7 @@ serde_json = "1.0" serde_with = "3.16" serial_test = "3.2" sha3 = "0.10.8" -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } solana-account-decoder = { version = "4.0" } solana-account-decoder-client-types = { version = "4.0" } solana-account-info = { version = "3.1" } @@ -228,7 +228,7 @@ solana-program = "3.0" solana-program-error = { version = "3.0" } solana-program-option = { version = "3.0" } solana-program-pack = { version = "3.0" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } solana-pubkey = { version = "4.1" } solana-pubsub-client = { version = "4.0" } solana-rent = { version = "3.0" } @@ -256,14 +256,15 @@ solana-transaction = { version = "3.0" } [workspace.dependencies.solana-svm] features = ["dev-context-only-utils"] git = "https://github.com/magicblock-labs/magicblock-svm.git" -rev = "b275f82" +rev = "782e0b00" [patch.crates-io] -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } solana-storage-proto = { path = "storage-proto" } -solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } + solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } +#solana-svm = { path = "../magicblock-svm/svm" } +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } # Fork is used to enable `disable_manual_compaction` usage # Fork is based on commit d4e9e16 of rocksdb (parent commit of 0.23.0 release) # without patching update isn't possible due to conflict with solana deps diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index 9c29bd7c9..a2ddbee13 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -72,17 +72,6 @@ pub enum MagicBlockInstruction { /// seeds: `["outbox-intent", intent_id.to_le_bytes()]` AcceptScheduleCommits, - /// Sets or advances the execution stage of an outbox intent. - /// Must be called before sending the L1 transaction. - /// - /// # Account references - /// - **0.** `[SIGNER]` Validator Authority - /// - **1.** `[WRITE]` Outbox intent PDA, seeds: `["outbox-intent", intent_id.to_le_bytes()]` - SetIntentExecutionStage { - intent_id: u64, - stage: ExecutionStage, - }, - /// Records the attempt to realize a scheduled commit on chain. /// Closes the associated outbox intent PDA account. /// @@ -327,6 +316,17 @@ pub enum MagicBlockInstruction { authority: Pubkey, instructions: Vec, }, + + /// Sets or advances the execution stage of an outbox intent. + /// Must be called before sending the L1 transaction. + /// + /// # Account references + /// - **0.** `[SIGNER]` Validator Authority + /// - **1.** `[WRITE]` Outbox intent PDA, seeds: `["outbox-intent", intent_id.to_le_bytes()]` + SetIntentExecutionStage { + intent_id: u64, + stage: ExecutionStage, + }, } impl MagicBlockInstruction { diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index af4be4759..f43f29282 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -12,6 +12,7 @@ use solana_pubkey::Pubkey; use crate::{ intent_bundles::outbox_intent_bundles::OutboxIntentBundle, + magic_scheduled_base_intent::ScheduledIntentBundle, schedule_transactions, utils::accounts::{ get_instruction_account_with_idx, get_instruction_pubkey_with_idx, @@ -21,7 +22,8 @@ use crate::{ }; const VALIDATOR_AUTHORITY_IDX: u16 = 0; -const MAGIC_CONTEXT_IDX: u16 = VALIDATOR_AUTHORITY_IDX + 1; +const MAGIC_PROGRAM_ID: u16 = VALIDATOR_AUTHORITY_IDX + 1; +const MAGIC_CONTEXT_IDX: u16 = MAGIC_PROGRAM_ID + 1; const VAULT_IDX: u16 = MAGIC_CONTEXT_IDX + 1; const INTENT_PDAS_OFFSET: u16 = VAULT_IDX + 1; @@ -74,6 +76,9 @@ fn validate( // Validate authority let transaction_context = &*invoke_context.transaction_context; + // TODO(edwin): add check for vault? + // TODO(edwin): add check for magic-program + let provided_validator_auth = get_instruction_pubkey_with_idx( transaction_context, VALIDATOR_AUTHORITY_IDX, @@ -127,10 +132,7 @@ fn verify_intent_pda( fn pop_scheduled_intents( invoke_context: &InvokeContext, -) -> Result< - Vec, - InstructionError, -> { +) -> Result, InstructionError> { let transaction_context = &*invoke_context.transaction_context; let num_ix_accounts = transaction_context .get_current_instruction_context()? @@ -189,7 +191,7 @@ fn create_outbox_account_cpi( invoke_context: &mut InvokeContext, validator_auth: Pubkey, pda: Pubkey, - outbox_account: OutboxIntentBundle, + mut outbox_account: OutboxIntentBundle, ) -> Result<(), InstructionError> { let intent_id = outbox_account.inner.id; let data = outbox_account.try_to_bytes().map_err(|_| { diff --git a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs index 6bad10e8a..80cf94d0c 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs @@ -130,7 +130,10 @@ pub fn process_scheduled_commit_sent( } const VALIDATOR_IDX: u16 = 0; - const CLOSING_PDA_IDX: u16 = 1; + // TODO(edwin): check presense of those + const MAGIC_PROGRAM_IDX: u16 = VALIDATOR_IDX + 1; + const MAGIC_VAULT_IDX: u16 = MAGIC_PROGRAM_IDX + 1; + const CLOSING_PDA_IDX: u16 = MAGIC_VAULT_IDX + 1; let transaction_context = &invoke_context.transaction_context; let ix_ctx = transaction_context.get_current_instruction_context()?; @@ -273,6 +276,7 @@ pub fn process_scheduled_commit_sent( "ScheduledCommitSent error message: {}", error_message ); + // TODO(edwin): fails to close intent then Err(InstructionError::Custom(INTENT_FAILED_CODE)) } else { Ok(()) diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 4026dbcc5..d5279e2f7 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,8 +1,9 @@ -use std::ops::Deref; +use std::{mem, ops::Deref}; use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; use magicblock_magic_program_api::outbox::{ExecutionStage, TwoStageProgress}; use serde::{Deserialize, Serialize}; +use solana_signature::Signature; use crate::magic_scheduled_base_intent::ScheduledIntentBundle; @@ -27,12 +28,27 @@ impl OutboxIntentBundle { self.status.apply_stage_transition(stage) } - pub fn try_to_bytes(&self) -> Result, bincode::Error> { - let body = bincode::serialize(self)?; - let mut out = - Vec::with_capacity(OUTBOX_INTENT_DISCRIMINATOR.len() + body.len()); - out.extend_from_slice(&OUTBOX_INTENT_DISCRIMINATOR); - out.extend_from_slice(&body); + pub(crate) fn try_to_bytes(&mut self) -> Result, bincode::Error> { + const DISCRIMINATOR_LEN: usize = OUTBOX_INTENT_DISCRIMINATOR.len(); + + // Replace current status with max size one + // Needed in order to estimate higher bound of account size to allocate + let prev_status = mem::replace( + &mut self.status, + OutboxIntentBundleStatus::max_size_variant(), + ); + let max_body_size = bincode::serialized_size(self)? as usize; + + // Allocate array of higher bound size + let mut out = vec![0u8; DISCRIMINATOR_LEN + max_body_size]; + out[..DISCRIMINATOR_LEN].copy_from_slice(&OUTBOX_INTENT_DISCRIMINATOR); + + // Switch status back + self.status = prev_status; + bincode::serialize_into( + std::io::Cursor::new(&mut out[DISCRIMINATOR_LEN..]), + self, + )?; Ok(out) } @@ -56,6 +72,15 @@ pub enum OutboxIntentBundleStatus { } impl OutboxIntentBundleStatus { + fn max_size_variant() -> Self { + Self::Executing(ExecutionStage::TwoStage( + TwoStageProgress::Finalizing { + commit: Signature::default(), + finalize: Signature::default(), + }, + )) + } + // TODO(edwin): split into is_valid_transition and apply_transaction fn apply_stage_transition( &mut self, diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index 24bdc52ef..a181be6f4 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -127,8 +127,9 @@ impl InstructionUtils { ) -> Instruction { let account_metas = vec![ AccountMeta::new(validator_authority_id(), true), - AccountMeta::new(outbox_intent_pda(scheduled_commit_id), false), + AccountMeta::new_readonly(crate::id(), false), AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), + AccountMeta::new(outbox_intent_pda(scheduled_commit_id), false), ]; Instruction::new_with_bincode( crate::id(), @@ -152,8 +153,10 @@ impl InstructionUtils { intent_ids: impl IntoIterator, ) -> Instruction { let mut account_metas = vec![ - AccountMeta::new_readonly(validator_authority_id(), true), + AccountMeta::new(validator_authority_id(), true), + AccountMeta::new_readonly(crate::id(), false), AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), + AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), ]; // Add outbox intent accounts diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index e17ec82d8..c9bf766ef 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -809,7 +809,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -3894,7 +3894,7 @@ dependencies = [ "solana-system-interface 2.0.0", "static_assertions", "strum", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -3925,7 +3925,7 @@ dependencies = [ "solana-system-interface 2.0.0", "static_assertions", "strum", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "solana-account" version = "3.4.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "bincode", "qualifier_attr", @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "base64 0.22.1", "bincode", @@ -8729,7 +8729,7 @@ dependencies = [ [[package]] name = "solana-svm" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "ahash 0.8.12", "log", @@ -9045,7 +9045,7 @@ dependencies = [ [[package]] name = "solana-transaction-context" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=782e0b00#782e0b0088d1fd7cf076c2524ebd714daf32e128" dependencies = [ "bincode", "qualifier_attr", diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index 47d49750d..bd3493de3 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -72,7 +72,7 @@ schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" shlex = "1.3.0" -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } solana-address-lookup-table-interface = "3.0" solana-commitment-config = "3.1.1" solana-compute-budget-interface = "3.0" @@ -113,9 +113,9 @@ url = "2.5.0" solana-storage-proto = { path = "../storage-proto" } # same reason as above rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } +solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } # Fix libsodium for ARM builds - upstream crates.io version breaks Linux ARM binaries libsodium-rs = { git = "https://github.com/jedisct1/libsodium-rs.git", rev = "0397a6c5785233f9f2ac91f3eedc3cceb74e0060" } diff --git a/test-integration/test-schedule-intent/tests/common/mod.rs b/test-integration/test-schedule-intent/tests/common/mod.rs index fc0be184d..b0f1d29d2 100644 --- a/test-integration/test-schedule-intent/tests/common/mod.rs +++ b/test-integration/test-schedule-intent/tests/common/mod.rs @@ -14,7 +14,8 @@ pub const LABEL: &str = "I am a label"; pub fn setup_payer(ctx: &IntegrationTestContext) -> Keypair { let payer = Keypair::new(); - ctx.airdrop_chain(&payer.pubkey(), LAMPORTS_PER_SOL).unwrap(); + ctx.airdrop_chain(&payer.pubkey(), LAMPORTS_PER_SOL) + .unwrap(); let ix = dlp_api::instruction_builder::top_up_ephemeral_balance( payer.pubkey(), @@ -76,13 +77,13 @@ pub fn add_to_counter( ctx.wait_for_next_slot_ephem().unwrap(); let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; - let counter_before = - ctx.fetch_ephem_account_struct::(counter_pda) - .unwrap_or(FlexiCounter { - count: 0, - updates: 0, - label: LABEL.to_string(), - }); + let counter_before = ctx + .fetch_ephem_account_struct::(counter_pda) + .unwrap_or(FlexiCounter { + count: 0, + updates: 0, + label: LABEL.to_string(), + }); let ix = create_add_ix(payer.pubkey(), value); ctx.send_and_confirm_instructions_with_payer_ephem(&[ix], payer) @@ -115,9 +116,11 @@ pub fn assert_counters( .iter() .map(|c| { if is_base { - ctx.fetch_chain_account_struct::(c.pda).unwrap() + ctx.fetch_chain_account_struct::(c.pda) + .unwrap() } else { - ctx.fetch_ephem_account_struct::(c.pda).unwrap() + ctx.fetch_ephem_account_struct::(c.pda) + .unwrap() } }) .collect::>(); diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index cd3c46e09..7a2d5b7e1 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -1,9 +1,8 @@ mod common; -use common::*; - use std::sync::{Arc, Mutex, Once}; use async_trait::async_trait; +use common::*; use integration_test_tools::{ loaded_accounts::DLP_TEST_AUTHORITY_BYTES, IntegrationTestContext, }; @@ -12,7 +11,7 @@ use magicblock_committor_service::{ accepted_intent_executor::AcceptedIntentExecutor, intent_execution_client::IntentExecutionClient, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, - IntentExecutorCtx, + IntentExecutor, IntentExecutorCtx, }, outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::OutboxIntentBundlesReader, @@ -27,16 +26,16 @@ use magicblock_magic_program_api::{ instruction::MagicBlockInstruction, outbox::ExecutionStage, MAGIC_CONTEXT_PUBKEY, }; -use magicblock_committor_service::intent_executor::IntentExecutor; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, outbox_intent_bundles::OutboxIntentBundle, - validator::init_validator_authority, - MagicContext, SentCommit, + validator::init_validator_authority, MagicContext, SentCommit, }; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; -use program_flexi_counter::{instruction::create_intent_bundle_ix, state::FlexiCounter}; +use program_flexi_counter::{ + instruction::create_intent_bundle_ix, state::FlexiCounter, +}; use solana_rpc_client::nonblocking::rpc_client::RpcClient as AsyncRpcClient; use solana_sdk::{ instruction::{AccountMeta, Instruction}, @@ -45,7 +44,6 @@ use solana_sdk::{ transaction::Transaction, }; - // --------------------------------------------------------------------------- // NoopCallbackScheduler // --------------------------------------------------------------------------- @@ -60,7 +58,10 @@ impl ActionsCallbackScheduler for NoopCallbackScheduler { _signature: Option, _result: ActionResult, ) -> Vec> { - callbacks.iter().map(|_| Ok(Signature::new_unique())).collect() + callbacks + .iter() + .map(|_| Ok(Signature::new_unique())) + .collect() } } @@ -117,7 +118,10 @@ impl OutboxClient for TestOutboxClient { intent_id: u64, stage: ExecutionStage, ) -> Result<(), Self::Error> { - self.stage_calls.lock().unwrap().push((intent_id, stage.clone())); + self.stage_calls + .lock() + .unwrap() + .push((intent_id, stage.clone())); let blockhash = self .ephem_rpc .get_latest_blockhash() @@ -125,9 +129,15 @@ impl OutboxClient for TestOutboxClient { .map_err(InternalOutboxClientError::RpcClientError)?; let ix = Instruction::new_with_bincode( magicblock_magic_program_api::id(), - &MagicBlockInstruction::SetIntentExecutionStage { intent_id, stage }, + &MagicBlockInstruction::SetIntentExecutionStage { + intent_id, + stage, + }, vec![ - AccountMeta::new_readonly(self.validator_keypair.pubkey(), true), + AccountMeta::new_readonly( + self.validator_keypair.pubkey(), + true, + ), AccountMeta::new(outbox_intent_pda(intent_id), false), ], ); @@ -171,9 +181,7 @@ fn ensure_validator_authority() -> Keypair { } fn read_next_intent_id(ctx: &IntegrationTestContext) -> u64 { - let data = ctx - .fetch_ephem_account_data(MAGIC_CONTEXT_PUBKEY) - .unwrap(); + let data = ctx.fetch_ephem_account_data(MAGIC_CONTEXT_PUBKEY).unwrap(); MagicContext::intent_id(&data) } @@ -213,7 +221,10 @@ fn schedule_and_accept( Some(&payer.pubkey()), ); let (_, confirmed) = ctx - .send_and_confirm_transaction_ephem(&mut tx, &[payer, &validator_keypair]) + .send_and_confirm_transaction_ephem( + &mut tx, + &[payer, &validator_keypair], + ) .unwrap(); assert!(confirmed, "schedule_and_accept tx not confirmed"); @@ -283,11 +294,7 @@ async fn test_accepted_executor_outbox_flow() { Box::new(executor).execute(outbox_bundle.inner).await; let _ = cleanup_handle.clean().await; - assert!( - result.inner.is_ok(), - "Executor failed: {:?}", - result.inner - ); + assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); let calls = stage_calls.lock().unwrap(); assert!( !calls.is_empty(), diff --git a/test-integration/test-schedule-intent/tests/test_schedule_intents.rs b/test-integration/test-schedule-intent/tests/test_schedule_intents.rs index e309ef483..b4a7329cc 100644 --- a/test-integration/test-schedule-intent/tests/test_schedule_intents.rs +++ b/test-integration/test-schedule-intent/tests/test_schedule_intents.rs @@ -1,6 +1,5 @@ mod common; use common::*; - use integration_test_tools::{ conversions::stringify_simulation_result, loaded_accounts::DLP_TEST_AUTHORITY_BYTES, IntegrationTestContext, @@ -15,7 +14,8 @@ use program_flexi_counter::{ }; use solana_rpc_client_api::config::RpcSimulateTransactionConfig; use solana_sdk::{ - pubkey::Pubkey, signature::Keypair, signer::Signer, transaction::Transaction, + pubkey::Pubkey, signature::Keypair, signer::Signer, + transaction::Transaction, }; use test_kit::init_logger; use tracing::*; @@ -643,7 +643,6 @@ fn test_intent_bundle_commit_and_commit_finalize() { assert_eq!(owner_finalize, delegation_program_id()); } - fn schedule_intent( ctx: &IntegrationTestContext, payers: &[&Keypair], @@ -864,4 +863,3 @@ fn dump_ephem_simulation( } } } - diff --git a/test-integration/test-tools/src/transactions.rs b/test-integration/test-tools/src/transactions.rs index a78e0fe64..18e5c6d8e 100644 --- a/test-integration/test-tools/src/transactions.rs +++ b/test-integration/test-tools/src/transactions.rs @@ -3,7 +3,7 @@ use std::{thread::sleep, time::Duration}; use solana_commitment_config::CommitmentConfig; -use solana_rpc_client::rpc_client::RpcClient; +use solana_rpc_client::rpc_client::{RpcClient, SerializableTransaction}; use solana_rpc_client_api::{ client_error, config::{RpcSendTransactionConfig, RpcSimulateTransactionConfig}, From e9d2e8271506b8c877b391aeabf67aa75fa17d0f Mon Sep 17 00:00:00 2001 From: taco-paco Date: Thu, 25 Jun 2026 16:04:54 +0700 Subject: [PATCH 61/97] fix: new tests --- .../magicblock/src/utils/instruction_utils.rs | 5 +- test-integration/Cargo.lock | 1 + .../test-schedule-intent/Cargo.toml | 1 + .../tests/test_outbox_flow.rs | 373 +++++++++++------- .../test-tools/src/transactions.rs | 1 + 5 files changed, 238 insertions(+), 143 deletions(-) diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index a181be6f4..3d5fed070 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -36,8 +36,7 @@ impl InstructionUtils { Self::into_transaction(payer, ix, recent_blockhash) } - #[cfg(test)] - pub(crate) fn schedule_commit_instruction( + pub fn schedule_commit_instruction( payer: &Pubkey, pdas: Vec, ) -> Instruction { @@ -149,7 +148,7 @@ impl InstructionUtils { Self::into_transaction(&validator_authority(), ix, recent_blockhash) } - pub(crate) fn accept_scheduled_commits_instruction( + pub fn accept_scheduled_commits_instruction( intent_ids: impl IntoIterator, ) -> Instruction { let mut account_metas = vec![ diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index c9bf766ef..0d00c316f 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -9953,6 +9953,7 @@ dependencies = [ name = "test-schedule-intent" version = "0.0.0" dependencies = [ + "anyhow", "async-trait", "ephemeral-rollups-sdk", "integration-test-tools", diff --git a/test-integration/test-schedule-intent/Cargo.toml b/test-integration/test-schedule-intent/Cargo.toml index 9f31b4d95..ca3a94212 100644 --- a/test-integration/test-schedule-intent/Cargo.toml +++ b/test-integration/test-schedule-intent/Cargo.toml @@ -7,6 +7,7 @@ version.workspace = true log = "0.4.29" [dev-dependencies] +anyhow = { workspace = true } async-trait = { workspace = true } ephemeral-rollups-sdk = { workspace = true } integration-test-tools = { workspace = true } diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 7a2d5b7e1..1c429f624 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -1,8 +1,13 @@ mod common; -use std::sync::{Arc, Mutex, Once}; +use std::{ + sync::{Arc, Mutex, Once}, + time::Duration, +}; +use anyhow::anyhow; use async_trait::async_trait; use common::*; +use ephemeral_rollups_sdk::{compat, ephem::MagicIntentBundleBuilder}; use integration_test_tools::{ loaded_accounts::DLP_TEST_AUTHORITY_BYTES, IntegrationTestContext, }; @@ -27,6 +32,7 @@ use magicblock_magic_program_api::{ MAGIC_CONTEXT_PUBKEY, }; use magicblock_program::{ + instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, outbox_intent_bundles::OutboxIntentBundle, validator::init_validator_authority, MagicContext, SentCommit, @@ -34,23 +40,228 @@ use magicblock_program::{ use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use program_flexi_counter::{ - instruction::create_intent_bundle_ix, state::FlexiCounter, + instruction::{ + create_intent_bundle_commit_and_finalize_ix, create_intent_bundle_ix, + }, + state::FlexiCounter, +}; +use solana_rpc_client::{ + nonblocking::rpc_client::RpcClient as AsyncRpcClient, + rpc_client::SerializableTransaction, }; -use solana_rpc_client::nonblocking::rpc_client::RpcClient as AsyncRpcClient; +use solana_rpc_client_api::client_error; use solana_sdk::{ instruction::{AccountMeta, Instruction}, + pubkey::Pubkey, signature::{Keypair, Signature}, signer::Signer, transaction::Transaction, }; -// --------------------------------------------------------------------------- -// NoopCallbackScheduler -// --------------------------------------------------------------------------- +type TestIntentExecutorCtx = IntentExecutorCtx< + TransactionPreparatorImpl, + RpcTaskInfoFetcher, + NoopCallbackScheduler, + TestOutboxClient, +>; + +struct TestEnv { + ctx: IntegrationTestContext, + validator_authority: Keypair, + chain_mb_client: MagicblockRpcClient, + intent_client: IntentExecutionClient, + outbox_client: Arc, + table_mania: TableMania, + task_info_fetcher: Arc>, + stage_calls: Arc>>, +} + +impl TestEnv { + async fn setup() -> Self { + let validator_authority = ensure_validator_authority(); + let ctx = IntegrationTestContext::try_new().unwrap(); + let chain_rpc = Arc::new(ctx.try_chain_client_async().unwrap()); + let ephem_rpc = Arc::new(ctx.try_ephem_client_async().unwrap()); + let chain_mb_rpc = MagicblockRpcClient::new(chain_rpc); + let ephem_mb_rpc = MagicblockRpcClient::new(ephem_rpc.clone()); + + let intent_client = IntentExecutionClient::new(chain_mb_rpc.clone()); + let outbox_client = Arc::new(TestOutboxClient::new( + ephem_rpc, + validator_authority.insecure_clone(), + )); + let stage_calls = outbox_client.stage_calls.clone(); + + let gc_config = GarbageCollectorConfig::default(); + let table_mania = TableMania::new( + chain_mb_rpc.clone(), + &validator_authority, + Some(gc_config), + ); + let compute_budget = ComputeBudgetConfig::new(1_000_000); + let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( + RpcTaskInfoFetcher::new(chain_mb_rpc.clone()), + )); + + Self { + ctx, + validator_authority, + chain_mb_client: chain_mb_rpc, + intent_client, + outbox_client, + table_mania, + task_info_fetcher, + + stage_calls, + } + } + + fn executor_ctx(&self, actions_timeout: Duration) -> TestIntentExecutorCtx { + TestIntentExecutorCtx { + intent_client: self.intent_client.clone(), + transaction_preparator: self.transaction_preparator(), + task_info_fetcher: self.task_info_fetcher.clone(), + outbox_client: self.outbox_client.clone(), + actions_callback_executor: NoopCallbackScheduler, + actions_timeout, + } + } + + fn transaction_preparator(&self) -> TransactionPreparatorImpl { + let compute_budget = ComputeBudgetConfig::new(1_000_000); + TransactionPreparatorImpl::new( + self.chain_mb_client.clone(), + self.table_mania.clone(), + compute_budget, + ) + } +} + +/// Schedyles commit of counters directly via magic-program using validator authority +fn schedule(ctx: &IntegrationTestContext, counters: &[Pubkey]) { + let validator_keypair = ensure_validator_authority(); + + let schedule_ix = schedule_commit_instruction( + &validator_keypair.pubkey(), + counters.to_vec(), + ); + let mut tx = Transaction::new_with_payer( + &[schedule_ix], + Some(&validator_keypair.pubkey()), + ); + let sig = ctx + .send_transaction_ephem(&mut tx, &[&validator_keypair]) + .unwrap(); + println!("schedule sig: {}", sig); +} + +/// Tries to accept intent +fn steal_accept_intent( + ctx: &IntegrationTestContext, + intent_id: u64, +) -> Result<(), anyhow::Error> { + let validator_keypair = ensure_validator_authority(); + + let accept_ix = + InstructionUtils::accept_scheduled_commits_instruction([intent_id]); + let mut tx = Transaction::new_with_payer( + &[accept_ix], + Some(&validator_keypair.pubkey()), + ); + + let (sig, confirmed) = + ctx.send_and_confirm_transaction_ephem(&mut tx, &[&validator_keypair])?; + + if !confirmed { + Err(anyhow!("tx not confirmed: {}", sig)) + } else { + println!("accept sig: {}", sig); + Ok(()) + } +} + +fn steal_schedule_accept_intent( + ctx: &IntegrationTestContext, + counters: &[Pubkey], +) -> u64 { + const MAX_ATTEMPTS: u8 = 5; + + let mut attempt = 0; + loop { + let intent_id = read_next_intent_id(ctx); + schedule(ctx, counters); + let result = steal_accept_intent(&ctx, intent_id); + match result { + Ok(()) => return intent_id, + Err(err) => { + println!("Failed to steal"); + + if attempt >= MAX_ATTEMPTS { + panic!("Failed to steal intent: {}", err); + } + attempt += 1; + } + } + } +} + +fn schedule_and_accept_2( + ctx: &IntegrationTestContext, + counters: &[Pubkey], +) -> OutboxIntentBundle { + ctx.wait_for_next_slot_ephem().unwrap(); + + let intent_id = steal_schedule_accept_intent(&ctx, counters); + let pda = outbox_intent_pda(intent_id); + let data = ctx.fetch_ephem_account_data(pda).unwrap(); + OutboxIntentBundle::try_from_bytes(&data).unwrap() +} + +/// AcceptedIntentExecutor drives the full commit flow using TestOutboxClient. +/// Verifies: executor succeeds, set_intent_execution_stage is called, +/// and the committed value lands on the base chain. +#[tokio::test(flavor = "multi_thread")] +async fn test_accepted_executor_outbox_flow() { + let test_env = TestEnv::setup().await; + + let payer = setup_payer(&test_env.ctx); + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + + init_counter(&test_env.ctx, &payer); + delegate_counter(&test_env.ctx, &payer); + add_to_counter(&test_env.ctx, &payer, 42); + + let outbox_bundle = schedule_and_accept_2(&test_env.ctx, &[counter_pda]); + let intent_id = outbox_bundle.inner.id; + + let executor_ctx = test_env.executor_ctx(DEFAULT_ACTIONS_TIMEOUT); + let executor = AcceptedIntentExecutor::new(executor_ctx); + + let (result, cleanup_handle) = + Box::new(executor).execute(outbox_bundle.inner).await; + let _ = cleanup_handle.clean().await; + + assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); + let calls = test_env.stage_calls.lock().unwrap(); + assert!( + !calls.is_empty(), + "Expected at least one set_intent_execution_stage call" + ); + assert!( + calls.iter().all(|(id, _)| *id == intent_id), + "Stage calls contain unexpected intent ids" + ); + + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + let counter = test_env + .ctx + .fetch_chain_account_struct::(counter_pda) + .unwrap(); + assert_eq!(counter.count, 42); +} #[derive(Clone, Default)] struct NoopCallbackScheduler; - impl ActionsCallbackScheduler for NoopCallbackScheduler { fn schedule( &self, @@ -65,12 +276,7 @@ impl ActionsCallbackScheduler for NoopCallbackScheduler { } } -// --------------------------------------------------------------------------- -// TestOutboxClient -// --------------------------------------------------------------------------- - struct TestOutboxReader; - #[async_trait] impl OutboxIntentBundlesReader for TestOutboxReader { type Error = std::convert::Infallible; @@ -167,10 +373,6 @@ impl OutboxClient for TestOutboxClient { } } -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - fn ensure_validator_authority() -> Keypair { static ONCE: Once = Once::new(); let kp = Keypair::try_from(&DLP_TEST_AUTHORITY_BYTES[..]).unwrap(); @@ -185,129 +387,20 @@ fn read_next_intent_id(ctx: &IntegrationTestContext) -> u64 { MagicContext::intent_id(&data) } -/// Sends a single ER transaction: [schedule_commit_ix, accept_ix]. -/// Both instructions land atomically — MagicContext is drained within the -/// same tx so CommittorService never sees the intent. -/// Returns the OutboxIntentBundle in Accepted stage. -fn schedule_and_accept( - ctx: &IntegrationTestContext, - payer: &Keypair, -) -> OutboxIntentBundle { - ctx.wait_for_next_slot_ephem().unwrap(); - - let validator_keypair = ensure_validator_authority(); - let intent_id = read_next_intent_id(ctx); - - let destination = Keypair::new(); - let schedule_ix = create_intent_bundle_ix( - vec![payer.pubkey()], - vec![], - destination.pubkey(), - vec![], - 100_000, - ); - let accept_ix = Instruction::new_with_bincode( +pub fn schedule_commit_instruction( + payer: &Pubkey, + pdas: Vec, +) -> Instruction { + let mut account_metas = vec![ + AccountMeta::new(*payer, true), + AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), + ]; + for pubkey in &pdas { + account_metas.push(AccountMeta::new_readonly(*pubkey, false)); + } + Instruction::new_with_bincode( magicblock_magic_program_api::id(), - &MagicBlockInstruction::AcceptScheduleCommits, - vec![ - AccountMeta::new_readonly(validator_keypair.pubkey(), true), - AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), - AccountMeta::new(outbox_intent_pda(intent_id), false), - ], - ); - - let mut tx = Transaction::new_with_payer( - &[schedule_ix, accept_ix], - Some(&payer.pubkey()), - ); - let (_, confirmed) = ctx - .send_and_confirm_transaction_ephem( - &mut tx, - &[payer, &validator_keypair], - ) - .unwrap(); - assert!(confirmed, "schedule_and_accept tx not confirmed"); - - let pda = outbox_intent_pda(intent_id); - let data = ctx.fetch_ephem_account_data(pda).unwrap(); - OutboxIntentBundle::try_from_bytes(&data).unwrap() -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -/// AcceptedIntentExecutor drives the full commit flow using TestOutboxClient. -/// Verifies: executor succeeds, set_intent_execution_stage is called, -/// and the committed value lands on the base chain. -#[tokio::test] -#[ignore] -async fn test_accepted_executor_outbox_flow() { - let ctx = IntegrationTestContext::try_new().unwrap(); - let payer = setup_payer(&ctx); - - init_counter(&ctx, &payer); - delegate_counter(&ctx, &payer); - add_to_counter(&ctx, &payer, 42); - - let outbox_bundle = schedule_and_accept(&ctx, &payer); - let intent_id = outbox_bundle.inner.id; - - let validator_keypair = ensure_validator_authority(); - let chain_rpc = Arc::new(ctx.try_chain_client_async().unwrap()); - let ephem_rpc = Arc::new(ctx.try_ephem_client_async().unwrap()); - let chain_mb_rpc = MagicblockRpcClient::new(chain_rpc); - let ephem_mb_rpc = MagicblockRpcClient::new(ephem_rpc.clone()); - - let outbox_client = Arc::new(TestOutboxClient::new( - ephem_rpc, - validator_keypair.insecure_clone(), - )); - let stage_calls = outbox_client.stage_calls.clone(); - - let gc_config = GarbageCollectorConfig::default(); - let table_mania = TableMania::new( - chain_mb_rpc.clone(), - &validator_keypair, - Some(gc_config), - ); - let compute_budget = ComputeBudgetConfig::new(1_000_000); - let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( - RpcTaskInfoFetcher::new(ephem_mb_rpc), - )); - let transaction_preparator = TransactionPreparatorImpl::new( - chain_mb_rpc.clone(), - table_mania, - compute_budget, - ); - - let executor = AcceptedIntentExecutor::new(IntentExecutorCtx { - intent_client: IntentExecutionClient::new(chain_mb_rpc), - transaction_preparator, - task_info_fetcher, - outbox_client, - actions_callback_executor: NoopCallbackScheduler, - actions_timeout: DEFAULT_ACTIONS_TIMEOUT, - }); - - let (result, cleanup_handle) = - Box::new(executor).execute(outbox_bundle.inner).await; - let _ = cleanup_handle.clean().await; - - assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); - let calls = stage_calls.lock().unwrap(); - assert!( - !calls.is_empty(), - "Expected at least one set_intent_execution_stage call" - ); - assert!( - calls.iter().all(|(id, _)| *id == intent_id), - "Stage calls contain unexpected intent ids" - ); - - let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; - let counter = ctx - .fetch_chain_account_struct::(counter_pda) - .unwrap(); - assert_eq!(counter.count, 42); + &MagicBlockInstruction::ScheduleCommit, + account_metas, + ) } diff --git a/test-integration/test-tools/src/transactions.rs b/test-integration/test-tools/src/transactions.rs index 18e5c6d8e..61036f5e1 100644 --- a/test-integration/test-tools/src/transactions.rs +++ b/test-integration/test-tools/src/transactions.rs @@ -57,6 +57,7 @@ pub fn send_transaction( ) -> Result { let blockhash = rpc_client.get_latest_blockhash()?; tx.try_sign(signers, blockhash)?; + println!("trtr: {}", tx.signatures[0]); let sig = rpc_client.send_transaction_with_config( tx, RpcSendTransactionConfig { From 59f1981f770ac3bdccbb94711f78d15aa34545a9 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Thu, 25 Jun 2026 18:48:31 +0700 Subject: [PATCH 62/97] feat: crash-restart test --- magicblock-committor-service/Cargo.toml | 1 + .../src/intent_executor/mod.rs | 2 +- .../src/outbox_client.rs | 2 +- .../service/outbox_intent_bundles_reader.rs | 23 ++++- .../process_accept_scheduled_commits.rs | 2 +- .../intent_bundles/outbox_intent_bundles.rs | 34 ++++--- .../test-schedule-intent/Cargo.toml | 2 +- .../tests/test_outbox_flow.rs | 92 ++++++++++++++----- 8 files changed, 115 insertions(+), 43 deletions(-) diff --git a/magicblock-committor-service/Cargo.toml b/magicblock-committor-service/Cargo.toml index ba2692959..f59f290fe 100644 --- a/magicblock-committor-service/Cargo.toml +++ b/magicblock-committor-service/Cargo.toml @@ -59,6 +59,7 @@ tokio-util = { workspace = true } tokio-stream = { workspace = true } [dev-dependencies] +magicblock-program = { workspace = true, features = ["dev-context-only-utils"] } solana-signature = { workspace = true, features = ["rand"] } rand = { workspace = true } tempfile = { workspace = true } diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 22c370356..78b6969dc 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -49,7 +49,7 @@ pub trait IntentExecutor: Send + Sync + 'static { ) -> (IntentExecutionResult, CleanupHandle); } -fn build_stage_intent_executor( +pub fn build_stage_intent_executor( ctx: IntentExecutorCtx, status: OutboxIntentBundleStatus, ) -> Box> diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index 6211ff289..e35796ce8 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -26,7 +26,7 @@ use solana_rpc_client_api::{ use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tracing::{debug, error}; - +use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use crate::service::outbox_intent_bundles_reader::{ InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, }; diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index 6ee2f9d04..ef677f4a3 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -6,8 +6,8 @@ use std::{ }; use async_trait::async_trait; -use magicblock_accounts_db::{error::AccountsDbError, AccountsDb}; -use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; +use magicblock_accounts_db::{error::AccountsDbError, traits::AccountsBank, AccountsDb}; +use magicblock_core::intent::outbox::{outbox_intent_pda, OUTBOX_INTENT_DISCRIMINATOR}; use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::{AccountSharedData, ReadableAccount}; use tracing::warn; @@ -22,6 +22,12 @@ pub trait OutboxIntentBundlesReader: Send + 'static { &mut self, n: usize, ) -> Result, Self::Error>; + + /// Fetches a single outbox intent by id. Returns `None` if the account does not exist. + async fn fetch_outbox_intent( + &self, + intent_id: u64, + ) -> Result, Self::Error>; } pub struct InternalOutboxIntentBundlesReader { @@ -163,6 +169,17 @@ impl OutboxIntentBundlesReader for InternalOutboxIntentBundlesReader { let take = n.min(self.buffer.len()); Ok(self.buffer.drain(..take).collect()) } + + async fn fetch_outbox_intent( + &self, + intent_id: u64, + ) -> Result, Self::Error> { + let pda = outbox_intent_pda(intent_id); + let Some(account) = self.accounts_db.get_account(&pda) else { + return Ok(None); + }; + Ok(Some(OutboxIntentBundle::try_from_bytes(account.data())?)) + } } #[derive(thiserror::Error, Debug)] @@ -171,6 +188,8 @@ pub enum OutboxIntentBundlesReaderError { ReadExceedsCapacityError(usize, usize), #[error("AccountsDbError: {0}")] AccountsDbError(#[from] AccountsDbError), + #[error("BincodeError: {0}")] + BincodeError(#[from] bincode::Error), } pub type OutboxIntentBundlesReaderResult = diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index f43f29282..a1571fca0 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -191,7 +191,7 @@ fn create_outbox_account_cpi( invoke_context: &mut InvokeContext, validator_auth: Pubkey, pda: Pubkey, - mut outbox_account: OutboxIntentBundle, + outbox_account: OutboxIntentBundle, ) -> Result<(), InstructionError> { let intent_id = outbox_account.inner.id; let data = outbox_account.try_to_bytes().map_err(|_| { diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index d5279e2f7..3fe6628b4 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,4 +1,4 @@ -use std::{mem, ops::Deref}; +use std::ops::Deref; use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; use magicblock_magic_program_api::outbox::{ExecutionStage, TwoStageProgress}; @@ -10,6 +10,7 @@ use crate::magic_scheduled_base_intent::ScheduledIntentBundle; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OutboxIntentBundle { pub inner: ScheduledIntentBundle, + // TODO(edwin): define visibility pub status: OutboxIntentBundleStatus, } @@ -21,30 +22,35 @@ impl OutboxIntentBundle { } } - pub fn apply_stage_transition( + pub(crate) fn apply_stage_transition( &mut self, stage: ExecutionStage, ) -> Result<(), &'static str> { self.status.apply_stage_transition(stage) } - pub(crate) fn try_to_bytes(&mut self) -> Result, bincode::Error> { + #[cfg(not(feature = "dev-context-only-utils"))] + pub(crate) fn try_to_bytes(&self) -> Result, bincode::Error> { + self.try_to_bytes_impl() + } + + #[cfg(feature = "dev-context-only-utils")] + pub fn try_to_bytes(&self) -> Result, bincode::Error> { + self.try_to_bytes_impl() + } + + fn try_to_bytes_impl(&self) -> Result, bincode::Error> { const DISCRIMINATOR_LEN: usize = OUTBOX_INTENT_DISCRIMINATOR.len(); - // Replace current status with max size one - // Needed in order to estimate higher bound of account size to allocate - let prev_status = mem::replace( - &mut self.status, - OutboxIntentBundleStatus::max_size_variant(), - ); - let max_body_size = bincode::serialized_size(self)? as usize; + // bincode serializes structs as field concatenation, so max body size + // is inner size + worst-case status size (TwoStage::Finalizing with 2 sigs) + let max_body_size = (bincode::serialized_size(&self.inner)? + + bincode::serialized_size( + &OutboxIntentBundleStatus::max_size_variant(), + )?) as usize; - // Allocate array of higher bound size let mut out = vec![0u8; DISCRIMINATOR_LEN + max_body_size]; out[..DISCRIMINATOR_LEN].copy_from_slice(&OUTBOX_INTENT_DISCRIMINATOR); - - // Switch status back - self.status = prev_status; bincode::serialize_into( std::io::Cursor::new(&mut out[DISCRIMINATOR_LEN..]), self, diff --git a/test-integration/test-schedule-intent/Cargo.toml b/test-integration/test-schedule-intent/Cargo.toml index ca3a94212..ba36a22dc 100644 --- a/test-integration/test-schedule-intent/Cargo.toml +++ b/test-integration/test-schedule-intent/Cargo.toml @@ -11,7 +11,7 @@ anyhow = { workspace = true } async-trait = { workspace = true } ephemeral-rollups-sdk = { workspace = true } integration-test-tools = { workspace = true } -magicblock-committor-service = { workspace = true } +magicblock-committor-service = { workspace = true, features = ["dev-context-only-utils"] } magicblock-core = { workspace = true } magicblock-delegation-program-api = { workspace = true } magicblock-magic-program-api = { workspace = true } diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 1c429f624..3a569bd6c 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -14,9 +14,10 @@ use integration_test_tools::{ use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, + build_stage_intent_executor, intent_execution_client::IntentExecutionClient, task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, - IntentExecutor, IntentExecutorCtx, + ExecutionOutput, IntentExecutor, IntentExecutorCtx, }, outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::OutboxIntentBundlesReader, @@ -34,8 +35,9 @@ use magicblock_magic_program_api::{ use magicblock_program::{ instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, - validator::init_validator_authority, MagicContext, SentCommit, + outbox_intent_bundles::{OutboxIntentBundle, OutboxIntentBundleStatus}, + validator::init_validator_authority, + MagicContext, SentCommit, }; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; @@ -205,7 +207,7 @@ fn steal_schedule_accept_intent( } } -fn schedule_and_accept_2( +fn schedule_and_accept( ctx: &IntegrationTestContext, counters: &[Pubkey], ) -> OutboxIntentBundle { @@ -221,7 +223,7 @@ fn schedule_and_accept_2( /// Verifies: executor succeeds, set_intent_execution_stage is called, /// and the committed value lands on the base chain. #[tokio::test(flavor = "multi_thread")] -async fn test_accepted_executor_outbox_flow() { +async fn test_pickup_executed_intent() { let test_env = TestEnv::setup().await; let payer = setup_payer(&test_env.ctx); @@ -230,29 +232,60 @@ async fn test_accepted_executor_outbox_flow() { init_counter(&test_env.ctx, &payer); delegate_counter(&test_env.ctx, &payer); add_to_counter(&test_env.ctx, &payer, 42); + let outbox_bundle = schedule_and_accept(&test_env.ctx, &[counter_pda]); - let outbox_bundle = schedule_and_accept_2(&test_env.ctx, &[counter_pda]); - let intent_id = outbox_bundle.inner.id; - + // Execute intent let executor_ctx = test_env.executor_ctx(DEFAULT_ACTIONS_TIMEOUT); let executor = AcceptedIntentExecutor::new(executor_ctx); - - let (result, cleanup_handle) = - Box::new(executor).execute(outbox_bundle.inner).await; - let _ = cleanup_handle.clean().await; - + let (result, cleanup_handle) = Box::new(executor) + .execute(outbox_bundle.inner.clone()) + .await; assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); - let calls = test_env.stage_calls.lock().unwrap(); + + cleanup_handle.clean().await.expect("cleanup failed"); + + // Simulates shutdown/crash after successful execution + // Validator could sent tx and then crash + // On restart validator will extract outbox bundles + let outbox_bundle = test_env + .outbox_client + .outbox_reader() + .fetch_outbox_intent(outbox_bundle.inner.id) + .await + .expect("fetch failed") + .expect("outbox bundle not found"); + + let ExecutionOutput::SingleStage(signature) = result.inner.unwrap() else { + panic!("Unexpected execution strategy"); + }; assert!( - !calls.is_empty(), - "Expected at least one set_intent_execution_stage call" + matches!( + outbox_bundle.status, + OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( + signature + )) + ), + "Invalid outbox state" ); - assert!( - calls.iter().all(|(id, _)| *id == intent_id), - "Stage calls contain unexpected intent ids" + // Builder executor + let executor = build_stage_intent_executor( + test_env.executor_ctx(DEFAULT_ACTIONS_TIMEOUT), + outbox_bundle.status, + ); + let (result, cleanup_handle) = Box::new(executor) + .execute(outbox_bundle.inner.clone()) + .await; + let ExecutionOutput::SingleStage(retried_signature) = result.inner.unwrap() + else { + panic!("Unexpected execution strategy"); + }; + assert_eq!( + signature, retried_signature, + "Execution shouldn't be retried" ); + cleanup_handle.clean().await.expect("cleanup failed"); - let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + // Validate on chain state let counter = test_env .ctx .fetch_chain_account_struct::(counter_pda) @@ -276,10 +309,11 @@ impl ActionsCallbackScheduler for NoopCallbackScheduler { } } -struct TestOutboxReader; +struct TestOutboxReader(Arc); + #[async_trait] impl OutboxIntentBundlesReader for TestOutboxReader { - type Error = std::convert::Infallible; + type Error = anyhow::Error; async fn read( &mut self, @@ -287,6 +321,18 @@ impl OutboxIntentBundlesReader for TestOutboxReader { ) -> Result, Self::Error> { Ok(vec![]) } + + async fn fetch_outbox_intent( + &self, + intent_id: u64, + ) -> Result, Self::Error> { + let pda = outbox_intent_pda(intent_id); + let mut accounts = self.0.get_multiple_accounts(&[pda]).await?; + let Some(account) = accounts.pop().flatten() else { + return Ok(None); + }; + Ok(Some(OutboxIntentBundle::try_from_bytes(&account.data)?)) + } } struct TestOutboxClient { @@ -369,7 +415,7 @@ impl OutboxClient for TestOutboxClient { } fn outbox_reader(&self) -> Self::OutboxReader { - TestOutboxReader + TestOutboxReader(self.ephem_rpc.clone()) } } From f968d8260ca4df8bee14dab38ed838409834d580 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 15:55:07 +0700 Subject: [PATCH 63/97] feat: added more tests --- .../accepted_intent_executor.rs | 2 +- .../strategy_executor/utils.rs | 23 +- .../src/outbox_client.rs | 6 +- .../service/outbox_intent_bundles_reader.rs | 8 +- test-integration/Cargo.lock | 1 + test-integration/Cargo.toml | 1 + .../flexi-counter/src/processor/callback.rs | 5 +- .../test-schedule-intent/Cargo.toml | 1 + .../test-schedule-intent/tests/common/mod.rs | 2 + .../tests/test_outbox_flow.rs | 719 +++++++++++++++++- .../tests/test_schedule_intents.rs | 6 - 11 files changed, 710 insertions(+), 64 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 207b1f2c9..0840240c5 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -157,7 +157,7 @@ where &self.ctx, &self.authority, intent_bundle, - None, + None, // No pending signature transaction_strategy, execution_report, || self.time_left(), diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index bdd0d63e1..29ec5c52e 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -407,14 +407,21 @@ pub(in crate::intent_executor) fn handle_actions_result( where A: ActionsCallbackScheduler, { - let (callbacks, junk) = if result.is_ok() { - let callbacks = transaction_strategy.extract_action_callbacks(); - (callbacks, TransactionStrategy::default()) - } else { - let mut removed_actions = - transaction_strategy.remove_actions(authority); - let callbacks = removed_actions.extract_action_callbacks(); - (callbacks, removed_actions) + let (callbacks, junk) = match result { + // TODO(edwin): add in PR description + // Timeout may be triggered right after tx was sent + // It could also be triggered prior + // In order to give simple contract to user we keep actions in both cases + Ok(()) | Err(ActionError::TimeoutError) => { + let callbacks = transaction_strategy.extract_action_callbacks(); + (callbacks, TransactionStrategy::default()) + } + _ => { + let mut removed_actions = + transaction_strategy.remove_actions(authority); + let callbacks = removed_actions.extract_action_callbacks(); + (callbacks, removed_actions) + } }; if !callbacks.is_empty() { let result = callback_scheduler.schedule(callbacks, signature, result); diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index e35796ce8..8ed7fc7e6 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -13,8 +13,8 @@ use magicblock_core::{ use magicblock_program::{ instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, - register_scheduled_commit_sent, MagicContext, Pubkey, SentCommit, - MAGIC_CONTEXT_PUBKEY, + outbox_intent_bundles::OutboxIntentBundle, register_scheduled_commit_sent, + MagicContext, Pubkey, SentCommit, MAGIC_CONTEXT_PUBKEY, }; use solana_account::{Account, ReadableAccount}; use solana_rpc_client::{ @@ -26,7 +26,7 @@ use solana_rpc_client_api::{ use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tracing::{debug, error}; -use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; + use crate::service::outbox_intent_bundles_reader::{ InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, }; diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs index ef677f4a3..ee6cdf252 100644 --- a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs +++ b/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs @@ -6,8 +6,12 @@ use std::{ }; use async_trait::async_trait; -use magicblock_accounts_db::{error::AccountsDbError, traits::AccountsBank, AccountsDb}; -use magicblock_core::intent::outbox::{outbox_intent_pda, OUTBOX_INTENT_DISCRIMINATOR}; +use magicblock_accounts_db::{ + error::AccountsDbError, traits::AccountsBank, AccountsDb, +}; +use magicblock_core::intent::outbox::{ + outbox_intent_pda, OUTBOX_INTENT_DISCRIMINATOR, +}; use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::{AccountSharedData, ReadableAccount}; use tracing::warn; diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 0d00c316f..8a13eda3b 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -9966,6 +9966,7 @@ dependencies = [ "magicblock-rpc-client", "magicblock-table-mania", "program-flexi-counter", + "serde_json", "solana-commitment-config", "solana-rpc-client", "solana-rpc-client-api", diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index bd3493de3..c6d315ab1 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -70,6 +70,7 @@ rayon = "1.10.0" rkyv = "0.7.45" schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" +serde_json = "1.0" serial_test = "3.2.0" shlex = "1.3.0" solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "782e0b00" } diff --git a/test-integration/programs/flexi-counter/src/processor/callback.rs b/test-integration/programs/flexi-counter/src/processor/callback.rs index 6f29e595e..447fba307 100644 --- a/test-integration/programs/flexi-counter/src/processor/callback.rs +++ b/test-integration/programs/flexi-counter/src/processor/callback.rs @@ -80,9 +80,8 @@ pub fn process_transfer_action_handler( /// counter PDA retains the pre-payment as the cost of the deal. /// /// On failure (`ok == false`): moves `amount` lamports from the counter PDA -/// back to the payer on Base (refund). The counter PDA can be written because -/// at callback time the account has already been undelegated and is owned by -/// the flexi-counter program again. +/// back to the payer on ER (refund). The counter PDA can be written because +/// at callback time the account is still delegated /// /// Accounts: /// 0. [signer, readonly] validator authority (auto-prepended by magic program) diff --git a/test-integration/test-schedule-intent/Cargo.toml b/test-integration/test-schedule-intent/Cargo.toml index ba36a22dc..8a97ad5ea 100644 --- a/test-integration/test-schedule-intent/Cargo.toml +++ b/test-integration/test-schedule-intent/Cargo.toml @@ -20,6 +20,7 @@ magicblock-rpc-client = { workspace = true } magicblock-table-mania = { workspace = true } program-flexi-counter = { workspace = true, features = ["no-entrypoint"] } solana-commitment-config = { workspace = true } +serde_json = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-sdk = { workspace = true } diff --git a/test-integration/test-schedule-intent/tests/common/mod.rs b/test-integration/test-schedule-intent/tests/common/mod.rs index b0f1d29d2..b033cb72a 100644 --- a/test-integration/test-schedule-intent/tests/common/mod.rs +++ b/test-integration/test-schedule-intent/tests/common/mod.rs @@ -11,6 +11,8 @@ use solana_sdk::{ }; pub const LABEL: &str = "I am a label"; +pub const BASE_ACTION_FEE: u64 = 5000; +pub const CALLBACK_FEE: u64 = 5000; pub fn setup_payer(ctx: &IntegrationTestContext) -> Keypair { let payer = Keypair::new(); diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 3a569bd6c..f0034861f 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -26,10 +26,12 @@ use magicblock_committor_service::{ }; use magicblock_core::{ intent::{outbox::outbox_intent_pda, BaseActionCallback}, - traits::{ActionResult, ActionsCallbackScheduler, CallbackScheduleError}, + traits::{ActionError, ActionResult, ActionsCallbackScheduler, CallbackScheduleError}, }; use magicblock_magic_program_api::{ - instruction::MagicBlockInstruction, outbox::ExecutionStage, + args::{CommitAndUndelegateArgs, CommitTypeArgs, MagicIntentBundleArgs, UndelegateTypeArgs}, + instruction::MagicBlockInstruction, + outbox::{ExecutionStage, TwoStageProgress}, MAGIC_CONTEXT_PUBKEY, }; use magicblock_program::{ @@ -44,14 +46,17 @@ use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use program_flexi_counter::{ instruction::{ create_intent_bundle_commit_and_finalize_ix, create_intent_bundle_ix, + create_transfer_intent_ix, }, state::FlexiCounter, }; use solana_rpc_client::{ + http_sender::HttpSender, nonblocking::rpc_client::RpcClient as AsyncRpcClient, - rpc_client::SerializableTransaction, + rpc_client::{RpcClientConfig, SerializableTransaction}, + rpc_sender::{RpcSender, RpcTransportStats}, }; -use solana_rpc_client_api::client_error; +use solana_rpc_client_api::{client_error, request::RpcRequest}; use solana_sdk::{ instruction::{AccountMeta, Instruction}, pubkey::Pubkey, @@ -60,10 +65,12 @@ use solana_sdk::{ transaction::Transaction, }; +type CallbackRecord = (Vec, Option, ActionResult); + type TestIntentExecutorCtx = IntentExecutorCtx< TransactionPreparatorImpl, RpcTaskInfoFetcher, - NoopCallbackScheduler, + RecordingCallbackScheduler, TestOutboxClient, >; @@ -71,11 +78,11 @@ struct TestEnv { ctx: IntegrationTestContext, validator_authority: Keypair, chain_mb_client: MagicblockRpcClient, + ephem_rpc: Arc, intent_client: IntentExecutionClient, - outbox_client: Arc, table_mania: TableMania, task_info_fetcher: Arc>, - stage_calls: Arc>>, + callback_scheduler: RecordingCallbackScheduler, } impl TestEnv { @@ -85,22 +92,14 @@ impl TestEnv { let chain_rpc = Arc::new(ctx.try_chain_client_async().unwrap()); let ephem_rpc = Arc::new(ctx.try_ephem_client_async().unwrap()); let chain_mb_rpc = MagicblockRpcClient::new(chain_rpc); - let ephem_mb_rpc = MagicblockRpcClient::new(ephem_rpc.clone()); let intent_client = IntentExecutionClient::new(chain_mb_rpc.clone()); - let outbox_client = Arc::new(TestOutboxClient::new( - ephem_rpc, - validator_authority.insecure_clone(), - )); - let stage_calls = outbox_client.stage_calls.clone(); - let gc_config = GarbageCollectorConfig::default(); let table_mania = TableMania::new( chain_mb_rpc.clone(), &validator_authority, Some(gc_config), ); - let compute_budget = ComputeBudgetConfig::new(1_000_000); let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( RpcTaskInfoFetcher::new(chain_mb_rpc.clone()), )); @@ -109,24 +108,34 @@ impl TestEnv { ctx, validator_authority, chain_mb_client: chain_mb_rpc, + ephem_rpc, intent_client, - outbox_client, table_mania, task_info_fetcher, - - stage_calls, + callback_scheduler: RecordingCallbackScheduler::default(), } } - fn executor_ctx(&self, actions_timeout: Duration) -> TestIntentExecutorCtx { - TestIntentExecutorCtx { + fn executor_ctx_builder( + &self, + actions_timeout: Duration, + ) -> TestIntentExecutorCtxBuilder { + let ctx = TestIntentExecutorCtx { intent_client: self.intent_client.clone(), transaction_preparator: self.transaction_preparator(), task_info_fetcher: self.task_info_fetcher.clone(), - outbox_client: self.outbox_client.clone(), - actions_callback_executor: NoopCallbackScheduler, + outbox_client: self.outbox_client().into(), + actions_callback_executor: self.callback_scheduler.clone(), actions_timeout, - } + }; + TestIntentExecutorCtxBuilder { ctx } + } + + fn outbox_client(&self) -> TestOutboxClient { + TestOutboxClient::new( + self.ephem_rpc.clone(), + self.validator_authority.insecure_clone(), + ) } fn transaction_preparator(&self) -> TransactionPreparatorImpl { @@ -137,10 +146,45 @@ impl TestEnv { compute_budget, ) } + + fn intent_client_with_send_sleep( + &self, + sleep_duration: Duration, + ) -> IntentExecutionClient { + let sender = SleepyRpcSender { + inner: HttpSender::new(IntegrationTestContext::url_chain()), + sleep_duration, + }; + let rpc_client = Arc::new(AsyncRpcClient::new_sender( + sender, + RpcClientConfig::with_commitment(self.ctx.commitment), + )); + IntentExecutionClient::new(MagicblockRpcClient::new(rpc_client)) + } +} + +pub struct TestIntentExecutorCtxBuilder { + ctx: TestIntentExecutorCtx, +} + +impl TestIntentExecutorCtxBuilder { + fn with_outbox_client(mut self, value: Arc) -> Self { + self.ctx.outbox_client = value; + self + } + + fn with_intent_client(mut self, value: IntentExecutionClient) -> Self { + self.ctx.intent_client = value; + self + } + + fn build(self) -> TestIntentExecutorCtx { + self.ctx + } } /// Schedyles commit of counters directly via magic-program using validator authority -fn schedule(ctx: &IntegrationTestContext, counters: &[Pubkey]) { +fn schedule_commit_finalize(ctx: &IntegrationTestContext, counters: &[Pubkey]) { let validator_keypair = ensure_validator_authority(); let schedule_ix = schedule_commit_instruction( @@ -154,7 +198,28 @@ fn schedule(ctx: &IntegrationTestContext, counters: &[Pubkey]) { let sig = ctx .send_transaction_ephem(&mut tx, &[&validator_keypair]) .unwrap(); - println!("schedule sig: {}", sig); + println!("schedule_commit_finalize sig: {}", sig); +} + +fn schedule_intent_with_callback( + ctx: &IntegrationTestContext, + payer: &Keypair, + destination: Pubkey, + amount: u64, +) { + let schedule_ix = create_transfer_intent_ix( + payer.pubkey(), + destination, + ctx.ephem_validator_identity.unwrap(), + amount, + false, + 100_000, + ); + let mut tx = + Transaction::new_with_payer(&[schedule_ix], Some(&payer.pubkey())); + let sig = ctx.send_transaction_ephem(&mut tx, &[payer]).unwrap(); + + println!("schedule_intent_with_callback sig: {}", sig); } /// Tries to accept intent @@ -184,14 +249,14 @@ fn steal_accept_intent( fn steal_schedule_accept_intent( ctx: &IntegrationTestContext, - counters: &[Pubkey], + schedule: impl Fn(&IntegrationTestContext), ) -> u64 { const MAX_ATTEMPTS: u8 = 5; let mut attempt = 0; loop { let intent_id = read_next_intent_id(ctx); - schedule(ctx, counters); + schedule(ctx); let result = steal_accept_intent(&ctx, intent_id); match result { Ok(()) => return intent_id, @@ -209,11 +274,11 @@ fn steal_schedule_accept_intent( fn schedule_and_accept( ctx: &IntegrationTestContext, - counters: &[Pubkey], + schedule: impl Fn(&IntegrationTestContext), ) -> OutboxIntentBundle { ctx.wait_for_next_slot_ephem().unwrap(); - let intent_id = steal_schedule_accept_intent(&ctx, counters); + let intent_id = steal_schedule_accept_intent(&ctx, schedule); let pda = outbox_intent_pda(intent_id); let data = ctx.fetch_ephem_account_data(pda).unwrap(); OutboxIntentBundle::try_from_bytes(&data).unwrap() @@ -223,6 +288,7 @@ fn schedule_and_accept( /// Verifies: executor succeeds, set_intent_execution_stage is called, /// and the committed value lands on the base chain. #[tokio::test(flavor = "multi_thread")] +#[ignore] async fn test_pickup_executed_intent() { let test_env = TestEnv::setup().await; @@ -232,10 +298,15 @@ async fn test_pickup_executed_intent() { init_counter(&test_env.ctx, &payer); delegate_counter(&test_env.ctx, &payer); add_to_counter(&test_env.ctx, &payer, 42); - let outbox_bundle = schedule_and_accept(&test_env.ctx, &[counter_pda]); + + let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { + schedule_commit_finalize(ctx, &[counter_pda]) + }); // Execute intent - let executor_ctx = test_env.executor_ctx(DEFAULT_ACTIONS_TIMEOUT); + let executor_ctx = test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .build(); let executor = AcceptedIntentExecutor::new(executor_ctx); let (result, cleanup_handle) = Box::new(executor) .execute(outbox_bundle.inner.clone()) @@ -248,7 +319,7 @@ async fn test_pickup_executed_intent() { // Validator could sent tx and then crash // On restart validator will extract outbox bundles let outbox_bundle = test_env - .outbox_client + .outbox_client() .outbox_reader() .fetch_outbox_intent(outbox_bundle.inner.id) .await @@ -269,7 +340,9 @@ async fn test_pickup_executed_intent() { ); // Builder executor let executor = build_stage_intent_executor( - test_env.executor_ctx(DEFAULT_ACTIONS_TIMEOUT), + test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .build(), outbox_bundle.status, ); let (result, cleanup_handle) = Box::new(executor) @@ -293,19 +366,439 @@ async fn test_pickup_executed_intent() { assert_eq!(counter.count, 42); } +#[tokio::test(flavor = "multi_thread")] +#[ignore] +async fn test_pickup_failed_intent() { + let test_env = TestEnv::setup().await; + + let payer = setup_payer(&test_env.ctx); + let counter_pda = FlexiCounter::pda(&payer.pubkey()).0; + + init_counter(&test_env.ctx, &payer); + delegate_counter(&test_env.ctx, &payer); + add_to_counter(&test_env.ctx, &payer, 42); + let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { + schedule_commit_finalize(ctx, &[counter_pda]); + }); + let intent_id = outbox_bundle.id; + + // Create executor that will fail before chain execution + let mut failing_outbox = test_env.outbox_client(); + failing_outbox.with_fail_execution_stage(true); + + let executor_ctx = test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .with_outbox_client(failing_outbox.into()) + .build(); + + let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let (result, cleanup_handle) = executor.execute(outbox_bundle.inner).await; + assert!(result + .inner + .unwrap_err() + .to_string() + .contains("set_intent_execution_stage failed")); + cleanup_handle + .clean() + .await + .expect("Fail must succeed even after failure"); + + // Verify chain status is still accepted + let chain_outbox_bundle = test_env + .outbox_client() + .outbox_reader() + .fetch_outbox_intent(intent_id) + .await + .expect("fetch suceeds") + .expect("outbox intent exists"); + assert_eq!( + chain_outbox_bundle.status, + OutboxIntentBundleStatus::Accepted, + "Invalid outbox state" + ); + + // Build executor that will drive execution to success + let executor_ctx = test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .build(); + let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let (result, cleanup_handle) = + executor.execute(chain_outbox_bundle.inner).await; + let ExecutionOutput::SingleStage(_) = + result.inner.expect("execution succeeded") + else { + panic!("Unexpected execution strategy"); + }; + cleanup_handle.clean().await.expect("cleanup failed"); + + // Validate on chain state + let counter = test_env + .ctx + .fetch_chain_account_struct::(counter_pda) + .unwrap(); + assert_eq!(counter.count, 42); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_pickup_after_timeout() { + const ACTIONS_TIMEOUT: Duration = Duration::from_millis(15); + /// Larger than `ACTIONS_TIMEOUT` in order to trigger timeout int `execute_with_timeout` + const SET_EXECUTION_STAGE_SLEEP: Duration = Duration::from_millis(20); + const TRANSFER_AMOUNT: u64 = 900_000; + + let test_env = TestEnv::setup().await; + + let payer = setup_payer(&test_env.ctx); + let chain_payer = setup_payer(&test_env.ctx); + + init_counter(&test_env.ctx, &payer); + delegate_counter(&test_env.ctx, &payer); + add_to_counter(&test_env.ctx, &payer, 42); + test_env.ctx.delegate_account(&chain_payer, &payer).unwrap(); + + let destination = Keypair::new(); + let payer_balance_before = test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { + schedule_intent_with_callback(ctx, &payer, destination.pubkey(), TRANSFER_AMOUNT); + }); + + let mut slow_outbox_client = test_env.outbox_client(); + slow_outbox_client.with_set_execution_stage_sleep(SET_EXECUTION_STAGE_SLEEP); + let executor_ctx = test_env + .executor_ctx_builder(ACTIONS_TIMEOUT) + .with_outbox_client(slow_outbox_client.into()) + .build(); + + let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; + assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); + cleanup_handle.clean().await.expect("cleanup failed"); + + // Callback was triggered with TimeoutError (set_intent_execution_stage exceeded ACTIONS_TIMEOUT) + let callback_calls = test_env.callback_scheduler.recorded_calls(); + assert_eq!(callback_calls.len(), 1, "Expected exactly one callback invocation"); + let (callbacks, signature, cb_result) = &callback_calls[0]; + assert!(!callbacks.is_empty(), "Callback list must not be empty"); + assert!(signature.is_none(), "No signature on timeout"); + assert!( + matches!(cb_result, Err(ActionError::TimeoutError)), + "Expected TimeoutError, got: {:?}", + cb_result + ); + + // Despite the callback timeout the commit tx still lands on chain + let payer_balance_after = test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + assert_eq!( + payer_balance_after + BASE_ACTION_FEE + CALLBACK_FEE + TRANSFER_AMOUNT, + payer_balance_before, + "Payer fees not deducted correctly" + ); + let dest_balance = test_env + .ctx + .fetch_chain_account_balance(&destination.pubkey()) + .unwrap(); + assert_eq!(dest_balance, TRANSFER_AMOUNT, "Destination did not receive funds"); +} + +/// Similar to `test_pickup_after_timeout` but the slow-down is at the RpcSender +/// level rather than at `set_intent_execution_stage`. +/// +/// Timeline inside `stage_execution_loop`: +/// 1. getLatestBlockhash → HTTP response + SEND_SLEEP (~80ms) → ok +/// 2. set_intent_execution_stage (TestOutboxClient, instant) +/// 3. pending_signature = Some(sig) +/// 4. sendTransaction → HTTP response (tx IS on chain) + SEND_SLEEP starts +/// → ACTIONS_TIMEOUT fires during that sleep +/// +/// ACTIONS_TIMEOUT (100ms) sits between one SEND_SLEEP (80ms) and two (160ms), +/// so the tx is always submitted before the timeout, but the callback gets +/// TimeoutError with no signature. On re-execution the executor finds +/// pending_signature set, confirms the tx succeeded, and returns without +/// re-submitting. +/// TODO(edwin): add same but where tx wasn't actually sent and we timedout +#[tokio::test(flavor = "multi_thread")] +async fn test_pick_up_after_tx_submission() { + const SEND_SLEEP: Duration = Duration::from_millis(80); + const ACTIONS_TIMEOUT: Duration = Duration::from_millis(100); + const TRANSFER_AMOUNT: u64 = 900_000; + + let test_env = TestEnv::setup().await; + + let payer = setup_payer(&test_env.ctx); + let chain_payer = setup_payer(&test_env.ctx); + + init_counter(&test_env.ctx, &payer); + delegate_counter(&test_env.ctx, &payer); + add_to_counter(&test_env.ctx, &payer, 42); + test_env.ctx.delegate_account(&chain_payer, &payer).unwrap(); + + let destination = Keypair::new(); + let payer_balance_before = + test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { + schedule_intent_with_callback(ctx, &payer, destination.pubkey(), TRANSFER_AMOUNT); + }); + + let slow_intent_client = test_env.intent_client_with_send_sleep(SEND_SLEEP); + let executor_ctx = test_env + .executor_ctx_builder(ACTIONS_TIMEOUT) + .with_intent_client(slow_intent_client) + .build(); + + let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; + assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); + cleanup_handle.clean().await.expect("cleanup failed"); + + // Callback triggered with TimeoutError — tx was submitted before timeout. + let callback_calls = test_env.callback_scheduler.recorded_calls(); + assert_eq!(callback_calls.len(), 1, "Expected exactly one callback invocation"); + let (callbacks, signature, cb_result) = &callback_calls[0]; + assert!(!callbacks.is_empty(), "Callback list must not be empty"); + assert!(signature.is_none(), "No signature on timeout"); + assert!( + matches!(cb_result, Err(ActionError::TimeoutError)), + "Expected TimeoutError, got: {:?}", + cb_result + ); + + // Despite the callback timeout the commit tx still lands on chain. + let payer_balance_after = + test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + assert_eq!( + payer_balance_after + BASE_ACTION_FEE + CALLBACK_FEE + TRANSFER_AMOUNT, + payer_balance_before, + "Payer fees not deducted correctly" + ); + let dest_balance = test_env + .ctx + .fetch_chain_account_balance(&destination.pubkey()) + .unwrap(); + assert_eq!(dest_balance, TRANSFER_AMOUNT, "Destination did not receive funds"); +} + +/// Verifies recovery when the validator crashes after the commit tx lands on chain +/// but before the finalize stage is recorded in the outbox. +/// +/// Flow: +/// 1. AcceptedIntentExecutor runs TwoStage; commit stage completes (tx on chain, +/// outbox updated to TwoStage::Committing). +/// 2. set_intent_execution_stage for the Finalizing stage fails → executor errors +/// and finalize tx is never sent. +/// 3. On restart, build_stage_intent_executor sees TwoStage::Committing, confirms +/// the commit sig on chain, then executes the finalize stage. +#[tokio::test(flavor = "multi_thread")] +async fn test_pickup_after_committing() { + const COUNTER_COUNT: usize = 9; + + let test_env = TestEnv::setup().await; + let counters = setup_counters(&test_env.ctx, COUNTER_COUNT); + let counter_pdas: Vec = + counters.iter().map(|(_, pda)| *pda).collect(); + + let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { + schedule_commit_and_undelegate_bundle(ctx, &counter_pdas); + }); + + // Fail only when the executor tries to record the Finalizing stage. + // The commit stage (Committing) is allowed through so the commit tx is sent. + let mut failing_outbox = test_env.outbox_client(); + failing_outbox.with_fail_finalizing(true); + let stage_calls = failing_outbox.stage_calls.clone(); + + let executor_ctx = test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .with_outbox_client(failing_outbox.into()) + .build(); + + let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let (result, cleanup_handle) = + executor.execute(outbox_bundle.inner.clone()).await; + assert!( + result.inner.is_err(), + "Expected failure on set_intent_execution_stage(Finalizing)" + ); + cleanup_handle.clean().await.expect("cleanup after failure"); + + // Commit stage was recorded; outbox is TwoStage::Committing + let calls = stage_calls.lock().unwrap(); + assert_eq!(calls.len(), 1, "Only the commit stage should be recorded"); + let commit_sig = match &calls[0].1 { + ExecutionStage::TwoStage(TwoStageProgress::Committing(sig)) => *sig, + other => panic!("Expected Committing stage, got {:?}", other), + }; + drop(calls); + + // Re-fetch outbox and confirm it reflects TwoStage::Committing + let outbox_bundle = test_env + .outbox_client() + .outbox_reader() + .fetch_outbox_intent(outbox_bundle.inner.id) + .await + .expect("fetch succeeded") + .expect("outbox bundle present"); + assert!( + matches!( + &outbox_bundle.status, + OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( + TwoStageProgress::Committing(_) + )) + ), + "Expected TwoStage::Committing in outbox, got {:?}", + outbox_bundle.status + ); + + // Recovery: build_stage_intent_executor detects the commit sig on chain and + // runs the finalize stage without re-submitting the commit tx. + let executor = build_stage_intent_executor( + test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .build(), + outbox_bundle.status, + ); + let (result, cleanup_handle) = + Box::new(executor).execute(outbox_bundle.inner.clone()).await; + let ExecutionOutput::TwoStage { + commit_signature, + finalize_signature: _, + } = result.inner.expect("recovery execution succeeded") + else { + panic!("Expected TwoStage output"); + }; + assert_eq!( + commit_signature, commit_sig, + "Commit sig must not change on recovery" + ); + cleanup_handle.clean().await.expect("cleanup after recovery"); + + // Counters should be finalized (undelegated) on chain + for (_, pda) in &counters { + let counter = test_env + .ctx + .fetch_chain_account_struct::(*pda) + .unwrap(); + assert!(counter.count > 0, "counter should be committed on chain"); + } +} + +/// Verifies recovery when the validator crashes after both the commit and +/// finalize txs have landed but before the service can act on the result. +/// +/// Flow: +/// 1. AcceptedIntentExecutor runs TwoStage to completion; outbox is updated to +/// TwoStage::Finalizing with both signatures. +/// 2. On restart, build_stage_intent_executor sees TwoStage::Finalizing, confirms +/// the finalize sig on chain, and returns the same signatures without +/// re-submitting anything. +#[tokio::test(flavor = "multi_thread")] +async fn test_pickup_after_finalizing() { + const COUNTER_COUNT: usize = 9; + + let test_env = TestEnv::setup().await; + let counters = setup_counters(&test_env.ctx, COUNTER_COUNT); + let counter_pdas: Vec = + counters.iter().map(|(_, pda)| *pda).collect(); + + let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { + schedule_commit_and_undelegate_bundle(ctx, &counter_pdas); + }); + + // Run to completion with a normal outbox client + let executor_ctx = test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .build(); + let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let (result, cleanup_handle) = + executor.execute(outbox_bundle.inner.clone()).await; + let ExecutionOutput::TwoStage { + commit_signature, + finalize_signature, + } = result.inner.expect("first execution succeeded") + else { + panic!("Expected TwoStage output"); + }; + cleanup_handle.clean().await.expect("cleanup after first run"); + + // Outbox should now show TwoStage::Finalizing + let outbox_bundle = test_env + .outbox_client() + .outbox_reader() + .fetch_outbox_intent(outbox_bundle.inner.id) + .await + .expect("fetch succeeded") + .expect("outbox bundle present"); + assert!( + matches!( + &outbox_bundle.status, + OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( + TwoStageProgress::Finalizing { .. } + )) + ), + "Expected TwoStage::Finalizing in outbox, got {:?}", + outbox_bundle.status + ); + + // Recovery: build_stage_intent_executor detects the finalize sig on chain and + // returns the same signatures without re-executing either stage. + let executor = build_stage_intent_executor( + test_env + .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .build(), + outbox_bundle.status, + ); + let (result, cleanup_handle) = + Box::new(executor).execute(outbox_bundle.inner.clone()).await; + let ExecutionOutput::TwoStage { + commit_signature: retried_commit, + finalize_signature: retried_finalize, + } = result.inner.expect("recovery execution succeeded") + else { + panic!("Expected TwoStage output"); + }; + assert_eq!( + retried_commit, commit_signature, + "Commit sig must not change on recovery" + ); + assert_eq!( + retried_finalize, finalize_signature, + "Finalize sig must not change on recovery" + ); + cleanup_handle.clean().await.expect("cleanup after recovery"); + + // Counters are finalized and undelegated on chain + for (_, pda) in &counters { + let counter = test_env + .ctx + .fetch_chain_account_struct::(*pda) + .unwrap(); + assert!(counter.count > 0, "counter should be committed on chain"); + } +} + #[derive(Clone, Default)] -struct NoopCallbackScheduler; -impl ActionsCallbackScheduler for NoopCallbackScheduler { +struct RecordingCallbackScheduler { + calls: Arc>>, +} + +impl RecordingCallbackScheduler { + fn recorded_calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} + +impl ActionsCallbackScheduler for RecordingCallbackScheduler { fn schedule( &self, callbacks: Vec, - _signature: Option, - _result: ActionResult, + signature: Option, + result: ActionResult, ) -> Vec> { - callbacks - .iter() - .map(|_| Ok(Signature::new_unique())) - .collect() + let count = callbacks.len(); + self.calls + .lock() + .unwrap() + .push((callbacks, signature, result)); + (0..count).map(|_| Ok(Signature::new_unique())).collect() } } @@ -339,6 +832,11 @@ struct TestOutboxClient { ephem_rpc: Arc, validator_keypair: Keypair, pub stage_calls: Arc>>, + + fail_set_execution_stage: bool, + set_execution_stage_sleep: Option, + fail_committing: bool, + fail_finalizing: bool, } impl TestOutboxClient { @@ -347,8 +845,28 @@ impl TestOutboxClient { ephem_rpc, validator_keypair, stage_calls: Default::default(), + fail_set_execution_stage: false, + set_execution_stage_sleep: None, + fail_committing: false, + fail_finalizing: false, } } + + fn with_fail_execution_stage(&mut self, value: bool) { + self.fail_set_execution_stage = value + } + + fn with_set_execution_stage_sleep(&mut self, value: Duration) { + self.set_execution_stage_sleep = Some(value); + } + + fn with_fail_committing(&mut self, value: bool) { + self.fail_committing = value; + } + + fn with_fail_finalizing(&mut self, value: bool) { + self.fail_finalizing = value; + } } #[async_trait] @@ -370,6 +888,31 @@ impl OutboxClient for TestOutboxClient { intent_id: u64, stage: ExecutionStage, ) -> Result<(), Self::Error> { + if let Some(sleep_duration) = self.set_execution_stage_sleep { + tokio::time::sleep(sleep_duration).await; + }; + + let should_fail = self.fail_set_execution_stage + || matches!( + &stage, + ExecutionStage::TwoStage(TwoStageProgress::Committing(_)) + if self.fail_committing + ) + || matches!( + &stage, + ExecutionStage::TwoStage(TwoStageProgress::Finalizing { .. }) + if self.fail_finalizing + ); + + if should_fail { + return Err(Self::Error::RpcClientError( + client_error::ErrorKind::Custom( + "set_intent_execution_stage failed".to_string(), + ) + .into(), + )); + } + self.stage_calls .lock() .unwrap() @@ -419,6 +962,32 @@ impl OutboxClient for TestOutboxClient { } } +struct SleepyRpcSender { + inner: HttpSender, + sleep_duration: Duration, +} + +#[async_trait] +impl RpcSender for SleepyRpcSender { + async fn send( + &self, + request: RpcRequest, + params: serde_json::Value, + ) -> solana_rpc_client_api::client_error::Result { + let result = self.inner.send(request, params).await; + tokio::time::sleep(self.sleep_duration).await; + result + } + + fn get_transport_stats(&self) -> RpcTransportStats { + self.inner.get_transport_stats() + } + + fn url(&self) -> String { + self.inner.url() + } +} + fn ensure_validator_authority() -> Keypair { static ONCE: Once = Once::new(); let kp = Keypair::try_from(&DLP_TEST_AUTHORITY_BYTES[..]).unwrap(); @@ -450,3 +1019,71 @@ pub fn schedule_commit_instruction( account_metas, ) } + +/// Schedules a CommitAndUndelegate bundle via ScheduleIntentBundle signed by the +/// validator authority. Using CommitAndUndelegate (vs CommitFinalize) generates +/// separate commit_tasks + finalize_tasks so build_execution_strategy selects +/// TwoStage execution. With 5 accounts the combined SingleStage tx needs ALTs +/// while each individual stage fits without them, guaranteeing TwoStage. +fn schedule_commit_and_undelegate_bundle_instruction( + payer: &Pubkey, + counter_pdas: &[Pubkey], +) -> Instruction { + // Indices are 1-based into the instruction account list: + // [0]=payer, [1]=magic_context, [2..]=counter PDAs + let indices: Vec = (2u8..2 + counter_pdas.len() as u8).collect(); + let args = MagicIntentBundleArgs { + commit: None, + commit_and_undelegate: Some(CommitAndUndelegateArgs { + commit_type: CommitTypeArgs::Standalone(indices), + undelegate_type: UndelegateTypeArgs::Standalone, + }), + commit_finalize: None, + commit_finalize_and_undelegate: None, + standalone_actions: vec![], + }; + let mut account_metas = vec![ + AccountMeta::new(*payer, true), + AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), + ]; + account_metas + .extend(counter_pdas.iter().map(|pk| AccountMeta::new(*pk, false))); + Instruction::new_with_bincode( + magicblock_magic_program_api::id(), + &MagicBlockInstruction::ScheduleIntentBundle(args), + account_metas, + ) +} + +fn schedule_commit_and_undelegate_bundle( + ctx: &IntegrationTestContext, + counter_pdas: &[Pubkey], +) { + let validator_keypair = ensure_validator_authority(); + let ix = schedule_commit_and_undelegate_bundle_instruction( + &validator_keypair.pubkey(), + counter_pdas, + ); + let mut tx = + Transaction::new_with_payer(&[ix], Some(&validator_keypair.pubkey())); + let sig = ctx + .send_transaction_ephem(&mut tx, &[&validator_keypair]) + .unwrap(); + println!("schedule_commit_and_undelegate_bundle sig: {}", sig); +} + +fn setup_counters( + ctx: &IntegrationTestContext, + n: usize, +) -> Vec<(Keypair, Pubkey)> { + (0..n) + .map(|i| { + let payer = setup_payer(ctx); + init_counter(ctx, &payer); + delegate_counter(ctx, &payer); + add_to_counter(ctx, &payer, (i + 1) as u8); + let pda = FlexiCounter::pda(&payer.pubkey()).0; + (payer, pda) + }) + .collect() +} diff --git a/test-integration/test-schedule-intent/tests/test_schedule_intents.rs b/test-integration/test-schedule-intent/tests/test_schedule_intents.rs index b4a7329cc..bc7471e87 100644 --- a/test-integration/test-schedule-intent/tests/test_schedule_intents.rs +++ b/test-integration/test-schedule-intent/tests/test_schedule_intents.rs @@ -517,9 +517,6 @@ fn test_transfer_intent_success() { // Cross-chain transfer amount // Also min rent for destination account as well const AMOUNT: u64 = 890_880; - // Fees charged from payer - const BASE_ACTION_FEE: u64 = 5000; - const CALLBACK_FEE: u64 = 5000; const CHARGED_AMOUNT: u64 = AMOUNT + BASE_ACTION_FEE + CALLBACK_FEE; init_logger!(); @@ -557,9 +554,6 @@ fn test_transfer_intent_failure_refunds_payer() { // Cross-chain transfer amount // Also min rent for destination account as well const AMOUNT: u64 = 890_880; - // Fees charged from payer - const BASE_ACTION_FEE: u64 = 5000; - const CALLBACK_FEE: u64 = 5000; init_logger!(); From bf226e0aa2d75fe2a2bb5c4d277e823323461501 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 15:55:32 +0700 Subject: [PATCH 64/97] fix: fmt --- .../tests/test_outbox_flow.rs | 105 +++++++++++++----- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index f0034861f..4409a81ff 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -26,10 +26,16 @@ use magicblock_committor_service::{ }; use magicblock_core::{ intent::{outbox::outbox_intent_pda, BaseActionCallback}, - traits::{ActionError, ActionResult, ActionsCallbackScheduler, CallbackScheduleError}, + traits::{ + ActionError, ActionResult, ActionsCallbackScheduler, + CallbackScheduleError, + }, }; use magicblock_magic_program_api::{ - args::{CommitAndUndelegateArgs, CommitTypeArgs, MagicIntentBundleArgs, UndelegateTypeArgs}, + args::{ + CommitAndUndelegateArgs, CommitTypeArgs, MagicIntentBundleArgs, + UndelegateTypeArgs, + }, instruction::MagicBlockInstruction, outbox::{ExecutionStage, TwoStageProgress}, MAGIC_CONTEXT_PUBKEY, @@ -65,7 +71,8 @@ use solana_sdk::{ transaction::Transaction, }; -type CallbackRecord = (Vec, Option, ActionResult); +type CallbackRecord = + (Vec, Option, ActionResult); type TestIntentExecutorCtx = IntentExecutorCtx< TransactionPreparatorImpl, @@ -457,26 +464,40 @@ async fn test_pickup_after_timeout() { test_env.ctx.delegate_account(&chain_payer, &payer).unwrap(); let destination = Keypair::new(); - let payer_balance_before = test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + let payer_balance_before = test_env + .ctx + .fetch_ephem_account_balance(&payer.pubkey()) + .unwrap(); let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { - schedule_intent_with_callback(ctx, &payer, destination.pubkey(), TRANSFER_AMOUNT); + schedule_intent_with_callback( + ctx, + &payer, + destination.pubkey(), + TRANSFER_AMOUNT, + ); }); let mut slow_outbox_client = test_env.outbox_client(); - slow_outbox_client.with_set_execution_stage_sleep(SET_EXECUTION_STAGE_SLEEP); + slow_outbox_client + .with_set_execution_stage_sleep(SET_EXECUTION_STAGE_SLEEP); let executor_ctx = test_env .executor_ctx_builder(ACTIONS_TIMEOUT) .with_outbox_client(slow_outbox_client.into()) .build(); let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); - let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; + let (result, cleanup_handle) = + executor.execute(outbox_bundle.inner.clone()).await; assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); cleanup_handle.clean().await.expect("cleanup failed"); // Callback was triggered with TimeoutError (set_intent_execution_stage exceeded ACTIONS_TIMEOUT) let callback_calls = test_env.callback_scheduler.recorded_calls(); - assert_eq!(callback_calls.len(), 1, "Expected exactly one callback invocation"); + assert_eq!( + callback_calls.len(), + 1, + "Expected exactly one callback invocation" + ); let (callbacks, signature, cb_result) = &callback_calls[0]; assert!(!callbacks.is_empty(), "Callback list must not be empty"); assert!(signature.is_none(), "No signature on timeout"); @@ -487,7 +508,10 @@ async fn test_pickup_after_timeout() { ); // Despite the callback timeout the commit tx still lands on chain - let payer_balance_after = test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + let payer_balance_after = test_env + .ctx + .fetch_ephem_account_balance(&payer.pubkey()) + .unwrap(); assert_eq!( payer_balance_after + BASE_ACTION_FEE + CALLBACK_FEE + TRANSFER_AMOUNT, payer_balance_before, @@ -497,7 +521,10 @@ async fn test_pickup_after_timeout() { .ctx .fetch_chain_account_balance(&destination.pubkey()) .unwrap(); - assert_eq!(dest_balance, TRANSFER_AMOUNT, "Destination did not receive funds"); + assert_eq!( + dest_balance, TRANSFER_AMOUNT, + "Destination did not receive funds" + ); } /// Similar to `test_pickup_after_timeout` but the slow-down is at the RpcSender @@ -533,10 +560,17 @@ async fn test_pick_up_after_tx_submission() { test_env.ctx.delegate_account(&chain_payer, &payer).unwrap(); let destination = Keypair::new(); - let payer_balance_before = - test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + let payer_balance_before = test_env + .ctx + .fetch_ephem_account_balance(&payer.pubkey()) + .unwrap(); let outbox_bundle = schedule_and_accept(&test_env.ctx, |ctx| { - schedule_intent_with_callback(ctx, &payer, destination.pubkey(), TRANSFER_AMOUNT); + schedule_intent_with_callback( + ctx, + &payer, + destination.pubkey(), + TRANSFER_AMOUNT, + ); }); let slow_intent_client = test_env.intent_client_with_send_sleep(SEND_SLEEP); @@ -546,13 +580,18 @@ async fn test_pick_up_after_tx_submission() { .build(); let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); - let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; + let (result, cleanup_handle) = + executor.execute(outbox_bundle.inner.clone()).await; assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); cleanup_handle.clean().await.expect("cleanup failed"); // Callback triggered with TimeoutError — tx was submitted before timeout. let callback_calls = test_env.callback_scheduler.recorded_calls(); - assert_eq!(callback_calls.len(), 1, "Expected exactly one callback invocation"); + assert_eq!( + callback_calls.len(), + 1, + "Expected exactly one callback invocation" + ); let (callbacks, signature, cb_result) = &callback_calls[0]; assert!(!callbacks.is_empty(), "Callback list must not be empty"); assert!(signature.is_none(), "No signature on timeout"); @@ -563,8 +602,10 @@ async fn test_pick_up_after_tx_submission() { ); // Despite the callback timeout the commit tx still lands on chain. - let payer_balance_after = - test_env.ctx.fetch_ephem_account_balance(&payer.pubkey()).unwrap(); + let payer_balance_after = test_env + .ctx + .fetch_ephem_account_balance(&payer.pubkey()) + .unwrap(); assert_eq!( payer_balance_after + BASE_ACTION_FEE + CALLBACK_FEE + TRANSFER_AMOUNT, payer_balance_before, @@ -574,7 +615,10 @@ async fn test_pick_up_after_tx_submission() { .ctx .fetch_chain_account_balance(&destination.pubkey()) .unwrap(); - assert_eq!(dest_balance, TRANSFER_AMOUNT, "Destination did not receive funds"); + assert_eq!( + dest_balance, TRANSFER_AMOUNT, + "Destination did not receive funds" + ); } /// Verifies recovery when the validator crashes after the commit tx lands on chain @@ -656,8 +700,9 @@ async fn test_pickup_after_committing() { .build(), outbox_bundle.status, ); - let (result, cleanup_handle) = - Box::new(executor).execute(outbox_bundle.inner.clone()).await; + let (result, cleanup_handle) = Box::new(executor) + .execute(outbox_bundle.inner.clone()) + .await; let ExecutionOutput::TwoStage { commit_signature, finalize_signature: _, @@ -669,7 +714,10 @@ async fn test_pickup_after_committing() { commit_signature, commit_sig, "Commit sig must not change on recovery" ); - cleanup_handle.clean().await.expect("cleanup after recovery"); + cleanup_handle + .clean() + .await + .expect("cleanup after recovery"); // Counters should be finalized (undelegated) on chain for (_, pda) in &counters { @@ -717,7 +765,10 @@ async fn test_pickup_after_finalizing() { else { panic!("Expected TwoStage output"); }; - cleanup_handle.clean().await.expect("cleanup after first run"); + cleanup_handle + .clean() + .await + .expect("cleanup after first run"); // Outbox should now show TwoStage::Finalizing let outbox_bundle = test_env @@ -746,8 +797,9 @@ async fn test_pickup_after_finalizing() { .build(), outbox_bundle.status, ); - let (result, cleanup_handle) = - Box::new(executor).execute(outbox_bundle.inner.clone()).await; + let (result, cleanup_handle) = Box::new(executor) + .execute(outbox_bundle.inner.clone()) + .await; let ExecutionOutput::TwoStage { commit_signature: retried_commit, finalize_signature: retried_finalize, @@ -763,7 +815,10 @@ async fn test_pickup_after_finalizing() { retried_finalize, finalize_signature, "Finalize sig must not change on recovery" ); - cleanup_handle.clean().await.expect("cleanup after recovery"); + cleanup_handle + .clean() + .await + .expect("cleanup after recovery"); // Counters are finalized and undelegated on chain for (_, pda) in &counters { From 252fc50cca0878ae60d8a25d9eda2e3eb1625551 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 16:31:40 +0700 Subject: [PATCH 65/97] fix: if we start from Finalizing that succeded some account could be undelegated, so we need to check if signature exists before hand --- .../two_stage_intent_executor.rs | 88 ++++++++++++++----- magicblock-magic-program-api/src/outbox.rs | 14 +++ 2 files changed, 78 insertions(+), 24 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index f60deec8a..f4d844323 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -1,4 +1,7 @@ -use std::time::{Duration, Instant}; +use std::{ + ops::ControlFlow, + time::{Duration, Instant}, +}; use async_trait::async_trait; use magicblock_core::traits::ActionsCallbackScheduler; @@ -16,7 +19,9 @@ use crate::{ error::{IntentExecutorError, IntentExecutorResult}, strategy_executor::{ two_stage::{Committed, Initialized, TwoStageStrategyExecutor}, - utils::{execute_with_timeout, FinalizeStage}, + utils::{ + check_pending_signature, execute_with_timeout, FinalizeStage, + }, }, task_info_fetcher::{ResetType, TaskInfoFetcher}, utils::{build_commit_finalize_tasks, execute_two_stage_flow}, @@ -127,7 +132,7 @@ where &mut self, intent: ScheduledIntentBundle, commit_signature: Signature, - pending_finalize_signature: Signature, + pending_finalize_signature: Option, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { // Commit succeeded so we skip those tasks all together @@ -146,7 +151,7 @@ where let committed_state = Committed::new( commit_signature, finalize_strategy, - Some(pending_finalize_signature), + pending_finalize_signature, ); let mut finalize_strategy_executor = TwoStageStrategyExecutor::committed( @@ -175,6 +180,60 @@ where finalize_signature: finalized_stage.finalize_signature, }) } + + async fn execute_inner( + &mut self, + intent: ScheduledIntentBundle, + execution_report: &mut IntentExecutionReport, + ) -> IntentExecutorResult { + let pending_signature = self.stage.pending_signature(); + let flow = + check_pending_signature(&self.ctx.intent_client, pending_signature) + .await?; + + match (&self.stage, flow) { + // Signature wasn't confirmed - need to reexecute from commit + (TwoStageProgress::Committing(_), ControlFlow::Continue(())) => { + self.execute_committing_intent( + intent, + *pending_signature, // TODO(edwin): should be none or nothing as we checked + execution_report, + ) + .await + } + // Signature confirmed - commit was executed, finalizing... + (TwoStageProgress::Committing(_), ControlFlow::Break(())) => { + self.execute_finalizing_intent( + intent, + *pending_signature, + None, + execution_report, + ) + .await + } + // Finalize didn't occur - execute + ( + TwoStageProgress::Finalizing { commit, .. }, + ControlFlow::Continue(()), + ) => { + self.execute_finalizing_intent( + intent, + *commit, + Some(*pending_signature), // TODO(edwin): stage_execution_loop will check twice then + execution_report, + ) + .await + } + // Finilize was executed - return + ( + TwoStageProgress::Finalizing { commit, finalize }, + ControlFlow::Break(()), + ) => Ok(ExecutionOutput::TwoStage { + commit_signature: *commit, + finalize_signature: *finalize, + }), + } + } } #[async_trait] @@ -197,32 +256,13 @@ where let pubkeys = intent.get_all_committed_pubkeys(); let mut execution_report = IntentExecutionReport::default(); - let result = match &self.stage { - TwoStageProgress::Committing(signature) => { - self.execute_committing_intent( - intent, - *signature, - &mut execution_report, - ) - .await - } - TwoStageProgress::Finalizing { commit, finalize } => { - self.execute_finalizing_intent( - intent, - *commit, - *finalize, - &mut execution_report, - ) - .await - } - }; + let result = self.execute_inner(intent, &mut execution_report).await; if is_undelegate { self.ctx .task_info_fetcher .reset(ResetType::Specific(&pubkeys)); } - self.close_buffers = result.is_ok(); self.junk = execution_report.junk; let result = IntentExecutionResult { diff --git a/magicblock-magic-program-api/src/outbox.rs b/magicblock-magic-program-api/src/outbox.rs index dd10716ee..7a0a548f2 100644 --- a/magicblock-magic-program-api/src/outbox.rs +++ b/magicblock-magic-program-api/src/outbox.rs @@ -56,6 +56,13 @@ impl ExecutionStage { Ok(()) } + + pub fn pending_signature(&self) -> &Signature { + match self { + Self::SingleStage(signature) => signature, + Self::TwoStage(value) => value.pending_signature(), + } + } } impl TwoStageProgress { @@ -108,4 +115,11 @@ impl TwoStageProgress { *self = new_state; Ok(()) } + + pub fn pending_signature(&self) -> &Signature { + match self { + Self::Committing(signature) => signature, + Self::Finalizing { finalize, .. } => finalize, + } + } } From 9f93dbdcc0336c3395865f0355f72d2d92a59f7d Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 16:59:12 +0700 Subject: [PATCH 66/97] feat: simplify state and avoid double query request of check_pending_signature --- .../accepted_intent_executor.rs | 3 +- .../single_stage_intent_executor.rs | 58 ++++++++++++------- .../strategy_executor/single_stage.rs | 3 +- .../strategy_executor/two_stage.rs | 9 ++- .../two_stage_intent_executor.rs | 31 +++------- .../src/intent_executor/utils.rs | 5 +- test-integration/Cargo.lock | 1 + .../test-schedule-intent/Cargo.toml | 1 + .../tests/test_outbox_flow.rs | 9 ++- 9 files changed, 62 insertions(+), 58 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 0840240c5..4411c6591 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -157,7 +157,6 @@ where &self.ctx, &self.authority, intent_bundle, - None, // No pending signature transaction_strategy, execution_report, || self.time_left(), @@ -172,7 +171,7 @@ where finalize_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - let state = Initialized::new(commit_strategy, finalize_strategy, None); + let state = Initialized::new(commit_strategy, finalize_strategy); execute_two_stage_flow( &self.ctx, state, diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 1f52968e6..72dcb3725 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -1,4 +1,7 @@ -use std::time::{Duration, Instant}; +use std::{ + ops::ControlFlow, + time::{Duration, Instant}, +}; use async_trait::async_trait; use magicblock_core::traits::ActionsCallbackScheduler; @@ -14,6 +17,7 @@ use crate::{ intent_executor::{ cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, + strategy_executor::utils::check_pending_signature, task_info_fetcher::{ResetType, TaskInfoFetcher}, utils::{build_commit_finalize_tasks, execute_single_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, @@ -76,29 +80,43 @@ where intent_bundle: ScheduledIntentBundle, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - // It we're here so previous run determined this should be single stage - let (commit_tasks, finalize_tasks) = build_commit_finalize_tasks( - &intent_bundle, - &self.ctx.task_info_fetcher, + let flow = check_pending_signature( + &self.ctx.intent_client, + &self.pending_signature, ) .await?; + match flow { + // Intent was executed - return + ControlFlow::Break(()) => { + Ok(ExecutionOutput::SingleStage(self.pending_signature)) + } + ControlFlow::Continue(()) => { + // It we're here so previous run determined this should be single stage + let (commit_tasks, finalize_tasks) = + build_commit_finalize_tasks( + &intent_bundle, + &self.ctx.task_info_fetcher, + ) + .await?; - let single_stage_tasks = [commit_tasks, finalize_tasks].concat(); - let transaction_strategy = TaskStrategist::build_strategy( - single_stage_tasks, - &self.authority.pubkey(), - )?; + let single_stage_tasks = + [commit_tasks, finalize_tasks].concat(); + let transaction_strategy = TaskStrategist::build_strategy( + single_stage_tasks, + &self.authority.pubkey(), + )?; - execute_single_stage_flow( - &self.ctx, - &self.authority, - intent_bundle, - Some(self.pending_signature), - transaction_strategy, - execution_report, - || self.time_left(), - ) - .await + execute_single_stage_flow( + &self.ctx, + &self.authority, + intent_bundle, + transaction_strategy, + execution_report, + || self.time_left(), + ) + .await + } + } } } diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 3858b5f01..125c18075 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -50,7 +50,6 @@ where pub fn new( authority: Keypair, intent_id: u64, - pending_signature: Option, intent_client: IntentExecutionClient, task_info_fetcher: Arc>, outbox_client: Arc, @@ -61,7 +60,7 @@ where Self { authority, intent_id, - pending_signature, + pending_signature: None, intent_client, task_info_fetcher, outbox_client, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index 94c6949ea..7aeaa3836 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -42,12 +42,11 @@ impl Initialized { pub fn new( commit_strategy: TransactionStrategy, finalize_strategy: TransactionStrategy, - pending_signature: Option, ) -> Self { Self { commit_strategy, finalize_strategy, - pending_signature, + pending_signature: None, current_attempt: 0, } } @@ -68,12 +67,11 @@ impl Committed { pub fn new( commit_signature: Signature, finalize_strategy: TransactionStrategy, - pending_signature: Option, ) -> Self { Self { commit_signature, finalize_strategy, - pending_signature, + pending_signature: None, current_attempt: 0, } } @@ -226,7 +224,7 @@ where /// Transitions to next executor state pub fn done( - mut self, + #[allow(unused_mut)] mut self, commit_signature: Signature, ) -> TwoStageStrategyExecutor<'a, A, O, Committed> { #[cfg(feature = "dev-context-only-utils")] @@ -357,6 +355,7 @@ where } /// Transitions to next executor state + #[allow(unused_mut)] pub fn done(mut self, finalize_signature: Signature) -> Finalized { #[cfg(feature = "dev-context-only-utils")] self.execution_report diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index f4d844323..311db86ea 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -90,7 +90,6 @@ where async fn execute_committing_intent( &mut self, intent_bundle: ScheduledIntentBundle, - pending_signature: Signature, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { // This stage was chosen prior so we build tasks for it @@ -111,11 +110,7 @@ where &self.authority.pubkey(), )?; - let state = Initialized::new( - commit_stage, - finalize_stage, - Some(pending_signature), - ); + let state = Initialized::new(commit_stage, finalize_stage); execute_two_stage_flow( &self.ctx, state, @@ -132,7 +127,6 @@ where &mut self, intent: ScheduledIntentBundle, commit_signature: Signature, - pending_finalize_signature: Option, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { // Commit succeeded so we skip those tasks all together @@ -148,11 +142,8 @@ where &self.authority.pubkey(), )?; - let committed_state = Committed::new( - commit_signature, - finalize_strategy, - pending_finalize_signature, - ); + let committed_state = + Committed::new(commit_signature, finalize_strategy); let mut finalize_strategy_executor = TwoStageStrategyExecutor::committed( committed_state, @@ -194,19 +185,14 @@ where match (&self.stage, flow) { // Signature wasn't confirmed - need to reexecute from commit (TwoStageProgress::Committing(_), ControlFlow::Continue(())) => { - self.execute_committing_intent( - intent, - *pending_signature, // TODO(edwin): should be none or nothing as we checked - execution_report, - ) - .await + self.execute_committing_intent(intent, execution_report) + .await } // Signature confirmed - commit was executed, finalizing... - (TwoStageProgress::Committing(_), ControlFlow::Break(())) => { + (TwoStageProgress::Committing(commit), ControlFlow::Break(())) => { self.execute_finalizing_intent( intent, - *pending_signature, - None, + *commit, execution_report, ) .await @@ -219,12 +205,11 @@ where self.execute_finalizing_intent( intent, *commit, - Some(*pending_signature), // TODO(edwin): stage_execution_loop will check twice then execution_report, ) .await } - // Finilize was executed - return + // Finalize was executed - return ( TwoStageProgress::Finalizing { commit, finalize }, ControlFlow::Break(()), diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 55e2411f1..5b99ed8d5 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -51,7 +51,6 @@ pub(in crate::intent_executor) async fn execute_single_stage_flow( ctx: &IntentExecutorCtx, authority: &Keypair, intent_bundle: ScheduledIntentBundle, - pending_signature: Option, transaction_strategy: TransactionStrategy, execution_report: &mut IntentExecutionReport, time_left: impl Fn() -> Option, @@ -68,7 +67,6 @@ where let mut single_stage_executor = SingleStageStrategyExecutor::new( authority.insecure_clone(), intent_bundle.id, - pending_signature, ctx.intent_client.clone(), ctx.task_info_fetcher.clone(), ctx.outbox_client.clone(), @@ -125,8 +123,7 @@ where // Create state for two stage flow // Pending signature set to None as if we're here tx failed - let state = - two_stage::Initialized::new(commit_strategy, finalize_strategy, None); + let state = two_stage::Initialized::new(commit_strategy, finalize_strategy); execute_two_stage_flow( ctx, diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 8a13eda3b..b39cbd95b 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -9967,6 +9967,7 @@ dependencies = [ "magicblock-table-mania", "program-flexi-counter", "serde_json", + "serial_test", "solana-commitment-config", "solana-rpc-client", "solana-rpc-client-api", diff --git a/test-integration/test-schedule-intent/Cargo.toml b/test-integration/test-schedule-intent/Cargo.toml index 8a97ad5ea..cb2e279fd 100644 --- a/test-integration/test-schedule-intent/Cargo.toml +++ b/test-integration/test-schedule-intent/Cargo.toml @@ -24,6 +24,7 @@ serde_json = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-sdk = { workspace = true } +serial_test = { workspace = true } test-kit = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 4409a81ff..823ccc556 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -5,6 +5,7 @@ use std::{ }; use anyhow::anyhow; +use serial_test::serial; use async_trait::async_trait; use common::*; use ephemeral_rollups_sdk::{compat, ephem::MagicIntentBundleBuilder}; @@ -295,7 +296,7 @@ fn schedule_and_accept( /// Verifies: executor succeeds, set_intent_execution_stage is called, /// and the committed value lands on the base chain. #[tokio::test(flavor = "multi_thread")] -#[ignore] +#[serial] async fn test_pickup_executed_intent() { let test_env = TestEnv::setup().await; @@ -374,7 +375,7 @@ async fn test_pickup_executed_intent() { } #[tokio::test(flavor = "multi_thread")] -#[ignore] +#[serial] async fn test_pickup_failed_intent() { let test_env = TestEnv::setup().await; @@ -447,6 +448,7 @@ async fn test_pickup_failed_intent() { } #[tokio::test(flavor = "multi_thread")] +#[serial] async fn test_pickup_after_timeout() { const ACTIONS_TIMEOUT: Duration = Duration::from_millis(15); /// Larger than `ACTIONS_TIMEOUT` in order to trigger timeout int `execute_with_timeout` @@ -544,6 +546,7 @@ async fn test_pickup_after_timeout() { /// re-submitting. /// TODO(edwin): add same but where tx wasn't actually sent and we timedout #[tokio::test(flavor = "multi_thread")] +#[serial] async fn test_pick_up_after_tx_submission() { const SEND_SLEEP: Duration = Duration::from_millis(80); const ACTIONS_TIMEOUT: Duration = Duration::from_millis(100); @@ -632,6 +635,7 @@ async fn test_pick_up_after_tx_submission() { /// 3. On restart, build_stage_intent_executor sees TwoStage::Committing, confirms /// the commit sig on chain, then executes the finalize stage. #[tokio::test(flavor = "multi_thread")] +#[serial] async fn test_pickup_after_committing() { const COUNTER_COUNT: usize = 9; @@ -739,6 +743,7 @@ async fn test_pickup_after_committing() { /// the finalize sig on chain, and returns the same signatures without /// re-submitting anything. #[tokio::test(flavor = "multi_thread")] +#[serial] async fn test_pickup_after_finalizing() { const COUNTER_COUNT: usize = 9; From ca81cfccfd4542957116c98618d4137288bf6c53 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:18:36 +0700 Subject: [PATCH 67/97] refactor: IntentExecutionManager -> IntentEngineHandle --- magicblock-committor-service/src/committor_processor.rs | 8 ++++---- magicblock-committor-service/src/error.rs | 2 +- ...ntent_execution_manager.rs => intent_engine_handle.rs} | 8 +++----- .../db.rs | 0 .../intent_channerl.rs | 2 +- .../intent_execution_engine.rs | 5 ++--- .../intent_scheduler.rs | 0 magicblock-committor-service/src/intent_executor/utils.rs | 1 - magicblock-committor-service/src/lib.rs | 2 +- magicblock-committor-service/src/outbox_client.rs | 4 ++-- magicblock-committor-service/src/service.rs | 2 +- magicblock-committor-service/src/tasks/task_strategist.rs | 2 +- .../test-schedule-intent/tests/test_outbox_flow.rs | 2 +- 13 files changed, 17 insertions(+), 21 deletions(-) rename magicblock-committor-service/src/{intent_execution_manager.rs => intent_engine_handle.rs} (93%) rename magicblock-committor-service/src/{intent_execution_manager => intent_engine_handle}/db.rs (100%) rename magicblock-committor-service/src/{intent_execution_manager => intent_engine_handle}/intent_channerl.rs (98%) rename magicblock-committor-service/src/{intent_execution_manager => intent_engine_handle}/intent_execution_engine.rs (99%) rename magicblock-committor-service/src/{intent_execution_manager => intent_engine_handle}/intent_scheduler.rs (100%) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index a08da1afc..8475314cc 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -17,8 +17,8 @@ use tracing::{error, info, instrument}; use crate::{ config::ChainConfig, error::{CommittorServiceError, CommittorServiceResult}, - intent_execution_manager::{ - db::DummyDB, BroadcastedIntentExecutionResult, IntentExecutionManager, + intent_engine_handle::{ + db::DummyDB, BroadcastedIntentExecutionResult, IntentEngineHandle, }, intent_executor::{ error::IntentExecutorError, @@ -40,7 +40,7 @@ pub struct CommittorProcessor { authority: Keypair, _table_mania: TableMania, magic_rpc_client: MagicblockRpcClient, - commits_scheduler: IntentExecutionManager, + commits_scheduler: IntentEngineHandle, task_info_fetcher: Arc>, pending_result_listeners: Arc>>, } @@ -93,7 +93,7 @@ impl CommittorProcessor { let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( RpcTaskInfoFetcher::new(magic_block_rpc_client.clone()), )); - let commits_scheduler = IntentExecutionManager::new( + let commits_scheduler = IntentEngineHandle::new( magic_block_rpc_client.clone(), // TODO(edwin): use DumberDb DummyDB::new(), diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index c447cd42f..4e72d4aca 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,7 +1,7 @@ use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::intent_execution_manager::intent_channerl::IntentScheduleError; +use crate::intent_engine_handle::intent_channerl::IntentScheduleError; pub type CommittorServiceResult = Result; diff --git a/magicblock-committor-service/src/intent_execution_manager.rs b/magicblock-committor-service/src/intent_engine_handle.rs similarity index 93% rename from magicblock-committor-service/src/intent_execution_manager.rs rename to magicblock-committor-service/src/intent_engine_handle.rs index 2d519526a..1afd6072e 100644 --- a/magicblock-committor-service/src/intent_execution_manager.rs +++ b/magicblock-committor-service/src/intent_engine_handle.rs @@ -13,7 +13,7 @@ use magicblock_table_mania::TableMania; use tokio::sync::broadcast; use crate::{ - intent_execution_manager::{ + intent_engine_handle::{ db::DB, intent_channerl::{channel, IntentScheduleError, IntentScheduleHandle}, intent_execution_engine::{IntentExecutionEngine, ResultSubscriber}, @@ -26,14 +26,12 @@ use crate::{ outbox_client::OutboxClient, }; -// TODO(edwin): rename -pub struct IntentExecutionManager { - // TODO(edwin): add JoinHandle of +pub struct IntentEngineHandle { intent_schedule_handle: IntentScheduleHandle, result_subscriber: ResultSubscriber, } -impl IntentExecutionManager { +impl IntentEngineHandle { pub fn new( rpc_client: MagicblockRpcClient, db: D, diff --git a/magicblock-committor-service/src/intent_execution_manager/db.rs b/magicblock-committor-service/src/intent_engine_handle/db.rs similarity index 100% rename from magicblock-committor-service/src/intent_execution_manager/db.rs rename to magicblock-committor-service/src/intent_engine_handle/db.rs diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs b/magicblock-committor-service/src/intent_engine_handle/intent_channerl.rs similarity index 98% rename from magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs rename to magicblock-committor-service/src/intent_engine_handle/intent_channerl.rs index 94eb8e655..6d166b909 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_channerl.rs +++ b/magicblock-committor-service/src/intent_engine_handle/intent_channerl.rs @@ -13,7 +13,7 @@ use tokio::sync::{ }; use tokio_stream::{wrappers::ReceiverStream, Stream}; -use crate::intent_execution_manager::{db, db::DB}; +use crate::intent_engine_handle::{db, db::DB}; const POISONED_MSG: &str = "Dummy DB mutex poisoned"; diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs similarity index 99% rename from magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs rename to magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs index 361e55fb3..cf96a8de5 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs @@ -18,7 +18,7 @@ use tokio_stream::StreamExt; use tracing::{error, info, instrument, trace, warn}; use crate::{ - intent_execution_manager::{ + intent_engine_handle::{ db::DB, intent_channerl::{IntentScheduleError, IntentStream}, intent_scheduler::{IntentScheduler, POISONED_INNER_MSG}, @@ -29,7 +29,6 @@ use crate::{ strategy_executor::error::TransactionStrategyExecutionError, ExecutionOutput, IntentExecutionResult, IntentExecutor, }, - tasks::task_strategist::TransactionStrategy, transaction_preparator::TransactionPreparator, }; @@ -357,7 +356,7 @@ mod tests { use super::*; use crate::{ - intent_execution_manager::{ + intent_engine_handle::{ db::{DummyDB, DB}, intent_channerl::{channel, IntentScheduleHandle}, intent_scheduler::{create_test_intent, create_test_intent_bundle}, diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs b/magicblock-committor-service/src/intent_engine_handle/intent_scheduler.rs similarity index 100% rename from magicblock-committor-service/src/intent_execution_manager/intent_scheduler.rs rename to magicblock-committor-service/src/intent_engine_handle/intent_scheduler.rs diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 5b99ed8d5..045d105f1 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -4,7 +4,6 @@ use futures_util::future::join; use magicblock_core::traits::ActionsCallbackScheduler; use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; use solana_keypair::Keypair; -use solana_signature::Signature; use solana_signer::Signer; use crate::{ diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 3eef2a410..6c146a29d 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -3,7 +3,7 @@ mod compute_budget; pub mod config; mod consts; pub mod error; -pub mod intent_execution_manager; +pub mod intent_engine_handle; pub mod intent_executor; pub mod tasks; pub mod transaction_preparator; diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox_client.rs index 8ed7fc7e6..6211ff289 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox_client.rs @@ -13,8 +13,8 @@ use magicblock_core::{ use magicblock_program::{ instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, - outbox_intent_bundles::OutboxIntentBundle, register_scheduled_commit_sent, - MagicContext, Pubkey, SentCommit, MAGIC_CONTEXT_PUBKEY, + register_scheduled_commit_sent, MagicContext, Pubkey, SentCommit, + MAGIC_CONTEXT_PUBKEY, }; use solana_account::{Account, ReadableAccount}; use solana_rpc_client::{ diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index d7db56ec2..62ca18683 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -29,7 +29,7 @@ use tracing::{error, info, instrument}; use crate::{ committor_processor::CommittorProcessor, error::CommittorServiceResult, - intent_execution_manager::BroadcastedIntentExecutionResult, + intent_engine_handle::BroadcastedIntentExecutionResult, intent_executor::{error::IntentExecutorError, ExecutionOutput}, outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::{ diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 2b5803bd9..562fe7e09 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -396,7 +396,7 @@ mod tests { use super::*; use crate::{ - intent_execution_manager::intent_scheduler::create_test_intent, + intent_engine_handle::intent_scheduler::create_test_intent, intent_executor::task_info_fetcher::{ TaskInfoFetcher, TaskInfoFetcherResult, }, diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 823ccc556..827a7d517 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -5,7 +5,6 @@ use std::{ }; use anyhow::anyhow; -use serial_test::serial; use async_trait::async_trait; use common::*; use ephemeral_rollups_sdk::{compat, ephem::MagicIntentBundleBuilder}; @@ -57,6 +56,7 @@ use program_flexi_counter::{ }, state::FlexiCounter, }; +use serial_test::serial; use solana_rpc_client::{ http_sender::HttpSender, nonblocking::rpc_client::RpcClient as AsyncRpcClient, From d68bd00ae0e670a6892b01b7e186fb34d90ab6da Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:22:18 +0700 Subject: [PATCH 68/97] fix: fmt --- .../src/intent_engine_handle/intent_execution_engine.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs index cf96a8de5..dcb66c0fb 100644 --- a/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs @@ -17,6 +17,8 @@ use tokio::{ use tokio_stream::StreamExt; use tracing::{error, info, instrument, trace, warn}; +#[cfg(feature = "dev-context-only-utils")] +use crate::tasks::task_strategist::TransactionStrategy; use crate::{ intent_engine_handle::{ db::DB, From d0c9a1e7f35530792212ba54482f6633ddad010e Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:23:31 +0700 Subject: [PATCH 69/97] refactor: mod intent_engine_handle -> intent_engine --- magicblock-committor-service/src/committor_processor.rs | 2 +- magicblock-committor-service/src/error.rs | 2 +- .../src/{intent_engine_handle.rs => intent_engine.rs} | 2 +- .../src/{intent_engine_handle => intent_engine}/db.rs | 0 .../intent_channerl.rs | 2 +- .../intent_execution_engine.rs | 4 ++-- .../intent_scheduler.rs | 0 magicblock-committor-service/src/lib.rs | 2 +- magicblock-committor-service/src/service.rs | 2 +- magicblock-committor-service/src/tasks/task_strategist.rs | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename magicblock-committor-service/src/{intent_engine_handle.rs => intent_engine.rs} (98%) rename magicblock-committor-service/src/{intent_engine_handle => intent_engine}/db.rs (100%) rename magicblock-committor-service/src/{intent_engine_handle => intent_engine}/intent_channerl.rs (98%) rename magicblock-committor-service/src/{intent_engine_handle => intent_engine}/intent_execution_engine.rs (99%) rename magicblock-committor-service/src/{intent_engine_handle => intent_engine}/intent_scheduler.rs (100%) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 8475314cc..7e2c7fcbd 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -17,7 +17,7 @@ use tracing::{error, info, instrument}; use crate::{ config::ChainConfig, error::{CommittorServiceError, CommittorServiceResult}, - intent_engine_handle::{ + intent_engine::{ db::DummyDB, BroadcastedIntentExecutionResult, IntentEngineHandle, }, intent_executor::{ diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index 4e72d4aca..1580ffc9a 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,7 +1,7 @@ use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::intent_engine_handle::intent_channerl::IntentScheduleError; +use crate::intent_engine::intent_channerl::IntentScheduleError; pub type CommittorServiceResult = Result; diff --git a/magicblock-committor-service/src/intent_engine_handle.rs b/magicblock-committor-service/src/intent_engine.rs similarity index 98% rename from magicblock-committor-service/src/intent_engine_handle.rs rename to magicblock-committor-service/src/intent_engine.rs index 1afd6072e..e15c51541 100644 --- a/magicblock-committor-service/src/intent_engine_handle.rs +++ b/magicblock-committor-service/src/intent_engine.rs @@ -13,7 +13,7 @@ use magicblock_table_mania::TableMania; use tokio::sync::broadcast; use crate::{ - intent_engine_handle::{ + intent_engine::{ db::DB, intent_channerl::{channel, IntentScheduleError, IntentScheduleHandle}, intent_execution_engine::{IntentExecutionEngine, ResultSubscriber}, diff --git a/magicblock-committor-service/src/intent_engine_handle/db.rs b/magicblock-committor-service/src/intent_engine/db.rs similarity index 100% rename from magicblock-committor-service/src/intent_engine_handle/db.rs rename to magicblock-committor-service/src/intent_engine/db.rs diff --git a/magicblock-committor-service/src/intent_engine_handle/intent_channerl.rs b/magicblock-committor-service/src/intent_engine/intent_channerl.rs similarity index 98% rename from magicblock-committor-service/src/intent_engine_handle/intent_channerl.rs rename to magicblock-committor-service/src/intent_engine/intent_channerl.rs index 6d166b909..0abe6f6d6 100644 --- a/magicblock-committor-service/src/intent_engine_handle/intent_channerl.rs +++ b/magicblock-committor-service/src/intent_engine/intent_channerl.rs @@ -13,7 +13,7 @@ use tokio::sync::{ }; use tokio_stream::{wrappers::ReceiverStream, Stream}; -use crate::intent_engine_handle::{db, db::DB}; +use crate::intent_engine::{db, db::DB}; const POISONED_MSG: &str = "Dummy DB mutex poisoned"; diff --git a/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs similarity index 99% rename from magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs rename to magicblock-committor-service/src/intent_engine/intent_execution_engine.rs index dcb66c0fb..be9ae1d8f 100644 --- a/magicblock-committor-service/src/intent_engine_handle/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs @@ -20,7 +20,7 @@ use tracing::{error, info, instrument, trace, warn}; #[cfg(feature = "dev-context-only-utils")] use crate::tasks::task_strategist::TransactionStrategy; use crate::{ - intent_engine_handle::{ + intent_engine::{ db::DB, intent_channerl::{IntentScheduleError, IntentStream}, intent_scheduler::{IntentScheduler, POISONED_INNER_MSG}, @@ -358,7 +358,7 @@ mod tests { use super::*; use crate::{ - intent_engine_handle::{ + intent_engine::{ db::{DummyDB, DB}, intent_channerl::{channel, IntentScheduleHandle}, intent_scheduler::{create_test_intent, create_test_intent_bundle}, diff --git a/magicblock-committor-service/src/intent_engine_handle/intent_scheduler.rs b/magicblock-committor-service/src/intent_engine/intent_scheduler.rs similarity index 100% rename from magicblock-committor-service/src/intent_engine_handle/intent_scheduler.rs rename to magicblock-committor-service/src/intent_engine/intent_scheduler.rs diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 6c146a29d..1a43a7227 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -3,7 +3,7 @@ mod compute_budget; pub mod config; mod consts; pub mod error; -pub mod intent_engine_handle; +pub mod intent_engine; pub mod intent_executor; pub mod tasks; pub mod transaction_preparator; diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 62ca18683..c026263f6 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -29,7 +29,7 @@ use tracing::{error, info, instrument}; use crate::{ committor_processor::CommittorProcessor, error::CommittorServiceResult, - intent_engine_handle::BroadcastedIntentExecutionResult, + intent_engine::BroadcastedIntentExecutionResult, intent_executor::{error::IntentExecutorError, ExecutionOutput}, outbox_client::{InternalOutboxClientError, OutboxClient}, service::outbox_intent_bundles_reader::{ diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 562fe7e09..ed3684fd3 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -396,7 +396,7 @@ mod tests { use super::*; use crate::{ - intent_engine_handle::intent_scheduler::create_test_intent, + intent_engine::intent_scheduler::create_test_intent, intent_executor::task_info_fetcher::{ TaskInfoFetcher, TaskInfoFetcherResult, }, From 701e0c131b20475eab19af070430dfb744f80c4c Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:28:42 +0700 Subject: [PATCH 70/97] refactor: task_info_fetcher moved into tasks module alongside with task_builder & strategist --- magicblock-api/src/magic_sys_adapter.rs | 2 +- .../src/committor_processor.rs | 11 +++++------ magicblock-committor-service/src/intent_engine.rs | 2 +- .../src/intent_executor/accepted_intent_executor.rs | 2 +- .../src/intent_executor/intent_executor_factory.rs | 9 ++++----- .../src/intent_executor/mod.rs | 7 ++++--- .../intent_executor/single_stage_intent_executor.rs | 6 ++++-- .../src/intent_executor/strategy_executor/patcher.rs | 7 +++++-- .../strategy_executor/single_stage.rs | 6 ++++-- .../intent_executor/strategy_executor/two_stage.rs | 6 ++++-- .../src/intent_executor/strategy_executor/utils.rs | 2 +- .../src/intent_executor/two_stage_intent_executor.rs | 2 +- .../src/intent_executor/utils.rs | 2 +- magicblock-committor-service/src/tasks/mod.rs | 1 + .../src/tasks/task_builder.rs | 12 +++++------- .../{intent_executor => tasks}/task_info_fetcher.rs | 0 .../src/tasks/task_strategist.rs | 4 +--- .../test-committor-service/tests/common.rs | 12 ++++++------ .../tests/test_intent_executor.rs | 9 ++++----- .../test-schedule-intent/tests/test_outbox_flow.rs | 8 ++------ 20 files changed, 55 insertions(+), 55 deletions(-) rename magicblock-committor-service/src/{intent_executor => tasks}/task_info_fetcher.rs (100%) diff --git a/magicblock-api/src/magic_sys_adapter.rs b/magicblock-api/src/magic_sys_adapter.rs index 3347d4dfa..e214442fc 100644 --- a/magicblock-api/src/magic_sys_adapter.rs +++ b/magicblock-api/src/magic_sys_adapter.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use magicblock_committor_service::{ committor_processor::CommittorProcessor, - intent_executor::task_info_fetcher::TaskInfoFetcherResult, + tasks::task_info_fetcher::TaskInfoFetcherResult, }; use magicblock_core::{intent::CommittedAccount, traits::MagicSys}; use magicblock_metrics::metrics; diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 7e2c7fcbd..3aed74c50 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -21,14 +21,13 @@ use crate::{ db::DummyDB, BroadcastedIntentExecutionResult, IntentEngineHandle, }, intent_executor::{ - error::IntentExecutorError, - intent_executor_factory::ExecutorConfig, - task_info_fetcher::{ - CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, - TaskInfoFetcherResult, - }, + error::IntentExecutorError, intent_executor_factory::ExecutorConfig, }, outbox_client::OutboxClient, + tasks::task_info_fetcher::{ + CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, + TaskInfoFetcherResult, + }, }; const POISONED_MUTEX_MSG: &str = diff --git a/magicblock-committor-service/src/intent_engine.rs b/magicblock-committor-service/src/intent_engine.rs index e15c51541..ca19cb017 100644 --- a/magicblock-committor-service/src/intent_engine.rs +++ b/magicblock-committor-service/src/intent_engine.rs @@ -21,9 +21,9 @@ use crate::{ intent_executor::{ error::IntentExecutorError, intent_executor_factory::{ExecutorConfig, IntentExecutorBuilderImpl}, - task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }, outbox_client::OutboxClient, + tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }; pub struct IntentEngineHandle { diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 4411c6591..e126887c6 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -15,7 +15,6 @@ use crate::{ cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, strategy_executor::two_stage::Initialized, - task_info_fetcher::{ResetType, TaskInfoFetcher}, utils::{ build_commit_finalize_tasks, execute_single_stage_flow, execute_two_stage_flow, @@ -26,6 +25,7 @@ use crate::{ outbox_client::OutboxClient, tasks::{ task_builder::TasksBuilder, + task_info_fetcher::{ResetType, TaskInfoFetcher}, task_strategist::{ StrategyExecutionMode, TaskStrategist, TransactionStrategy, TwoStageExecutionMode, diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index 1731214a7..f7669ddbe 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -7,13 +7,12 @@ use magicblock_table_mania::TableMania; use crate::{ intent_executor::{ - build_stage_intent_executor, - error::IntentExecutorError, - intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, - IntentExecutor, IntentExecutorCtx, + build_stage_intent_executor, error::IntentExecutorError, + intent_execution_client::IntentExecutionClient, IntentExecutor, + IntentExecutorCtx, }, outbox_client::OutboxClient, + tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, ComputeBudgetConfig, }; diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 78b6969dc..2ea571272 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -5,7 +5,6 @@ pub mod intent_execution_client; pub(crate) mod intent_executor_factory; pub mod single_stage_intent_executor; pub mod strategy_executor; -pub mod task_info_fetcher; pub mod two_stage_intent_executor; pub mod utils; @@ -30,11 +29,13 @@ use crate::{ error::{IntentExecutorError, IntentExecutorResult}, intent_execution_client::IntentExecutionClient, single_stage_intent_executor::SingleStageIntentExecutor, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, two_stage_intent_executor::TwoStageIntentExecutor, }, outbox_client::OutboxClient, - tasks::task_strategist::TransactionStrategy, + tasks::{ + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + task_strategist::TransactionStrategy, + }, transaction_preparator::TransactionPreparator, }; diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 72dcb3725..73314ba56 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -18,13 +18,15 @@ use crate::{ cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, strategy_executor::utils::check_pending_signature, - task_info_fetcher::{ResetType, TaskInfoFetcher}, utils::{build_commit_finalize_tasks, execute_single_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, outbox_client::OutboxClient, - tasks::task_strategist::{TaskStrategist, TransactionStrategy}, + tasks::{ + task_info_fetcher::{ResetType, TaskInfoFetcher}, + task_strategist::{TaskStrategist, TransactionStrategy}, + }, transaction_preparator::TransactionPreparator, }; diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs index 3b11eb3bf..76e15c729 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/patcher.rs @@ -19,10 +19,13 @@ use crate::{ handle_undelegation_error, prepare_and_execute_strategy, }, }, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, IntentExecutionReport, }, - tasks::{task_strategist::TransactionStrategy, BaseTaskImpl, FinalizeTask}, + tasks::{ + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + task_strategist::TransactionStrategy, + BaseTaskImpl, FinalizeTask, + }, transaction_preparator::TransactionPreparator, }; diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 125c18075..5afa9d93e 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -18,11 +18,13 @@ use crate::{ handle_actions_result, stage_execution_loop, ExecutionState, }, }, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, IntentExecutionReport, }, outbox_client::OutboxClient, - tasks::task_strategist::TransactionStrategy, + tasks::{ + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + task_strategist::TransactionStrategy, + }, transaction_preparator::TransactionPreparator, }; diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index 7aeaa3836..d5b32c7de 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -19,11 +19,13 @@ use crate::{ handle_actions_result, stage_execution_loop, ExecutionState, }, }, - task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, IntentExecutionReport, }, outbox_client::OutboxClient, - tasks::task_strategist::TransactionStrategy, + tasks::{ + task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, + task_strategist::TransactionStrategy, + }, transaction_preparator::TransactionPreparator, }; diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index 29ec5c52e..eb0a66e7c 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -23,12 +23,12 @@ use crate::{ single_stage::SingleStageStrategyExecutor, two_stage::{Committed, Initialized, TwoStageStrategyExecutor}, }, - task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, IntentExecutionReport, }, outbox_client::OutboxClient, tasks::{ task_builder::TaskBuilderError, + task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, task_strategist::{TaskStrategist, TransactionStrategy}, BaseTaskImpl, }, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 311db86ea..268c3ca1e 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -23,7 +23,6 @@ use crate::{ check_pending_signature, execute_with_timeout, FinalizeStage, }, }, - task_info_fetcher::{ResetType, TaskInfoFetcher}, utils::{build_commit_finalize_tasks, execute_two_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, @@ -31,6 +30,7 @@ use crate::{ outbox_client::OutboxClient, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, + task_info_fetcher::{ResetType, TaskInfoFetcher}, task_strategist::{ TaskStrategist, TransactionStrategy, TwoStageExecutionMode, }, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 045d105f1..8d38ea4ac 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -18,12 +18,12 @@ use crate::{ FinalizeStage, SingleStage, }, }, - task_info_fetcher::TaskInfoFetcher, ExecutionOutput, IntentExecutionReport, IntentExecutorCtx, }, outbox_client::OutboxClient, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, + task_info_fetcher::TaskInfoFetcher, task_strategist::TransactionStrategy, BaseTaskImpl, }, diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 7640ff74e..d8fa1645d 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -19,6 +19,7 @@ use solana_pubkey::Pubkey; pub mod commit_finalize_task; pub mod commit_task; pub mod task_builder; +pub mod task_info_fetcher; pub mod task_strategist; pub mod utils; diff --git a/magicblock-committor-service/src/tasks/task_builder.rs b/magicblock-committor-service/src/tasks/task_builder.rs index 87d743e20..adf81fc13 100644 --- a/magicblock-committor-service/src/tasks/task_builder.rs +++ b/magicblock-committor-service/src/tasks/task_builder.rs @@ -12,15 +12,13 @@ use solana_pubkey::Pubkey; use solana_signature::Signature; use tracing::error; -use crate::{ - intent_executor::task_info_fetcher::{ +use crate::tasks::{ + commit_task::{CommitDelivery, CommitTask}, + task_info_fetcher::{ TaskInfoFetcher, TaskInfoFetcherError, TaskInfoFetcherResult, }, - tasks::{ - commit_task::{CommitDelivery, CommitTask}, - BaseActionTask, BaseActionTaskV1, BaseActionTaskV2, BaseTaskImpl, - CommitFinalizeTask, FinalizeTask, UndelegateTask, - }, + BaseActionTask, BaseActionTaskV1, BaseActionTaskV2, BaseTaskImpl, + CommitFinalizeTask, FinalizeTask, UndelegateTask, }; #[async_trait] diff --git a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs b/magicblock-committor-service/src/tasks/task_info_fetcher.rs similarity index 100% rename from magicblock-committor-service/src/intent_executor/task_info_fetcher.rs rename to magicblock-committor-service/src/tasks/task_info_fetcher.rs diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index ed3684fd3..df2a4256f 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -397,14 +397,12 @@ mod tests { use super::*; use crate::{ intent_engine::intent_scheduler::create_test_intent, - intent_executor::task_info_fetcher::{ - TaskInfoFetcher, TaskInfoFetcherResult, - }, tasks::{ commit_task::CommitTask, task_builder::{ TaskBuilderImpl, TasksBuilder, COMMIT_STATE_SIZE_THRESHOLD, }, + task_info_fetcher::{TaskInfoFetcher, TaskInfoFetcherResult}, BaseActionTask, BaseActionTaskV1, FinalizeTask, TaskStrategy, UndelegateTask, }, diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 13107c321..4f4b92c72 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -9,13 +9,9 @@ use std::{ use async_trait::async_trait; use magicblock_committor_service::{ intent_executor::{ - accepted_intent_executor::AcceptedIntentExecutor, - error::IntentExecutorError, + accepted_intent_executor::AcceptedIntentExecutor + , intent_execution_client::IntentExecutionClient, - task_info_fetcher::{ - CacheTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, - TaskInfoFetcherResult, - }, IntentExecutorCtx, }, outbox_client::{InternalOutboxClientError, OutboxClient}, @@ -45,6 +41,10 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; +use magicblock_committor_service::tasks::task_info_fetcher::{ + CacheTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, + TaskInfoFetcherResult, +}; // Helper function to create a test RPC client pub async fn create_test_client() -> MagicblockRpcClient { diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index fd8045053..1a23a7503 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -22,10 +22,6 @@ use magicblock_committor_service::{ two_stage::{Initialized, TwoStageStrategyExecutor}, utils::prepare_and_execute_strategy, }, - task_info_fetcher::{ - CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, - TaskInfoFetcherError, - }, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, @@ -69,7 +65,10 @@ use solana_sdk::{ transaction::{Transaction, TransactionError}, }; use solana_sdk_ids::system_program; - +use magicblock_committor_service::tasks::task_info_fetcher::{ + CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, + TaskInfoFetcherError, +}; use crate::{ common::{MockActionsCallbackExecutor, MockOutboxClient, TestFixture}, utils::{ diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 827a7d517..5455d61a6 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -7,7 +7,6 @@ use std::{ use anyhow::anyhow; use async_trait::async_trait; use common::*; -use ephemeral_rollups_sdk::{compat, ephem::MagicIntentBundleBuilder}; use integration_test_tools::{ loaded_accounts::DLP_TEST_AUTHORITY_BYTES, IntegrationTestContext, }; @@ -16,7 +15,6 @@ use magicblock_committor_service::{ accepted_intent_executor::AcceptedIntentExecutor, build_stage_intent_executor, intent_execution_client::IntentExecutionClient, - task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, ExecutionOutput, IntentExecutor, IntentExecutorCtx, }, outbox_client::{InternalOutboxClientError, OutboxClient}, @@ -50,10 +48,7 @@ use magicblock_program::{ use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use program_flexi_counter::{ - instruction::{ - create_intent_bundle_commit_and_finalize_ix, create_intent_bundle_ix, - create_transfer_intent_ix, - }, + instruction::create_transfer_intent_ix, state::FlexiCounter, }; use serial_test::serial; @@ -71,6 +66,7 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; +use magicblock_committor_service::tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}; type CallbackRecord = (Vec, Option, ActionResult); From c3ced7eb04260f4d8de0c86906cd3f449fece950 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:39:33 +0700 Subject: [PATCH 71/97] refactor: group outbox machinery together --- magicblock-api/src/magic_validator.rs | 5 +-- .../src/committor_processor.rs | 2 +- .../src/intent_engine.rs | 2 +- .../accepted_intent_executor.rs | 2 +- .../src/intent_executor/error.rs | 2 +- .../intent_executor_factory.rs | 2 +- .../src/intent_executor/mod.rs | 2 +- .../single_stage_intent_executor.rs | 2 +- .../strategy_executor/single_stage.rs | 2 +- .../strategy_executor/two_stage.rs | 2 +- .../strategy_executor/utils.rs | 2 +- .../two_stage_intent_executor.rs | 2 +- .../src/intent_executor/utils.rs | 2 +- magicblock-committor-service/src/lib.rs | 2 +- .../src/outbox/mod.rs | 2 ++ .../src/{ => outbox}/outbox_client.rs | 2 +- .../outbox_intent_bundles_reader.rs | 0 magicblock-committor-service/src/service.rs | 10 +++--- .../test-committor-service/tests/common.rs | 31 ++++++++++++------- .../tests/test_intent_executor.rs | 11 ++++--- .../tests/test_outbox_flow.rs | 15 ++++----- 21 files changed, 58 insertions(+), 44 deletions(-) create mode 100644 magicblock-committor-service/src/outbox/mod.rs rename magicblock-committor-service/src/{ => outbox}/outbox_client.rs (99%) rename magicblock-committor-service/src/{service => outbox}/outbox_intent_bundles_reader.rs (100%) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 79b7964ab..2126c596b 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -20,8 +20,9 @@ use magicblock_chainlink::{ }; use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, - outbox_client::InternalOutboxClient, service::IntentExecutionService, - ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, + outbox::outbox_client::InternalOutboxClient, + service::IntentExecutionService, ComputeBudgetConfig, + DEFAULT_ACTIONS_TIMEOUT, }; use magicblock_config::{ config::{ diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 3aed74c50..1843a216c 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -23,7 +23,7 @@ use crate::{ intent_executor::{ error::IntentExecutorError, intent_executor_factory::ExecutorConfig, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::task_info_fetcher::{ CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherResult, diff --git a/magicblock-committor-service/src/intent_engine.rs b/magicblock-committor-service/src/intent_engine.rs index ca19cb017..d86bac4d5 100644 --- a/magicblock-committor-service/src/intent_engine.rs +++ b/magicblock-committor-service/src/intent_engine.rs @@ -22,7 +22,7 @@ use crate::{ error::IntentExecutorError, intent_executor_factory::{ExecutorConfig, IntentExecutorBuilderImpl}, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }; diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index e126887c6..7fd8a33f1 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -22,7 +22,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_builder::TasksBuilder, task_info_fetcher::{ResetType, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index d906825b2..38373202f 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -6,7 +6,7 @@ use solana_signer::SignerError; use crate::{ intent_executor::strategy_executor::error::TransactionStrategyExecutionError, - outbox_client::InternalOutboxClientError, + outbox::outbox_client::InternalOutboxClientError, tasks::{ task_builder::TaskBuilderError, task_strategist::TaskStrategistError, }, diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index f7669ddbe..2af938866 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -11,7 +11,7 @@ use crate::{ intent_execution_client::IntentExecutionClient, IntentExecutor, IntentExecutorCtx, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, ComputeBudgetConfig, diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index 2ea571272..ae2d1d748 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -31,7 +31,7 @@ use crate::{ single_stage_intent_executor::SingleStageIntentExecutor, two_stage_intent_executor::TwoStageIntentExecutor, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, task_strategist::TransactionStrategy, diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 73314ba56..6c60a0b2d 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -22,7 +22,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_info_fetcher::{ResetType, TaskInfoFetcher}, task_strategist::{TaskStrategist, TransactionStrategy}, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 5afa9d93e..7570c0f26 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -20,7 +20,7 @@ use crate::{ }, IntentExecutionReport, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, task_strategist::TransactionStrategy, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index d5b32c7de..f22c6d74c 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -21,7 +21,7 @@ use crate::{ }, IntentExecutionReport, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, task_strategist::TransactionStrategy, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index eb0a66e7c..af17e4cdb 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -25,7 +25,7 @@ use crate::{ }, IntentExecutionReport, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_builder::TaskBuilderError, task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 268c3ca1e..89a02a8f7 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -27,7 +27,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::{ResetType, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 8d38ea4ac..73b7f26b3 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -20,7 +20,7 @@ use crate::{ }, ExecutionOutput, IntentExecutionReport, IntentExecutorCtx, }, - outbox_client::OutboxClient, + outbox::outbox_client::OutboxClient, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::TaskInfoFetcher, diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 1a43a7227..8bc5e3c14 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -10,7 +10,7 @@ pub mod transaction_preparator; pub mod transactions; pub(crate) mod utils; -pub mod outbox_client; +pub mod outbox; pub mod service; #[cfg(test)] pub mod test_utils; diff --git a/magicblock-committor-service/src/outbox/mod.rs b/magicblock-committor-service/src/outbox/mod.rs new file mode 100644 index 000000000..fbb67639f --- /dev/null +++ b/magicblock-committor-service/src/outbox/mod.rs @@ -0,0 +1,2 @@ +pub mod outbox_client; +pub mod outbox_intent_bundles_reader; diff --git a/magicblock-committor-service/src/outbox_client.rs b/magicblock-committor-service/src/outbox/outbox_client.rs similarity index 99% rename from magicblock-committor-service/src/outbox_client.rs rename to magicblock-committor-service/src/outbox/outbox_client.rs index 6211ff289..ef894ce8b 100644 --- a/magicblock-committor-service/src/outbox_client.rs +++ b/magicblock-committor-service/src/outbox/outbox_client.rs @@ -27,7 +27,7 @@ use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tracing::{debug, error}; -use crate::service::outbox_intent_bundles_reader::{ +use crate::outbox::outbox_intent_bundles_reader::{ InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, }; diff --git a/magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs b/magicblock-committor-service/src/outbox/outbox_intent_bundles_reader.rs similarity index 100% rename from magicblock-committor-service/src/service/outbox_intent_bundles_reader.rs rename to magicblock-committor-service/src/outbox/outbox_intent_bundles_reader.rs diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index c026263f6..6d5858eb2 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,5 +1,3 @@ -pub mod outbox_intent_bundles_reader; - use std::{ collections::{HashMap, HashSet}, future::Future, @@ -31,9 +29,11 @@ use crate::{ error::CommittorServiceResult, intent_engine::BroadcastedIntentExecutionResult, intent_executor::{error::IntentExecutorError, ExecutionOutput}, - outbox_client::{InternalOutboxClientError, OutboxClient}, - service::outbox_intent_bundles_reader::{ - OutboxIntentBundlesReader, OutboxIntentBundlesReaderError, + outbox::{ + outbox_client::{InternalOutboxClientError, OutboxClient}, + outbox_intent_bundles_reader::{ + OutboxIntentBundlesReader, OutboxIntentBundlesReaderError, + }, }, }; diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 4f4b92c72..784b0dd87 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -9,14 +9,20 @@ use std::{ use async_trait::async_trait; use magicblock_committor_service::{ intent_executor::{ - accepted_intent_executor::AcceptedIntentExecutor - , - intent_execution_client::IntentExecutionClient, - IntentExecutorCtx, + accepted_intent_executor::AcceptedIntentExecutor, + intent_execution_client::IntentExecutionClient, IntentExecutorCtx, + }, + outbox::{ + outbox_client::{InternalOutboxClientError, OutboxClient}, + outbox_intent_bundles_reader::OutboxIntentBundlesReader, + }, + tasks::{ + commit_task::{CommitDelivery, CommitTask}, + task_info_fetcher::{ + CacheTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, + TaskInfoFetcherResult, + }, }, - outbox_client::{InternalOutboxClientError, OutboxClient}, - service::outbox_intent_bundles_reader::OutboxIntentBundlesReader, - tasks::commit_task::{CommitDelivery, CommitTask}, transaction_preparator::{ delivery_preparator::DeliveryPreparator, TransactionPreparatorImpl, }, @@ -41,10 +47,6 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; -use magicblock_committor_service::tasks::task_info_fetcher::{ - CacheTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherError, - TaskInfoFetcherResult, -}; // Helper function to create a test RPC client pub async fn create_test_client() -> MagicblockRpcClient { @@ -156,6 +158,13 @@ impl OutboxIntentBundlesReader for MockOutboxReader { ) -> Result, Self::Error> { Ok(vec![]) } + + async fn fetch_outbox_intent( + &self, + intent_id: u64, + ) -> Result, Self::Error> { + Ok(None) + } } #[derive(Clone)] diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 1a23a7503..a53c5cac8 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -27,6 +27,10 @@ use magicblock_committor_service::{ }, tasks::{ task_builder::{TaskBuilderError, TaskBuilderImpl, TasksBuilder}, + task_info_fetcher::{ + CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, + TaskInfoFetcherError, + }, task_strategist::{TaskStrategist, TransactionStrategy}, }, transaction_preparator::{ @@ -65,10 +69,7 @@ use solana_sdk::{ transaction::{Transaction, TransactionError}, }; use solana_sdk_ids::system_program; -use magicblock_committor_service::tasks::task_info_fetcher::{ - CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, - TaskInfoFetcherError, -}; + use crate::{ common::{MockActionsCallbackExecutor, MockOutboxClient, TestFixture}, utils::{ @@ -1428,7 +1429,7 @@ async fn create_two_stage_executor<'a>( TaskStrategist::build_strategy(commit_tasks, authority).unwrap(); let finalize_strategy = TaskStrategist::build_strategy(finalize_tasks, authority).unwrap(); - let state = Initialized::new(commit_strategy, finalize_strategy, None); + let state = Initialized::new(commit_strategy, finalize_strategy); TwoStageStrategyExecutor::new( state, fixture.authority.insecure_clone(), diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 5455d61a6..962bf8a0b 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -14,11 +14,14 @@ use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, build_stage_intent_executor, - intent_execution_client::IntentExecutionClient, - ExecutionOutput, IntentExecutor, IntentExecutorCtx, + intent_execution_client::IntentExecutionClient, ExecutionOutput, + IntentExecutor, IntentExecutorCtx, }, - outbox_client::{InternalOutboxClientError, OutboxClient}, - service::outbox_intent_bundles_reader::OutboxIntentBundlesReader, + outbox::{ + outbox_client::{InternalOutboxClientError, OutboxClient}, + outbox_intent_bundles_reader::OutboxIntentBundlesReader, + }, + tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, }; @@ -48,8 +51,7 @@ use magicblock_program::{ use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use program_flexi_counter::{ - instruction::create_transfer_intent_ix, - state::FlexiCounter, + instruction::create_transfer_intent_ix, state::FlexiCounter, }; use serial_test::serial; use solana_rpc_client::{ @@ -66,7 +68,6 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; -use magicblock_committor_service::tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}; type CallbackRecord = (Vec, Option, ActionResult); From dd3702288f98b5294759724bc288479dc48517a1 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:42:45 +0700 Subject: [PATCH 72/97] refactor: merge consts & transactions into utils.rs --- magicblock-committor-service/src/consts.rs | 15 ------------ magicblock-committor-service/src/lib.rs | 4 +--- .../src/tasks/commit_finalize_task.rs | 2 +- .../src/tasks/commit_task.rs | 2 +- magicblock-committor-service/src/tasks/mod.rs | 14 +++++------ .../src/tasks/task_strategist.rs | 2 +- .../src/transactions.rs | 12 ---------- magicblock-committor-service/src/utils.rs | 24 +++++++++++++++++++ 8 files changed, 34 insertions(+), 41 deletions(-) delete mode 100644 magicblock-committor-service/src/consts.rs delete mode 100644 magicblock-committor-service/src/transactions.rs diff --git a/magicblock-committor-service/src/consts.rs b/magicblock-committor-service/src/consts.rs deleted file mode 100644 index 3fb495cd3..000000000 --- a/magicblock-committor-service/src/consts.rs +++ /dev/null @@ -1,15 +0,0 @@ -// https://solana.com/docs/core/transactions#transaction-size - -use magicblock_committor_program::{ - consts::MAX_INSTRUCTION_DATA_SIZE, - instruction::IX_WRITE_SIZE_WITHOUT_CHUNKS, -}; - -const BUDGET_SET_COMPUTE_UNIT_PRICE_BYTES: u16 = (1 + 8) * 8; -const BUDGET_SET_COMPUTE_UNIT_LIMIT_BYTES: u16 = (1 + 4) * 8; - -/// The maximum size of a chunk that can be written as part of a single transaction -pub(super) const MAX_WRITE_CHUNK_SIZE: u16 = MAX_INSTRUCTION_DATA_SIZE - - IX_WRITE_SIZE_WITHOUT_CHUNKS - - BUDGET_SET_COMPUTE_UNIT_PRICE_BYTES - - BUDGET_SET_COMPUTE_UNIT_LIMIT_BYTES; diff --git a/magicblock-committor-service/src/lib.rs b/magicblock-committor-service/src/lib.rs index 8bc5e3c14..82c8c4bb8 100644 --- a/magicblock-committor-service/src/lib.rs +++ b/magicblock-committor-service/src/lib.rs @@ -1,19 +1,17 @@ pub mod committor_processor; mod compute_budget; pub mod config; -mod consts; pub mod error; pub mod intent_engine; pub mod intent_executor; pub mod tasks; pub mod transaction_preparator; -pub mod transactions; -pub(crate) mod utils; pub mod outbox; pub mod service; #[cfg(test)] pub mod test_utils; +pub mod utils; pub use compute_budget::ComputeBudgetConfig; pub use config::DEFAULT_ACTIONS_TIMEOUT; diff --git a/magicblock-committor-service/src/tasks/commit_finalize_task.rs b/magicblock-committor-service/src/tasks/commit_finalize_task.rs index 6a174f582..0f8f98999 100644 --- a/magicblock-committor-service/src/tasks/commit_finalize_task.rs +++ b/magicblock-committor-service/src/tasks/commit_finalize_task.rs @@ -11,11 +11,11 @@ use solana_instruction::Instruction; use solana_pubkey::Pubkey; use crate::{ - consts::MAX_WRITE_CHUNK_SIZE, tasks::{ commit_task::{CommitBufferStage, CommitDelivery}, BaseTask, BaseTaskImpl, PreparationTask, }, + utils::MAX_WRITE_CHUNK_SIZE, }; /// A task that commits a delegated account's state to the base layer and finalizes it in the same diff --git a/magicblock-committor-service/src/tasks/commit_task.rs b/magicblock-committor-service/src/tasks/commit_task.rs index 4d4ee77d1..3ff694d34 100644 --- a/magicblock-committor-service/src/tasks/commit_task.rs +++ b/magicblock-committor-service/src/tasks/commit_task.rs @@ -10,8 +10,8 @@ use solana_instruction::Instruction; use solana_pubkey::Pubkey; use crate::{ - consts::MAX_WRITE_CHUNK_SIZE, tasks::{BaseTask, BaseTaskImpl, CleanupTask, PreparationTask}, + utils::MAX_WRITE_CHUNK_SIZE, }; /// Lifecycle stage of a buffer used for commit delivery. diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index d8fa1645d..68893a076 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -25,8 +25,11 @@ pub mod utils; pub use task_builder::TaskBuilderImpl; -use crate::tasks::{ - commit_finalize_task::CommitFinalizeTask, commit_task::CommitTask, +use crate::{ + tasks::{ + commit_finalize_task::CommitFinalizeTask, commit_task::CommitTask, + }, + utils::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, }; #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -723,12 +726,7 @@ fn test_close_buffer_limit() { use solana_transaction::Transaction; use tracing::info; - use crate::{ - test_utils, - transactions::{ - serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE, - }, - }; + use crate::{self, test_utils}; test_utils::init_test_logger(); diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index df2a4256f..ea7e9813f 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -7,7 +7,7 @@ use solana_signer::{Signer, SignerError}; use crate::{ tasks::{utils::TransactionUtils, BaseActionTask, BaseTask, BaseTaskImpl}, - transactions::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, + utils::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, }; #[derive(Default, Clone)] diff --git a/magicblock-committor-service/src/transactions.rs b/magicblock-committor-service/src/transactions.rs deleted file mode 100644 index 2c25c7d42..000000000 --- a/magicblock-committor-service/src/transactions.rs +++ /dev/null @@ -1,12 +0,0 @@ -use solana_packet::PACKET_DATA_SIZE; -use solana_rpc_client::rpc_client::SerializableTransaction; - -/// Maximum serialized transaction size that can be sent over the wire. -pub(crate) const MAX_TRANSACTION_WIRE_SIZE: usize = PACKET_DATA_SIZE; - -pub fn serialized_transaction_size( - transaction: &impl SerializableTransaction, -) -> usize { - // SAFETY: runs on transactions we already serialize before sending. - usize::try_from(bincode::serialized_size(transaction).unwrap()).unwrap() -} diff --git a/magicblock-committor-service/src/utils.rs b/magicblock-committor-service/src/utils.rs index 8b1378917..46bc392b0 100644 --- a/magicblock-committor-service/src/utils.rs +++ b/magicblock-committor-service/src/utils.rs @@ -1 +1,25 @@ +// https://solana.com/docs/core/transactions#transaction-size +use magicblock_committor_program::{ + consts::MAX_INSTRUCTION_DATA_SIZE, + instruction::IX_WRITE_SIZE_WITHOUT_CHUNKS, +}; +use solana_packet::PACKET_DATA_SIZE; +use solana_rpc_client::rpc_client::SerializableTransaction; + +const BUDGET_SET_COMPUTE_UNIT_PRICE_BYTES: u16 = (1 + 8) * 8; +const BUDGET_SET_COMPUTE_UNIT_LIMIT_BYTES: u16 = (1 + 4) * 8; +/// The maximum size of a chunk that can be written as part of a single transaction +pub(crate) const MAX_WRITE_CHUNK_SIZE: u16 = MAX_INSTRUCTION_DATA_SIZE + - IX_WRITE_SIZE_WITHOUT_CHUNKS + - BUDGET_SET_COMPUTE_UNIT_PRICE_BYTES + - BUDGET_SET_COMPUTE_UNIT_LIMIT_BYTES; +/// Maximum serialized transaction size that can be sent over the wire. +pub(crate) const MAX_TRANSACTION_WIRE_SIZE: usize = PACKET_DATA_SIZE; + +pub fn serialized_transaction_size( + transaction: &impl SerializableTransaction, +) -> usize { + // SAFETY: runs on transactions we already serialize before sending. + usize::try_from(bincode::serialized_size(transaction).unwrap()).unwrap() +} From 299cdd160965c862a073f31e32d01893457a3907 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Fri, 26 Jun 2026 18:48:34 +0700 Subject: [PATCH 73/97] wip --- .../src/intent_executor/intent_executor_factory.rs | 1 - magicblock-committor-service/src/tasks/mod.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index 2af938866..28fd30670 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -17,7 +17,6 @@ use crate::{ ComputeBudgetConfig, }; -// TODO(edwin): T could be removed if cleanuphandle is a trait pub trait IntentExecutorBuilder { fn create_instance( &self, diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 68893a076..2cca813d1 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -726,7 +726,7 @@ fn test_close_buffer_limit() { use solana_transaction::Transaction; use tracing::info; - use crate::{self, test_utils}; + use crate::test_utils; test_utils::init_test_logger(); From ab283367d6f2b3682cc5f24afb18b44c82779194 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 13:04:34 +0700 Subject: [PATCH 74/97] feat: move notify_commit_sent into the IntentExecution flow --- .../src/intent_executor/mod.rs | 8 +- .../src/intent_executor/utils.rs | 20 +- .../src/outbox/mod.rs | 30 +++ .../src/outbox/outbox_client.rs | 29 ++- .../src/outbox/utils.rs | 82 +++++++ magicblock-committor-service/src/service.rs | 228 +----------------- .../tests/test_outbox_flow.rs | 13 +- 7 files changed, 168 insertions(+), 242 deletions(-) create mode 100644 magicblock-committor-service/src/outbox/utils.rs diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index ae2d1d748..abf03d9c5 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -151,10 +151,16 @@ impl IntentExecutionReport { self.patched_errors.push(value); } - pub fn patched_errors(&self) -> &Vec { + pub fn patched_errors(&self) -> &[TransactionStrategyExecutionError] { &self.patched_errors } + pub fn callbacks_report( + &self, + ) -> &[Result] { + &self.callbacks_report + } + pub fn add_callback_report( &mut self, values: impl IntoIterator>, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index 73b7f26b3..c69a4f14a 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -20,7 +20,7 @@ use crate::{ }, ExecutionOutput, IntentExecutionReport, IntentExecutorCtx, }, - outbox::outbox_client::OutboxClient, + outbox::{outbox_client::OutboxClient, ScheduledBaseIntentMeta}, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::TaskInfoFetcher, @@ -61,6 +61,7 @@ where O: OutboxClient, O::Error: Into, { + let meta = ScheduledBaseIntentMeta::new(&intent_bundle); let committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); let mut single_stage_executor = SingleStageStrategyExecutor::new( @@ -107,7 +108,12 @@ where transaction_strategy.clone(), ); execution_report.dispose(transaction_strategy); - return res.map(ExecutionOutput::SingleStage); + let output = res.map(ExecutionOutput::SingleStage); + ctx.outbox_client + .notify_commit_sent(meta, &output, execution_report) + .await + .map_err(Into::into)?; + return output; } }; @@ -150,6 +156,7 @@ where O: OutboxClient, O::Error: Into, { + let meta = ScheduledBaseIntentMeta::new(&intent_bundle); let committed_pubkeys = intent_bundle.get_all_committed_pubkeys(); let mut executor = TwoStageStrategyExecutor::new( state, @@ -183,8 +190,13 @@ where .await?; let finalized_stage = finalize_executor.done(finalize_signature); - Ok(ExecutionOutput::TwoStage { + let output = ExecutionOutput::TwoStage { commit_signature: finalized_stage.commit_signature, finalize_signature: finalized_stage.finalize_signature, - }) + }; + ctx.outbox_client + .notify_commit_sent(meta, &Ok(output), execution_report) + .await + .map_err(Into::into)?; + Ok(output) } diff --git a/magicblock-committor-service/src/outbox/mod.rs b/magicblock-committor-service/src/outbox/mod.rs index fbb67639f..1e01369fb 100644 --- a/magicblock-committor-service/src/outbox/mod.rs +++ b/magicblock-committor-service/src/outbox/mod.rs @@ -1,2 +1,32 @@ +use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use solana_hash::Hash; +use solana_keypair::Address as Pubkey; +use solana_transaction::Transaction; + pub mod outbox_client; pub mod outbox_intent_bundles_reader; +pub(crate) mod utils; + +pub struct ScheduledBaseIntentMeta { + pub(crate) id: u64, + pub(crate) slot: u64, + pub(crate) blockhash: Hash, + pub(crate) payer: Pubkey, + pub(crate) included_pubkeys: Vec, + pub(crate) intent_sent_transaction: Transaction, + pub(crate) requested_undelegation: bool, +} + +impl ScheduledBaseIntentMeta { + pub(crate) fn new(intent: &ScheduledIntentBundle) -> Self { + Self { + id: intent.id, + slot: intent.slot, + blockhash: intent.blockhash, + payer: intent.payer, + included_pubkeys: intent.get_all_committed_pubkeys(), + intent_sent_transaction: intent.sent_transaction.clone(), + requested_undelegation: intent.has_undelegate_intent(), + } + } +} diff --git a/magicblock-committor-service/src/outbox/outbox_client.rs b/magicblock-committor-service/src/outbox/outbox_client.rs index ef894ce8b..c1cc69037 100644 --- a/magicblock-committor-service/src/outbox/outbox_client.rs +++ b/magicblock-committor-service/src/outbox/outbox_client.rs @@ -13,8 +13,7 @@ use magicblock_core::{ use magicblock_program::{ instruction_utils::InstructionUtils, magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, - register_scheduled_commit_sent, MagicContext, Pubkey, SentCommit, - MAGIC_CONTEXT_PUBKEY, + register_scheduled_commit_sent, MagicContext, Pubkey, MAGIC_CONTEXT_PUBKEY, }; use solana_account::{Account, ReadableAccount}; use solana_rpc_client::{ @@ -23,12 +22,20 @@ use solana_rpc_client::{ use solana_rpc_client_api::{ client_error, client_error::ErrorKind as RpcClientErrorKind, }; -use solana_transaction::Transaction; use solana_transaction_error::TransactionError; use tracing::{debug, error}; -use crate::outbox::outbox_intent_bundles_reader::{ - InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, +use crate::{ + intent_executor::{ + error::IntentExecutorResult, ExecutionOutput, IntentExecutionReport, + }, + outbox::{ + outbox_intent_bundles_reader::{ + InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, + }, + utils::build_sent_commit, + ScheduledBaseIntentMeta, + }, }; #[async_trait] @@ -58,8 +65,9 @@ pub trait OutboxClient: Send + Sync + 'static { /// Processes intent results, submitting them on chain(ER) async fn notify_commit_sent( &self, - sent_tx: Transaction, - sent_commit: SentCommit, + meta: ScheduledBaseIntentMeta, + result: &IntentExecutorResult, + execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error>; /// Returns reader capable of reading IntentBundles from Outbox @@ -216,9 +224,12 @@ impl OutboxClient for InternalOutboxClient { async fn notify_commit_sent( &self, - sent_tx: Transaction, - sent_commit: SentCommit, + meta: ScheduledBaseIntentMeta, + result: &IntentExecutorResult, + execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { + let (sent_tx, sent_commit) = + build_sent_commit(meta, result, execution_report); // TODO(edwin): is using handle directly here ok? This could require Chainlink mechanics register_scheduled_commit_sent(sent_commit); let txn = with_encoded(sent_tx).inspect_err(|err| { diff --git a/magicblock-committor-service/src/outbox/utils.rs b/magicblock-committor-service/src/outbox/utils.rs new file mode 100644 index 000000000..6ac9ffc6e --- /dev/null +++ b/magicblock-committor-service/src/outbox/utils.rs @@ -0,0 +1,82 @@ +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, Pubkey, SentCommit, +}; +use solana_hash::Hash; +use solana_transaction::Transaction; +use tracing::{error, info}; + +use crate::{ + intent_executor::{ + error::IntentExecutorResult, ExecutionOutput, IntentExecutionReport, + }, + outbox::ScheduledBaseIntentMeta, +}; + +pub(crate) fn build_sent_commit( + meta: ScheduledBaseIntentMeta, + result: &IntentExecutorResult, + execution_report: &IntentExecutionReport, +) -> (Transaction, SentCommit) { + let error_message = result.as_ref().err().map(|err| format!("{:?}", err)); + + let chain_signatures = match result { + Ok(output) => match output { + ExecutionOutput::SingleStage(sig) => vec![*sig], + ExecutionOutput::TwoStage { + commit_signature, + finalize_signature, + } => vec![*commit_signature, *finalize_signature], + }, + Err(err) => { + error!( + "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", + meta.id, meta.slot, meta.blockhash, err + ); + err.base_signatures() + .map(|(commit, finalize)| { + finalize.map(|f| vec![commit, f]).unwrap_or(vec![commit]) + }) + .unwrap_or_default() + } + }; + + let patched_errors = execution_report + .patched_errors() + .iter() + .map(|err| { + info!("Patched intent: {}. error was: {}", meta.id, err); + err.to_string() + }) + .collect(); + + let callbacks_scheduling_results = execution_report + .callbacks_report() + .iter() + .map(|r| match r { + Ok(sig) => format!("OK: {sig}"), + Err(err) => { + error!( + "Callback failed to schedule: {}. error: {}", + meta.id, err + ); + format!("ERR: {err}") + } + }) + .collect(); + + let sent_commit = SentCommit { + message_id: meta.id, + slot: meta.slot, + blockhash: meta.blockhash, + payer: meta.payer, + chain_signatures, + included_pubkeys: meta.included_pubkeys, + excluded_pubkeys: vec![], + requested_undelegation: meta.requested_undelegation, + error_message, + patched_errors, + callbacks_scheduling_results, + }; + + (meta.intent_sent_transaction, sent_commit) +} diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 6d5858eb2..1a8986eca 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -1,34 +1,23 @@ use std::{ - collections::{HashMap, HashSet}, - future::Future, - mem, - num::NonZeroUsize, - sync::{Arc, Mutex}, + collections::HashSet, future::Future, mem, num::NonZeroUsize, sync::Arc, time::Duration, }; use magicblock_account_cloner::ChainlinkCloner; use magicblock_chainlink::{ProdChainlink, ProdInnerChainlink}; use magicblock_metrics::metrics::{self}; -use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, - outbox_intent_bundles::OutboxIntentBundle, Pubkey, SentCommit, -}; -use solana_hash::Hash; -use solana_transaction::Transaction; +use magicblock_program::{outbox_intent_bundles::OutboxIntentBundle, Pubkey}; use tokio::{ - sync::broadcast, task, task::{JoinError, JoinHandle}, }; use tokio_util::sync::CancellationToken; -use tracing::{error, info, instrument}; +use tracing::error; use crate::{ committor_processor::CommittorProcessor, error::CommittorServiceResult, - intent_engine::BroadcastedIntentExecutionResult, - intent_executor::{error::IntentExecutorError, ExecutionOutput}, + intent_executor::error::IntentExecutorError, outbox::{ outbox_client::{InternalOutboxClientError, OutboxClient}, outbox_intent_bundles_reader::{ @@ -37,8 +26,6 @@ use crate::{ }, }; -const POISONED_MUTEX_MSG: &str = "ServiceInner intents_meta_map mutex poisoned"; - pub type InnerChainlinkImpl = ProdInnerChainlink; pub type ChainlinkImpl = ProdChainlink; @@ -116,8 +103,6 @@ pub struct ServiceInner { // TODO(edwin): can be removed if LatestBlocK moved into magicblock-core slot_interval: Duration, cancellation_token: CancellationToken, - /// Meta for ongoing executing intents - intents_meta_map: Arc>>, } impl ServiceInner @@ -144,21 +129,10 @@ where processor, slot_interval, cancellation_token, - intents_meta_map: Arc::new(Mutex::default()), } } - /// Starts 2 workers: one accepting intents, another awaiting and handling results fn start(self) -> JoinHandle<()> { - let result_subscriber = self.processor.subscribe_for_results(); - let cancellation_token = self.cancellation_token.clone(); - tokio::spawn(Self::result_processor( - result_subscriber, - cancellation_token, - self.intents_meta_map.clone(), - self.outbox_client.clone(), - )); - tokio::task::spawn(self.accept_worker()) } @@ -266,37 +240,21 @@ where return Ok(()); } - // Add metas for intent we schedule - let intent_ids: Vec; let pubkeys_being_undelegated = { - let mut intent_metas = - self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); let mut pubkeys_being_undelegated = HashSet::::new(); intent_bundles.iter().for_each(|intent| { - intent_metas - .insert(intent.id, ScheduledBaseIntentMeta::new(intent)); if let Some(undelegate) = intent.get_undelegate_intent_pubkeys() { pubkeys_being_undelegated.extend(undelegate); } }); - intent_ids = intent_bundles.iter().map(|b| b.id).collect(); pubkeys_being_undelegated.into_iter().collect::>() }; self.process_undelegation_requests(pubkeys_being_undelegated) .await; - let result = schedule(intent_bundles).await; - // If scheduling failed remove from map - if result.is_err() { - let mut intent_metas = - self.intents_meta_map.lock().expect(POISONED_MUTEX_MSG); - intent_ids.iter().for_each(|id| { - intent_metas.remove(id); - }); - } - result + schedule(intent_bundles).await } async fn process_undelegation_requests(&self, pubkeys: Vec) { @@ -333,182 +291,6 @@ where ); } } - - #[instrument(skip( - result_subscription, - cancellation_token, - intents_meta_map, - outbox_client - ))] - async fn result_processor( - mut result_subscription: broadcast::Receiver< - BroadcastedIntentExecutionResult, - >, - cancellation_token: CancellationToken, - intents_meta_map: Arc>>, - outbox_client: Arc, - ) { - loop { - let execution_result = tokio::select! { - biased; - _ = cancellation_token.cancelled() => { - info!("Shutting down"); - return; - } - execution_result = result_subscription.recv() => { - match execution_result { - Ok(result) => result, - Err(broadcast::error::RecvError::Closed) => { - info!("Intent execution service shut down"); - break; - } - Err(broadcast::error::RecvError::Lagged(skipped)) => { - // SAFETY: This shouldn't happen as our tx execution is faster than Intent execution on Base layer - // If this ever happens it requires investigation - error!(skipped_count = skipped, "Lagging behind intent execution"); - continue; - } - } - } - }; - - if let Err(err) = ServiceInner::::process_execution_result( - &outbox_client, - execution_result, - &intents_meta_map, - ) - .await - { - error!(error = ?err, "Failed process intent execution results"); - } - } - } - - async fn process_execution_result( - outbox_client: &Arc, - execution_result: BroadcastedIntentExecutionResult, - intents_meta_map: &Arc>>, - ) -> Result<(), O::Error> { - // Create IntentMeta - let intent_id = execution_result.id; - // Remove intent from metas - let Some(mut intent_meta) = intents_meta_map - .lock() - .expect(POISONED_MUTEX_MSG) - .remove(&intent_id) - else { - // Possible if we have duplicate Intents - // First one will remove id from map and second could fail. - // This should not happen and needs investigation! - error!(intent_id, "Failed to find intent metadata"); - return Ok(()); - }; - - let sent_transaction = - mem::take(&mut intent_meta.intent_sent_transaction); - let sent_commit = ServiceInner::::build_sent_commit( - intent_id, - intent_meta, - execution_result, - ); - outbox_client - .notify_commit_sent(sent_transaction, sent_commit) - .await?; - - Ok(()) - } - - fn build_sent_commit( - intent_id: u64, - intent_meta: ScheduledBaseIntentMeta, - result: BroadcastedIntentExecutionResult, - ) -> SentCommit { - let error_message = - result.as_ref().err().map(|err| format!("{:?}", err)); - let chain_signatures = match result.inner { - Ok(value) => match value { - ExecutionOutput::SingleStage(signature) => vec![signature], - ExecutionOutput::TwoStage { - commit_signature, - finalize_signature, - } => vec![commit_signature, finalize_signature], - }, - Err(err) => { - error!( - "Failed to commit intent: {}, slot: {}, blockhash: {}. {:?}", - intent_id, intent_meta.slot, intent_meta.blockhash, err - ); - err.base_signatures() - .map(|(commit, finalize)| { - finalize - .map(|finalize| vec![commit, finalize]) - .unwrap_or(vec![commit]) - }) - .unwrap_or_default() - } - }; - let patched_errors = result - .patched_errors - .iter() - .map(|err| { - info!("Patched intent: {}. error was: {}", intent_id, err); - err.to_string() - }) - .collect(); - - let callbacks_report = result - .callbacks_report - .iter() - .map(|r| match r { - Ok(sig) => { - format!("OK: {sig}") - } - Err(err) => { - error!( - "Callback failed to schedule: {}. error: {}", - intent_id, err - ); - format!("ERR: {err}") - } - }) - .collect(); - - SentCommit { - message_id: intent_id, - slot: intent_meta.slot, - blockhash: intent_meta.blockhash, - payer: intent_meta.payer, - chain_signatures, - included_pubkeys: intent_meta.included_pubkeys, - excluded_pubkeys: vec![], - requested_undelegation: intent_meta.requested_undelegation, - error_message, - patched_errors, - callbacks_scheduling_results: callbacks_report, - } - } -} - -struct ScheduledBaseIntentMeta { - slot: u64, - blockhash: Hash, - payer: Pubkey, - included_pubkeys: Vec, - intent_sent_transaction: Transaction, - requested_undelegation: bool, -} - -impl ScheduledBaseIntentMeta { - fn new(intent: &ScheduledIntentBundle) -> Self { - Self { - slot: intent.slot, - blockhash: intent.blockhash, - payer: intent.payer, - included_pubkeys: intent.get_all_committed_pubkeys(), - intent_sent_transaction: intent.sent_transaction.clone(), - requested_undelegation: intent.has_undelegate_intent(), - } - } } #[derive(thiserror::Error, Debug)] diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 962bf8a0b..b09cf8775 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -14,8 +14,9 @@ use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, build_stage_intent_executor, - intent_execution_client::IntentExecutionClient, ExecutionOutput, - IntentExecutor, IntentExecutorCtx, + error::IntentExecutorResult, + intent_execution_client::IntentExecutionClient, + ExecutionOutput, IntentExecutionReport, IntentExecutor, IntentExecutorCtx, }, outbox::{ outbox_client::{InternalOutboxClientError, OutboxClient}, @@ -46,7 +47,7 @@ use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, outbox_intent_bundles::{OutboxIntentBundle, OutboxIntentBundleStatus}, validator::init_validator_authority, - MagicContext, SentCommit, + MagicContext, }; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; @@ -68,6 +69,7 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; +use magicblock_committor_service::outbox::ScheduledBaseIntentMeta; type CallbackRecord = (Vec, Option, ActionResult); @@ -1008,8 +1010,9 @@ impl OutboxClient for TestOutboxClient { async fn notify_commit_sent( &self, - _sent_tx: Transaction, - _sent_commit: SentCommit, + _meta: ScheduledBaseIntentMeta, + _result: &IntentExecutorResult, + _execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { Ok(()) } From 2337630ac23b574ac0ce97f560575443c28cc4a0 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 13:07:44 +0700 Subject: [PATCH 75/97] refactor: file structs --- .../src/committor_processor.rs | 2 +- .../src/intent_engine.rs | 2 +- .../accepted_intent_executor.rs | 2 +- .../intent_executor_factory.rs | 2 +- .../src/intent_executor/mod.rs | 2 +- .../single_stage_intent_executor.rs | 2 +- .../strategy_executor/single_stage.rs | 2 +- .../strategy_executor/two_stage.rs | 2 +- .../strategy_executor/utils.rs | 2 +- .../two_stage_intent_executor.rs | 2 +- .../src/intent_executor/utils.rs | 2 +- .../src/outbox/mod.rs | 48 ++++++++++++++++++- .../src/outbox/outbox_client.rs | 38 +-------------- magicblock-committor-service/src/service.rs | 3 +- .../test-committor-service/tests/common.rs | 3 +- .../tests/test_outbox_flow.rs | 4 +- 16 files changed, 65 insertions(+), 53 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 1843a216c..0cfe94b42 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -23,7 +23,7 @@ use crate::{ intent_executor::{ error::IntentExecutorError, intent_executor_factory::ExecutorConfig, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::task_info_fetcher::{ CacheTaskInfoFetcher, RpcTaskInfoFetcher, TaskInfoFetcher, TaskInfoFetcherResult, diff --git a/magicblock-committor-service/src/intent_engine.rs b/magicblock-committor-service/src/intent_engine.rs index d86bac4d5..4d7efc375 100644 --- a/magicblock-committor-service/src/intent_engine.rs +++ b/magicblock-committor-service/src/intent_engine.rs @@ -22,7 +22,7 @@ use crate::{ error::IntentExecutorError, intent_executor_factory::{ExecutorConfig, IntentExecutorBuilderImpl}, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }; diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 7fd8a33f1..8e5935422 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -22,7 +22,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_builder::TasksBuilder, task_info_fetcher::{ResetType, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index 28fd30670..d64eeff99 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -11,7 +11,7 @@ use crate::{ intent_execution_client::IntentExecutionClient, IntentExecutor, IntentExecutorCtx, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, ComputeBudgetConfig, diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index abf03d9c5..fba9972b3 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -31,7 +31,7 @@ use crate::{ single_stage_intent_executor::SingleStageIntentExecutor, two_stage_intent_executor::TwoStageIntentExecutor, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, task_strategist::TransactionStrategy, diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 6c60a0b2d..285b1edd1 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -22,7 +22,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_info_fetcher::{ResetType, TaskInfoFetcher}, task_strategist::{TaskStrategist, TransactionStrategy}, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 7570c0f26..1452b8182 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -20,7 +20,7 @@ use crate::{ }, IntentExecutionReport, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, task_strategist::TransactionStrategy, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index f22c6d74c..d6da5f8b4 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -21,7 +21,7 @@ use crate::{ }, IntentExecutionReport, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_info_fetcher::{CacheTaskInfoFetcher, TaskInfoFetcher}, task_strategist::TransactionStrategy, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index af17e4cdb..d8ddf4be7 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -25,7 +25,7 @@ use crate::{ }, IntentExecutionReport, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_builder::TaskBuilderError, task_info_fetcher::{CacheTaskInfoFetcher, ResetType, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 89a02a8f7..7ae6e4449 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -27,7 +27,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox::outbox_client::OutboxClient, + outbox::OutboxClient, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::{ResetType, TaskInfoFetcher}, diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index c69a4f14a..d046359e8 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -20,7 +20,7 @@ use crate::{ }, ExecutionOutput, IntentExecutionReport, IntentExecutorCtx, }, - outbox::{outbox_client::OutboxClient, ScheduledBaseIntentMeta}, + outbox::{OutboxClient, ScheduledBaseIntentMeta}, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::TaskInfoFetcher, diff --git a/magicblock-committor-service/src/outbox/mod.rs b/magicblock-committor-service/src/outbox/mod.rs index 1e01369fb..335381b07 100644 --- a/magicblock-committor-service/src/outbox/mod.rs +++ b/magicblock-committor-service/src/outbox/mod.rs @@ -1,12 +1,58 @@ -use magicblock_program::magic_scheduled_base_intent::ScheduledIntentBundle; +use async_trait::async_trait; +use magicblock_program::{ + magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, +}; use solana_hash::Hash; use solana_keypair::Address as Pubkey; use solana_transaction::Transaction; +use crate::{ + intent_executor::{ + error::IntentExecutorResult, ExecutionOutput, IntentExecutionReport, + }, + outbox::outbox_intent_bundles_reader::OutboxIntentBundlesReader, +}; + pub mod outbox_client; pub mod outbox_intent_bundles_reader; pub(crate) mod utils; +#[async_trait] +pub trait OutboxClient: Send + Sync + 'static { + type Error: std::error::Error + Send; + /// Type that is able to read IntentBundles from Outbox + /// Can be via AccountsDB, RpcClient or any other means + type OutboxReader: OutboxIntentBundlesReader; + + /// Executes `Accept` tx and returns accepted intents + async fn accept_scheduled_intents( + &self, + ) -> Result< + Vec, + (Vec, Self::Error), + >; + + /// Sets execution stage for outbox intent + /// Note: intent has to be accepted prior + /// Calling with invalid state transitions will lead to `TransactionError` + async fn set_intent_execution_stage( + &self, + intent_id: u64, + stage: ExecutionStage, + ) -> Result<(), Self::Error>; + + /// Processes intent results, submitting them on chain(ER) + async fn notify_commit_sent( + &self, + meta: ScheduledBaseIntentMeta, + result: &IntentExecutorResult, + execution_report: &IntentExecutionReport, + ) -> Result<(), Self::Error>; + + /// Returns reader capable of reading IntentBundles from Outbox + fn outbox_reader(&self) -> Self::OutboxReader; +} + pub struct ScheduledBaseIntentMeta { pub(crate) id: u64, pub(crate) slot: u64, diff --git a/magicblock-committor-service/src/outbox/outbox_client.rs b/magicblock-committor-service/src/outbox/outbox_client.rs index c1cc69037..a082c29d6 100644 --- a/magicblock-committor-service/src/outbox/outbox_client.rs +++ b/magicblock-committor-service/src/outbox/outbox_client.rs @@ -34,46 +34,10 @@ use crate::{ InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, }, utils::build_sent_commit, - ScheduledBaseIntentMeta, + OutboxClient, ScheduledBaseIntentMeta, }, }; -#[async_trait] -pub trait OutboxClient: Send + Sync + 'static { - type Error: std::error::Error + Send; - /// Type that is able to read IntentBundles from Outbox - /// Can be via AccountsDB, RpcClient or any other means - type OutboxReader: OutboxIntentBundlesReader; - - /// Executes `Accept` tx and returns accepted intents - async fn accept_scheduled_intents( - &self, - ) -> Result< - Vec, - (Vec, Self::Error), - >; - - /// Sets execution stage for outbox intent - /// Note: intent has to be accepted prior - /// Calling with invalid state transitions will lead to `TransactionError` - async fn set_intent_execution_stage( - &self, - intent_id: u64, - stage: ExecutionStage, - ) -> Result<(), Self::Error>; - - /// Processes intent results, submitting them on chain(ER) - async fn notify_commit_sent( - &self, - meta: ScheduledBaseIntentMeta, - result: &IntentExecutorResult, - execution_report: &IntentExecutionReport, - ) -> Result<(), Self::Error>; - - /// Returns reader capable of reading IntentBundles from Outbox - fn outbox_reader(&self) -> Self::OutboxReader; -} - /// Implementation of `OutboxClient` that uses ER internals /// Potentially could be replaced with RPC base Client pub struct InternalOutboxClient { diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 1a8986eca..2ca96fae1 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -19,10 +19,11 @@ use crate::{ error::CommittorServiceResult, intent_executor::error::IntentExecutorError, outbox::{ - outbox_client::{InternalOutboxClientError, OutboxClient}, + outbox_client::InternalOutboxClientError, outbox_intent_bundles_reader::{ OutboxIntentBundlesReader, OutboxIntentBundlesReaderError, }, + OutboxClient, }, }; diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 784b0dd87..01e7b0c8d 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -13,7 +13,7 @@ use magicblock_committor_service::{ intent_execution_client::IntentExecutionClient, IntentExecutorCtx, }, outbox::{ - outbox_client::{InternalOutboxClientError, OutboxClient}, + outbox_client::InternalOutboxClientError, outbox_intent_bundles_reader::OutboxIntentBundlesReader, }, tasks::{ @@ -47,6 +47,7 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; +use magicblock_committor_service::outbox::OutboxClient; // Helper function to create a test RPC client pub async fn create_test_client() -> MagicblockRpcClient { diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index b09cf8775..c60124c99 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -19,7 +19,7 @@ use magicblock_committor_service::{ ExecutionOutput, IntentExecutionReport, IntentExecutor, IntentExecutorCtx, }, outbox::{ - outbox_client::{InternalOutboxClientError, OutboxClient}, + outbox_client::InternalOutboxClientError, outbox_intent_bundles_reader::OutboxIntentBundlesReader, }, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, @@ -69,7 +69,7 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; -use magicblock_committor_service::outbox::ScheduledBaseIntentMeta; +use magicblock_committor_service::outbox::{OutboxClient, ScheduledBaseIntentMeta}; type CallbackRecord = (Vec, Option, ActionResult); From ae6e17385ad31b8efb448b58d00e568bde39a03d Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 14:45:39 +0700 Subject: [PATCH 76/97] refactor: remove unnecessary members of structs + some test cleanup --- .../src/committor_processor.rs | 4 - .../accepted_intent_executor.rs | 28 +++--- .../src/intent_executor/error.rs | 1 - .../intent_execution_client.rs | 4 +- .../intent_executor_factory.rs | 5 +- .../src/intent_executor/mod.rs | 18 ++-- .../single_stage_intent_executor.rs | 26 ++--- .../strategy_executor/utils.rs | 7 +- .../two_stage_intent_executor.rs | 31 ++---- .../src/outbox/mod.rs | 14 +-- .../src/outbox/outbox_client.rs | 8 +- .../src/outbox/utils.rs | 5 +- magicblock-committor-service/src/service.rs | 12 +-- magicblock-committor-service/src/tasks/mod.rs | 7 +- magicblock-metrics/src/metrics/mod.rs | 13 +++ programs/magicblock/src/magic_context.rs | 6 -- .../test-committor-service/tests/common.rs | 39 ++++---- .../tests/test_intent_executor.rs | 42 ++++---- .../tests/test_outbox_flow.rs | 95 ++++++++++--------- .../test-tools/src/transactions.rs | 2 +- 20 files changed, 179 insertions(+), 188 deletions(-) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 0cfe94b42..961433c29 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -36,9 +36,7 @@ const POISONED_MUTEX_MSG: &str = type BundleResultListener = oneshot::Sender; pub struct CommittorProcessor { - authority: Keypair, _table_mania: TableMania, - magic_rpc_client: MagicblockRpcClient, commits_scheduler: IntentEngineHandle, task_info_fetcher: Arc>, pending_result_listeners: Arc>>, @@ -116,9 +114,7 @@ impl CommittorProcessor { )); Self { - authority, _table_mania: table_mania, - magic_rpc_client: magic_block_rpc_client, commits_scheduler, task_info_fetcher, pending_result_listeners, diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 8e5935422..4242714d6 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -38,14 +38,10 @@ use crate::{ pub struct AcceptedIntentExecutor { ctx: IntentExecutorCtx, authority: Keypair, + /// Timeout for Intent's actions + pub actions_timeout: Duration, /// Intent execution started at pub started_at: Instant, - /// Junk that needs to be cleaned up - junk: Vec, - /// Set to false on execution failure so cleanup only releases ALT - /// reservations without closing buffer PDAs (see race condition note in - /// intent_execution_engine) - close_buffers: bool, } impl AcceptedIntentExecutor @@ -56,14 +52,16 @@ where O: OutboxClient, O::Error: Into, { - pub fn new(ctx: IntentExecutorCtx) -> Self { + pub fn new( + ctx: IntentExecutorCtx, + actions_timeout: Duration, + ) -> Self { let authority = validator_authority(); Self { ctx, authority, + actions_timeout, started_at: Instant::now(), - junk: vec![], - close_buffers: true, } } @@ -141,9 +139,7 @@ where } fn time_left(&self) -> Option { - self.ctx - .actions_timeout - .checked_sub(self.started_at.elapsed()) + self.actions_timeout.checked_sub(self.started_at.elapsed()) } /// Starting execution from single stage @@ -226,8 +222,8 @@ where }); }); - self.close_buffers = result.is_ok(); - self.junk = execution_report.junk; + let close_buffers = result.is_ok(); + let junk = execution_report.junk; let result = IntentExecutionResult { inner: result, patched_errors: execution_report.patched_errors, @@ -238,8 +234,8 @@ where }; let cleanup_handle = CleanupHandle::new( self.authority, - self.junk, - self.close_buffers, + junk, + close_buffers, self.ctx.transaction_preparator, ); diff --git a/magicblock-committor-service/src/intent_executor/error.rs b/magicblock-committor-service/src/intent_executor/error.rs index 38373202f..a3eaca0eb 100644 --- a/magicblock-committor-service/src/intent_executor/error.rs +++ b/magicblock-committor-service/src/intent_executor/error.rs @@ -57,7 +57,6 @@ pub enum IntentExecutorError { OutboxClientError(#[from] InternalOutboxClientError), #[error("Failed to get pending signature status: {0}")] GetPendingSignatureStatusError(#[source] MagicBlockRpcClientError), - // TODO(edwin): remove once proper retries introduced #[error("TaskBuilderError: {0}")] TaskBuilderError(#[from] TaskBuilderError), #[error("FailedToCommitError: {err}")] diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 7a5c7b697..9b179248f 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -111,13 +111,12 @@ impl IntentExecutionClient { signatures: &[Signature], ) -> MagicBlockRpcClientResult>>> { - // TODO(edwin): add metrics/log for this one + let _timer = metrics::start_rpc_client_signature_history_timer(); let response = self .rpc_client .get_inner() .get_signature_statuses_with_history(signatures) .await - // TODO(edwin): map to more specific error? verify .map_err(MagicBlockRpcClientError::from)?; Ok(response .value @@ -231,7 +230,6 @@ struct IntentErrorMapper { transaction_error_mapper: TxMap, } -// TODO(edwin): probably could be removed impl SendErrorMapper for IntentErrorMapper where TxMap: TransactionErrorMapper< diff --git a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs index d64eeff99..acf5491a4 100644 --- a/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs +++ b/magicblock-committor-service/src/intent_executor/intent_executor_factory.rs @@ -14,7 +14,7 @@ use crate::{ outbox::OutboxClient, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, - ComputeBudgetConfig, + ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, }; pub trait IntentExecutorBuilder { @@ -61,8 +61,7 @@ where task_info_fetcher: self.task_info_fetcher.clone(), outbox_client: self.outbox_client.clone(), actions_callback_executor: self.actions_callback_executor.clone(), - actions_timeout: self.executor_config.actions_timeout, }; - build_stage_intent_executor(ctx, status) + build_stage_intent_executor(ctx, status, DEFAULT_ACTIONS_TIMEOUT) } } diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index fba9972b3..b47374bab 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -53,6 +53,7 @@ pub trait IntentExecutor: Send + Sync + 'static { pub fn build_stage_intent_executor( ctx: IntentExecutorCtx, status: OutboxIntentBundleStatus, + actions_timeout: Duration, ) -> Box> where T: TransactionPreparator, @@ -63,17 +64,21 @@ where { match status { OutboxIntentBundleStatus::Accepted => { - Box::new(AcceptedIntentExecutor::new(ctx)) + Box::new(AcceptedIntentExecutor::new(ctx, actions_timeout)) as Box + 'static> } OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( sig, - )) => Box::new(SingleStageIntentExecutor::new(ctx, sig)) - as Box + 'static>, + )) => { + Box::new(SingleStageIntentExecutor::new(ctx, actions_timeout, sig)) + as Box + 'static> + } OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( value, - )) => Box::new(TwoStageIntentExecutor::new(ctx, value)) - as Box + 'static>, + )) => { + Box::new(TwoStageIntentExecutor::new(ctx, actions_timeout, value)) + as Box + 'static> + } } } @@ -83,9 +88,6 @@ pub struct IntentExecutorCtx { pub task_info_fetcher: Arc>, pub outbox_client: Arc, pub actions_callback_executor: A, - // TODO(edwin): more like config field. exclude? - /// Timeout for Intent's actions - pub actions_timeout: Duration, } #[derive(Clone, Copy, Debug)] diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 285b1edd1..97446ed6c 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -25,7 +25,7 @@ use crate::{ outbox::OutboxClient, tasks::{ task_info_fetcher::{ResetType, TaskInfoFetcher}, - task_strategist::{TaskStrategist, TransactionStrategy}, + task_strategist::TaskStrategist, }, transaction_preparator::TransactionPreparator, }; @@ -37,14 +37,10 @@ pub struct SingleStageIntentExecutor { /// Intent Executor context ctx: IntentExecutorCtx, + /// Timeout for Intent's actions + pub actions_timeout: Duration, /// Intent execution started at pub started_at: Instant, - /// Junk that needs to be cleaned up - junk: Vec, - /// Set to false on execution failure so cleanup only releases ALT - /// reservations without closing buffer PDAs (see race condition note in - /// intent_execution_engine) - close_buffers: bool, } impl SingleStageIntentExecutor @@ -57,6 +53,7 @@ where { pub fn new( ctx: IntentExecutorCtx, + actions_timeout: Duration, pending_signature: Signature, ) -> Self { let authority = validator_authority(); @@ -65,16 +62,13 @@ where pending_signature, ctx, + actions_timeout, started_at: Instant::now(), - junk: vec![], - close_buffers: true, } } fn time_left(&self) -> Option { - self.ctx - .actions_timeout - .checked_sub(self.started_at.elapsed()) + self.actions_timeout.checked_sub(self.started_at.elapsed()) } pub async fn execute_inner( @@ -162,8 +156,8 @@ where }); }); - self.close_buffers = result.is_ok(); - self.junk = execution_report.junk; + let close_buffers = result.is_ok(); + let junk = execution_report.junk; let result = IntentExecutionResult { inner: result, patched_errors: execution_report.patched_errors, @@ -174,8 +168,8 @@ where }; let cleanup_handle = CleanupHandle::new( self.authority, - self.junk, - self.close_buffers, + junk, + close_buffers, self.ctx.transaction_preparator, ); diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index d8ddf4be7..941a53d3d 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -178,8 +178,8 @@ pub(in crate::intent_executor) async fn check_pending_signature( match statuses.get(0) { Some(Some(Ok(()))) => Ok(ControlFlow::Break(())), - // TODO(edwin): well, that is bizarre one None => { + // well, that is bizarre one warn!(pending_signature = ?sig, "RPC did not return status for signature"); Ok(ControlFlow::Continue(())) } @@ -431,7 +431,10 @@ where junk } -// TODO(edwin): docs +/// Executes a stage with an optional timeout that applies only to callbacks. +/// If the timeout fires, callbacks are triggered with `TimeoutError` immediately +/// (respecting the timeout contract on the user smart contract side), then +/// execution continues without a timeout to drive the intent to completion. pub(in crate::intent_executor) async fn execute_with_timeout( time_left: Option, mut executor: impl StageExecutor, diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 7ae6e4449..f77e59820 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -31,9 +31,7 @@ use crate::{ tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::{ResetType, TaskInfoFetcher}, - task_strategist::{ - TaskStrategist, TransactionStrategy, TwoStageExecutionMode, - }, + task_strategist::{TaskStrategist, TwoStageExecutionMode}, }, transaction_preparator::TransactionPreparator, }; @@ -45,14 +43,10 @@ pub struct TwoStageIntentExecutor { /// Intent Executor context ctx: IntentExecutorCtx, + /// Timeout for Intent's actions + pub actions_timeout: Duration, /// Intent execution started at pub started_at: Instant, - /// Junk that needs to be cleaned up - junk: Vec, - /// Set to false on execution failure so cleanup only releases ALT - /// reservations without closing buffer PDAs (see race condition note in - /// intent_execution_engine) - close_buffers: bool, } impl TwoStageIntentExecutor @@ -65,6 +59,7 @@ where { pub fn new( ctx: IntentExecutorCtx, + actions_timeout: Duration, stage: TwoStageProgress, ) -> Self { let authority = validator_authority(); @@ -73,17 +68,13 @@ where stage, ctx, - // TODO(edwin): deduce started_at properly + actions_timeout, started_at: Instant::now(), - junk: vec![], - close_buffers: true, } } fn time_left(&self) -> Option { - self.ctx - .actions_timeout - .checked_sub(self.started_at.elapsed()) + self.actions_timeout.checked_sub(self.started_at.elapsed()) } /// Picks up execution from commit stage signature @@ -234,10 +225,8 @@ where mut self: Box, intent: ScheduledIntentBundle, ) -> (IntentExecutionResult, CleanupHandle) { - // TODO(edwin): see if can be extracted into single utils // Duplicates AcceptedIntentExecutor::execute let is_undelegate = intent.has_undelegate_intent(); - // TODO(edwin): should validate non emptiness of pubkeys? Shouldn't be possible tho let pubkeys = intent.get_all_committed_pubkeys(); let mut execution_report = IntentExecutionReport::default(); @@ -248,8 +237,8 @@ where .task_info_fetcher .reset(ResetType::Specific(&pubkeys)); } - self.close_buffers = result.is_ok(); - self.junk = execution_report.junk; + let close_buffers = result.is_ok(); + let junk = execution_report.junk; let result = IntentExecutionResult { inner: result, patched_errors: execution_report.patched_errors, @@ -260,8 +249,8 @@ where }; let cleanup_handle = CleanupHandle::new( self.authority, - self.junk, - self.close_buffers, + junk, + close_buffers, self.ctx.transaction_preparator, ); diff --git a/magicblock-committor-service/src/outbox/mod.rs b/magicblock-committor-service/src/outbox/mod.rs index 335381b07..a6e877b6e 100644 --- a/magicblock-committor-service/src/outbox/mod.rs +++ b/magicblock-committor-service/src/outbox/mod.rs @@ -54,13 +54,13 @@ pub trait OutboxClient: Send + Sync + 'static { } pub struct ScheduledBaseIntentMeta { - pub(crate) id: u64, - pub(crate) slot: u64, - pub(crate) blockhash: Hash, - pub(crate) payer: Pubkey, - pub(crate) included_pubkeys: Vec, - pub(crate) intent_sent_transaction: Transaction, - pub(crate) requested_undelegation: bool, + pub id: u64, + pub slot: u64, + pub blockhash: Hash, + pub payer: Pubkey, + pub included_pubkeys: Vec, + pub intent_sent_transaction: Transaction, + pub requested_undelegation: bool, } impl ScheduledBaseIntentMeta { diff --git a/magicblock-committor-service/src/outbox/outbox_client.rs b/magicblock-committor-service/src/outbox/outbox_client.rs index a082c29d6..163cf1db5 100644 --- a/magicblock-committor-service/src/outbox/outbox_client.rs +++ b/magicblock-committor-service/src/outbox/outbox_client.rs @@ -30,11 +30,8 @@ use crate::{ error::IntentExecutorResult, ExecutionOutput, IntentExecutionReport, }, outbox::{ - outbox_intent_bundles_reader::{ - InternalOutboxIntentBundlesReader, OutboxIntentBundlesReader, - }, - utils::build_sent_commit, - OutboxClient, ScheduledBaseIntentMeta, + outbox_intent_bundles_reader::InternalOutboxIntentBundlesReader, + utils::build_sent_commit, OutboxClient, ScheduledBaseIntentMeta, }, }; @@ -44,7 +41,6 @@ pub struct InternalOutboxClient { /// Provides access to MagicContext accounts_db: Arc, /// RPC client for sending accept transactions to the ER - // TODO(edwin): check if needs to be Arc rpc_client: Arc, /// Internal endpoint for scheduling ER TXs transaction_scheduler: TransactionSchedulerHandle, diff --git a/magicblock-committor-service/src/outbox/utils.rs b/magicblock-committor-service/src/outbox/utils.rs index 6ac9ffc6e..12ef4f26c 100644 --- a/magicblock-committor-service/src/outbox/utils.rs +++ b/magicblock-committor-service/src/outbox/utils.rs @@ -1,7 +1,4 @@ -use magicblock_program::{ - magic_scheduled_base_intent::ScheduledIntentBundle, Pubkey, SentCommit, -}; -use solana_hash::Hash; +use magicblock_program::SentCommit; use solana_transaction::Transaction; use tracing::{error, info}; diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 2ca96fae1..601e2b712 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -141,13 +141,10 @@ where // Reschedule existing outbox intents first // We need to ensure that accounts in outbox a scheduled before // we accept new incoming Intents - self.reschedule_intents() - .await - .inspect_err(|err| { - error!(error = ?err, "Failed to reschedule pending bundles") - }) - // TODO(edwin): early shutdown or cleanup errors to avoid this - .expect("Failed to reschedule intents"); + if let Err(err) = self.reschedule_intents().await { + // TODO(edwin): alerts + error!(error = ?err, "Failed to reschedule pending bundles") + } let mut interval = tokio::time::interval(self.slot_interval); loop { @@ -193,7 +190,6 @@ where return Ok(()); } - // TODO(edwin): use status let read_len = intent_bundles_chunk.len(); // Schedule without initial persistence as bundle already exists in db let result = self diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 2cca813d1..304e7cf83 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -25,11 +25,8 @@ pub mod utils; pub use task_builder::TaskBuilderImpl; -use crate::{ - tasks::{ - commit_finalize_task::CommitFinalizeTask, commit_task::CommitTask, - }, - utils::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, +use crate::tasks::{ + commit_finalize_task::CommitFinalizeTask, commit_task::CommitTask, }; #[derive(Clone, Copy, PartialEq, Eq, Debug)] diff --git a/magicblock-metrics/src/metrics/mod.rs b/magicblock-metrics/src/metrics/mod.rs index c4b01f1a7..f11d19589 100644 --- a/magicblock-metrics/src/metrics/mod.rs +++ b/magicblock-metrics/src/metrics/mod.rs @@ -419,6 +419,14 @@ lazy_static::lazy_static! { "Number of signatures included in batched getSignatureStatuses requests" ).unwrap(); + static ref RPC_CLIENT_SIGNATURE_HISTORY_DURATION_SECONDS: Histogram = Histogram::with_opts( + HistogramOpts::new( + "rpc_client_signature_history_duration_seconds", + "Time in seconds spent on getSignatureStatuses-with-history (restart recovery)" + ) + .buckets(vec![0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0]) + ).unwrap(); + static ref COMMITTOR_INTENT_TASK_PREPARATION_TIME: HistogramVec = HistogramVec::new( HistogramOpts::new( @@ -640,6 +648,7 @@ pub(crate) fn register() { register!(RPC_CLIENT_SIGNATURE_WS_FALLBACK_COUNT); register!(RPC_CLIENT_SIGNATURE_STATUS_BATCH_COUNT); register!(RPC_CLIENT_SIGNATURE_STATUS_BATCH_SIGNATURES_COUNT); + register!(RPC_CLIENT_SIGNATURE_HISTORY_DURATION_SECONDS); register!(CONNECTED_PUBSUB_CLIENTS_GAUGE); register!(CONNECTED_DIRECT_PUBSUB_CLIENTS_GAUGE); register!(PUBSUB_CLIENT_UPTIME_GAUGE); @@ -929,6 +938,10 @@ pub fn inc_rpc_client_signature_status_batch_signatures_count(count: u64) { RPC_CLIENT_SIGNATURE_STATUS_BATCH_SIGNATURES_COUNT.inc_by(count) } +pub fn start_rpc_client_signature_history_timer() -> HistogramTimer { + RPC_CLIENT_SIGNATURE_HISTORY_DURATION_SECONDS.start_timer() +} + pub fn set_connected_pubsub_clients_count(count: usize) { CONNECTED_PUBSUB_CLIENTS_GAUGE.set(count as i64); } diff --git a/programs/magicblock/src/magic_context.rs b/programs/magicblock/src/magic_context.rs index 7daad1ad5..03caa1110 100644 --- a/programs/magicblock/src/magic_context.rs +++ b/programs/magicblock/src/magic_context.rs @@ -53,12 +53,6 @@ impl MagicContext { self.scheduled_base_intents.push(base_intent); } - pub(crate) fn take_scheduled_commits( - &mut self, - ) -> Vec { - mem::take(&mut self.scheduled_base_intents) - } - pub(crate) fn take_front_scheduled_commits( &mut self, n: usize, diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 01e7b0c8d..790279a55 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -10,11 +10,14 @@ use async_trait::async_trait; use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, - intent_execution_client::IntentExecutionClient, IntentExecutorCtx, + error::IntentExecutorResult, + intent_execution_client::IntentExecutionClient, ExecutionOutput, + IntentExecutionReport, IntentExecutorCtx, }, outbox::{ outbox_client::InternalOutboxClientError, - outbox_intent_bundles_reader::OutboxIntentBundlesReader, + outbox_intent_bundles_reader::OutboxIntentBundlesReader, OutboxClient, + ScheduledBaseIntentMeta, }, tasks::{ commit_task::{CommitDelivery, CommitTask}, @@ -34,7 +37,7 @@ use magicblock_core::{ }; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, outbox::ExecutionStage, - outbox_intent_bundles::OutboxIntentBundle, SentCommit, + outbox_intent_bundles::OutboxIntentBundle, }; use magicblock_rpc_client::MagicblockRpcClient; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; @@ -45,9 +48,7 @@ use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ signature::{Keypair, Signature}, signer::Signer, - transaction::Transaction, }; -use magicblock_committor_service::outbox::OutboxClient; // Helper function to create a test RPC client pub async fn create_test_client() -> MagicblockRpcClient { @@ -127,14 +128,19 @@ impl TestFixture { MockActionsCallbackExecutor, MockOutboxClient, > { - AcceptedIntentExecutor::new(IntentExecutorCtx { - intent_client: IntentExecutionClient::new(self.rpc_client.clone()), - transaction_preparator: self.create_transaction_preparator(), - task_info_fetcher: self.create_task_info_fetcher(), - outbox_client: Arc::new(MockOutboxClient), - actions_callback_executor: MockActionsCallbackExecutor::default(), - actions_timeout: DEFAULT_ACTIONS_TIMEOUT, - }) + AcceptedIntentExecutor::new( + IntentExecutorCtx { + intent_client: IntentExecutionClient::new( + self.rpc_client.clone(), + ), + transaction_preparator: self.create_transaction_preparator(), + task_info_fetcher: self.create_task_info_fetcher(), + outbox_client: Arc::new(MockOutboxClient), + actions_callback_executor: MockActionsCallbackExecutor::default( + ), + }, + DEFAULT_ACTIONS_TIMEOUT, + ) } #[allow(dead_code)] @@ -162,7 +168,7 @@ impl OutboxIntentBundlesReader for MockOutboxReader { async fn fetch_outbox_intent( &self, - intent_id: u64, + _intent_id: u64, ) -> Result, Self::Error> { Ok(None) } @@ -195,8 +201,9 @@ impl OutboxClient for MockOutboxClient { async fn notify_commit_sent( &self, - _sent_tx: Transaction, - _sent_commit: SentCommit, + _meta: ScheduledBaseIntentMeta, + _result: &IntentExecutorResult, + _execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { Ok(()) } diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index a53c5cac8..2fb167e71 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -119,16 +119,18 @@ impl TestEnv { } let callback_executor = MockActionsCallbackExecutor::default(); - let intent_executor = AcceptedIntentExecutor::new(IntentExecutorCtx { - intent_client: IntentExecutionClient::new( - fixture.rpc_client.clone(), - ), - transaction_preparator, - task_info_fetcher: task_info_fetcher.clone(), - outbox_client: Arc::new(MockOutboxClient), - actions_callback_executor: callback_executor.clone(), - actions_timeout: DEFAULT_ACTIONS_TIMEOUT, - }); + let intent_executor = AcceptedIntentExecutor::new( + IntentExecutorCtx { + intent_client: IntentExecutionClient::new( + fixture.rpc_client.clone(), + ), + transaction_preparator, + task_info_fetcher: task_info_fetcher.clone(), + outbox_client: Arc::new(MockOutboxClient), + actions_callback_executor: callback_executor.clone(), + }, + DEFAULT_ACTIONS_TIMEOUT, + ); Self { fixture, @@ -1268,14 +1270,18 @@ async fn test_action_callback_fired_on_timeout() { let task_info_fetcher = Arc::new(CacheTaskInfoFetcher::new( RpcTaskInfoFetcher::new(fixture.rpc_client.clone()), )); - let intent_executor = AcceptedIntentExecutor::new(IntentExecutorCtx { - intent_client: IntentExecutionClient::new(fixture.rpc_client.clone()), - transaction_preparator: fixture.create_transaction_preparator(), - task_info_fetcher, - outbox_client: Arc::new(MockOutboxClient), - actions_callback_executor: callback_executor.clone(), - actions_timeout: Duration::ZERO, - }); + let intent_executor = AcceptedIntentExecutor::new( + IntentExecutorCtx { + intent_client: IntentExecutionClient::new( + fixture.rpc_client.clone(), + ), + transaction_preparator: fixture.create_transaction_preparator(), + task_info_fetcher, + outbox_client: Arc::new(MockOutboxClient), + actions_callback_executor: callback_executor.clone(), + }, + Duration::ZERO, + ); let scheduled_intent = create_scheduled_intent(base_intent); let (res, cleanup_handle) = diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index c60124c99..ef79a8ed4 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -13,14 +13,14 @@ use integration_test_tools::{ use magicblock_committor_service::{ intent_executor::{ accepted_intent_executor::AcceptedIntentExecutor, - build_stage_intent_executor, - error::IntentExecutorResult, - intent_execution_client::IntentExecutionClient, - ExecutionOutput, IntentExecutionReport, IntentExecutor, IntentExecutorCtx, + build_stage_intent_executor, error::IntentExecutorResult, + intent_execution_client::IntentExecutionClient, ExecutionOutput, + IntentExecutionReport, IntentExecutor, IntentExecutorCtx, }, outbox::{ outbox_client::InternalOutboxClientError, - outbox_intent_bundles_reader::OutboxIntentBundlesReader, + outbox_intent_bundles_reader::OutboxIntentBundlesReader, OutboxClient, + ScheduledBaseIntentMeta, }, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, @@ -58,7 +58,7 @@ use serial_test::serial; use solana_rpc_client::{ http_sender::HttpSender, nonblocking::rpc_client::RpcClient as AsyncRpcClient, - rpc_client::{RpcClientConfig, SerializableTransaction}, + rpc_client::RpcClientConfig, rpc_sender::{RpcSender, RpcTransportStats}, }; use solana_rpc_client_api::{client_error, request::RpcRequest}; @@ -69,7 +69,6 @@ use solana_sdk::{ signer::Signer, transaction::Transaction, }; -use magicblock_committor_service::outbox::{OutboxClient, ScheduledBaseIntentMeta}; type CallbackRecord = (Vec, Option, ActionResult); @@ -123,17 +122,13 @@ impl TestEnv { } } - fn executor_ctx_builder( - &self, - actions_timeout: Duration, - ) -> TestIntentExecutorCtxBuilder { + fn executor_ctx_builder(&self) -> TestIntentExecutorCtxBuilder { let ctx = TestIntentExecutorCtx { intent_client: self.intent_client.clone(), transaction_preparator: self.transaction_preparator(), task_info_fetcher: self.task_info_fetcher.clone(), outbox_client: self.outbox_client().into(), actions_callback_executor: self.callback_scheduler.clone(), - actions_timeout, }; TestIntentExecutorCtxBuilder { ctx } } @@ -311,10 +306,9 @@ async fn test_pickup_executed_intent() { }); // Execute intent - let executor_ctx = test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) - .build(); - let executor = AcceptedIntentExecutor::new(executor_ctx); + let executor_ctx = test_env.executor_ctx_builder().build(); + let executor = + AcceptedIntentExecutor::new(executor_ctx, DEFAULT_ACTIONS_TIMEOUT); let (result, cleanup_handle) = Box::new(executor) .execute(outbox_bundle.inner.clone()) .await; @@ -347,10 +341,9 @@ async fn test_pickup_executed_intent() { ); // Builder executor let executor = build_stage_intent_executor( - test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) - .build(), + test_env.executor_ctx_builder().build(), outbox_bundle.status, + DEFAULT_ACTIONS_TIMEOUT, ); let (result, cleanup_handle) = Box::new(executor) .execute(outbox_bundle.inner.clone()) @@ -394,11 +387,14 @@ async fn test_pickup_failed_intent() { failing_outbox.with_fail_execution_stage(true); let executor_ctx = test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .executor_ctx_builder() .with_outbox_client(failing_outbox.into()) .build(); - let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let executor = Box::new(AcceptedIntentExecutor::new( + executor_ctx, + DEFAULT_ACTIONS_TIMEOUT, + )); let (result, cleanup_handle) = executor.execute(outbox_bundle.inner).await; assert!(result .inner @@ -425,10 +421,11 @@ async fn test_pickup_failed_intent() { ); // Build executor that will drive execution to success - let executor_ctx = test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) - .build(); - let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let executor_ctx = test_env.executor_ctx_builder().build(); + let executor = Box::new(AcceptedIntentExecutor::new( + executor_ctx, + DEFAULT_ACTIONS_TIMEOUT, + )); let (result, cleanup_handle) = executor.execute(chain_outbox_bundle.inner).await; let ExecutionOutput::SingleStage(_) = @@ -482,11 +479,12 @@ async fn test_pickup_after_timeout() { slow_outbox_client .with_set_execution_stage_sleep(SET_EXECUTION_STAGE_SLEEP); let executor_ctx = test_env - .executor_ctx_builder(ACTIONS_TIMEOUT) + .executor_ctx_builder() .with_outbox_client(slow_outbox_client.into()) .build(); - let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let executor = + Box::new(AcceptedIntentExecutor::new(executor_ctx, ACTIONS_TIMEOUT)); let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); @@ -577,11 +575,12 @@ async fn test_pick_up_after_tx_submission() { let slow_intent_client = test_env.intent_client_with_send_sleep(SEND_SLEEP); let executor_ctx = test_env - .executor_ctx_builder(ACTIONS_TIMEOUT) + .executor_ctx_builder() .with_intent_client(slow_intent_client) .build(); - let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let executor = + Box::new(AcceptedIntentExecutor::new(executor_ctx, ACTIONS_TIMEOUT)); let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); @@ -654,11 +653,14 @@ async fn test_pickup_after_committing() { let stage_calls = failing_outbox.stage_calls.clone(); let executor_ctx = test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) + .executor_ctx_builder() .with_outbox_client(failing_outbox.into()) .build(); - let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let executor = Box::new(AcceptedIntentExecutor::new( + executor_ctx, + DEFAULT_ACTIONS_TIMEOUT, + )); let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; assert!( @@ -698,10 +700,9 @@ async fn test_pickup_after_committing() { // Recovery: build_stage_intent_executor detects the commit sig on chain and // runs the finalize stage without re-submitting the commit tx. let executor = build_stage_intent_executor( - test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) - .build(), + test_env.executor_ctx_builder().build(), outbox_bundle.status, + DEFAULT_ACTIONS_TIMEOUT, ); let (result, cleanup_handle) = Box::new(executor) .execute(outbox_bundle.inner.clone()) @@ -756,10 +757,11 @@ async fn test_pickup_after_finalizing() { }); // Run to completion with a normal outbox client - let executor_ctx = test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) - .build(); - let executor = Box::new(AcceptedIntentExecutor::new(executor_ctx)); + let executor_ctx = test_env.executor_ctx_builder().build(); + let executor = Box::new(AcceptedIntentExecutor::new( + executor_ctx, + DEFAULT_ACTIONS_TIMEOUT, + )); let (result, cleanup_handle) = executor.execute(outbox_bundle.inner.clone()).await; let ExecutionOutput::TwoStage { @@ -796,10 +798,9 @@ async fn test_pickup_after_finalizing() { // Recovery: build_stage_intent_executor detects the finalize sig on chain and // returns the same signatures without re-executing either stage. let executor = build_stage_intent_executor( - test_env - .executor_ctx_builder(DEFAULT_ACTIONS_TIMEOUT) - .build(), + test_env.executor_ctx_builder().build(), outbox_bundle.status, + DEFAULT_ACTIONS_TIMEOUT, ); let (result, cleanup_handle) = Box::new(executor) .execute(outbox_bundle.inner.clone()) @@ -891,6 +892,7 @@ struct TestOutboxClient { ephem_rpc: Arc, validator_keypair: Keypair, pub stage_calls: Arc>>, + pub sent_commits: Arc>>, fail_set_execution_stage: bool, set_execution_stage_sleep: Option, @@ -904,6 +906,7 @@ impl TestOutboxClient { ephem_rpc, validator_keypair, stage_calls: Default::default(), + sent_commits: Default::default(), fail_set_execution_stage: false, set_execution_stage_sleep: None, fail_committing: false, @@ -1010,10 +1013,16 @@ impl OutboxClient for TestOutboxClient { async fn notify_commit_sent( &self, - _meta: ScheduledBaseIntentMeta, - _result: &IntentExecutorResult, + meta: ScheduledBaseIntentMeta, + result: &IntentExecutorResult, _execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { + let succeeded = result.is_ok(); + self.ephem_rpc + .send_and_confirm_transaction(&meta.intent_sent_transaction) + .await + .map_err(InternalOutboxClientError::RpcClientError)?; + self.sent_commits.lock().unwrap().push((meta.id, succeeded)); Ok(()) } diff --git a/test-integration/test-tools/src/transactions.rs b/test-integration/test-tools/src/transactions.rs index 61036f5e1..fe689c39a 100644 --- a/test-integration/test-tools/src/transactions.rs +++ b/test-integration/test-tools/src/transactions.rs @@ -3,7 +3,7 @@ use std::{thread::sleep, time::Duration}; use solana_commitment_config::CommitmentConfig; -use solana_rpc_client::rpc_client::{RpcClient, SerializableTransaction}; +use solana_rpc_client::rpc_client::RpcClient; use solana_rpc_client_api::{ client_error, config::{RpcSendTransactionConfig, RpcSimulateTransactionConfig}, From 6d9e3c329a1575b46314bcc2165dc02a1ac2d3cd Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 14:48:00 +0700 Subject: [PATCH 77/97] refactor: remove unused --- .../tests/test_ix_commit_local.rs | 1 + .../test-schedule-intent/tests/common/mod.rs | 3 +++ .../tests/test_outbox_flow.rs | 16 +++++----------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 0c11bbf08..dda3b9e35 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -49,6 +49,7 @@ mod utils; // ----------------- type ExpectedStrategies = HashMap; +#[allow(unused)] pub struct AccountCommitInfo { data: Vec, commit_nonce: u64, diff --git a/test-integration/test-schedule-intent/tests/common/mod.rs b/test-integration/test-schedule-intent/tests/common/mod.rs index b033cb72a..d12312893 100644 --- a/test-integration/test-schedule-intent/tests/common/mod.rs +++ b/test-integration/test-schedule-intent/tests/common/mod.rs @@ -104,11 +104,13 @@ pub fn add_to_counter( ) } +#[allow(unused)] pub struct ExpectedCounter { pub pda: Pubkey, pub expected: u64, } +#[allow(unused)] pub fn assert_counters( ctx: &IntegrationTestContext, expected_counters: &[ExpectedCounter], @@ -132,6 +134,7 @@ pub fn assert_counters( } } +#[allow(unused)] pub fn verify_undelegation_in_ephem_via_owner( pubkeys: &[Pubkey], ctx: &IntegrationTestContext, diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index ef79a8ed4..4b6ad84d6 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -330,13 +330,11 @@ async fn test_pickup_executed_intent() { let ExecutionOutput::SingleStage(signature) = result.inner.unwrap() else { panic!("Unexpected execution strategy"); }; - assert!( - matches!( - outbox_bundle.status, - OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( - signature - )) - ), + assert_eq!( + outbox_bundle.status, + OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( + signature + )), "Invalid outbox state" ); // Builder executor @@ -922,10 +920,6 @@ impl TestOutboxClient { self.set_execution_stage_sleep = Some(value); } - fn with_fail_committing(&mut self, value: bool) { - self.fail_committing = value; - } - fn with_fail_finalizing(&mut self, value: bool) { self.fail_finalizing = value; } From 46d44b777472fc5c055d5ee5fdd49cf27e793fc3 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 14:59:40 +0700 Subject: [PATCH 78/97] fix: test compilation --- magicblock-committor-service/src/tasks/mod.rs | 1 + .../src/instruction.rs | 3 +- .../process_accept_scheduled_commits.rs | 2 +- .../outbox/process_scheduled_commit_sent.rs | 14 ++----- .../schedule/process_schedule_commit_tests.rs | 42 ++++++++++++++++--- .../magicblock/src/utils/instruction_utils.rs | 2 +- 6 files changed, 44 insertions(+), 20 deletions(-) diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 304e7cf83..3527db99d 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -717,6 +717,7 @@ mod serialization_safety_test { #[test] fn test_close_buffer_limit() { + use crate::utils::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}; use solana_compute_budget_interface::ComputeBudgetInstruction; use solana_keypair::Keypair; use solana_signer::Signer; diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index a2ddbee13..b437b9ef7 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -68,8 +68,7 @@ pub enum MagicBlockInstruction { /// # Account references /// - **0.** `[SIGNER]` Validator Authority /// - **1.** `[WRITE]` Magic Context Account containing the initially scheduled commits - /// - **2..n** `[WRITE]` Outbox intent PDAs, one per accepted intent, - /// seeds: `["outbox-intent", intent_id.to_le_bytes()]` + /// - **2..n** `[WRITE]` Outbox intent PDAs, one per accepted intent, seeds: `["outbox-intent", intent_id.to_le_bytes()]` AcceptScheduleCommits, /// Records the attempt to realize a scheduled commit on chain. diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index a1571fca0..9b5defbcf 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -94,7 +94,7 @@ fn validate( } // Validate authority is a signer - if !signers.contains(&validator_auth) { + if !signers.contains(validator_auth) { ic_msg!( invoke_context, "AcceptScheduledCommits ERR: validator authority pubkey {} not in signers", diff --git a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs index 80cf94d0c..d264f4b0e 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs @@ -377,8 +377,6 @@ mod tests { ensure_started_validator(&mut account_data, None); let mut ix = InstructionUtils::scheduled_commit_sent_instruction( - &crate::id(), - &validator::validator_authority_id(), commit.message_id, ); ix.accounts[1].is_signer = false; @@ -413,11 +411,10 @@ mod tests { }; ensure_started_validator(&mut account_data, None); - let ix = InstructionUtils::scheduled_commit_sent_instruction( - &crate::id(), - &fake_validator.pubkey(), + let mut ix = InstructionUtils::scheduled_commit_sent_instruction( commit.message_id, ); + ix.accounts[0].pubkey = fake_validator.pubkey(); let transaction_accounts = transaction_accounts_from_map(&ix, &mut account_data); process_instruction( @@ -448,11 +445,10 @@ mod tests { }; ensure_started_validator(&mut account_data, None); - let ix = InstructionUtils::scheduled_commit_sent_instruction( - &fake_program.pubkey(), - &validator::validator_authority_id(), + let mut ix = InstructionUtils::scheduled_commit_sent_instruction( commit.message_id, ); + ix.accounts[1].pubkey = fake_program.pubkey(); let transaction_accounts = transaction_accounts_from_map(&ix, &mut account_data); @@ -478,8 +474,6 @@ mod tests { ensure_started_validator(&mut account_data, None); let ix = InstructionUtils::scheduled_commit_sent_instruction( - &crate::id(), - &validator::validator_authority_id(), commit.message_id, ); diff --git a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs index 4dd93ddd6..144e69358 100644 --- a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs +++ b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs @@ -363,7 +363,12 @@ mod tests { &payer, program, committee, ); - let ix = InstructionUtils::accept_scheduled_commits_instruction(); + let intent_ids = bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id); + let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -551,7 +556,12 @@ mod tests { &payer, program, committee, ); - let ix = InstructionUtils::accept_scheduled_commits_instruction(); + let intent_ids = bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id); + let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -636,8 +646,13 @@ mod tests { 1, ); + let intent_ids = bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id); let ix_accept = - InstructionUtils::accept_scheduled_commits_instruction(); + InstructionUtils::accept_scheduled_commits_instruction(intent_ids); let (mut account_data2, mut transaction_accounts2) = prepare_transaction_with_single_committee( &payer, @@ -782,8 +797,13 @@ mod tests { 1, ); + let intent_ids = bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id); let ix_accept = - InstructionUtils::accept_scheduled_commits_instruction(); + InstructionUtils::accept_scheduled_commits_instruction(intent_ids); let (mut account_data2, mut transaction_accounts2) = prepare_transaction_with_single_committee( &payer, @@ -893,7 +913,12 @@ mod tests { (true, true, true), ); - let ix = InstructionUtils::accept_scheduled_commits_instruction(); + let intent_ids = bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id); + let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -1008,7 +1033,12 @@ mod tests { (true, true, true), ); - let ix = InstructionUtils::accept_scheduled_commits_instruction(); + let intent_ids = bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id); + let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index 3d5fed070..dba5b6c02 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -161,7 +161,7 @@ impl InstructionUtils { // Add outbox intent accounts let outbox_intent_metas = intent_ids .into_iter() - .map(|intent_id| outbox_intent_pda(intent_id)) + .map(outbox_intent_pda) .map(|intent_pda| AccountMeta::new(intent_pda, false)); account_metas.extend(outbox_intent_metas); From 2343721c1ad84dcfa3a701667da6a1f5087197b0 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 21:07:57 +0700 Subject: [PATCH 79/97] fix: unit tests --- magicblock-committor-service/src/tasks/mod.rs | 6 +- programs/magicblock/src/intent_bundles/mod.rs | 2 - .../process_accept_scheduled_commits.rs | 1 - .../outbox/process_scheduled_commit_sent.rs | 151 +++++++---- .../schedule/process_schedule_commit_tests.rs | 240 ++++++++++-------- programs/magicblock/src/lib.rs | 1 - 6 files changed, 237 insertions(+), 164 deletions(-) diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 3527db99d..46e1bf83d 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -717,14 +717,16 @@ mod serialization_safety_test { #[test] fn test_close_buffer_limit() { - use crate::utils::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}; use solana_compute_budget_interface::ComputeBudgetInstruction; use solana_keypair::Keypair; use solana_signer::Signer; use solana_transaction::Transaction; use tracing::info; - use crate::test_utils; + use crate::{ + test_utils, + utils::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE}, + }; test_utils::init_test_logger(); diff --git a/programs/magicblock/src/intent_bundles/mod.rs b/programs/magicblock/src/intent_bundles/mod.rs index 06991aa80..01c902946 100644 --- a/programs/magicblock/src/intent_bundles/mod.rs +++ b/programs/magicblock/src/intent_bundles/mod.rs @@ -1,7 +1,5 @@ pub mod magic_scheduled_base_intent; pub mod outbox; -// TODO(edwin): rename? pub mod outbox_intent_bundles; -// TODO(edwin): rename? mod process_execute_callback; pub mod schedule; diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index 9b5defbcf..dc09795cf 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -76,7 +76,6 @@ fn validate( // Validate authority let transaction_context = &*invoke_context.transaction_context; - // TODO(edwin): add check for vault? // TODO(edwin): add check for magic-program let provided_validator_auth = get_instruction_pubkey_with_idx( diff --git a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs index d264f4b0e..a3ceb86b1 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_scheduled_commit_sent.rs @@ -117,7 +117,7 @@ fn get_scheduled_commit(id: u64) -> Option { pub fn process_scheduled_commit_sent( signers: HashSet, invoke_context: &mut InvokeContext, - commit_id: u64, + intent_id: u64, ) -> Result<(), InstructionError> { let mode = coordination_mode::CoordinationMode::current(); if !mode.should_schedule_intents() { @@ -129,8 +129,59 @@ pub fn process_scheduled_commit_sent( return Ok(()); } + let (validator_authority_id, expected_pda) = + validate(&signers, invoke_context, intent_id)?; + + // Only after we passed all checks do we remove the commit from the global hashmap + // Otherwise a malicious actor could remove a commit from the hashmap without + // signing as the validator + let commit = match SENT_COMMITS.write() { + Ok(mut commits) => match commits.remove(&intent_id) { + Some(commit) => commit, + None => { + ic_msg!( + invoke_context, + "ScheduleCommitSent ERR: commit with id {} not found", + intent_id + ); + return Err(InstructionError::Custom( + custom_error_codes::CANNOT_FIND_SCHEDULED_COMMIT, + )); + } + }, + Err(err) => { + ic_msg!( + invoke_context, + "ScheduleCommitSent ERR: failed to lock SENT_COMMITS: {}", + err + ); + return Err(InstructionError::Custom( + custom_error_codes::UNABLE_TO_UNLOCK_SENT_COMMITS, + )); + } + }; + + // Log data + log_sent_commit(invoke_context, &commit); + commit + .error_message + .map(|_| Err(InstructionError::Custom(INTENT_FAILED_CODE))) + .unwrap_or(Ok(()))?; + + // Close Outbox intent + close_outbox_account_cpi( + invoke_context, + validator_authority_id, + expected_pda, + ) +} + +fn validate( + signers: &HashSet, + invoke_context: &InvokeContext, + intent_id: u64, +) -> Result<(Pubkey, Pubkey), InstructionError> { const VALIDATOR_IDX: u16 = 0; - // TODO(edwin): check presense of those const MAGIC_PROGRAM_IDX: u16 = VALIDATOR_IDX + 1; const MAGIC_VAULT_IDX: u16 = MAGIC_PROGRAM_IDX + 1; const CLOSING_PDA_IDX: u16 = MAGIC_VAULT_IDX + 1; @@ -160,6 +211,22 @@ pub fn process_scheduled_commit_sent( return Err(InstructionError::IncorrectAuthority); } + // Assert magic program account + let magic_program_pubkey = get_instruction_pubkey_with_idx( + transaction_context, + MAGIC_PROGRAM_IDX, + )?; + if *magic_program_pubkey != crate::id() { + ic_msg!( + invoke_context, + "ScheduleCommitSent ERR: account at idx {} is {}, expected magic program {}", + MAGIC_PROGRAM_IDX, + magic_program_pubkey, + crate::id() + ); + return Err(InstructionError::IncorrectProgramId); + } + // Assert signers if !signers.contains(&validator_authority_id) { ic_msg!( @@ -172,7 +239,7 @@ pub fn process_scheduled_commit_sent( // Validate outbox intent PDA let provided_pda = get_instruction_pubkey_with_idx(transaction_context, CLOSING_PDA_IDX)?; - let expected_pda = outbox_intent_pda(commit_id); + let expected_pda = outbox_intent_pda(intent_id); if *provided_pda != expected_pda { ic_msg!( invoke_context, @@ -180,40 +247,18 @@ pub fn process_scheduled_commit_sent( CLOSING_PDA_IDX, provided_pda, expected_pda, - commit_id + intent_id ); return Err(InstructionError::InvalidArgument); } - // Only after we passed all checks do we remove the commit from the global hashmap - // Otherwise a malicious actor could remove a commit from the hashmap without - // signing as the validator - let commit = match SENT_COMMITS.write() { - Ok(mut commits) => match commits.remove(&commit_id) { - Some(commit) => commit, - None => { - ic_msg!( - invoke_context, - "ScheduleCommitSent ERR: commit with id {} not found", - commit_id - ); - return Err(InstructionError::Custom( - custom_error_codes::CANNOT_FIND_SCHEDULED_COMMIT, - )); - } - }, - Err(err) => { - ic_msg!( - invoke_context, - "ScheduleCommitSent ERR: failed to lock SENT_COMMITS: {}", - err - ); - return Err(InstructionError::Custom( - custom_error_codes::UNABLE_TO_UNLOCK_SENT_COMMITS, - )); - } - }; + Ok((validator_authority_id, expected_pda)) +} +fn log_sent_commit( + invoke_context: &InvokeContext, + commit: &SentCommitPrintable, +) { ic_msg!( invoke_context, "ScheduledCommitSent id: {}, slot: {}, blockhash: {}", @@ -221,13 +266,11 @@ pub fn process_scheduled_commit_sent( commit.slot, commit.blockhash, ); - ic_msg!( invoke_context, "ScheduledCommitSent payer: {}", commit.payer ); - ic_msg!( invoke_context, "ScheduledCommitSent included: [{}]", @@ -246,11 +289,9 @@ pub fn process_scheduled_commit_sent( sig ); } - if commit.requested_undelegation { - ic_msg!(invoke_context, "ScheduledCommitSent requested undelegation",); + ic_msg!(invoke_context, "ScheduledCommitSent requested undelegation"); } - for (idx, error) in commit.patched_errors.iter().enumerate() { ic_msg!( invoke_context, @@ -259,7 +300,6 @@ pub fn process_scheduled_commit_sent( error ); } - for (idx, report) in commit.callbacks_scheduling_results.iter().enumerate() { ic_msg!( @@ -269,26 +309,13 @@ pub fn process_scheduled_commit_sent( report ); } - - let result = if let Some(error_message) = commit.error_message { + if let Some(error_message) = &commit.error_message { ic_msg!( invoke_context, "ScheduledCommitSent error message: {}", error_message ); - // TODO(edwin): fails to close intent then - Err(InstructionError::Custom(INTENT_FAILED_CODE)) - } else { - Ok(()) - }; - - close_outbox_account_cpi( - invoke_context, - validator_authority_id, - expected_pda, - )?; - - result + } } fn close_outbox_account_cpi( @@ -324,7 +351,6 @@ mod tests { use crate::{ instruction_utils::InstructionUtils, test_utils::{ensure_started_validator, process_instruction}, - validator, }; fn single_acc_commit(commit_id: u64) -> SentCommit { @@ -379,7 +405,7 @@ mod tests { let mut ix = InstructionUtils::scheduled_commit_sent_instruction( commit.message_id, ); - ix.accounts[1].is_signer = false; + ix.accounts[0].is_signer = false; let transaction_accounts = transaction_accounts_from_map(&ix, &mut account_data); @@ -469,7 +495,20 @@ mod tests { fn test_registered_all_checks_out() { let commit = setup_registered_commit(); - let mut account_data = HashMap::new(); + let pda = outbox_intent_pda(commit.message_id); + let mut pda_account = AccountSharedData::new(0, 0, &crate::id()); + pda_account.set_ephemeral(true); + + let mut account_data = { + let mut map = HashMap::new(); + // Pre-fund vault so CloseEphemeralAccount CPI can refund sponsor + let mut vault = AccountSharedData::new(10_000, 0, &crate::id()); + vault.set_ephemeral(true); + map.insert(EPHEMERAL_VAULT_PUBKEY, vault); + // Add outbox PDA as existing ephemeral account (created by accept) + map.insert(pda, pda_account); + map + }; ensure_started_validator(&mut account_data, None); diff --git a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs index 144e69358..0785c9cc3 100644 --- a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs +++ b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs @@ -20,14 +20,13 @@ use solana_sdk_ids::{system_program, sysvar::clock}; use solana_signer::Signer; use crate::{ + intent_bundles::outbox_intent_bundles::OutboxIntentBundle, magic_context::MagicContext, magic_scheduled_base_intent::{ ScheduledIntentBundle, ACTUAL_COMMIT_LIMIT, COMMIT_FEE_LAMPORTS, }, magic_sys::COMMIT_LIMIT, - schedule_transactions::{ - magic_fee_vault_pubkey, transaction_scheduler::TransactionScheduler, - }, + schedule_transactions::magic_fee_vault_pubkey, test_utils::{ ensure_started_validator, process_instruction, process_instruction_with_logs, StubNonces, @@ -161,7 +160,6 @@ fn find_magic_context_account( fn assert_non_accepted_actions<'a>( processed_scheduled: &'a [AccountSharedData], - payer: &Pubkey, expected_non_accepted_commits: usize, ) -> &'a AccountSharedData { let magic_context_acc = find_magic_context_account(processed_scheduled) @@ -169,34 +167,68 @@ fn assert_non_accepted_actions<'a>( let magic_context = bincode::deserialize::(magic_context_acc.data()).unwrap(); - let accepted_scheduled_actions = - TransactionScheduler::default().get_scheduled_actions_by_payer(payer); assert_eq!( magic_context.scheduled_base_intents.len(), expected_non_accepted_commits ); - assert_eq!(accepted_scheduled_actions.len(), 0); magic_context_acc } fn assert_accepted_actions( processed_accepted: &[AccountSharedData], - payer: &Pubkey, - expected_scheduled_actions: usize, + pre_accept_magic_context: &AccountSharedData, + expected_accepted_count: usize, ) -> Vec { - let magic_context_acc = find_magic_context_account(processed_accepted) + let post_magic_context_acc = find_magic_context_account(processed_accepted) .expect("magic context account not found"); - let magic_context = - bincode::deserialize::(magic_context_acc.data()).unwrap(); - - let scheduled_actions = - TransactionScheduler::default().get_scheduled_actions_by_payer(payer); + let post_magic_context = + bincode::deserialize::(post_magic_context_acc.data()) + .unwrap(); + assert_eq!(post_magic_context.scheduled_base_intents.len(), 0); + + let pre_magic_context = + bincode::deserialize::(pre_accept_magic_context.data()) + .unwrap(); + let accepted_intents = pre_magic_context.scheduled_base_intents; + assert_eq!(accepted_intents.len(), expected_accepted_count); + + for intent in &accepted_intents { + let expected = OutboxIntentBundle::accepted(intent.clone()); + let actual = processed_accepted + .iter() + .filter(|acc| acc.owner() == &crate::id() && acc.ephemeral()) + .filter_map(|acc| { + OutboxIntentBundle::try_from_bytes(acc.data()).ok() + }) + .find(|bundle| bundle.inner.id == intent.id) + .unwrap_or_else(|| { + panic!( + "outbox PDA for intent {} not found in processed_accepted", + intent.id + ) + }); + assert_eq!(actual, expected); + } - assert_eq!(magic_context.scheduled_base_intents.len(), 0); - assert_eq!(scheduled_actions.len(), expected_scheduled_actions); + accepted_intents +} - scheduled_actions +/// Pre-populates uninitialized outbox intent PDA accounts into `account_data`. +/// The accept instruction creates these accounts via CPI (`CreateEphemeralAccount`), +/// which requires them to be present in the transaction context as uninitialized +/// system-owned entries — exactly what `validate_new_ephemeral` expects. +/// The first 4 accounts (validator, program, magic_context, vault) are already +/// in `account_data`, so we skip them. +fn ensure_outbox_pda_accounts_exist( + account_data: &mut HashMap, + accept_ix: &Instruction, +) { + for acc_meta in accept_ix.accounts.iter().skip(4) { + account_data.entry(acc_meta.pubkey).or_insert_with(|| { + AccountSharedData::new(0, 0, &system_program::id()) + }); + } } fn extend_transaction_accounts_from_ix( @@ -347,11 +379,8 @@ mod tests { // At this point the intent to commit was added to the magic context account, // but not yet accepted - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); (processed_scheduled.clone(), magic_context_acc.clone()) }; @@ -363,12 +392,17 @@ mod tests { &payer, program, committee, ); - let intent_ids = bincode::deserialize::(magic_context_acc.data()) - .unwrap() - .scheduled_base_intents - .into_iter() - .map(|i| i.id); - let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); + let intent_ids = + bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id) + .collect::>(); + let ix = InstructionUtils::accept_scheduled_commits_instruction( + intent_ids.into_iter(), + ); + ensure_outbox_pda_accounts_exist(&mut account_data, &ix); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -386,7 +420,7 @@ mod tests { // At this point the intended commits were accepted and moved to the global let scheduled_intents = assert_accepted_actions( &processed_accepted, - &payer.pubkey(), + &magic_context_acc, 1, ); @@ -479,11 +513,8 @@ mod tests { Ok(()), ); - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); let magic_context = bincode::deserialize::(magic_context_acc.data()) .unwrap(); @@ -540,11 +571,8 @@ mod tests { // At this point the intent to commit was added to the magic context account, // but not yet accepted - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); (processed_scheduled.clone(), magic_context_acc.clone()) }; @@ -556,12 +584,17 @@ mod tests { &payer, program, committee, ); - let intent_ids = bincode::deserialize::(magic_context_acc.data()) - .unwrap() - .scheduled_base_intents - .into_iter() - .map(|i| i.id); - let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); + let intent_ids = + bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id) + .collect::>(); + let ix = InstructionUtils::accept_scheduled_commits_instruction( + intent_ids.into_iter(), + ); + ensure_outbox_pda_accounts_exist(&mut account_data, &ix); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -579,7 +612,7 @@ mod tests { // At this point the intended commits were accepted and moved to the global let scheduled_commits = assert_accepted_actions( &processed_accepted, - &payer.pubkey(), + &magic_context_acc, 1, ); @@ -640,25 +673,26 @@ mod tests { ); // Extract magic context and then accept scheduled commits - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); - let intent_ids = bincode::deserialize::(magic_context_acc.data()) - .unwrap() - .scheduled_base_intents - .into_iter() - .map(|i| i.id); - let ix_accept = - InstructionUtils::accept_scheduled_commits_instruction(intent_ids); + let intent_ids = + bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id) + .collect::>(); + let ix_accept = InstructionUtils::accept_scheduled_commits_instruction( + intent_ids.into_iter(), + ); let (mut account_data2, mut transaction_accounts2) = prepare_transaction_with_single_committee( &payer, Pubkey::new_unique(), ata_pubkey, ); + ensure_outbox_pda_accounts_exist(&mut account_data2, &ix_accept); extend_transaction_accounts_from_ix_adding_magic_context( &ix_accept, magic_context_acc, @@ -673,7 +707,7 @@ mod tests { ); let scheduled = - assert_accepted_actions(&processed_accepted, &payer.pubkey(), 1); + assert_accepted_actions(&processed_accepted, magic_context_acc, 1); // Verify the committed pubkey remapped to eATA assert_eq!( scheduled[0].intent_bundle.get_all_committed_pubkeys(), @@ -728,11 +762,8 @@ mod tests { ix.accounts, Ok(()), ); - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); let magic_context = bincode::deserialize::(magic_context_acc.data()) .unwrap(); @@ -791,25 +822,26 @@ mod tests { ); // Extract magic context and then accept scheduled commits - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); - let intent_ids = bincode::deserialize::(magic_context_acc.data()) - .unwrap() - .scheduled_base_intents - .into_iter() - .map(|i| i.id); - let ix_accept = - InstructionUtils::accept_scheduled_commits_instruction(intent_ids); + let intent_ids = + bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id) + .collect::>(); + let ix_accept = InstructionUtils::accept_scheduled_commits_instruction( + intent_ids.into_iter(), + ); let (mut account_data2, mut transaction_accounts2) = prepare_transaction_with_single_committee( &payer, Pubkey::new_unique(), ata_pubkey, ); + ensure_outbox_pda_accounts_exist(&mut account_data2, &ix_accept); extend_transaction_accounts_from_ix_adding_magic_context( &ix_accept, magic_context_acc, @@ -824,7 +856,7 @@ mod tests { ); let scheduled = - assert_accepted_actions(&processed_accepted, &payer.pubkey(), 1); + assert_accepted_actions(&processed_accepted, magic_context_acc, 1); // Verify the committed pubkey remapped to eATA assert_eq!( scheduled[0].intent_bundle.get_all_committed_pubkeys(), @@ -885,11 +917,8 @@ mod tests { // At this point the intent to commit was added to the magic context account, // but not yet accepted - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); ( processed_scheduled.clone(), @@ -913,12 +942,17 @@ mod tests { (true, true, true), ); - let intent_ids = bincode::deserialize::(magic_context_acc.data()) - .unwrap() - .scheduled_base_intents - .into_iter() - .map(|i| i.id); - let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); + let intent_ids = + bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id) + .collect::>(); + let ix = InstructionUtils::accept_scheduled_commits_instruction( + intent_ids.into_iter(), + ); + ensure_outbox_pda_accounts_exist(&mut accounts_data, &ix); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -936,7 +970,7 @@ mod tests { // At this point the intended commits were accepted and moved to the global let scheduled_commits = assert_accepted_actions( &processed_accepted, - &payer.pubkey(), + &magic_context_acc, 1, ); @@ -1005,11 +1039,8 @@ mod tests { // At this point the intent to commit was added to the magic context account, // but not yet accepted - let magic_context_acc = assert_non_accepted_actions( - &processed_scheduled, - &payer.pubkey(), - 1, - ); + let magic_context_acc = + assert_non_accepted_actions(&processed_scheduled, 1); ( processed_scheduled.clone(), @@ -1033,12 +1064,17 @@ mod tests { (true, true, true), ); - let intent_ids = bincode::deserialize::(magic_context_acc.data()) - .unwrap() - .scheduled_base_intents - .into_iter() - .map(|i| i.id); - let ix = InstructionUtils::accept_scheduled_commits_instruction(intent_ids); + let intent_ids = + bincode::deserialize::(magic_context_acc.data()) + .unwrap() + .scheduled_base_intents + .into_iter() + .map(|i| i.id) + .collect::>(); + let ix = InstructionUtils::accept_scheduled_commits_instruction( + intent_ids.into_iter(), + ); + ensure_outbox_pda_accounts_exist(&mut accounts_data, &ix); extend_transaction_accounts_from_ix_adding_magic_context( &ix, &magic_context_acc, @@ -1056,7 +1092,7 @@ mod tests { // At this point the intended commits were accepted and moved to the global let scheduled_commits = assert_accepted_actions( &processed_accepted, - &payer.pubkey(), + &magic_context_acc, 1, ); diff --git a/programs/magicblock/src/lib.rs b/programs/magicblock/src/lib.rs index a2de797e7..5820b3535 100644 --- a/programs/magicblock/src/lib.rs +++ b/programs/magicblock/src/lib.rs @@ -11,7 +11,6 @@ pub mod magicblock_processor; pub mod test_utils; mod utils; pub mod validator; -// TODO(edwin): safe to just rename or will break integrations? pub use intent_bundles::{ magic_scheduled_base_intent, outbox_intent_bundles, schedule as schedule_transactions, From e26504108d6e3b0e9b1fc45779d792f88ce0dacd Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 21:14:28 +0700 Subject: [PATCH 80/97] refactor: todo removals --- .../process_accept_scheduled_commits.rs | 21 ++++++++++++++----- .../intent_bundles/outbox_intent_bundles.rs | 1 - programs/magicblock/src/magic_context.rs | 1 - 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index dc09795cf..bc5bbc9a8 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -74,10 +74,23 @@ fn validate( MAGIC_CONTEXT_IDX, )?; - // Validate authority let transaction_context = &*invoke_context.transaction_context; - // TODO(edwin): add check for magic-program + // Assert magic program account + let magic_program_pubkey = + get_instruction_pubkey_with_idx(transaction_context, MAGIC_PROGRAM_ID)?; + if *magic_program_pubkey != crate::id() { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: account at idx {} is {}, expected magic program {}", + MAGIC_PROGRAM_ID, + magic_program_pubkey, + crate::id() + ); + return Err(InstructionError::IncorrectProgramId); + } + + // Assert validator authority let provided_validator_auth = get_instruction_pubkey_with_idx( transaction_context, VALIDATOR_AUTHORITY_IDX, @@ -110,7 +123,6 @@ fn verify_intent_pda( intent_id: u64, pda_idx: u16, ) -> Result { - // TODO(edwin): add check that acount doesn't exist? let transaction_context = &*invoke_context.transaction_context; let provided = get_instruction_pubkey_with_idx(transaction_context, pda_idx)?; @@ -206,10 +218,9 @@ fn create_outbox_account_cpi( invoke_context, validator_auth, pda, - data.len() as u32, // TODO(edwin): fix cast + data.len() as u32, )?; - // TODO(edwin): simplify/ // Move intent data in new account let transaction_context = &*invoke_context.transaction_context; let tx_idx = transaction_context diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 3fe6628b4..f07dfa142 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -87,7 +87,6 @@ impl OutboxIntentBundleStatus { )) } - // TODO(edwin): split into is_valid_transition and apply_transaction fn apply_stage_transition( &mut self, stage: ExecutionStage, diff --git a/programs/magicblock/src/magic_context.rs b/programs/magicblock/src/magic_context.rs index 03caa1110..8105486bb 100644 --- a/programs/magicblock/src/magic_context.rs +++ b/programs/magicblock/src/magic_context.rs @@ -66,7 +66,6 @@ impl MagicContext { const ID_OFFSET: usize = 0; const ID_END: usize = mem::size_of::(); - // TODO(edwin): should exist, return Option and cast to error let Some(raw_id) = data.get(ID_OFFSET..ID_END) else { return 0; }; From 1b5363b5ce95639a15cab16a8fc2c28e2a6e3a98 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 21:47:48 +0700 Subject: [PATCH 81/97] feat: replace DummyDb with more ram friendly one. DummyDb test-only now --- magicblock-api/src/magic_sys_adapter.rs | 5 +- magicblock-api/src/magic_validator.rs | 14 ++- .../src/committor_processor.rs | 12 +-- .../src/intent_engine.rs | 8 +- .../src/intent_engine/db.rs | 101 ++++++++---------- .../src/intent_engine/intent_channerl.rs | 10 +- .../intent_engine/intent_execution_engine.rs | 6 +- magicblock-committor-service/src/service.rs | 17 +-- .../tests/test_ix_commit_local.rs | 7 +- 9 files changed, 91 insertions(+), 89 deletions(-) diff --git a/magicblock-api/src/magic_sys_adapter.rs b/magicblock-api/src/magic_sys_adapter.rs index e214442fc..d42a41748 100644 --- a/magicblock-api/src/magic_sys_adapter.rs +++ b/magicblock-api/src/magic_sys_adapter.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use magicblock_committor_service::{ committor_processor::CommittorProcessor, + intent_engine::db::DummyIntentBacklog, tasks::task_info_fetcher::TaskInfoFetcherResult, }; use magicblock_core::{intent::CommittedAccount, traits::MagicSys}; @@ -13,7 +14,7 @@ use tracing::error; #[derive(Clone)] pub struct MagicSysAdapter { handle: tokio::runtime::Handle, - committor_processor: Arc, + committor_processor: Arc>, } impl MagicSysAdapter { @@ -28,7 +29,7 @@ impl MagicSysAdapter { pub fn new( handle: tokio::runtime::Handle, - committor_processor: Arc, + committor_processor: Arc>, ) -> Self { Self { handle, diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 2126c596b..0b275426c 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -20,6 +20,7 @@ use magicblock_chainlink::{ }; use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, + intent_engine::db::DummyIntentBacklog, outbox::outbox_client::InternalOutboxClient, service::IntentExecutionService, ComputeBudgetConfig, DEFAULT_ACTIONS_TIMEOUT, @@ -98,8 +99,12 @@ type InnerChainlinkImpl = ProdInnerChainlink; type ChainlinkImpl = ProdChainlink; -type IntentExecutionServiceImpl = - IntentExecutionService>; +type CommittorProcessorImpl = CommittorProcessor; + +type IntentExecutionServiceImpl = IntentExecutionService< + InternalOutboxClient, + DummyIntentBacklog, +>; // ----------------- // MagicValidator @@ -245,6 +250,7 @@ impl MagicValidator { let processor = Self::init_committor_processor( &config, ledger.latest_block(), + &accountsdb, &outbox_client, &shared_chain_slot, ); @@ -476,9 +482,10 @@ impl MagicValidator { pub fn init_committor_processor( config: &ValidatorParams, latest_block: &LatestBlock, + accounts_db: &Arc, outbox_client: &Arc>, shared_chain_slot: &Option>, - ) -> CommittorProcessor { + ) -> CommittorProcessorImpl { let authority = config.validator.keypair.insecure_clone(); let base_chain_config = ChainConfig { rpc_uri: config.rpc_url().to_owned(), @@ -503,6 +510,7 @@ impl MagicValidator { authority, base_chain_config, shared_chain_slot.clone(), + DummyIntentBacklog::new(accounts_db.clone()), outbox_client.clone(), actions_callback_executor, ) diff --git a/magicblock-committor-service/src/committor_processor.rs b/magicblock-committor-service/src/committor_processor.rs index 961433c29..98416d4ef 100644 --- a/magicblock-committor-service/src/committor_processor.rs +++ b/magicblock-committor-service/src/committor_processor.rs @@ -18,7 +18,7 @@ use crate::{ config::ChainConfig, error::{CommittorServiceError, CommittorServiceResult}, intent_engine::{ - db::DummyDB, BroadcastedIntentExecutionResult, IntentEngineHandle, + db::BacklogDB, BroadcastedIntentExecutionResult, IntentEngineHandle, }, intent_executor::{ error::IntentExecutorError, intent_executor_factory::ExecutorConfig, @@ -35,18 +35,19 @@ const POISONED_MUTEX_MSG: &str = type BundleResultListener = oneshot::Sender; -pub struct CommittorProcessor { +pub struct CommittorProcessor { _table_mania: TableMania, - commits_scheduler: IntentEngineHandle, + commits_scheduler: IntentEngineHandle, task_info_fetcher: Arc>, pending_result_listeners: Arc>>, } -impl CommittorProcessor { +impl CommittorProcessor { pub fn new( authority: Keypair, chain_config: ChainConfig, chain_slot: Option>, + db: D, outbox_client: Arc, actions_callback_executor: A, ) -> Self @@ -92,8 +93,7 @@ impl CommittorProcessor { )); let commits_scheduler = IntentEngineHandle::new( magic_block_rpc_client.clone(), - // TODO(edwin): use DumberDb - DummyDB::new(), + db, task_info_fetcher.clone(), outbox_client, table_mania.clone(), diff --git a/magicblock-committor-service/src/intent_engine.rs b/magicblock-committor-service/src/intent_engine.rs index 4d7efc375..0a297eead 100644 --- a/magicblock-committor-service/src/intent_engine.rs +++ b/magicblock-committor-service/src/intent_engine.rs @@ -1,4 +1,4 @@ -pub(crate) mod db; +pub mod db; pub mod intent_channerl; mod intent_execution_engine; pub mod intent_scheduler; @@ -14,7 +14,7 @@ use tokio::sync::broadcast; use crate::{ intent_engine::{ - db::DB, + db::BacklogDB, intent_channerl::{channel, IntentScheduleError, IntentScheduleHandle}, intent_execution_engine::{IntentExecutionEngine, ResultSubscriber}, }, @@ -26,12 +26,12 @@ use crate::{ tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, }; -pub struct IntentEngineHandle { +pub struct IntentEngineHandle { intent_schedule_handle: IntentScheduleHandle, result_subscriber: ResultSubscriber, } -impl IntentEngineHandle { +impl IntentEngineHandle { pub fn new( rpc_client: MagicblockRpcClient, db: D, diff --git a/magicblock-committor-service/src/intent_engine/db.rs b/magicblock-committor-service/src/intent_engine/db.rs index 70084fb94..5ad9cab90 100644 --- a/magicblock-committor-service/src/intent_engine/db.rs +++ b/magicblock-committor-service/src/intent_engine/db.rs @@ -1,8 +1,4 @@ -use std::{ - cell::RefCell, - collections::VecDeque, - sync::{Arc, Mutex}, -}; +use std::{cell::RefCell, collections::VecDeque, sync::Arc}; /// DB for storing intents that overflow committor channel use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; @@ -11,9 +7,7 @@ use magicblock_metrics::metrics; use magicblock_program::outbox_intent_bundles::OutboxIntentBundle; use solana_account::ReadableAccount; -const POISONED_MUTEX_MSG: &str = "Dummy db mutex poisoned"; - -pub trait DB: Send + 'static { +pub trait BacklogDB: Send + 'static { fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, @@ -28,27 +22,29 @@ pub trait DB: Send + 'static { fn is_empty(&self) -> bool; } -pub(crate) struct DummyDB { - db: Mutex>, +pub struct DummyIntentBacklog { + accounts_db: Arc, + queue: RefCell>, } -impl DummyDB { - pub fn new() -> Self { +impl DummyIntentBacklog { + pub fn new(accounts_db: Arc) -> Self { Self { - db: Mutex::new(VecDeque::new()), + accounts_db, + queue: RefCell::new(VecDeque::new()), } } } -impl DB for DummyDB { +impl BacklogDB for DummyIntentBacklog { fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, ) -> DBResult<()> { - let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); - db.push_back(intent_bundle); + let mut queue = self.queue.borrow_mut(); + queue.push_back(intent_bundle.inner.id); - metrics::set_committor_intents_backlog_count(db.len() as i64); + metrics::set_committor_intents_backlog_count(queue.len() as i64); Ok(()) } @@ -56,49 +52,57 @@ impl DB for DummyDB { &self, intent_bundles: Vec, ) -> DBResult<()> { - let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); - db.extend(intent_bundles); + let mut queue = self.queue.borrow_mut(); + queue.extend(intent_bundles.into_iter().map(|el| el.inner.id)); - metrics::set_committor_intents_backlog_count(db.len() as i64); + metrics::set_committor_intents_backlog_count(queue.len() as i64); Ok(()) } fn pop_intent_bundle(&self) -> DBResult> { - let mut db = self.db.lock().expect(POISONED_MUTEX_MSG); - let res = db.pop_front(); + let mut queue = self.queue.borrow_mut(); + let Some(id) = queue.pop_front() else { + return Ok(None); + }; + metrics::set_committor_intents_backlog_count(queue.len() as i64); + drop(queue); - metrics::set_committor_intents_backlog_count(db.len() as i64); - Ok(res) + let intent_pda = outbox_intent_pda(id); + let account = self + .accounts_db + .get_account(&intent_pda) + .ok_or(Error::IntentNotFoundError(id))?; + + let outbox_intent = OutboxIntentBundle::try_from_bytes(account.data())?; + Ok(Some(outbox_intent)) } fn is_empty(&self) -> bool { - self.db.lock().expect(POISONED_MUTEX_MSG).is_empty() + self.queue.borrow().is_empty() } } -pub struct DumberDB { - accounts_db: Arc, - queue: RefCell>, +#[cfg(any(test, feature = "dev-context-only-utils"))] +pub struct DummyDB { + db: std::sync::Mutex>, } -impl DumberDB { - pub fn new(accounts_db: Arc) -> Self { +#[cfg(any(test, feature = "dev-context-only-utils"))] +impl DummyDB { + pub fn new() -> Self { Self { - accounts_db, - queue: RefCell::new(VecDeque::new()), + db: std::sync::Mutex::new(VecDeque::new()), } } } -impl DB for DumberDB { +#[cfg(any(test, feature = "dev-context-only-utils"))] +impl BacklogDB for DummyDB { fn store_intent_bundle( &self, intent_bundle: OutboxIntentBundle, ) -> DBResult<()> { - let mut queue = self.queue.borrow_mut(); - queue.push_back(intent_bundle.inner.id); - - metrics::set_committor_intents_backlog_count(queue.len() as i64); + self.db.lock().unwrap().push_back(intent_bundle); Ok(()) } @@ -106,33 +110,16 @@ impl DB for DumberDB { &self, intent_bundles: Vec, ) -> DBResult<()> { - let mut queue = self.queue.borrow_mut(); - queue.extend(intent_bundles.into_iter().map(|el| el.inner.id)); - - metrics::set_committor_intents_backlog_count(queue.len() as i64); + self.db.lock().unwrap().extend(intent_bundles); Ok(()) } fn pop_intent_bundle(&self) -> DBResult> { - let mut queue = self.queue.borrow_mut(); - let Some(id) = queue.pop_front() else { - return Ok(None); - }; - metrics::set_committor_intents_backlog_count(queue.len() as i64); - drop(queue); - - let intent_pda = outbox_intent_pda(id); - let account = self - .accounts_db - .get_account(&intent_pda) - .ok_or(Error::IntentNotFoundError(id))?; - - let outbox_intent = OutboxIntentBundle::try_from_bytes(account.data())?; - Ok(Some(outbox_intent)) + Ok(self.db.lock().unwrap().pop_front()) } fn is_empty(&self) -> bool { - self.queue.borrow().is_empty() + self.db.lock().unwrap().is_empty() } } diff --git a/magicblock-committor-service/src/intent_engine/intent_channerl.rs b/magicblock-committor-service/src/intent_engine/intent_channerl.rs index 0abe6f6d6..baa2ad571 100644 --- a/magicblock-committor-service/src/intent_engine/intent_channerl.rs +++ b/magicblock-committor-service/src/intent_engine/intent_channerl.rs @@ -13,7 +13,7 @@ use tokio::sync::{ }; use tokio_stream::{wrappers::ReceiverStream, Stream}; -use crate::intent_engine::{db, db::DB}; +use crate::intent_engine::{db, db::BacklogDB}; const POISONED_MSG: &str = "Dummy DB mutex poisoned"; @@ -23,7 +23,7 @@ pub struct IntentScheduleHandle { sender: Sender, } -impl IntentScheduleHandle { +impl IntentScheduleHandle { pub fn new(db: Arc>, sender: Sender) -> Self { Self { db, sender } } @@ -71,7 +71,7 @@ pub struct IntentStream { stream: ReceiverStream, } -impl IntentStream { +impl IntentStream { pub fn new( db: Arc>, receiver: Receiver, @@ -83,7 +83,7 @@ impl IntentStream { } } -impl Stream for IntentStream { +impl Stream for IntentStream { type Item = Result; fn poll_next( @@ -105,7 +105,7 @@ impl Stream for IntentStream { } } -pub(crate) fn channel( +pub(crate) fn channel( db: &Arc>, buffer: usize, ) -> (IntentScheduleHandle, IntentStream) { diff --git a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs index be9ae1d8f..7fd7be8b9 100644 --- a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs @@ -21,7 +21,7 @@ use tracing::{error, info, instrument, trace, warn}; use crate::tasks::task_strategist::TransactionStrategy; use crate::{ intent_engine::{ - db::DB, + db::BacklogDB, intent_channerl::{IntentScheduleError, IntentStream}, intent_scheduler::{IntentScheduler, POISONED_INNER_MSG}, }, @@ -104,7 +104,7 @@ pub(crate) struct IntentExecutionEngine { impl IntentExecutionEngine where - D: DB, + D: BacklogDB, T: TransactionPreparator, F: IntentExecutorBuilder + Send + Sync + 'static, { @@ -359,7 +359,7 @@ mod tests { use super::*; use crate::{ intent_engine::{ - db::{DummyDB, DB}, + db::{BacklogDB, DummyDB}, intent_channerl::{channel, IntentScheduleHandle}, intent_scheduler::{create_test_intent, create_test_intent_bundle}, }, diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index 601e2b712..cb278b229 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -17,6 +17,7 @@ use tracing::error; use crate::{ committor_processor::CommittorProcessor, error::CommittorServiceResult, + intent_engine::db::BacklogDB, intent_executor::error::IntentExecutorError, outbox::{ outbox_client::InternalOutboxClientError, @@ -30,14 +31,14 @@ use crate::{ pub type InnerChainlinkImpl = ProdInnerChainlink; pub type ChainlinkImpl = ProdChainlink; -pub enum IntentExecutionService { - Created(ServiceInner), +pub enum IntentExecutionService { + Created(ServiceInner), Started(JoinHandle<()>), Stopped, Error, } -impl IntentExecutionService +impl IntentExecutionService where O: OutboxClient, // OutboxClient errors should be convertible to Service errors @@ -51,7 +52,7 @@ where pub fn new( chainlink: Arc, intent_client: Arc, - processor: Arc, + processor: Arc>, slot_interval: Duration, cancellation_token: CancellationToken, ) -> Self { @@ -93,20 +94,20 @@ where } } -pub struct ServiceInner { +pub struct ServiceInner { /// Chainlink for notifying of undelegations chainlink: Arc, /// ER client specific for Intent needs. Could be switched to RpcClient outbox_client: Arc, /// Processor of accepted intents - processor: Arc, + processor: Arc>, /// Time interval to scrape MagicContext(ER slot interval) // TODO(edwin): can be removed if LatestBlocK moved into magicblock-core slot_interval: Duration, cancellation_token: CancellationToken, } -impl ServiceInner +impl ServiceInner where O: OutboxClient, // OutboxClient errors should be convertible to Service errors @@ -120,7 +121,7 @@ where pub fn new( chainlink: Arc, outbox_client: Arc, - processor: Arc, + processor: Arc>, slot_interval: Duration, cancellation_token: CancellationToken, ) -> Self { diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index dda3b9e35..e455cb1cb 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -4,6 +4,7 @@ use borsh::to_vec; use magicblock_committor_service::{ committor_processor::CommittorProcessor, config::ChainConfig, + intent_engine::db::DummyDB, intent_executor::{error::IntentExecutorError, ExecutionOutput}, tasks::{ commit_task::CommitDelivery, task_strategist::TransactionStrategy, @@ -351,6 +352,7 @@ async fn commit_single_account( validator_auth.insecure_clone(), ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), None, + DummyDB::new(), Arc::new(common::MockOutboxClient), common::MockActionsCallbackExecutor::default(), )); @@ -432,6 +434,7 @@ async fn commit_book_order_account( validator_auth.insecure_clone(), ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), None, + DummyDB::new(), Arc::new(common::MockOutboxClient), common::MockActionsCallbackExecutor::default(), )); @@ -904,6 +907,7 @@ async fn commit_multiple_accounts( validator_auth.insecure_clone(), ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), None, + DummyDB::new(), Arc::new(common::MockOutboxClient), common::MockActionsCallbackExecutor::default(), )); @@ -972,6 +976,7 @@ async fn execute_intent_bundle( validator_auth.insecure_clone(), ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), None, + DummyDB::new(), Arc::new(common::MockOutboxClient), common::MockActionsCallbackExecutor::default(), )); @@ -1045,7 +1050,7 @@ async fn execute_intent_bundle( // Test Executor // ----------------- async fn ix_commit_local( - processor: Arc, + processor: Arc>, intent_bundles: Vec, expected_strategies: ExpectedStrategies, program_id: Pubkey, From a0d584886023f49e202a9619acd03ba210af083b Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 21:53:30 +0700 Subject: [PATCH 82/97] fix: some lints --- magicblock-committor-service/src/intent_engine/db.rs | 11 +++++++++-- .../src/intent_engine/intent_execution_engine.rs | 2 +- .../intent_executor/strategy_executor/single_stage.rs | 1 + .../src/intent_executor/strategy_executor/utils.rs | 5 +++-- .../src/intent_executor/utils.rs | 4 ++-- .../schedule/process_schedule_commit_tests.rs | 6 +++--- 6 files changed, 19 insertions(+), 10 deletions(-) diff --git a/magicblock-committor-service/src/intent_engine/db.rs b/magicblock-committor-service/src/intent_engine/db.rs index 5ad9cab90..fde645843 100644 --- a/magicblock-committor-service/src/intent_engine/db.rs +++ b/magicblock-committor-service/src/intent_engine/db.rs @@ -88,14 +88,21 @@ pub struct DummyDB { } #[cfg(any(test, feature = "dev-context-only-utils"))] -impl DummyDB { - pub fn new() -> Self { +impl Default for DummyDB { + fn default() -> Self { Self { db: std::sync::Mutex::new(VecDeque::new()), } } } +#[cfg(any(test, feature = "dev-context-only-utils"))] +impl DummyDB { + pub fn new() -> Self { + Self::default() + } +} + #[cfg(any(test, feature = "dev-context-only-utils"))] impl BacklogDB for DummyDB { fn store_intent_bundle( diff --git a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs index 7fd7be8b9..baa7415b1 100644 --- a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs @@ -117,7 +117,7 @@ where MAX_EXECUTORS as usize, )), inner: Arc::new(Mutex::new(IntentScheduler::new())), - _phantom_data: PhantomData::default(), + _phantom_data: PhantomData, } } diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 1452b8182..9eaf15f72 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -49,6 +49,7 @@ where O: OutboxClient, O::Error: Into, { + #[allow(clippy::too_many_arguments)] pub fn new( authority: Keypair, intent_id: u64, diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index 941a53d3d..fb32c5a93 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -46,6 +46,7 @@ pub(in crate::intent_executor) struct ExecutionState<'a> { pub execution_report: &'a mut IntentExecutionReport, } +#[allow(clippy::too_many_arguments)] pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( authority: &Keypair, intent_client: &IntentExecutionClient, @@ -145,7 +146,7 @@ pub async fn prepare_transaction( transaction_strategy: &mut TransactionStrategy, ) -> Result { let mut prepared_message = transaction_preparator - .prepare_for_strategy(&authority, transaction_strategy) + .prepare_for_strategy(authority, transaction_strategy) .await?; // Get latest blockhash(Part of preparation I guess) @@ -176,7 +177,7 @@ pub(in crate::intent_executor) async fn check_pending_signature( .await .map_err(IntentExecutorError::GetPendingSignatureStatusError)?; - match statuses.get(0) { + match statuses.first() { Some(Some(Ok(()))) => Ok(ControlFlow::Break(())), None => { // well, that is bizarre one diff --git a/magicblock-committor-service/src/intent_executor/utils.rs b/magicblock-committor-service/src/intent_executor/utils.rs index d046359e8..b575830ad 100644 --- a/magicblock-committor-service/src/intent_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/utils.rs @@ -37,9 +37,9 @@ pub(in crate::intent_executor) async fn build_commit_finalize_tasks< task_info_fetcher: &Arc, ) -> IntentExecutorResult<(Vec, Vec)> { let commit_tasks_fut = - TaskBuilderImpl::commit_tasks(&task_info_fetcher, &intent_bundle); + TaskBuilderImpl::commit_tasks(task_info_fetcher, intent_bundle); let finalize_tasks_fut = - TaskBuilderImpl::finalize_tasks(&task_info_fetcher, &intent_bundle); + TaskBuilderImpl::finalize_tasks(task_info_fetcher, intent_bundle); let (commit_tasks, finalize_tasks) = join(commit_tasks_fut, finalize_tasks_fut).await; diff --git a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs index 0785c9cc3..c5f6f1839 100644 --- a/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs +++ b/programs/magicblock/src/intent_bundles/schedule/process_schedule_commit_tests.rs @@ -158,10 +158,10 @@ fn find_magic_context_account( .find(|acc| acc.owner() == &crate::id() && acc.lamports() == u64::MAX) } -fn assert_non_accepted_actions<'a>( - processed_scheduled: &'a [AccountSharedData], +fn assert_non_accepted_actions( + processed_scheduled: &[AccountSharedData], expected_non_accepted_commits: usize, -) -> &'a AccountSharedData { +) -> &AccountSharedData { let magic_context_acc = find_magic_context_account(processed_scheduled) .expect("magic context account not found"); let magic_context = From db043d97ed41ee524798a97ffed4e5abe95a45c3 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 29 Jun 2026 21:54:30 +0700 Subject: [PATCH 83/97] fix: lint --- magicblock-api/src/magic_validator.rs | 2 +- magicblock-committor-service/src/service.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 0b275426c..fadde077a 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -242,7 +242,7 @@ impl MagicValidator { &config, &accountsdb, &dispatch.transaction_scheduler, - &ledger.latest_block(), + ledger.latest_block(), ); Arc::new(val) }; diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index cb278b229..ae42e2a3d 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -301,6 +301,6 @@ pub enum IntentExecutionServiceError { IntentRpcClientError(#[from] InternalOutboxClientError), #[error("OutboxReaderError")] OutboxReaderError(#[from] OutboxIntentBundlesReaderError), - #[error("asd")] - ASdError(#[from] IntentExecutorError), + #[error("IntentExecutorError: {0}")] + IntentExecutorError(#[from] Box), } From e5d20bcbd0eaa1e4649d7069fe209f38821c7cb4 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 1 Jul 2026 13:48:28 +0700 Subject: [PATCH 84/97] fix: address regression --- .../src/outbox/mod.rs | 24 +++++++++++++++++-- .../src/outbox/outbox_client.rs | 19 ++++++++++----- .../src/outbox/utils.rs | 5 ++-- magicblock-committor-service/src/service.rs | 8 ++++++- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/magicblock-committor-service/src/outbox/mod.rs b/magicblock-committor-service/src/outbox/mod.rs index a6e877b6e..71d3ff3b5 100644 --- a/magicblock-committor-service/src/outbox/mod.rs +++ b/magicblock-committor-service/src/outbox/mod.rs @@ -59,7 +59,7 @@ pub struct ScheduledBaseIntentMeta { pub blockhash: Hash, pub payer: Pubkey, pub included_pubkeys: Vec, - pub intent_sent_transaction: Transaction, + pub intent_sent_transaction: IntentSentTransaction, pub requested_undelegation: bool, } @@ -71,8 +71,28 @@ impl ScheduledBaseIntentMeta { blockhash: intent.blockhash, payer: intent.payer, included_pubkeys: intent.get_all_committed_pubkeys(), - intent_sent_transaction: intent.sent_transaction.clone(), + intent_sent_transaction: if intent + .sent_transaction + .signatures + .is_empty() + { + IntentSentTransaction::Recovered + } else { + IntentSentTransaction::Known(intent.sent_transaction.clone()) + }, requested_undelegation: intent.has_undelegate_intent(), } } } + +/// Tracks the `ScheduledCommitSent` transaction for an in-flight intent. +/// +/// Bundles recovered from the on-chain outbox have a stale `sent_transaction` +/// (original blockhash is expired by restart time). `Recovered` signals that +/// the transaction must be rebuilt at notification time with a fresh ER blockhash. +#[derive(Default)] +pub enum IntentSentTransaction { + Known(Transaction), + #[default] + Recovered, +} diff --git a/magicblock-committor-service/src/outbox/outbox_client.rs b/magicblock-committor-service/src/outbox/outbox_client.rs index 163cf1db5..be85fdba3 100644 --- a/magicblock-committor-service/src/outbox/outbox_client.rs +++ b/magicblock-committor-service/src/outbox/outbox_client.rs @@ -1,4 +1,4 @@ -use std::{num::NonZeroUsize, sync::Arc, time::Duration}; +use std::{mem, num::NonZeroUsize, sync::Arc, time::Duration}; use async_trait::async_trait; use backoff::{future::retry, ExponentialBackoff}; @@ -31,7 +31,8 @@ use crate::{ }, outbox::{ outbox_intent_bundles_reader::InternalOutboxIntentBundlesReader, - utils::build_sent_commit, OutboxClient, ScheduledBaseIntentMeta, + utils::build_sent_commit, IntentSentTransaction, OutboxClient, + ScheduledBaseIntentMeta, }, }; @@ -184,15 +185,21 @@ impl OutboxClient for InternalOutboxClient { async fn notify_commit_sent( &self, - meta: ScheduledBaseIntentMeta, + mut meta: ScheduledBaseIntentMeta, result: &IntentExecutorResult, execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { - let (sent_tx, sent_commit) = - build_sent_commit(meta, result, execution_report); + let tx = match mem::take(&mut meta.intent_sent_transaction) { + IntentSentTransaction::Known(tx) => tx, + IntentSentTransaction::Recovered => { + let blockhash = self.latest_block_provider.blockhash(); + InstructionUtils::scheduled_commit_sent(meta.id, blockhash) + } + }; + let sent_commit = build_sent_commit(meta, result, execution_report); // TODO(edwin): is using handle directly here ok? This could require Chainlink mechanics register_scheduled_commit_sent(sent_commit); - let txn = with_encoded(sent_tx).inspect_err(|err| { + let txn = with_encoded(tx).inspect_err(|err| { // Unreachable case, all intent transactions are smaller than 64KB by construction error!(error = ?err, "Failed to bincode intent transaction"); })?; diff --git a/magicblock-committor-service/src/outbox/utils.rs b/magicblock-committor-service/src/outbox/utils.rs index 12ef4f26c..db4e05fe4 100644 --- a/magicblock-committor-service/src/outbox/utils.rs +++ b/magicblock-committor-service/src/outbox/utils.rs @@ -1,5 +1,4 @@ use magicblock_program::SentCommit; -use solana_transaction::Transaction; use tracing::{error, info}; use crate::{ @@ -13,7 +12,7 @@ pub(crate) fn build_sent_commit( meta: ScheduledBaseIntentMeta, result: &IntentExecutorResult, execution_report: &IntentExecutionReport, -) -> (Transaction, SentCommit) { +) -> SentCommit { let error_message = result.as_ref().err().map(|err| format!("{:?}", err)); let chain_signatures = match result { @@ -75,5 +74,5 @@ pub(crate) fn build_sent_commit( callbacks_scheduling_results, }; - (meta.intent_sent_transaction, sent_commit) + sent_commit } diff --git a/magicblock-committor-service/src/service.rs b/magicblock-committor-service/src/service.rs index ae42e2a3d..8786a25d6 100644 --- a/magicblock-committor-service/src/service.rs +++ b/magicblock-committor-service/src/service.rs @@ -7,6 +7,7 @@ use magicblock_account_cloner::ChainlinkCloner; use magicblock_chainlink::{ProdChainlink, ProdInnerChainlink}; use magicblock_metrics::metrics::{self}; use magicblock_program::{outbox_intent_bundles::OutboxIntentBundle, Pubkey}; +use solana_transaction::Transaction; use tokio::{ task, task::{JoinError, JoinHandle}, @@ -183,10 +184,15 @@ where let mut outbox_bundles_reader = self.outbox_client.outbox_reader(); loop { // Read by chunks in order not to overload `IntentExecutionEngine` - let intent_bundles_chunk = outbox_bundles_reader + let mut intent_bundles_chunk = outbox_bundles_reader .read(RESCHEDULE_CHUNK_SIZE.get()) .await .map_err(Into::into)?; + // Original blockhash is stale after restart; signal recovery so + // notify_commit_sent rebuilds the tx with a fresh ER blockhash. + intent_bundles_chunk.iter_mut().for_each(|b| { + b.inner.sent_transaction = Transaction::default() + }); if intent_bundles_chunk.is_empty() { return Ok(()); } From 8b622a57d14bb979f423b3377815acf93d4ddf67 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 1 Jul 2026 15:26:48 +0700 Subject: [PATCH 85/97] fix: test lint --- magicblock-committor-service/src/outbox/utils.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/magicblock-committor-service/src/outbox/utils.rs b/magicblock-committor-service/src/outbox/utils.rs index db4e05fe4..1e1f930a0 100644 --- a/magicblock-committor-service/src/outbox/utils.rs +++ b/magicblock-committor-service/src/outbox/utils.rs @@ -60,7 +60,7 @@ pub(crate) fn build_sent_commit( }) .collect(); - let sent_commit = SentCommit { + SentCommit { message_id: meta.id, slot: meta.slot, blockhash: meta.blockhash, @@ -72,7 +72,5 @@ pub(crate) fn build_sent_commit( error_message, patched_errors, callbacks_scheduling_results, - }; - - sent_commit + } } From 69000b7d117cfd44452bbf254de73fac55e77752 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 1 Jul 2026 15:40:21 +0700 Subject: [PATCH 86/97] fix: lint --- .../tests/test_outbox_flow.rs | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 4b6ad84d6..0bbc46a91 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -19,8 +19,8 @@ use magicblock_committor_service::{ }, outbox::{ outbox_client::InternalOutboxClientError, - outbox_intent_bundles_reader::OutboxIntentBundlesReader, OutboxClient, - ScheduledBaseIntentMeta, + outbox_intent_bundles_reader::OutboxIntentBundlesReader, + IntentSentTransaction, OutboxClient, ScheduledBaseIntentMeta, }, tasks::task_info_fetcher::{CacheTaskInfoFetcher, RpcTaskInfoFetcher}, transaction_preparator::TransactionPreparatorImpl, @@ -259,7 +259,7 @@ fn steal_schedule_accept_intent( loop { let intent_id = read_next_intent_id(ctx); schedule(ctx); - let result = steal_accept_intent(&ctx, intent_id); + let result = steal_accept_intent(ctx, intent_id); match result { Ok(()) => return intent_id, Err(err) => { @@ -280,7 +280,7 @@ fn schedule_and_accept( ) -> OutboxIntentBundle { ctx.wait_for_next_slot_ephem().unwrap(); - let intent_id = steal_schedule_accept_intent(&ctx, schedule); + let intent_id = steal_schedule_accept_intent(ctx, schedule); let pda = outbox_intent_pda(intent_id); let data = ctx.fetch_ephem_account_data(pda).unwrap(); OutboxIntentBundle::try_from_bytes(&data).unwrap() @@ -668,13 +668,15 @@ async fn test_pickup_after_committing() { cleanup_handle.clean().await.expect("cleanup after failure"); // Commit stage was recorded; outbox is TwoStage::Committing - let calls = stage_calls.lock().unwrap(); - assert_eq!(calls.len(), 1, "Only the commit stage should be recorded"); - let commit_sig = match &calls[0].1 { - ExecutionStage::TwoStage(TwoStageProgress::Committing(sig)) => *sig, - other => panic!("Expected Committing stage, got {:?}", other), + let commit_sig = { + let calls = stage_calls.lock().unwrap(); + assert_eq!(calls.len(), 1, "Only the commit stage should be recorded"); + let commit_sig = match &calls[0].1 { + ExecutionStage::TwoStage(TwoStageProgress::Committing(sig)) => *sig, + other => panic!("Expected Committing stage, got {:?}", other), + }; + commit_sig }; - drop(calls); // Re-fetch outbox and confirm it reflects TwoStage::Committing let outbox_bundle = test_env @@ -1011,9 +1013,13 @@ impl OutboxClient for TestOutboxClient { result: &IntentExecutorResult, _execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { + let IntentSentTransaction::Known(ref tx) = meta.intent_sent_transaction + else { + panic!("should be known"); + }; let succeeded = result.is_ok(); self.ephem_rpc - .send_and_confirm_transaction(&meta.intent_sent_transaction) + .send_and_confirm_transaction(tx) .await .map_err(InternalOutboxClientError::RpcClientError)?; self.sent_commits.lock().unwrap().push((meta.id, succeeded)); From 966132e02094f96a6903d7b1d14009a73709b05b Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 6 Jul 2026 15:18:57 +0700 Subject: [PATCH 87/97] fix: typo --- magicblock-committor-service/src/error.rs | 2 +- magicblock-committor-service/src/intent_engine.rs | 4 ++-- .../intent_engine/{intent_channerl.rs => intent_channel.rs} | 0 .../src/intent_engine/intent_execution_engine.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename magicblock-committor-service/src/intent_engine/{intent_channerl.rs => intent_channel.rs} (100%) diff --git a/magicblock-committor-service/src/error.rs b/magicblock-committor-service/src/error.rs index 1580ffc9a..eef33ba9a 100644 --- a/magicblock-committor-service/src/error.rs +++ b/magicblock-committor-service/src/error.rs @@ -1,7 +1,7 @@ use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::intent_engine::intent_channerl::IntentScheduleError; +use crate::intent_engine::intent_channel::IntentScheduleError; pub type CommittorServiceResult = Result; diff --git a/magicblock-committor-service/src/intent_engine.rs b/magicblock-committor-service/src/intent_engine.rs index 0a297eead..361f49395 100644 --- a/magicblock-committor-service/src/intent_engine.rs +++ b/magicblock-committor-service/src/intent_engine.rs @@ -1,5 +1,5 @@ pub mod db; -pub mod intent_channerl; +pub mod intent_channel; mod intent_execution_engine; pub mod intent_scheduler; @@ -15,7 +15,7 @@ use tokio::sync::broadcast; use crate::{ intent_engine::{ db::BacklogDB, - intent_channerl::{channel, IntentScheduleError, IntentScheduleHandle}, + intent_channel::{channel, IntentScheduleError, IntentScheduleHandle}, intent_execution_engine::{IntentExecutionEngine, ResultSubscriber}, }, intent_executor::{ diff --git a/magicblock-committor-service/src/intent_engine/intent_channerl.rs b/magicblock-committor-service/src/intent_engine/intent_channel.rs similarity index 100% rename from magicblock-committor-service/src/intent_engine/intent_channerl.rs rename to magicblock-committor-service/src/intent_engine/intent_channel.rs diff --git a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs index baa7415b1..073df7bf0 100644 --- a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs @@ -22,7 +22,7 @@ use crate::tasks::task_strategist::TransactionStrategy; use crate::{ intent_engine::{ db::BacklogDB, - intent_channerl::{IntentScheduleError, IntentStream}, + intent_channel::{IntentScheduleError, IntentStream}, intent_scheduler::{IntentScheduler, POISONED_INNER_MSG}, }, intent_executor::{ @@ -360,7 +360,7 @@ mod tests { use crate::{ intent_engine::{ db::{BacklogDB, DummyDB}, - intent_channerl::{channel, IntentScheduleHandle}, + intent_channel::{channel, IntentScheduleHandle}, intent_scheduler::{create_test_intent, create_test_intent_bundle}, }, intent_executor::{ From 1e170090a10130bfd8c7f8673a31e97bb2860fcd Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 6 Jul 2026 15:40:59 +0700 Subject: [PATCH 88/97] fix: cleanup fix: IntentStream --- .../src/intent_engine/intent_channel.rs | 19 ++++++++++++------- .../src/intent_executor/cleanup_handle.rs | 8 ++++++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/magicblock-committor-service/src/intent_engine/intent_channel.rs b/magicblock-committor-service/src/intent_engine/intent_channel.rs index baa2ad571..541873668 100644 --- a/magicblock-committor-service/src/intent_engine/intent_channel.rs +++ b/magicblock-committor-service/src/intent_engine/intent_channel.rs @@ -11,7 +11,7 @@ use tokio::sync::{ mpsc, mpsc::{error::TrySendError, Receiver, Sender}, }; -use tokio_stream::{wrappers::ReceiverStream, Stream}; +use tokio_stream::{wrappers::ReceiverStream, Stream, StreamExt}; use crate::intent_engine::{db, db::BacklogDB}; @@ -95,13 +95,18 @@ impl Stream for IntentStream { // That means we have backlog // prior to using channel again we have to clean it all first if !db.is_empty() { - // Some(T) always will be returned here as per check above - let el = db.pop_intent_bundle(); - return Poll::Ready(el.transpose()); + // Before starting to clean backlog we need to clean channel first + if let Poll::Ready(item) = this.stream.poll_next(cx) { + Poll::Ready(item.map(Ok)) + } else { + // Some(T) always will be returned here as per check above + let el = db.pop_intent_bundle(); + Poll::Ready(el.transpose()) + } + } else { + let item = ready!(this.stream.poll_next(cx)); + Poll::Ready(item.map(Ok)) } - - let item = ready!(this.stream.poll_next(cx)); - Poll::Ready(item.map(Ok)) } } diff --git a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs index 43d7806fa..c10479336 100644 --- a/magicblock-committor-service/src/intent_executor/cleanup_handle.rs +++ b/magicblock-committor-service/src/intent_executor/cleanup_handle.rs @@ -1,4 +1,4 @@ -use futures_util::future::try_join_all; +use futures_util::future::join_all; use solana_keypair::Keypair; use crate::{ @@ -44,6 +44,10 @@ impl CleanupHandle { ) }); - try_join_all(cleanup_futs).await.map(|_| ()) + join_all(cleanup_futs) + .await + .into_iter() + .find_map(Result::err) + .map_or(Ok(()), Err) } } From 7e064b811465bc0b0a94df7c300f1d0a7f75866d Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 6 Jul 2026 18:07:15 +0700 Subject: [PATCH 89/97] fix: test fix: close outbox when it was fulfilled on previous run --- .../single_stage_intent_executor.rs | 19 +++++++++++--- .../two_stage_intent_executor.rs | 26 ++++++++++++++----- .../tests/test_intent_executor.rs | 2 +- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 97446ed6c..f51eb070e 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -22,7 +22,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox::OutboxClient, + outbox::{OutboxClient, ScheduledBaseIntentMeta}, tasks::{ task_info_fetcher::{ResetType, TaskInfoFetcher}, task_strategist::TaskStrategist, @@ -82,9 +82,22 @@ where ) .await?; match flow { - // Intent was executed - return + // Intent was already executed on a previous run - notify the + // outbox so it isn't left pending and rediscovered again. ControlFlow::Break(()) => { - Ok(ExecutionOutput::SingleStage(self.pending_signature)) + let output = + Ok(ExecutionOutput::SingleStage(self.pending_signature)); + self.ctx + .outbox_client + .notify_commit_sent( + ScheduledBaseIntentMeta::new(&intent_bundle), + &output, + execution_report, + ) + .await + .map_err(Into::into)?; + + output } ControlFlow::Continue(()) => { // It we're here so previous run determined this should be single stage diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index f77e59820..2ab7925c5 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -27,7 +27,7 @@ use crate::{ ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, }, - outbox::OutboxClient, + outbox::{OutboxClient, ScheduledBaseIntentMeta}, tasks::{ task_builder::{TaskBuilderImpl, TasksBuilder}, task_info_fetcher::{ResetType, TaskInfoFetcher}, @@ -200,14 +200,28 @@ where ) .await } - // Finalize was executed - return + // Finalize was already executed on a previous run - notify the + // outbox so it isn't left pending and rediscovered again. ( TwoStageProgress::Finalizing { commit, finalize }, ControlFlow::Break(()), - ) => Ok(ExecutionOutput::TwoStage { - commit_signature: *commit, - finalize_signature: *finalize, - }), + ) => { + let output = Ok(ExecutionOutput::TwoStage { + commit_signature: *commit, + finalize_signature: *finalize, + }); + self.ctx + .outbox_client + .notify_commit_sent( + ScheduledBaseIntentMeta::new(&intent), + &output, + execution_report, + ) + .await + .map_err(Into::into)?; + + output + } } } } diff --git a/test-integration/test-committor-service/tests/test_intent_executor.rs b/test-integration/test-committor-service/tests/test_intent_executor.rs index 2fb167e71..d6dc14152 100644 --- a/test-integration/test-committor-service/tests/test_intent_executor.rs +++ b/test-integration/test-committor-service/tests/test_intent_executor.rs @@ -1288,7 +1288,7 @@ async fn test_action_callback_fired_on_timeout() { Box::new(intent_executor).execute(scheduled_intent).await; assert!(res.inner.is_ok()); - assert!(res.patched_errors.is_empty()); + assert_eq!(res.patched_errors.len(), 1, "Action was supposed to fail"); assert_eq!(res.callbacks_report.len(), 1, "1 callback scheduled"); assert!(res.callbacks_report[0].is_ok(), "mock returns Ok(sig)"); From 1a5c05a08fdfdd76431c49a5e83b96f33de30607 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 6 Jul 2026 18:23:44 +0700 Subject: [PATCH 90/97] fix: pubkey reset --- .../accepted_intent_executor.rs | 15 +++++++++----- .../single_stage_intent_executor.rs | 15 +++++++++----- .../two_stage_intent_executor.rs | 20 ++++++++++++++----- .../magic_scheduled_base_intent.rs | 14 +++++++++++++ 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs index 4242714d6..6f41c7555 100644 --- a/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/accepted_intent_executor.rs @@ -196,20 +196,25 @@ where base_intent: ScheduledIntentBundle, ) -> (IntentExecutionResult, CleanupHandle) { self.started_at = Instant::now(); - let is_undelegate = base_intent.has_undelegate_intent(); let pubkeys = base_intent.get_all_committed_pubkeys(); + let undelegated_pubkeys = base_intent.get_undelegated_pubkeys(); let mut execution_report = IntentExecutionReport::default(); let result = self.execute_inner(base_intent, &mut execution_report).await; if !pubkeys.is_empty() { - // Reset TaskInfoFetcher, as cache could become invalid - // NOTE: if undelegation was removed - we still reset - // We assume its safe since all consecutive commits will fail - if result.is_err() || is_undelegate { + if result.is_err() { + // We can't know what landed on chain, resync everything self.ctx .task_info_fetcher .reset(ResetType::Specific(&pubkeys)); + } else if !undelegated_pubkeys.is_empty() { + // Only undelegated accounts' nonces become stale. Keep the + // rest cached: a chain re-fetch can race the just-landed + // finalize and reuse a nonce (buffer PDA collision). + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&undelegated_pubkeys)); } } diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index f51eb070e..72330d0fd 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -143,20 +143,25 @@ where base_intent: ScheduledIntentBundle, ) -> (IntentExecutionResult, CleanupHandle) { self.started_at = Instant::now(); - let is_undelegate = base_intent.has_undelegate_intent(); let pubkeys = base_intent.get_all_committed_pubkeys(); + let undelegated_pubkeys = base_intent.get_undelegated_pubkeys(); let mut execution_report = IntentExecutionReport::default(); let result = self.execute_inner(base_intent, &mut execution_report).await; if !pubkeys.is_empty() { - // Reset TaskInfoFetcher, as cache could become invalid - // NOTE: if undelegation was removed - we still reset - // We assume its safe since all consecutive commits will fail - if result.is_err() || is_undelegate { + if result.is_err() { + // We can't know what landed on chain, resync everything self.ctx .task_info_fetcher .reset(ResetType::Specific(&pubkeys)); + } else if !undelegated_pubkeys.is_empty() { + // Only undelegated accounts' nonces become stale. Keep the + // rest cached: a chain re-fetch can race the just-landed + // finalize and reuse a nonce (buffer PDA collision). + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&undelegated_pubkeys)); } } diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 2ab7925c5..90e5c1f9b 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -240,16 +240,26 @@ where intent: ScheduledIntentBundle, ) -> (IntentExecutionResult, CleanupHandle) { // Duplicates AcceptedIntentExecutor::execute - let is_undelegate = intent.has_undelegate_intent(); let pubkeys = intent.get_all_committed_pubkeys(); + let undelegated_pubkeys = intent.get_undelegated_pubkeys(); let mut execution_report = IntentExecutionReport::default(); let result = self.execute_inner(intent, &mut execution_report).await; - if is_undelegate { - self.ctx - .task_info_fetcher - .reset(ResetType::Specific(&pubkeys)); + if !pubkeys.is_empty() { + if result.is_err() { + // We can't know what landed on chain, resync everything + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&pubkeys)); + } else if !undelegated_pubkeys.is_empty() { + // Only undelegated accounts' nonces become stale. Keep the + // rest cached: a chain re-fetch can race the just-landed + // finalize and reuse a nonce (buffer PDA collision). + self.ctx + .task_info_fetcher + .reset(ResetType::Specific(&undelegated_pubkeys)); + } } let close_buffers = result.is_ok(); let junk = execution_report.junk; diff --git a/programs/magicblock/src/intent_bundles/magic_scheduled_base_intent.rs b/programs/magicblock/src/intent_bundles/magic_scheduled_base_intent.rs index 88050bf62..9d585601f 100644 --- a/programs/magicblock/src/intent_bundles/magic_scheduled_base_intent.rs +++ b/programs/magicblock/src/intent_bundles/magic_scheduled_base_intent.rs @@ -182,6 +182,20 @@ impl ScheduledIntentBundle { self.intent_bundle.get_undelegate_intent_pubkeys() } + /// Returns pubkeys of accounts whose commit nonce becomes stale once + /// undelegated, i.e. those undelegated directly or via commit-finalize. + pub fn get_undelegated_pubkeys(&self) -> Vec { + self.intent_bundle + .get_undelegate_intent_pubkeys() + .into_iter() + .chain( + self.intent_bundle + .get_commit_finalize_and_undelegate_intent_pubkeys(), + ) + .flatten() + .collect() + } + pub fn has_undelegate_intent(&self) -> bool { self.intent_bundle.has_undelegate_intent() } From 7d849f4d5cd855808dbe342976520881df93eb10 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Mon, 6 Jul 2026 22:12:41 +0700 Subject: [PATCH 91/97] fix: execute_finalizing_intent flow to close outbox --- .../two_stage_intent_executor.rs | 12 +- mimd-0025.md | 293 ------------------ 2 files changed, 10 insertions(+), 295 deletions(-) delete mode 100644 mimd-0025.md diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index 90e5c1f9b..d01e5c65d 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -120,6 +120,7 @@ where commit_signature: Signature, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { + let meta = ScheduledBaseIntentMeta::new(&intent); // Commit succeeded so we skip those tasks all together let finalize_tasks = TaskBuilderImpl::finalize_tasks( &self.ctx.task_info_fetcher, @@ -157,10 +158,17 @@ where let finalized_stage = finalize_strategy_executor.done(finalize_signature); - Ok(ExecutionOutput::TwoStage { + let result = Ok(ExecutionOutput::TwoStage { commit_signature: finalized_stage.commit_signature, finalize_signature: finalized_stage.finalize_signature, - }) + }); + self.ctx + .outbox_client + .notify_commit_sent(meta, &result, execution_report) + .await + .map_err(Into::into)?; + + result } async fn execute_inner( diff --git a/mimd-0025.md b/mimd-0025.md deleted file mode 100644 index 8a3bf2f5a..000000000 --- a/mimd-0025.md +++ /dev/null @@ -1,293 +0,0 @@ -# Intent Execution Durability - -## Motivation - -When a user schedules a commit, they are delegating an account to the ER with the expectation that its state will be persisted to the base chain. The validator is obligated to fulfill this — an unexecuted commit is a protocol-level breach of trust. - -Today, that obligation is not enforced past the accept boundary. Once an Intent is accepted it leaves `AccountsDB` and lives exclusively in process memory. A validator crash anywhere between acceptance and L1 confirmation silently drops the Intent. There is no recovery, no retry, no report — the commitment is gone. - -This proposal closes that gap by anchoring accepted Intents back in `AccountsDB` for their entire execution lifecycle. - ---- - -## Problem - -`Accepted` intents leave `AccountsDB` and exist only in the in-memory `TransactionScheduler` global and the `CommittorService` mpsc channel. Neither survives a process restart. - -### Current flow - -``` -1. User calls ScheduleCommit / ScheduleIntentBundle - → Intent stored in MagicContext (AccountsDB, survives restart) -2. Validator calls AcceptScheduledCommits - → Intents popped from MagicContext - → Moved into TransactionScheduler (in-memory global, lost on restart) -3. CommittorService picks up intents, executes them on L1 -4. ScheduledCommitSent ER tx logs the result -``` - -The window between step 2 and step 4 is unprotected. Any crash drops all in-flight intents. - ---- - -## Proposed Solution - -This proposal is similar to the **outbox pattern**, where `AccountsDB` itself serves as the durable outbox. Rather than handing an Intent directly to the `CommittorService` and hoping for the best, the validator first writes the Intent into `AccountsDB` as a dedicated ephemeral account. The `CommittorService` then processes it from there and only closes the account once L1 execution is confirmed. A crash at any point leaves the account in `AccountsDB`; on restart the validator calls `getProgramAccounts` with a discriminator filter, reads each account's status, and resumes accordingly. - -Additionally, just before sending a transaction to L1, the validator records the transaction signature back into the `OutboxIntentBundle` via `SetIntentExecutionStage`. This lets the restart logic distinguish "prepared but not yet sent" from "sent — check the signature on L1." - -**No stage is executed twice.** Because `SetIntentExecutionStage` is committed to `AccountsDB` before the L1 transaction is submitted, any restart that finds `status = Executing(...)` will always query L1 for the stored signature first. A confirmed & successful tx signature advances or closes the intent. The write-before-send ordering is the invariant that makes this hold. - -### New execution flow - -``` -1. ScheduleCommit / ScheduleIntentBundle -2. AcceptScheduledCommits — moves Intent from MagicContext into a PDA with status Accepted -3. IntentExecutionService takes Intent for execution -4. SetIntentExecutionStage(signature) — updates Intent status with the constructed L1 signature -5. Send TX to Base -6. Close Intent account with report -``` - -#### Crash 1→2 - -Intent is still in `MagicContext`. The next `AcceptScheduledCommits` call picks it up and retries. No state is lost. - -#### Crash 2→3 - -`OutboxIntentBundle` exists with `status = Accepted`. On restart: discovered via `getProgramAccounts`, rescheduled for full execution. - -#### Crash 3→4 - -`OutboxIntentBundle` exists with `status = Accepted` — indistinguishable from crash 2→3. The L1 transaction was never sent, so the Intent can safely be re-prepared and re-executed. - -#### Crash 4→5 - -`OutboxIntentBundle` exists with `status = Executing(signature)`. On restart: check `signature` on L1. -- Confirmed & successful → close Intent account -- !(Confirmed & successful) → reschedule - -#### Crash 5→6 - -Same as crash 4→5: `status = Executing(signature)`. Check signature, close or reschedule accordingly. - -> **Note:** Two-stage execution (commit + finalize) is a complication of step 5. Each stage has its own signature stored separately in the `OutboxIntentBundle`. This is expanded in [Instruction changes](#new-setintentexecutionstage--intent_id-stage-executionstage-). - ---- - -## Account Design - -### `OutboxIntentBundle` ephemeral account - -**Seeds:** `["outbox-intent", intent_id.to_le_bytes()]` -**Owner:** MagicBlock builtin program - -```rust -pub struct OutboxIntentBundle { - pub inner: ScheduledIntentBundle, - pub status: OutboxIntentBundleStatus, -} - -pub enum OutboxIntentBundleStatus { - Accepted, - Executing(ExecutionStage), -} - -pub enum ExecutionStage { - SingleStage(Signature), - TwoStage(TwoStageProgress), -} - -pub enum TwoStageProgress { - Committing(Signature), - Finalizing { commit: Signature, finalize: Signature }, -} -``` - ---- - -## Instruction Changes - -### Modified: `AcceptScheduledCommits` - -Removes up to N intents from the front of `MagicContext.scheduled_base_intents`. For each intent an `OutboxIntentBundle` ephemeral account is created with `status = Accepted`. N is bounded by the transaction account limit: every new account must be passed as writable in the same transaction. - -**Accounts:** -``` -[validator_auth (signer), magic_context (writable), - new_pda_0 (writable), new_pda_1 (writable), ...] -``` - -**Logic (after existing pop from MagicContext):** for each accepted intent, create an `OutboxIntentBundle` ephemeral account with `status = Accepted`. - ---- - -### New: `SetIntentExecutionStage { intent_id, stage: ExecutionStage }` - -Sets or advances the execution stage of an intent. Handles the initial transition from `Accepted`, signature replacement on retry, and advancement from `Committing` to `Finalizing` — all through a single instruction with a unified set of transition rules. - -**Accounts:** `[validator_auth (signer), intent_pda (writable)]` - -**Transitions:** - -| From | To | Notes | -|---|---|---| -| `Accepted` | `Executing(SingleStage(sig))` | | -| `Accepted` | `Executing(TwoStage(Committing(sig)))` | | -| `Executing(SingleStage(_))` | `Executing(SingleStage(new_sig))` | Signature replacement on retry | -| `Executing(TwoStage(Committing(_)))` | `Executing(TwoStage(Committing(new_sig)))` | Signature replacement on retry | -| `Executing(TwoStage(Committing(commit)))` | `Executing(TwoStage(Finalizing { commit, finalize: sig }))` | Advance to finalize phase | -| `Executing(TwoStage(Finalizing { commit, _ }))` | `Executing(TwoStage(Finalizing { commit, finalize: new_sig }))` | Signature replacement on retry; `commit` must match stored value | - -**Checks:** -- From `Accepted`: rejects `TwoStage(Finalizing { .. })` — cannot skip the commit phase. -- From `Executing(_)`: the execution type (`SingleStage` vs `TwoStage`) cannot change. -- From `Executing(TwoStage(Finalizing { .. }))`: rejects `TwoStage(Committing(_))` — **downgrade not permitted**. Reaching `Finalizing` means the commit transaction was confirmed on L1; going back to `Committing` would re-commit an already-committed account. - ---- - -### New: `CloseOutboxIntent { intent_id }` - -Deallocates the `OutboxIntentBundle` ephemeral account. Called after L1 execution is confirmed. - -**Accounts:** -``` -[validator_auth (signer), closing_pda (writable)] -``` - -**Logic:** -1. Wipe and deallocate `closing_pda` - ---- - -## Restart Recovery - -### Discovery - -``` -1. getProgramAccounts(magic_program_id, filters=[discriminator(OutboxIntentBundle)]) - → returns all open intent accounts -2. Deserialize each account, sort by inner.id ascending -``` - -Cost: one RPC call returning the set of intents that were accepted and not yet closed. No scanning of closed IDs, no range guessing, no dependency on total historical intent count. - -### Classification and recovery - -```rust -match intent.status { - Accepted => - reschedule_full(intent.inner), - - Executing(SingleStage(sig)) => - match get_signature_status(sig).await { - Succeeded => close_outbox_intent(intent.inner.id).await, - _ => reschedule_full(intent.inner), - }, - - Executing(TwoStage(Committing(sig))) => - match get_signature_status(sig).await { - Succeeded => reschedule_finalize_only(intent.inner), - _ => reschedule_full(intent.inner), - }, - - Executing(TwoStage(Finalizing { finalize, .. })) => - match get_signature_status(finalize).await { - Succeeded => close_outbox_intent(intent.inner.id).await, - _ => reschedule_finalize_only(intent.inner), - }, -} -``` - ---- - -## Intent Discovery: `getProgramAccounts` - -The MagicBlock builtin program owns `MagicContext` (one singleton) and one `OutboxIntentBundle` per open intent. A `getProgramAccounts` call filtered by the `OutboxIntentBundle` discriminator returns all intent accounts that were accepted and have not yet been closed. After deserialization, accounts are sorted by `inner.id` to restore processing order. - -Each account's `status` field determines how recovery proceeds — an account with `status = Accepted` means the intent has not yet been sent to L1 and needs full execution; an account with `status = Executing(...)` requires checking the stored signature(s) on L1 before deciding whether to close or reschedule. - ---- - -## Considered Alternatives - -### Option A: Linked list (`OutboxIntentList` + `prev_id`/`next_id`) - -Maintain an `OutboxIntentList` singleton (head + tail) and embed `prev_id`/`next_id` pointers in each `OutboxIntentBundle`. On restart, traverse from head to tail. - -**Problem — unnecessary complexity:** `getProgramAccounts` already returns the same set of open intents in one call. The linked list duplicates this with extra fields per account, a shared singleton written on every accept and every close, and more complex close logic (relinking neighbors). - -**Verdict:** Superseded by `getProgramAccounts`. - ---- - -### Option B: Watermark + range scan - -Store `accepted_intent_watermark: u64` in `MagicContext`. On restart, scan intent PDAs for IDs in the range `[watermark, intent_id)`. - -**Mechanism:** -- Watermark advances only when the intent at the watermark position closes -- On restart: batch-fetch PDAs via `getMultipleAccounts` for the range - -**Problem — stuck watermark:** If a single intent fails persistently (e.g., an integrator bug causes repeated execution failure), the watermark is stuck at that intent's ID. Every subsequent restart must scan from that ID through the entire current range. Other intents in the range are discoverable, but the scan grows unboundedly until the stuck intent is resolved. - -Furthermore, the watermark advancement logic requires careful coordination: advancing past gaps means scanning forward for the next live PDA, which requires either additional instructions or off-chain bookkeeping. - -**Verdict:** Functionally correct but operationally fragile. A single stuck intent degrades restart performance for all subsequent restarts. - ---- - -### Option C: Bitmap pages - -Group intent IDs into pages of fixed size (e.g., 256). Each page is a PDA storing a bitmap of which intent IDs within its range are still active. - -```rust -// PDA: ["outbox-intent-page", page_id.to_le_bytes()] -struct ActiveIntentPage { - page_id: u64, - base_intent_id: u64, - active_bitmap: [u64; 4], // 256 bits - active_count: u16, -} -``` - -On restart: scan page PDAs from `min_active_page` to `current_page`, extract set bits, fetch the corresponding intent PDAs. - -**Problem — hot shared account:** Every `CloseOutboxIntent` for any intent in page range [0, 255] writes to the same page 0 PDA. Every `AcceptScheduledCommits` that allocates into page 0 also writes to page 0. All operations in the same page range serialize on a single shared account. - -**Problem — page watermark inherits the stuck-intent problem:** `min_active_page` cannot advance past page N until all 256 intents in page N close. One stuck intent holds up the entire page's retirement. The restart scan cost is O(N/256) page fetches — better than a plain watermark, but the underlying problem remains. - -**Problem — two-level indirection on restart:** Pages tell you which IDs were accepted; you still need a second round of PDA fetches to get the actual intent data. - -**Verdict:** Better than a plain watermark for restart scan efficiency (coarser granularity), but introduces a shared hot-account bottleneck and does not fully eliminate the stuck-intent scan problem. - ---- - -### Comparison - -| | `getProgramAccounts` | Linked list | Bitmap pages | Watermark scan | -|---|---|---|---|---| -| Restart discovery | One filtered RPC call | O(open) traversal | O(N/PAGE\_SIZE) + O(open) | O(gap from watermark) | -| Stuck-intent impact on restart | None | None | Blocks page retirement | Blocks watermark advancement | -| Close instruction complexity | Wipe account (2 accounts) | Relink 2 neighbors (3–5 accounts) | Flip 1 bit (2 accounts) | Wipe account (2 accounts) | -| Shared hot account | None | `OutboxIntentList` singleton | Page PDA (up to 256 intents) | None | -| Extra state per intent | 0 | +16 bytes (prev\_id, next\_id) | ~0.2 bytes in page bitmap | 0 | - -**`getProgramAccounts` is chosen** for its minimal account structure, no shared hot accounts, simplest close logic, and correct stuck-intent behavior. - ---- - -## Implementation Scope - -| Component | Change | -|---|---| -| `magicblock-magic-program-api/src/pda.rs` | Add `outbox_intent_pda(id)` | -| `magicblock-magic-program-api/src/instruction.rs` | Add `SetIntentExecutionStage`, `CloseOutboxIntent` variants | -| `programs/magicblock/src/` | Add `OutboxIntentBundle`, `OutboxIntentBundleStatus` types | -| `programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs` | Create intent ephemeral accounts | -| `programs/magicblock/src/schedule_transactions/` (new files) | `process_set_intent_execution_stage.rs`, `process_close_outbox_intent.rs` | -| `programs/magicblock/src/magicblock_processor.rs` | Wire new instruction variants | -| `magicblock-committor-service/src/` (new file) | `restart_loader.rs` — `getProgramAccounts` fetch, sort by id, classification, recovery dispatch | -| `magicblock-committor-service/src/intent_executor/` | Call `SetIntentExecutionStage` before L1 submission | -| `magicblock-accounts/src/scheduled_commits_processor.rs` | Replace `register_scheduled_commit_sent` with `CloseOutboxIntent` call | - From ebf8fd44ded07444f4289da4665fb18086ef12bf Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 7 Jul 2026 16:01:18 +0700 Subject: [PATCH 92/97] fix: ambiguity between not sent and maybe awaiting processed state --- Cargo.lock | 1 + .../src/intent_engine/intent_channel.rs | 2 +- .../intent_execution_client.rs | 15 ++ .../single_stage_intent_executor.rs | 33 ++--- .../strategy_executor/single_stage.rs | 8 +- .../strategy_executor/two_stage.rs | 20 +-- .../strategy_executor/utils.rs | 139 +++++++++++++----- .../two_stage_intent_executor.rs | 33 ++--- magicblock-core/Cargo.toml | 4 +- magicblock-magic-program-api/Cargo.toml | 1 + magicblock-magic-program-api/src/outbox.rs | 31 ++-- .../intent_bundles/outbox_intent_bundles.rs | 10 +- test-integration/Cargo.lock | 1 + .../tests/test_outbox_flow.rs | 15 +- 14 files changed, 200 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5908b7e93..7945ae5bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3897,6 +3897,7 @@ dependencies = [ "bincode", "const-crypto", "serde", + "solana-hash 4.2.0", "solana-program 2.3.0", "solana-program 3.0.0", "solana-signature 2.3.0", diff --git a/magicblock-committor-service/src/intent_engine/intent_channel.rs b/magicblock-committor-service/src/intent_engine/intent_channel.rs index 541873668..89b1589ff 100644 --- a/magicblock-committor-service/src/intent_engine/intent_channel.rs +++ b/magicblock-committor-service/src/intent_engine/intent_channel.rs @@ -11,7 +11,7 @@ use tokio::sync::{ mpsc, mpsc::{error::TrySendError, Receiver, Sender}, }; -use tokio_stream::{wrappers::ReceiverStream, Stream, StreamExt}; +use tokio_stream::{wrappers::ReceiverStream, Stream}; use crate::intent_engine::{db, db::BacklogDB}; diff --git a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs index 9b179248f..54974e3c5 100644 --- a/magicblock-committor-service/src/intent_executor/intent_execution_client.rs +++ b/magicblock-committor-service/src/intent_executor/intent_execution_client.rs @@ -125,6 +125,21 @@ impl IntentExecutionClient { .collect()) } + /// Returns whether `blockhash` is still within its valid window, i.e. + /// whether a transaction built with it could still land. Used to tell + /// apart "not landed yet, may still land" from "guaranteed dead" when + /// a pending signature isn't found on-chain. + pub(in crate::intent_executor) async fn is_blockhash_valid( + &self, + blockhash: &Hash, + ) -> MagicBlockRpcClientResult { + self.rpc_client + .get_inner() + .is_blockhash_valid(blockhash, self.rpc_client.commitment()) + .await + .map_err(MagicBlockRpcClientError::from) + } + pub(in crate::intent_executor) async fn get_latest_blockhash( &self, ) -> MagicBlockRpcClientResult { diff --git a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs index 72330d0fd..3090c86da 100644 --- a/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/single_stage_intent_executor.rs @@ -1,23 +1,19 @@ -use std::{ - ops::ControlFlow, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use async_trait::async_trait; use magicblock_core::traits::ActionsCallbackScheduler; use magicblock_program::{ magic_scheduled_base_intent::ScheduledIntentBundle, - validator::validator_authority, + outbox::PendingTransaction, validator::validator_authority, }; use solana_keypair::Keypair; -use solana_signature::Signature; use solana_signer::Signer; use crate::{ intent_executor::{ cleanup_handle::CleanupHandle, error::{IntentExecutorError, IntentExecutorResult}, - strategy_executor::utils::check_pending_signature, + strategy_executor::utils::resolve_pending_signature, utils::{build_commit_finalize_tasks, execute_single_stage_flow}, ExecutionOutput, IntentExecutionReport, IntentExecutionResult, IntentExecutor, IntentExecutorCtx, @@ -32,8 +28,8 @@ use crate::{ pub struct SingleStageIntentExecutor { authority: Keypair, - /// Signature of committing - pending_signature: Signature, + /// data of pending stage + pending_transaction: PendingTransaction, /// Intent Executor context ctx: IntentExecutorCtx, @@ -54,12 +50,12 @@ where pub fn new( ctx: IntentExecutorCtx, actions_timeout: Duration, - pending_signature: Signature, + pending_transaction: PendingTransaction, ) -> Self { let authority = validator_authority(); Self { authority, - pending_signature, + pending_transaction, ctx, actions_timeout, @@ -76,17 +72,18 @@ where intent_bundle: ScheduledIntentBundle, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - let flow = check_pending_signature( + let succeeded = resolve_pending_signature( &self.ctx.intent_client, - &self.pending_signature, + &self.pending_transaction, ) .await?; - match flow { + match succeeded { // Intent was already executed on a previous run - notify the // outbox so it isn't left pending and rediscovered again. - ControlFlow::Break(()) => { - let output = - Ok(ExecutionOutput::SingleStage(self.pending_signature)); + true => { + let output = Ok(ExecutionOutput::SingleStage( + self.pending_transaction.signature, + )); self.ctx .outbox_client .notify_commit_sent( @@ -99,7 +96,7 @@ where output } - ControlFlow::Continue(()) => { + false => { // It we're here so previous run determined this should be single stage let (commit_tasks, finalize_tasks) = build_commit_finalize_tasks( diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs index 9eaf15f72..eabf603c8 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/single_stage.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; -use magicblock_program::outbox::ExecutionStage; +use magicblock_program::outbox::{ExecutionStage, PendingTransaction}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_signature::Signature; @@ -30,7 +30,7 @@ use crate::{ pub struct SingleStageStrategyExecutor<'a, F, A, O> { current_attempt: u8, - pending_signature: Option, + pending_transaction: Option, execution_report: &'a mut IntentExecutionReport, authority: Keypair, @@ -63,7 +63,7 @@ where Self { authority, intent_id, - pending_signature: None, + pending_transaction: None, intent_client, task_info_fetcher, outbox_client, @@ -97,7 +97,7 @@ where let execution_state = ExecutionState { current_attempt: &mut self.current_attempt, transaction_strategy: &mut self.transaction_strategy, - pending_signature: &mut self.pending_signature, + pending_transaction: &mut self.pending_transaction, execution_report: self.execution_report, }; let result = stage_execution_loop( diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index d6da5f8b4..360ce997b 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -1,7 +1,9 @@ use std::{mem, sync::Arc}; use magicblock_core::traits::{ActionError, ActionsCallbackScheduler}; -use magicblock_program::outbox::{ExecutionStage, TwoStageProgress}; +use magicblock_program::outbox::{ + ExecutionStage, PendingTransaction, TwoStageProgress, +}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_signature::Signature; @@ -34,9 +36,9 @@ pub struct Initialized { commit_strategy: TransactionStrategy, /// Finalize stage strategy finalize_strategy: TransactionStrategy, - /// Signature pending confirmation + /// Transaction pending confirmation /// Sources: Outbox with status Committing, timeout - pending_signature: Option, + pending_transaction: Option, current_attempt: u8, } @@ -48,7 +50,7 @@ impl Initialized { Self { commit_strategy, finalize_strategy, - pending_signature: None, + pending_transaction: None, current_attempt: 0, } } @@ -61,7 +63,7 @@ pub struct Committed { finalize_strategy: TransactionStrategy, /// Signature pending confirmation /// Sources: Outbox with status Finalizing, timeout - pending_signature: Option, + pending_transaction: Option, current_attempt: u8, } @@ -73,7 +75,7 @@ impl Committed { Self { commit_signature, finalize_strategy, - pending_signature: None, + pending_transaction: None, current_attempt: 0, } } @@ -146,7 +148,7 @@ where let execution_state = ExecutionState { current_attempt: &mut self.state.current_attempt, transaction_strategy: &mut self.state.commit_strategy, - pending_signature: &mut self.state.pending_signature, + pending_transaction: &mut self.state.pending_transaction, execution_report: self.execution_report, }; let commit_stage_patcher = CommitStagePatcher { @@ -245,7 +247,7 @@ where state: Committed { commit_signature, finalize_strategy: self.state.finalize_strategy, - pending_signature: None, + pending_transaction: None, current_attempt: 0, }, } @@ -293,7 +295,7 @@ where let execution_state = ExecutionState { current_attempt: &mut self.state.current_attempt, transaction_strategy: &mut self.state.finalize_strategy, - pending_signature: &mut self.state.pending_signature, + pending_transaction: &mut self.state.pending_transaction, execution_report: self.execution_report, }; let finalize_stage_patcher = FinalizeStagePatcher { diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs index fb32c5a93..eb9dcffc9 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/utils.rs @@ -4,14 +4,14 @@ use async_trait::async_trait; use magicblock_core::traits::{ ActionError, ActionResult, ActionsCallbackScheduler, }; -use magicblock_program::outbox::ExecutionStage; +use magicblock_program::outbox::{ExecutionStage, PendingTransaction}; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_signature::Signature; use solana_transaction::versioned::VersionedTransaction; -use tokio::time::timeout; -use tracing::{error, info, warn}; +use tokio::time::{sleep, timeout}; +use tracing::{error, info}; use crate::{ intent_executor::{ @@ -39,10 +39,14 @@ use crate::{ const STAGE_LOOP_CEILING: u8 = 10; +/// How long to wait between re-checks of a pending signature that is still +/// within its blockhash's valid window (i.e. it might still land). +const PENDING_SIGNATURE_POLL_INTERVAL: Duration = Duration::from_secs(2); + pub(in crate::intent_executor) struct ExecutionState<'a> { pub current_attempt: &'a mut u8, pub transaction_strategy: &'a mut TransactionStrategy, - pub pending_signature: &'a mut Option, + pub pending_transaction: &'a mut Option, pub execution_report: &'a mut IntentExecutionReport, } @@ -54,7 +58,7 @@ pub(in crate::intent_executor) async fn stage_execution_loop<'a, T, O, P>( transaction_preparator: &T, mut patcher: P, intent_id: u64, - make_outbox_stage: impl Fn(Signature) -> ExecutionStage, + make_outbox_stage: impl Fn(PendingTransaction) -> ExecutionStage, map_preparation_err: fn(TransactionPreparatorError) -> IntentExecutorError, state: ExecutionState<'a>, ) -> IntentExecutorResult> @@ -65,14 +69,20 @@ where P: Patcher, { loop { - if let Some(ref sig) = state.pending_signature { - let flow = check_pending_signature(intent_client, sig).await?; - // Pending signature corresponds to succeeded transaction - break - if let ControlFlow::Break(()) = flow { - break Ok(Ok(*sig)); + if let Some(ref pending) = state.pending_transaction { + match check_pending_signature(intent_client, pending).await? { + PendingSignatureStatus::Succeeded => { + break Ok(Ok(pending.signature)); + } + // May still land - don't resend, just re-check later + PendingSignatureStatus::Pending => { + sleep(PENDING_SIGNATURE_POLL_INTERVAL).await; + continue; + } + PendingSignatureStatus::Rejected => { + *state.pending_transaction = None; + } } - - *state.pending_signature = None; } *state.current_attempt += 1; @@ -88,18 +98,19 @@ where .map_err(map_preparation_err)?; // Record in outbox before sending - let signature = prepared_transaction.get_signature(); + let pending = PendingTransaction { + signature: *prepared_transaction.get_signature(), + blockhash: *prepared_transaction.get_recent_blockhash(), + }; outbox_client - .set_intent_execution_stage( - intent_id, - make_outbox_stage(*signature), - ) + .set_intent_execution_stage(intent_id, make_outbox_stage(pending)) .await .map_err(Into::into)?; // Now record locally signature of tx we about to send // Precaution for timeout in between - *state.pending_signature = Some(*signature); + *state.pending_transaction = Some(pending); + let signature = &pending.signature; // Send signed tx let execution_result = intent_client @@ -110,7 +121,7 @@ where .await; // Result returned, cleanup pending signature - *state.pending_signature = None; + *state.pending_transaction = None; let execution_err = match execution_result { Ok(()) => return Ok(Ok(*signature)), @@ -163,35 +174,85 @@ pub async fn prepare_transaction( Ok(transaction) } -/// Checks a `pending_signature` loaded from the outbox against the full -/// transaction history. Returns `Break(sig)` if the tx succeeded — the caller -/// can short-circuit and skip re-execution. Returns `Continue` for every other -/// outcome (tx failed, never sent, or RPC returned no entry), letting the -/// normal execution loop proceed. +pub(in crate::intent_executor) enum PendingSignatureStatus { + /// The transaction landed successfully. + Succeeded, + /// Not found on-chain yet, but its blockhash is still within its valid + /// window - it may still land. Callers must not resend. + Pending, + /// The transaction landed and failed, or its blockhash has expired + /// without landing - guaranteed dead, safe to send a new transaction. + Rejected, +} + +/// Checks a pending signature loaded from the outbox against the full +/// transaction history. A signature that isn't found on-chain is ambiguous +/// on its own — Solana's RPC can't tell "never sent" apart from "sent but +/// not yet landed" — so we additionally check whether its blockhash is +/// still valid to decide between `Pending` (may still land) and `Rejected` +/// (guaranteed dead). +/// +/// Blockhash validity is checked *before* signature status, not after. +/// Validity is monotonic (once expired, it stays expired), so a +/// slightly-stale "expired" reading is still safe to act on later. But +/// "signature not found" is a snapshot that can flip to "found" at any +/// later instant — if we checked it first and the blockhash-expired check +/// second, the tx could land in the gap between the two calls and we'd +/// wrongly conclude `Rejected`, causing a double execution. pub(in crate::intent_executor) async fn check_pending_signature( client: &IntentExecutionClient, - sig: &Signature, -) -> IntentExecutorResult> { + pending: &PendingTransaction, +) -> IntentExecutorResult { + let blockhash_still_valid = client + .is_blockhash_valid(&pending.blockhash) + .await + .map_err(IntentExecutorError::GetPendingSignatureStatusError)?; + let statuses = client - .get_signature_statuses_with_history(std::slice::from_ref(sig)) + .get_signature_statuses_with_history(std::slice::from_ref( + &pending.signature, + )) .await .map_err(IntentExecutorError::GetPendingSignatureStatusError)?; match statuses.first() { - Some(Some(Ok(()))) => Ok(ControlFlow::Break(())), - None => { - // well, that is bizarre one - warn!(pending_signature = ?sig, "RPC did not return status for signature"); - Ok(ControlFlow::Continue(())) + Some(Some(Ok(()))) => Ok(PendingSignatureStatus::Succeeded), + Some(Some(Err(err))) => { + info!(pending_signature = ?pending.signature, error = ?err, "Transaction corresponding to pending signature failed"); + Ok(PendingSignatureStatus::Rejected) } - // Any case below means tx failed in execution we continue attempts - Some(None) => { - info!(pending_signature = ?sig, "Transaction corresponding to pending signature was never sent"); - Ok(ControlFlow::Continue(())) + // Not found on-chain: could be genuinely never sent, or simply not + // landed yet. + None | Some(None) => { + if blockhash_still_valid { + Ok(PendingSignatureStatus::Pending) + } else { + info!(pending_signature = ?pending.signature, "Pending transaction's blockhash expired without landing"); + Ok(PendingSignatureStatus::Rejected) + } } - Some(Some(Err(err))) => { - info!(pending_signature = ?sig, error = ?err, "Transaction corresponding to pending signature failed"); - Ok(ControlFlow::Continue(())) + } +} + +/// Waits until a pending signature resolves definitively, sleeping and +/// re-checking while `Pending` (naturally bounded by the blockhash's own +/// expiry window). Returns `true` if the transaction landed successfully, +/// `false` if it's guaranteed dead (failed on-chain or blockhash expired). +/// Callers that don't already sit inside a retry loop (e.g. resuming after +/// a restart) should use this instead of calling `check_pending_signature` +/// directly, to avoid acting on a `Pending` result as if it were safe to +/// resend. +pub(in crate::intent_executor) async fn resolve_pending_signature( + client: &IntentExecutionClient, + pending: &PendingTransaction, +) -> IntentExecutorResult { + loop { + match check_pending_signature(client, pending).await? { + PendingSignatureStatus::Succeeded => return Ok(true), + PendingSignatureStatus::Rejected => return Ok(false), + PendingSignatureStatus::Pending => { + sleep(PENDING_SIGNATURE_POLL_INTERVAL).await; + } } } } diff --git a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs index d01e5c65d..2d703ca15 100644 --- a/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs +++ b/magicblock-committor-service/src/intent_executor/two_stage_intent_executor.rs @@ -1,7 +1,4 @@ -use std::{ - ops::ControlFlow, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use async_trait::async_trait; use magicblock_core::traits::ActionsCallbackScheduler; @@ -20,7 +17,7 @@ use crate::{ strategy_executor::{ two_stage::{Committed, Initialized, TwoStageStrategyExecutor}, utils::{ - check_pending_signature, execute_with_timeout, FinalizeStage, + execute_with_timeout, resolve_pending_signature, FinalizeStage, }, }, utils::{build_commit_finalize_tasks, execute_two_stage_flow}, @@ -176,31 +173,28 @@ where intent: ScheduledIntentBundle, execution_report: &mut IntentExecutionReport, ) -> IntentExecutorResult { - let pending_signature = self.stage.pending_signature(); - let flow = - check_pending_signature(&self.ctx.intent_client, pending_signature) + let pending = *self.stage.pending_transaction(); + let succeeded = + resolve_pending_signature(&self.ctx.intent_client, &pending) .await?; - match (&self.stage, flow) { + match (&self.stage, succeeded) { // Signature wasn't confirmed - need to reexecute from commit - (TwoStageProgress::Committing(_), ControlFlow::Continue(())) => { + (TwoStageProgress::Committing(_), false) => { self.execute_committing_intent(intent, execution_report) .await } // Signature confirmed - commit was executed, finalizing... - (TwoStageProgress::Committing(commit), ControlFlow::Break(())) => { + (TwoStageProgress::Committing(commit), true) => { self.execute_finalizing_intent( intent, - *commit, + commit.signature, execution_report, ) .await } // Finalize didn't occur - execute - ( - TwoStageProgress::Finalizing { commit, .. }, - ControlFlow::Continue(()), - ) => { + (TwoStageProgress::Finalizing { commit, .. }, false) => { self.execute_finalizing_intent( intent, *commit, @@ -210,13 +204,10 @@ where } // Finalize was already executed on a previous run - notify the // outbox so it isn't left pending and rediscovered again. - ( - TwoStageProgress::Finalizing { commit, finalize }, - ControlFlow::Break(()), - ) => { + (TwoStageProgress::Finalizing { commit, finalize }, true) => { let output = Ok(ExecutionOutput::TwoStage { commit_signature: *commit, - finalize_signature: *finalize, + finalize_signature: finalize.signature, }); self.ctx .outbox_client diff --git a/magicblock-core/Cargo.toml b/magicblock-core/Cargo.toml index fe621634b..e1cd89dff 100644 --- a/magicblock-core/Cargo.toml +++ b/magicblock-core/Cargo.toml @@ -17,7 +17,7 @@ bytes = { workspace = true } serde = { workspace = true, features = ["derive"] } solana-account = { workspace = true } -solana-account-decoder = { workspace = true } +solana-account-decoder = { workspace = true, features = ["agave-unstable-api"] } solana-clock = { workspace = true } solana-hash = { workspace = true } solana-message = { workspace = true } @@ -27,7 +27,7 @@ solana-signature = { workspace = true } solana-transaction = { workspace = true, features = ["blake3", "verify"] } solana-transaction-context = { workspace = true } solana-transaction-error = { workspace = true } -solana-transaction-status-client-types = { workspace = true } +solana-transaction-status-client-types = { workspace = true, features = ["agave-unstable-api"] } magicblock-magic-program-api = { workspace = true } spl-token = { workspace = true } spl-token-2022 = { workspace = true } diff --git a/magicblock-magic-program-api/Cargo.toml b/magicblock-magic-program-api/Cargo.toml index 4c6358131..d8d62fef2 100644 --- a/magicblock-magic-program-api/Cargo.toml +++ b/magicblock-magic-program-api/Cargo.toml @@ -18,6 +18,7 @@ backward-compat = [ [dependencies] solana-program = { workspace = true } solana-program-compat = { package = "solana-program", version = ">=2.0, <3", default-features = false, optional = true } +solana-hash = { workspace = true, features = ["serde"] } solana-signature = { workspace = true, features = ["serde"] } solana-signature-compat = { package = "solana-signature", version = ">=2.0, <3", default-features = false, features = ["serde"], optional = true } bincode = "^1.3.3" diff --git a/magicblock-magic-program-api/src/outbox.rs b/magicblock-magic-program-api/src/outbox.rs index 7a0a548f2..5cce7f1c6 100644 --- a/magicblock-magic-program-api/src/outbox.rs +++ b/magicblock-magic-program-api/src/outbox.rs @@ -1,18 +1,29 @@ use serde::{Deserialize, Serialize}; +use solana_hash::Hash; use solana_signature::Signature; +/// A transaction that was sent but not yet confirmed, along with the +/// blockhash it was built with. The blockhash is needed on recovery to +/// tell apart "may still land" (blockhash still valid) from "guaranteed +/// dead" (blockhash expired) when the signature isn't found on-chain. +#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub struct PendingTransaction { + pub signature: Signature, + pub blockhash: Hash, +} + #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] pub enum ExecutionStage { - SingleStage(Signature), + SingleStage(PendingTransaction), TwoStage(TwoStageProgress), } #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] pub enum TwoStageProgress { - Committing(Signature), + Committing(PendingTransaction), Finalizing { commit: Signature, - finalize: Signature, + finalize: PendingTransaction, }, } @@ -57,10 +68,10 @@ impl ExecutionStage { Ok(()) } - pub fn pending_signature(&self) -> &Signature { + pub fn pending_transaction(&self) -> &PendingTransaction { match self { - Self::SingleStage(signature) => signature, - Self::TwoStage(value) => value.pending_signature(), + Self::SingleStage(pending) => pending, + Self::TwoStage(value) => value.pending_transaction(), } } } @@ -77,10 +88,10 @@ impl TwoStageProgress { } // Commit was successfully executed and now we move on to Finalizing ( - Self::Committing(this_sig), + Self::Committing(this_pending), Self::Finalizing { commit, finalize }, ) => { - if this_sig != &commit { + if this_pending.signature != commit { return Err( "commit signature mismatch on advance to Finalizing", ); @@ -116,9 +127,9 @@ impl TwoStageProgress { Ok(()) } - pub fn pending_signature(&self) -> &Signature { + pub fn pending_transaction(&self) -> &PendingTransaction { match self { - Self::Committing(signature) => signature, + Self::Committing(pending) => pending, Self::Finalizing { finalize, .. } => finalize, } } diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index f07dfa142..2d954a59d 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -1,8 +1,11 @@ use std::ops::Deref; use magicblock_core::intent::outbox::OUTBOX_INTENT_DISCRIMINATOR; -use magicblock_magic_program_api::outbox::{ExecutionStage, TwoStageProgress}; +use magicblock_magic_program_api::outbox::{ + ExecutionStage, PendingTransaction, TwoStageProgress, +}; use serde::{Deserialize, Serialize}; +use solana_hash::Hash; use solana_signature::Signature; use crate::magic_scheduled_base_intent::ScheduledIntentBundle; @@ -82,7 +85,10 @@ impl OutboxIntentBundleStatus { Self::Executing(ExecutionStage::TwoStage( TwoStageProgress::Finalizing { commit: Signature::default(), - finalize: Signature::default(), + finalize: PendingTransaction { + signature: Signature::default(), + blockhash: Hash::default(), + }, }, )) } diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index f96fa7c97..392d64905 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -3989,6 +3989,7 @@ dependencies = [ "bincode", "const-crypto", "serde", + "solana-hash 4.2.0", "solana-program 3.0.0", "solana-signature", ] diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 0bbc46a91..ff0bc7326 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -330,13 +330,14 @@ async fn test_pickup_executed_intent() { let ExecutionOutput::SingleStage(signature) = result.inner.unwrap() else { panic!("Unexpected execution strategy"); }; - assert_eq!( - outbox_bundle.status, + match &outbox_bundle.status { OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( - signature - )), - "Invalid outbox state" - ); + pending, + )) => { + assert_eq!(pending.signature, signature, "Invalid outbox state"); + } + other => panic!("Invalid outbox state: {other:?}"), + } // Builder executor let executor = build_stage_intent_executor( test_env.executor_ctx_builder().build(), @@ -715,7 +716,7 @@ async fn test_pickup_after_committing() { panic!("Expected TwoStage output"); }; assert_eq!( - commit_signature, commit_sig, + commit_signature, commit_sig.signature, "Commit sig must not change on recovery" ); cleanup_handle From 9dd00f63051089e2ce1493d43928129d5df5f16d Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 7 Jul 2026 16:14:03 +0700 Subject: [PATCH 93/97] fix: docs + visability --- .../intent_engine/intent_execution_engine.rs | 5 +++-- .../src/instruction.rs | 15 ++++++++------ .../intent_bundles/outbox_intent_bundles.rs | 7 +++++-- .../tests/test_outbox_flow.rs | 20 +++++++++---------- .../test-tools/src/transactions.rs | 1 - 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs index 073df7bf0..d4a74a601 100644 --- a/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_engine/intent_execution_engine.rs @@ -168,8 +168,9 @@ where .expect(SEMAPHORE_CLOSED_MSG); // Spawn executor - let executor = - self.executor_builder.create_instance(intent.status.clone()); + let executor = self + .executor_builder + .create_instance(intent.status().clone()); let inner = self.inner.clone(); let handle = tokio::spawn(Self::execute( diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index b4ab629c0..ec958181b 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -63,12 +63,14 @@ pub enum MagicBlockInstruction { /// with `status = Accepted`. This is the second part of scheduling a commit. /// /// N is determined by the number of writable PDA accounts provided beyond - /// the two fixed accounts. It is run at the start of the slot. + /// the four fixed accounts. It is run at the start of the slot. /// /// # Account references - /// - **0.** `[SIGNER]` Validator Authority - /// - **1.** `[WRITE]` Magic Context Account containing the initially scheduled commits - /// - **2..n** `[WRITE]` Outbox intent PDAs, one per accepted intent, seeds: `["outbox-intent", intent_id.to_le_bytes()]` + /// - **0.** `[WRITE, SIGNER]` Validator Authority + /// - **1.** `[]` Magic Program + /// - **2.** `[WRITE]` Magic Context Account containing the initially scheduled commits + /// - **3.** `[WRITE]` Ephemeral Vault + /// - **4..n** `[WRITE]` Outbox intent PDAs, one per accepted intent, seeds: `["outbox-intent", intent_id.to_le_bytes()]` AcceptScheduleCommits, /// Records the attempt to realize a scheduled commit on chain. @@ -80,8 +82,9 @@ pub enum MagicBlockInstruction { /// /// # Account references /// - **0.** `[WRITE, SIGNER]` Validator Authority (receives rent refund from close) - /// - **1.** `[WRITE]` Outbox intent PDA to close, seeds: `["outbox-intent", intent_id.to_le_bytes()]` - /// - **2.** `[WRITE]` Ephemeral vault + /// - **1.** `[]` Magic Program + /// - **2.** `[WRITE]` Ephemeral Vault + /// - **3.** `[WRITE]` Outbox intent PDA to close, seeds: `["outbox-intent", intent_id.to_le_bytes()]` ScheduledCommitSent(u64), /// Schedules execution of a single *base intent*. diff --git a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs index 2d954a59d..6a2e2b16f 100644 --- a/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs +++ b/programs/magicblock/src/intent_bundles/outbox_intent_bundles.rs @@ -13,8 +13,7 @@ use crate::magic_scheduled_base_intent::ScheduledIntentBundle; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OutboxIntentBundle { pub inner: ScheduledIntentBundle, - // TODO(edwin): define visibility - pub status: OutboxIntentBundleStatus, + status: OutboxIntentBundleStatus, } impl OutboxIntentBundle { @@ -25,6 +24,10 @@ impl OutboxIntentBundle { } } + pub fn status(&self) -> &OutboxIntentBundleStatus { + &self.status + } + pub(crate) fn apply_stage_transition( &mut self, stage: ExecutionStage, diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index ff0bc7326..ff7db90fb 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -330,7 +330,7 @@ async fn test_pickup_executed_intent() { let ExecutionOutput::SingleStage(signature) = result.inner.unwrap() else { panic!("Unexpected execution strategy"); }; - match &outbox_bundle.status { + match outbox_bundle.status() { OutboxIntentBundleStatus::Executing(ExecutionStage::SingleStage( pending, )) => { @@ -341,7 +341,7 @@ async fn test_pickup_executed_intent() { // Builder executor let executor = build_stage_intent_executor( test_env.executor_ctx_builder().build(), - outbox_bundle.status, + outbox_bundle.status().clone(), DEFAULT_ACTIONS_TIMEOUT, ); let (result, cleanup_handle) = Box::new(executor) @@ -414,8 +414,8 @@ async fn test_pickup_failed_intent() { .expect("fetch suceeds") .expect("outbox intent exists"); assert_eq!( - chain_outbox_bundle.status, - OutboxIntentBundleStatus::Accepted, + chain_outbox_bundle.status(), + &OutboxIntentBundleStatus::Accepted, "Invalid outbox state" ); @@ -689,20 +689,20 @@ async fn test_pickup_after_committing() { .expect("outbox bundle present"); assert!( matches!( - &outbox_bundle.status, + outbox_bundle.status(), OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( TwoStageProgress::Committing(_) )) ), "Expected TwoStage::Committing in outbox, got {:?}", - outbox_bundle.status + outbox_bundle.status() ); // Recovery: build_stage_intent_executor detects the commit sig on chain and // runs the finalize stage without re-submitting the commit tx. let executor = build_stage_intent_executor( test_env.executor_ctx_builder().build(), - outbox_bundle.status, + outbox_bundle.status().clone(), DEFAULT_ACTIONS_TIMEOUT, ); let (result, cleanup_handle) = Box::new(executor) @@ -787,20 +787,20 @@ async fn test_pickup_after_finalizing() { .expect("outbox bundle present"); assert!( matches!( - &outbox_bundle.status, + outbox_bundle.status(), OutboxIntentBundleStatus::Executing(ExecutionStage::TwoStage( TwoStageProgress::Finalizing { .. } )) ), "Expected TwoStage::Finalizing in outbox, got {:?}", - outbox_bundle.status + outbox_bundle.status() ); // Recovery: build_stage_intent_executor detects the finalize sig on chain and // returns the same signatures without re-executing either stage. let executor = build_stage_intent_executor( test_env.executor_ctx_builder().build(), - outbox_bundle.status, + outbox_bundle.status().clone(), DEFAULT_ACTIONS_TIMEOUT, ); let (result, cleanup_handle) = Box::new(executor) diff --git a/test-integration/test-tools/src/transactions.rs b/test-integration/test-tools/src/transactions.rs index fe689c39a..a78e0fe64 100644 --- a/test-integration/test-tools/src/transactions.rs +++ b/test-integration/test-tools/src/transactions.rs @@ -57,7 +57,6 @@ pub fn send_transaction( ) -> Result { let blockhash = rpc_client.get_latest_blockhash()?; tx.try_sign(signers, blockhash)?; - println!("trtr: {}", tx.signatures[0]); let sig = rpc_client.send_transaction_with_config( tx, RpcSendTransactionConfig { From 75153bbb16e75b57050304358e0775ab01cc594b Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 7 Jul 2026 16:23:06 +0700 Subject: [PATCH 94/97] fix: channel --- .../src/intent_engine/intent_channel.rs | 9 ++++--- .../process_accept_scheduled_commits.rs | 25 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/magicblock-committor-service/src/intent_engine/intent_channel.rs b/magicblock-committor-service/src/intent_engine/intent_channel.rs index 89b1589ff..48ded0b61 100644 --- a/magicblock-committor-service/src/intent_engine/intent_channel.rs +++ b/magicblock-committor-service/src/intent_engine/intent_channel.rs @@ -95,9 +95,12 @@ impl Stream for IntentStream { // That means we have backlog // prior to using channel again we have to clean it all first if !db.is_empty() { - // Before starting to clean backlog we need to clean channel first - if let Poll::Ready(item) = this.stream.poll_next(cx) { - Poll::Ready(item.map(Ok)) + // Before starting to clean backlog we need to clean channel first. + // A closed channel (`Ready(None)`) must NOT end the stream here - + // backlog still has items to drain, and the channel closing + // doesn't mean there's no more work left. + if let Poll::Ready(Some(item)) = this.stream.poll_next(cx) { + Poll::Ready(Some(Ok(item))) } else { // Some(T) always will be returned here as per check above let el = db.pop_intent_bundle(); diff --git a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs index bc5bbc9a8..a650bd572 100644 --- a/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs +++ b/programs/magicblock/src/intent_bundles/outbox/process_accept_scheduled_commits.rs @@ -151,15 +151,22 @@ fn pop_scheduled_intents( as usize; // Assert enough accounts - let Some(num_accept_intents) = - num_ix_accounts.checked_sub(INTENT_PDAS_OFFSET as usize) - else { - ic_msg!( - invoke_context, - "AcceptScheduledCommits ERR: not enough accounts to accept intents ({}), need validator authority, magic context, vault, and at least one outbox intent PDA", - num_ix_accounts - ); - return Err(InstructionError::MissingAccount); + let num_accept_intents = match num_ix_accounts + .checked_sub(INTENT_PDAS_OFFSET as usize) + { + Some(0) => { + // No outbox intent PDAs provided - nothing to accept + return Ok(vec![]); + } + Some(count) => count, + None => { + ic_msg!( + invoke_context, + "AcceptScheduledCommits ERR: not enough accounts to accept intents ({}), need validator authority, magic context, vault, and at least one outbox intent PDA", + num_ix_accounts + ); + return Err(InstructionError::MissingAccount); + } }; let magic_context_acc = get_instruction_account_with_idx( From 36fbae15b213e461620e7c3e7c95fdca670b5e0d Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 7 Jul 2026 21:04:31 +0700 Subject: [PATCH 95/97] fix: tests --- .../strategy_executor/two_stage.rs | 10 ++-- .../tests/test_outbox_flow.rs | 49 ++++++------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs index 360ce997b..af1a6f284 100644 --- a/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs +++ b/magicblock-committor-service/src/intent_executor/strategy_executor/two_stage.rs @@ -168,7 +168,11 @@ where transaction_preparator, commit_stage_patcher, self.intent_id, - |sig| ExecutionStage::TwoStage(TwoStageProgress::Committing(sig)), + |pending_tx| { + ExecutionStage::TwoStage(TwoStageProgress::Committing( + pending_tx, + )) + }, IntentExecutorError::FailedCommitPreparationError, execution_state, ) @@ -311,10 +315,10 @@ where transaction_preparator, finalize_stage_patcher, self.intent_id, - |sig| { + |pending_tx| { ExecutionStage::TwoStage(TwoStageProgress::Finalizing { commit: commit_signature, - finalize: sig, + finalize: pending_tx, }) }, IntentExecutorError::FailedFinalizePreparationError, diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index ff7db90fb..91caef10c 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -82,7 +82,6 @@ type TestIntentExecutorCtx = IntentExecutorCtx< struct TestEnv { ctx: IntegrationTestContext, - validator_authority: Keypair, chain_mb_client: MagicblockRpcClient, ephem_rpc: Arc, intent_client: IntentExecutionClient, @@ -112,7 +111,6 @@ impl TestEnv { Self { ctx, - validator_authority, chain_mb_client: chain_mb_rpc, ephem_rpc, intent_client, @@ -134,10 +132,7 @@ impl TestEnv { } fn outbox_client(&self) -> TestOutboxClient { - TestOutboxClient::new( - self.ephem_rpc.clone(), - self.validator_authority.insecure_clone(), - ) + TestOutboxClient::new(self.ephem_rpc.clone()) } fn transaction_preparator(&self) -> TransactionPreparatorImpl { @@ -306,13 +301,22 @@ async fn test_pickup_executed_intent() { }); // Execute intent - let executor_ctx = test_env.executor_ctx_builder().build(); + let outbox_client = Arc::new(test_env.outbox_client()); + let executor_ctx = test_env + .executor_ctx_builder() + .with_outbox_client(outbox_client.clone()) + .build(); let executor = AcceptedIntentExecutor::new(executor_ctx, DEFAULT_ACTIONS_TIMEOUT); let (result, cleanup_handle) = Box::new(executor) .execute(outbox_bundle.inner.clone()) .await; assert!(result.inner.is_ok(), "Executor failed: {:?}", result.inner); + assert_eq!( + outbox_client.sent_commits.lock().unwrap().as_slice(), + [(outbox_bundle.inner.id, true)], + "notify_commit_sent should have been called once with success" + ); cleanup_handle.clean().await.expect("cleanup failed"); @@ -891,7 +895,6 @@ impl OutboxIntentBundlesReader for TestOutboxReader { struct TestOutboxClient { ephem_rpc: Arc, - validator_keypair: Keypair, pub stage_calls: Arc>>, pub sent_commits: Arc>>, @@ -902,10 +905,9 @@ struct TestOutboxClient { } impl TestOutboxClient { - fn new(ephem_rpc: Arc, validator_keypair: Keypair) -> Self { + fn new(ephem_rpc: Arc) -> Self { Self { ephem_rpc, - validator_keypair, stage_calls: Default::default(), sent_commits: Default::default(), fail_set_execution_stage: false, @@ -981,25 +983,8 @@ impl OutboxClient for TestOutboxClient { .get_latest_blockhash() .await .map_err(InternalOutboxClientError::RpcClientError)?; - let ix = Instruction::new_with_bincode( - magicblock_magic_program_api::id(), - &MagicBlockInstruction::SetIntentExecutionStage { - intent_id, - stage, - }, - vec![ - AccountMeta::new_readonly( - self.validator_keypair.pubkey(), - true, - ), - AccountMeta::new(outbox_intent_pda(intent_id), false), - ], - ); - let tx = Transaction::new_signed_with_payer( - &[ix], - Some(&self.validator_keypair.pubkey()), - &[&self.validator_keypair], - blockhash, + let tx = InstructionUtils::set_intent_execution_stage( + blockhash, intent_id, stage, ); self.ephem_rpc .send_and_confirm_transaction(&tx) @@ -1014,15 +999,11 @@ impl OutboxClient for TestOutboxClient { result: &IntentExecutorResult, _execution_report: &IntentExecutionReport, ) -> Result<(), Self::Error> { - let IntentSentTransaction::Known(ref tx) = meta.intent_sent_transaction + let IntentSentTransaction::Known(_) = meta.intent_sent_transaction else { panic!("should be known"); }; let succeeded = result.is_ok(); - self.ephem_rpc - .send_and_confirm_transaction(tx) - .await - .map_err(InternalOutboxClientError::RpcClientError)?; self.sent_commits.lock().unwrap().push((meta.id, succeeded)); Ok(()) } From 1a93f67e5c44960909bb1c7ccfc7d63c111d2f03 Mon Sep 17 00:00:00 2001 From: taco-paco Date: Tue, 7 Jul 2026 21:13:54 +0700 Subject: [PATCH 96/97] fix: coderabbit --- programs/magicblock/src/magic_context.rs | 21 +++++++------------ .../tests/test_outbox_flow.rs | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/programs/magicblock/src/magic_context.rs b/programs/magicblock/src/magic_context.rs index 8105486bb..a7505ee7b 100644 --- a/programs/magicblock/src/magic_context.rs +++ b/programs/magicblock/src/magic_context.rs @@ -62,37 +62,32 @@ impl MagicContext { } /// Returns `intent_id` store in `MagicContext` without deserializing whole account - pub fn intent_id(data: &[u8]) -> u64 { + pub fn intent_id(data: &[u8]) -> Option { const ID_OFFSET: usize = 0; const ID_END: usize = mem::size_of::(); - let Some(raw_id) = data.get(ID_OFFSET..ID_END) else { - return 0; - }; + let raw_id = data.get(ID_OFFSET..ID_END)?; let mut buf = [0; mem::size_of::()]; buf.copy_from_slice(raw_id); - u64::from_le_bytes(buf) + Some(u64::from_le_bytes(buf)) } - pub fn scheduled_intents_len(data: &[u8]) -> u64 { + pub fn scheduled_intents_len(data: &[u8]) -> Option { const LEN_OFFSET: usize = mem::size_of::(); const LEN_END: usize = LEN_OFFSET + mem::size_of::(); if is_zeroed(data) { - return 0; + return None; } - let Some(raw_len) = data.get(LEN_OFFSET..LEN_END) else { - return 0; - }; - + let raw_len = data.get(LEN_OFFSET..LEN_END)?; let mut len = [0; mem::size_of::()]; len.copy_from_slice(raw_len); - u64::from_le_bytes(len) + Some(u64::from_le_bytes(len)) } pub fn has_scheduled_intents(data: &[u8]) -> bool { - Self::scheduled_intents_len(data) != 0 + Self::scheduled_intents_len(data).unwrap_or(0) != 0 } } diff --git a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs index 91caef10c..4f31a561e 100644 --- a/test-integration/test-schedule-intent/tests/test_outbox_flow.rs +++ b/test-integration/test-schedule-intent/tests/test_outbox_flow.rs @@ -1050,7 +1050,7 @@ fn ensure_validator_authority() -> Keypair { fn read_next_intent_id(ctx: &IntegrationTestContext) -> u64 { let data = ctx.fetch_ephem_account_data(MAGIC_CONTEXT_PUBKEY).unwrap(); - MagicContext::intent_id(&data) + MagicContext::intent_id(&data).unwrap() } pub fn schedule_commit_instruction( From 5d7a22bae4315eebef75db89dae8e3248d19e88f Mon Sep 17 00:00:00 2001 From: taco-paco Date: Wed, 8 Jul 2026 11:59:06 +0700 Subject: [PATCH 97/97] feat: upd .lock --- Cargo.lock | 80 ++++++++++++++-------------- test-integration/Cargo.lock | 102 ++++++++++++++++++------------------ 2 files changed, 91 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7945ae5bb..fd021c30f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,7 +280,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -291,7 +291,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -791,7 +791,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -811,7 +811,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -1939,7 +1939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2403,7 +2403,7 @@ dependencies = [ [[package]] name = "guinea" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "magicblock-magic-program-api", @@ -3453,7 +3453,7 @@ dependencies = [ [[package]] name = "magicblock-account-cloner" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "bincode", @@ -3480,7 +3480,7 @@ dependencies = [ [[package]] name = "magicblock-accounts" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "magicblock-committor-service", @@ -3493,7 +3493,7 @@ dependencies = [ [[package]] name = "magicblock-accounts-db" -version = "0.13.3" +version = "0.13.4" dependencies = [ "flate2", "lmdb-rkv", @@ -3515,7 +3515,7 @@ dependencies = [ [[package]] name = "magicblock-aml" -version = "0.13.3" +version = "0.13.4" dependencies = [ "futures-util", "magicblock-config", @@ -3533,7 +3533,7 @@ dependencies = [ [[package]] name = "magicblock-aperture" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "agave-geyser-plugin-interface", @@ -3584,7 +3584,7 @@ dependencies = [ [[package]] name = "magicblock-api" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "anyhow", @@ -3647,7 +3647,7 @@ dependencies = [ [[package]] name = "magicblock-chainlink" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "arc-swap", @@ -3705,7 +3705,7 @@ dependencies = [ [[package]] name = "magicblock-committor-program" -version = "0.13.3" +version = "0.13.4" dependencies = [ "borsh", "paste", @@ -3720,7 +3720,7 @@ dependencies = [ [[package]] name = "magicblock-committor-service" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "backoff", @@ -3770,7 +3770,7 @@ dependencies = [ [[package]] name = "magicblock-config" -version = "0.13.3" +version = "0.13.4" dependencies = [ "clap", "derive_more", @@ -3790,7 +3790,7 @@ dependencies = [ [[package]] name = "magicblock-core" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "bytes", @@ -3850,7 +3850,7 @@ dependencies = [ [[package]] name = "magicblock-ledger" -version = "0.13.3" +version = "0.13.4" dependencies = [ "arc-swap", "bincode", @@ -3892,7 +3892,7 @@ dependencies = [ [[package]] name = "magicblock-magic-program-api" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "const-crypto", @@ -3906,7 +3906,7 @@ dependencies = [ [[package]] name = "magicblock-metrics" -version = "0.13.3" +version = "0.13.4" dependencies = [ "http-body-util", "hyper", @@ -3921,7 +3921,7 @@ dependencies = [ [[package]] name = "magicblock-processor" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "agave-precompiles", @@ -3974,7 +3974,7 @@ dependencies = [ [[package]] name = "magicblock-program" -version = "0.13.3" +version = "0.13.4" dependencies = [ "assert_matches", "bincode", @@ -4015,7 +4015,7 @@ dependencies = [ [[package]] name = "magicblock-replicator" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-nats", "bincode", @@ -4041,7 +4041,7 @@ dependencies = [ [[package]] name = "magicblock-rpc-client" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "futures-util", @@ -4068,7 +4068,7 @@ dependencies = [ [[package]] name = "magicblock-services" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "futures-util", @@ -4089,7 +4089,7 @@ dependencies = [ [[package]] name = "magicblock-table-mania" -version = "0.13.3" +version = "0.13.4" dependencies = [ "ed25519-dalek 1.0.1", "magicblock-metrics", @@ -4116,7 +4116,7 @@ dependencies = [ [[package]] name = "magicblock-task-scheduler" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "chrono", @@ -4143,7 +4143,7 @@ dependencies = [ [[package]] name = "magicblock-tui-client" -version = "0.13.3" +version = "0.13.4" dependencies = [ "chrono", "clap", @@ -4165,7 +4165,7 @@ dependencies = [ [[package]] name = "magicblock-validator" -version = "0.13.3" +version = "0.13.4" dependencies = [ "console-subscriber", "magicblock-api", @@ -4181,7 +4181,7 @@ dependencies = [ [[package]] name = "magicblock-validator-admin" -version = "0.13.3" +version = "0.13.4" dependencies = [ "magicblock-delegation-program-api", "magicblock-program", @@ -4198,7 +4198,7 @@ dependencies = [ [[package]] name = "magicblock-version" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "git-version", @@ -4429,7 +4429,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4997,7 +4997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph", @@ -5018,7 +5018,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -5720,7 +5720,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6274,7 +6274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8546,7 +8546,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "bs58", @@ -9621,12 +9621,12 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "test-kit" -version = "0.13.3" +version = "0.13.4" dependencies = [ "guinea", "magicblock-accounts-db", @@ -10570,7 +10570,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 392d64905..8339bc05a 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -289,7 +289,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -300,7 +300,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -809,7 +809,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -829,7 +829,7 @@ dependencies = [ "bitflags 2.13.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -2055,7 +2055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2521,10 +2521,10 @@ dependencies = [ [[package]] name = "guinea" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "serde", "solana-program 3.0.0", ] @@ -3520,14 +3520,14 @@ dependencies = [ [[package]] name = "magicblock-account-cloner" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "bincode", "magicblock-chainlink", "magicblock-core", "magicblock-ledger", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "magicblock-program", "rand 0.9.4", "solana-account 3.4.0", @@ -3547,7 +3547,7 @@ dependencies = [ [[package]] name = "magicblock-accounts" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "magicblock-committor-service", @@ -3560,12 +3560,12 @@ dependencies = [ [[package]] name = "magicblock-accounts-db" -version = "0.13.3" +version = "0.13.4" dependencies = [ "flate2", "lmdb-rkv", "magicblock-config", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "memmap2 0.9.10", "parking_lot", "reflink-copy", @@ -3580,7 +3580,7 @@ dependencies = [ [[package]] name = "magicblock-aml" -version = "0.13.3" +version = "0.13.4" dependencies = [ "futures-util", "magicblock-config", @@ -3596,7 +3596,7 @@ dependencies = [ [[package]] name = "magicblock-aperture" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "agave-geyser-plugin-interface", @@ -3642,7 +3642,7 @@ dependencies = [ [[package]] name = "magicblock-api" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "anyhow", @@ -3659,7 +3659,7 @@ dependencies = [ "magicblock-core", "magicblock-delegation-program-api 0.3.0", "magicblock-ledger", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "magicblock-metrics", "magicblock-processor", "magicblock-program", @@ -3705,7 +3705,7 @@ dependencies = [ [[package]] name = "magicblock-chainlink" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "arc-swap", @@ -3720,7 +3720,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-delegation-program-api 0.3.0", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "magicblock-metrics", "parking_lot", "scc", @@ -3761,7 +3761,7 @@ dependencies = [ [[package]] name = "magicblock-committor-program" -version = "0.13.3" +version = "0.13.4" dependencies = [ "borsh 1.7.0", "paste", @@ -3776,7 +3776,7 @@ dependencies = [ [[package]] name = "magicblock-committor-service" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-trait", "backoff", @@ -3823,7 +3823,7 @@ dependencies = [ [[package]] name = "magicblock-config" -version = "0.13.3" +version = "0.13.4" dependencies = [ "clap", "derive_more", @@ -3841,12 +3841,12 @@ dependencies = [ [[package]] name = "magicblock-core" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "bytes", "flume", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "serde", "solana-account 3.4.0", "solana-account-decoder", @@ -3933,7 +3933,7 @@ dependencies = [ [[package]] name = "magicblock-ledger" -version = "0.13.3" +version = "0.13.4" dependencies = [ "arc-swap", "bincode", @@ -3984,7 +3984,7 @@ dependencies = [ [[package]] name = "magicblock-magic-program-api" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "const-crypto", @@ -3996,7 +3996,7 @@ dependencies = [ [[package]] name = "magicblock-metrics" -version = "0.13.3" +version = "0.13.4" dependencies = [ "http-body-util", "hyper", @@ -4011,7 +4011,7 @@ dependencies = [ [[package]] name = "magicblock-processor" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "agave-precompiles", @@ -4057,13 +4057,13 @@ dependencies = [ [[package]] name = "magicblock-program" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "lazy_static", "magicblock-core", "magicblock-delegation-program-api 0.3.0", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "num-derive", "num-traits", "parking_lot", @@ -4093,7 +4093,7 @@ dependencies = [ [[package]] name = "magicblock-replicator" -version = "0.13.3" +version = "0.13.4" dependencies = [ "async-nats", "bincode", @@ -4118,7 +4118,7 @@ dependencies = [ [[package]] name = "magicblock-rpc-client" -version = "0.13.3" +version = "0.13.4" dependencies = [ "futures-util", "magicblock-metrics", @@ -4144,12 +4144,12 @@ dependencies = [ [[package]] name = "magicblock-services" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "futures-util", "magicblock-core", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "solana-instruction 3.3.0", "solana-keypair", "solana-message 3.1.0", @@ -4165,7 +4165,7 @@ dependencies = [ [[package]] name = "magicblock-table-mania" -version = "0.13.3" +version = "0.13.4" dependencies = [ "ed25519-dalek 1.0.1", "magicblock-metrics", @@ -4192,7 +4192,7 @@ dependencies = [ [[package]] name = "magicblock-task-scheduler" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "chrono", @@ -4218,7 +4218,7 @@ dependencies = [ [[package]] name = "magicblock-validator-admin" -version = "0.13.3" +version = "0.13.4" dependencies = [ "magicblock-delegation-program-api 0.3.0", "magicblock-program", @@ -4235,7 +4235,7 @@ dependencies = [ [[package]] name = "magicblock-version" -version = "0.13.3" +version = "0.13.4" dependencies = [ "agave-feature-set", "git-version", @@ -4478,7 +4478,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5048,7 +5048,7 @@ dependencies = [ "bincode", "borsh 1.7.0", "ephemeral-rollups-sdk", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "serde", "solana-program 3.0.0", "solana-system-interface 3.1.0", @@ -5070,7 +5070,7 @@ dependencies = [ "borsh 1.7.0", "ephemeral-rollups-sdk", "magicblock-delegation-program-api 0.3.0", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "rkyv 0.7.46", "solana-program 3.0.0", "solana-system-interface 3.1.0", @@ -5119,7 +5119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph", @@ -5140,7 +5140,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -5828,7 +5828,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6008,7 +6008,7 @@ dependencies = [ "integration-test-tools", "magicblock-core", "magicblock-delegation-program-api 0.3.0", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "magicblock-program", "program-schedulecommit", "rand 0.8.6", @@ -6029,7 +6029,7 @@ version = "0.0.0" dependencies = [ "integration-test-tools", "magicblock-delegation-program-api 0.3.0", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "program-schedulecommit", "program-schedulecommit-security", "schedulecommit-client", @@ -6450,7 +6450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8711,7 +8711,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "0.13.3" +version = "0.13.4" dependencies = [ "bincode", "bs58", @@ -9786,7 +9786,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9896,7 +9896,7 @@ dependencies = [ [[package]] name = "test-kit" -version = "0.13.3" +version = "0.13.4" dependencies = [ "guinea", "magicblock-accounts-db", @@ -10004,7 +10004,7 @@ dependencies = [ "magicblock-committor-service", "magicblock-core", "magicblock-delegation-program-api 0.3.0", - "magicblock-magic-program-api 0.13.3", + "magicblock-magic-program-api 0.13.4", "magicblock-program", "magicblock-rpc-client", "magicblock-table-mania", @@ -10983,7 +10983,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]]