From a5896d290f80614c05a7508e12c28376e6cbca30 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Mon, 29 Jun 2026 18:00:07 +0200 Subject: [PATCH 01/13] feat: replace task scheduler with hydra --- Cargo.lock | 2 + config.example.toml | 27 - magicblock-api/src/magic_validator.rs | 1 - magicblock-config/src/config/mod.rs | 2 - magicblock-config/src/config/scheduler.rs | 42 - magicblock-config/src/consts.rs | 7 - magicblock-config/src/lib.rs | 4 +- magicblock-config/src/tests.rs | 48 - .../src/instruction.rs | 11 - magicblock-magic-program-api/src/pda.rs | 9 - magicblock-processor/src/builtins.rs | 5 - magicblock-task-scheduler/Cargo.toml | 2 + magicblock-task-scheduler/src/db.rs | 489 +------ magicblock-task-scheduler/src/hydra.rs | 202 +++ magicblock-task-scheduler/src/lib.rs | 10 + magicblock-task-scheduler/src/service.rs | 1204 +++++------------ .../magicblock/src/magicblock_processor.rs | 64 +- programs/magicblock/src/schedule_task/mod.rs | 19 +- .../src/schedule_task/process_execute_task.rs | 381 ------ .../schedule_task/process_schedule_task.rs | 6 +- .../magicblock/src/utils/instruction_utils.rs | 45 +- test-integration/Cargo.lock | 2 + .../configs/schedule-task.devnet.toml | 4 + test-integration/programs/hydra/hydra.so | Bin 0 -> 24536 bytes .../test-ledger-restore/src/lib.rs | 7 +- .../test-task-scheduler/src/lib.rs | 148 +- .../tests/test_cancel_ongoing_task.rs | 95 +- .../tests/test_independent_authority.rs | 135 ++ .../tests/test_migration.rs | 65 + .../tests/test_reschedule_task.rs | 85 +- .../tests/test_schedule_error.rs | 173 --- .../tests/test_schedule_magic_cpi_crank.rs | 141 -- .../tests/test_schedule_task.rs | 70 +- .../tests/test_unauthorized_reschedule.rs | 200 --- .../tests/test_use_crank_signer.rs | 91 -- 35 files changed, 996 insertions(+), 2800 deletions(-) delete mode 100644 magicblock-config/src/config/scheduler.rs create mode 100644 magicblock-task-scheduler/src/hydra.rs delete mode 100644 programs/magicblock/src/schedule_task/process_execute_task.rs create mode 100755 test-integration/programs/hydra/hydra.so create mode 100644 test-integration/test-task-scheduler/tests/test_independent_authority.rs create mode 100644 test-integration/test-task-scheduler/tests/test_migration.rs delete mode 100644 test-integration/test-task-scheduler/tests/test_schedule_error.rs delete mode 100644 test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs delete mode 100644 test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs delete mode 100644 test-integration/test-task-scheduler/tests/test_use_crank_signer.rs diff --git a/Cargo.lock b/Cargo.lock index 9c2ab6345..0b7c5acd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4124,7 +4124,9 @@ dependencies = [ "solana-pubkey 4.1.0", "solana-rpc-client", "solana-rpc-client-api", + "solana-sha256-hasher 3.1.0", "solana-signature 3.3.0", + "solana-system-interface 3.1.0", "solana-transaction", "solana-transaction-error 3.1.0", "thiserror 2.0.18", diff --git a/config.example.toml b/config.example.toml index f9e12417b..1b05477a7 100644 --- a/config.example.toml +++ b/config.example.toml @@ -311,33 +311,6 @@ fqdn = "https://validator.example.com" # Env: MBV_CHAIN_OPERATION__CLAIM_FEES_FREQUENCY claim-fees-frequency = "24h" -# ============================================================================== -# Internal Task Scheduler -# ============================================================================== -[task-scheduler] -# If true, clears all pending scheduled tasks on startup. -# Default: false -# Env: MBV_TASK_SCHEDULER__RESET -reset = false - -# The minimum interval between task executions. -# Supports humantime (e.g. "10ms", "1s"). -# Default: "10ms" -# Env: MBV_TASK_SCHEDULER__MIN_INTERVAL -min-interval = "10ms" - -# How long failed task executions and failed scheduling records are retained. -# Supports humantime (e.g. "1h", "7d"). -# Default: "14d" -# Env: MBV_TASK_SCHEDULER__FAILED_TASK_RETENTION -failed-task-retention = "14d" - -# How often old failed task executions and failed scheduling records are deleted. -# Supports humantime (e.g. "1m", "1h"). -# Default: "1h" -# Env: MBV_TASK_SCHEDULER__FAILED_TASK_CLEANUP_INTERVAL -failed-task-cleanup-interval = "1h" - # ============================================================================== # Pre-loaded Programs # ============================================================================== diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index ea1585b1f..431ea3d89 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -413,7 +413,6 @@ impl MagicValidator { let step_start = Instant::now(); let task_scheduler = TaskSchedulerService::new( &task_scheduler_db_path, - &config.task_scheduler, config.aperture.listen.http(), dispatch .tasks_service diff --git a/magicblock-config/src/config/mod.rs b/magicblock-config/src/config/mod.rs index 5d9377d10..86e6fe3e2 100644 --- a/magicblock-config/src/config/mod.rs +++ b/magicblock-config/src/config/mod.rs @@ -7,7 +7,6 @@ pub mod ledger; pub mod lifecycle; pub mod metrics; pub mod program; -pub mod scheduler; pub mod validator; // Re-export types for backward compatibility and easier access @@ -21,5 +20,4 @@ pub use grpc::GrpcConfig; pub use ledger::LedgerConfig; pub use lifecycle::LifecycleMode; pub use program::LoadableProgram; -pub use scheduler::TaskSchedulerConfig; pub use validator::ValidatorConfig; diff --git a/magicblock-config/src/config/scheduler.rs b/magicblock-config/src/config/scheduler.rs deleted file mode 100644 index e27eadee6..000000000 --- a/magicblock-config/src/config/scheduler.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::time::Duration; - -use serde::{Deserialize, Serialize}; - -use crate::consts; - -/// Configuration for the internal task scheduler. -#[derive(Deserialize, Serialize, Debug, Clone)] -#[serde(rename_all = "kebab-case", deny_unknown_fields, default)] -pub struct TaskSchedulerConfig { - /// If true, clears all pending scheduled tasks on startup. - pub reset: bool, - /// The minimum interval between task executions. - /// Supports humantime (e.g. "10ms", "1s"). - #[serde(with = "humantime")] - pub min_interval: Duration, - /// How long failed task and scheduling records are retained. - /// Supports humantime (e.g. "1h", "7d"). - #[serde(with = "humantime")] - pub failed_task_retention: Duration, - /// How often failed task and scheduling records are cleaned up. - /// Supports humantime (e.g. "1m", "1h"). - #[serde(with = "humantime")] - pub failed_task_cleanup_interval: Duration, -} - -impl Default for TaskSchedulerConfig { - fn default() -> Self { - Self { - reset: false, - min_interval: Duration::from_millis( - consts::DEFAULT_TASK_SCHEDULER_MIN_INTERVAL_MILLIS, - ), - failed_task_retention: Duration::from_secs( - consts::DEFAULT_TASK_SCHEDULER_FAILED_TASK_RETENTION_SECS, - ), - failed_task_cleanup_interval: Duration::from_secs( - consts::DEFAULT_TASK_SCHEDULER_FAILED_TASK_CLEANUP_INTERVAL_SECS, - ), - } - } -} diff --git a/magicblock-config/src/consts.rs b/magicblock-config/src/consts.rs index e1452b7b4..7e8d07836 100644 --- a/magicblock-config/src/consts.rs +++ b/magicblock-config/src/consts.rs @@ -64,13 +64,6 @@ pub const DEFAULT_METRICS_ADDR: &str = "0.0.0.0:9000"; /// Default frequency of metrics collection in seconds pub const DEFAULT_METRICS_COLLECT_FREQUENCY_SEC: u64 = 30; -// Task Scheduler Defaults -pub const DEFAULT_TASK_SCHEDULER_MIN_INTERVAL_MILLIS: u64 = 10; -pub const DEFAULT_TASK_SCHEDULER_FAILED_TASK_RETENTION_SECS: u64 = - 14 * 24 * 60 * 60; // 14 days -pub const DEFAULT_TASK_SCHEDULER_FAILED_TASK_CLEANUP_INTERVAL_SECS: u64 = - 60 * 60; // 1 hour - // ChainLink Defaults /// Default delay in milliseconds between resubscribing to accounts after a pubsub reconnection pub const DEFAULT_RESUBSCRIPTION_DELAY_MS: u64 = 50; diff --git a/magicblock-config/src/lib.rs b/magicblock-config/src/lib.rs index f0136d376..79a394d9c 100644 --- a/magicblock-config/src/lib.rs +++ b/magicblock-config/src/lib.rs @@ -22,8 +22,7 @@ pub mod types; use crate::{ config::{ AccountsDbConfig, ChainLinkConfig, ChainOperationConfig, - CommittorConfig, LedgerConfig, LoadableProgram, TaskSchedulerConfig, - ValidatorConfig, + CommittorConfig, LedgerConfig, LoadableProgram, ValidatorConfig, }, types::Remote, }; @@ -66,7 +65,6 @@ pub struct ValidatorParams { pub ledger: LedgerConfig, pub chainlink: ChainLinkConfig, pub chain_operation: Option, - pub task_scheduler: TaskSchedulerConfig, pub programs: Vec, } diff --git a/magicblock-config/src/tests.rs b/magicblock-config/src/tests.rs index 1268d9bc8..8be2a6145 100644 --- a/magicblock-config/src/tests.rs +++ b/magicblock-config/src/tests.rs @@ -387,16 +387,6 @@ fn test_cli_ledger_reset_overrides_toml() { assert!(config.ledger.reset); } -#[test] -#[serial] -fn test_task_scheduler_bool_env() { - // Verify standard boolean parsing from Env vars work on nested fields - let _guard = EnvVarGuard::new("MBV_TASK_SCHEDULER__RESET", "true"); - - let config = run_cli(vec![]); - assert!(config.task_scheduler.reset); -} - #[test] #[parallel] fn test_example_config_full_coverage() { @@ -503,21 +493,6 @@ fn test_example_config_full_coverage() { // ======================================================================== // 11. Optional Sections // ======================================================================== - // Task scheduler reset should be false - assert!(!config.task_scheduler.reset); - assert_eq!( - config.task_scheduler.min_interval, - Duration::from_millis(10) - ); - assert_eq!( - config.task_scheduler.failed_task_retention, - Duration::from_secs(14 * 24 * 60 * 60) - ); - assert_eq!( - config.task_scheduler.failed_task_cleanup_interval, - Duration::from_secs(60 * 60) - ); - // The example file has the programs section with 2 entries assert_eq!( config.programs.len(), @@ -597,14 +572,6 @@ fn test_env_vars_full_coverage() { EnvVarGuard::new("MBV_CHAINLINK__RISK__CACHE_TTL", "45m"), EnvVarGuard::new("MBV_CHAINLINK__RISK__REQUEST_TIMEOUT", "3s"), EnvVarGuard::new("MBV_CHAINLINK__RISK__RISK_SCORE_THRESHOLD", "8"), - // --- Task Scheduler --- - EnvVarGuard::new("MBV_TASK_SCHEDULER__RESET", "true"), - EnvVarGuard::new("MBV_TASK_SCHEDULER__MIN_INTERVAL", "99ms"), - EnvVarGuard::new("MBV_TASK_SCHEDULER__FAILED_TASK_RETENTION", "2h"), - EnvVarGuard::new( - "MBV_TASK_SCHEDULER__FAILED_TASK_CLEANUP_INTERVAL", - "3m", - ), // --- Chain Operation (Optional Section) --- // Figment can instantiate optional structs if their fields are present EnvVarGuard::new("MBV_CHAIN_OPERATION__COUNTRY_CODE", "DE"), @@ -681,21 +648,6 @@ fn test_env_vars_full_coverage() { ); assert_eq!(config.chainlink.risk.risk_score_threshold, 8); - // Task Scheduler - assert!(config.task_scheduler.reset); - assert_eq!( - config.task_scheduler.min_interval, - Duration::from_millis(99) - ); - assert_eq!( - config.task_scheduler.failed_task_retention, - Duration::from_secs(2 * 60 * 60) - ); - assert_eq!( - config.task_scheduler.failed_task_cleanup_interval, - Duration::from_secs(3 * 60) - ); - // Chain Operation // Verify the optional struct was created and populated let chain_op = config diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index 208db91f6..9450f6061 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -300,17 +300,6 @@ pub enum MagicBlockInstruction { /// - **0.** `[SIGNER]` Validator Authority /// - **1.** `[WRITE]` Account to evict EvictAccount { pubkey: Pubkey }, - - /// Executes a crank - /// - /// # Account references - /// - **0.** `[SIGNER]` Validator authority - /// - **1.** `[]` Crank signer PDA - /// - **2..n** `[]` Accounts required by the embedded instructions - ExecuteCrank { - authority: Pubkey, - instructions: Vec, - }, } impl MagicBlockInstruction { diff --git a/magicblock-magic-program-api/src/pda.rs b/magicblock-magic-program-api/src/pda.rs index 234713ca5..21081b58e 100644 --- a/magicblock-magic-program-api/src/pda.rs +++ b/magicblock-magic-program-api/src/pda.rs @@ -1,14 +1,5 @@ use crate::Pubkey; -pub const CRANK_SEED: &[u8] = b"crank-executor"; -pub fn crank_signer_pda(authority: &Pubkey) -> Pubkey { - Pubkey::find_program_address( - &[CRANK_SEED, authority.as_ref()], - &crate::CRANK_PROGRAM_ID, - ) - .0 -} - /// Callback signer PDA info pub const CALLBACK_SEED: &[u8] = b"callback-executor"; const CALLBACK_SIGNER_PDA: ([u8; 32], u8) = diff --git a/magicblock-processor/src/builtins.rs b/magicblock-processor/src/builtins.rs index c7a500e05..2327be257 100644 --- a/magicblock-processor/src/builtins.rs +++ b/magicblock-processor/src/builtins.rs @@ -49,11 +49,6 @@ pub static BUILTINS: &[Builtin] = &[ name: "magicblock_program", entrypoint: magicblock_processor::Entrypoint::vm, }, - Builtin { - program_id: magicblock_program::CRANK_PROGRAM_ID, - name: "magicblock_crank_program", - entrypoint: magicblock_processor::CrankEntrypoint::vm, - }, Builtin { program_id: magicblock_program::CALLBACK_PROGRAM_ID, name: "magicblock_callback_program", diff --git a/magicblock-task-scheduler/Cargo.toml b/magicblock-task-scheduler/Cargo.toml index fa0ede3d6..842a932d7 100644 --- a/magicblock-task-scheduler/Cargo.toml +++ b/magicblock-task-scheduler/Cargo.toml @@ -22,7 +22,9 @@ solana-message = { workspace = true } solana-pubkey = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } +solana-sha256-hasher = { workspace = true } solana-signature = { workspace = true } +solana-system-interface = { workspace = true, features = ["bincode"] } solana-transaction = { workspace = true } solana-transaction-error = { workspace = true } thiserror = { workspace = true } diff --git a/magicblock-task-scheduler/src/db.rs b/magicblock-task-scheduler/src/db.rs index 3fc980dc8..e45abc6da 100644 --- a/magicblock-task-scheduler/src/db.rs +++ b/magicblock-task-scheduler/src/db.rs @@ -1,12 +1,11 @@ use std::{ - collections::HashMap, path::{Path, PathBuf}, sync::Arc, }; use chrono::Utc; use magicblock_program::args::ScheduleTaskRequest; -use rusqlite::{params, Connection, OptionalExtension}; +use rusqlite::{params, Connection}; use solana_instruction::Instruction; use solana_pubkey::Pubkey; use tokio::sync::Mutex; @@ -17,20 +16,16 @@ use crate::errors::TaskSchedulerResult; /// Uses i64 for all timestamps and IDs to avoid overflows #[derive(Debug, Clone, PartialEq, Eq)] pub struct DbTask { - /// Unique identifier for this task + /// Unique identifier for this task. pub id: i64, - /// Instructions to execute when triggered + /// Instructions to execute when triggered. pub instructions: Vec, - /// Authority that can modify or cancel this task + /// Authority that owns this task. pub authority: Pubkey, - /// How frequently the task should be executed, in milliseconds + /// How frequently the task should be executed, in milliseconds. pub execution_interval_millis: i64, /// Number of times this task still needs to be executed. pub executions_left: i64, - /// Timestamp of the last execution of this task in milliseconds since UNIX epoch - pub last_execution_millis: i64, - /// Timestamp of the latest persisted mutation of this task in milliseconds since UNIX epoch - pub updated_at: i64, } impl From for DbTask { @@ -41,122 +36,11 @@ impl From for DbTask { authority: task.authority, execution_interval_millis: task.execution_interval_millis, executions_left: task.iterations, - last_execution_millis: 0, - updated_at: 0, } } } -#[derive(Debug, Clone, Copy)] -pub struct CrankSuccessUpdate { - /// Task whose successful execution should be persisted. - pub task_id: i64, - /// Actual execution timestamp to store in `tasks.last_execution_millis`. - pub last_execution_millis: i64, - /// Optimistic concurrency token. The update is applied only if the current - /// row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, -} - -#[derive(Debug, Clone, Copy)] -pub struct CrankSuccessRemoval { - /// Task whose final successful execution should remove it from `tasks`. - pub task_id: i64, - /// Optimistic concurrency token. The removal is applied only if the current - /// row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, -} - -#[derive(Debug, Clone)] -pub struct CrankFailedMove { - /// Task to remove from `tasks` and append to `failed_tasks`. - pub task_id: i64, - /// Optimistic concurrency token. The move is applied only if the current - /// row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, - /// Error text persisted in `failed_tasks.error` when the move succeeds. - pub error: String, -} - -#[derive(Debug, Clone, Copy)] -pub struct CrankRetryCheck { - /// Task that failed but remains retryable. - pub task_id: i64, - /// Optimistic concurrency token. The retry is considered valid only if the - /// current row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, -} - -/// Applied metadata for a successful task execution that remains scheduled. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankSuccessUpdateCompletion { - /// Optimistic token supplied by the caller and matched against the previous - /// `tasks.updated_at` value. - pub expected_updated_at: i64, - /// New `tasks.updated_at` value written by the database transaction. - pub new_updated_at: i64, -} - -/// Applied metadata for a task removed after its final successful execution. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankSuccessRemovalCompletion { - /// Optimistic token supplied by the caller and matched against the previous - /// `tasks.updated_at` value before deleting the row. - pub expected_updated_at: i64, -} - -/// Applied metadata for a task moved from `tasks` to `failed_tasks`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankFailedMoveCompletion { - /// Optimistic token supplied by the caller and matched against the previous - /// `tasks.updated_at` value before moving the task. - pub expected_updated_at: i64, -} - -/// Applied metadata for a retryable failed task that still matches its DB row. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankRetryCheckCompletion { - /// Actual `tasks.updated_at` value read from the database. This equals the - /// requested optimistic token when the retry check succeeds. - pub current_updated_at: i64, -} - -/// Results for a crank batch transaction keyed by task ID. -/// -/// Each map contains only entries whose optimistic `expected_updated_at` token -/// matched the database state and whose corresponding DB operation succeeded. -#[derive(Debug, Default)] -pub struct CrankBatchCompletion { - /// Continued successful executions. Values include both the matched - /// optimistic token and the new `tasks.updated_at` written by the DB. - pub success_updates: HashMap, - /// Final successful executions removed from `tasks`. Values contain the - /// matched optimistic token used for the delete. - pub success_removals: HashMap, - /// Failed executions moved to `failed_tasks`. Values contain the matched - /// optimistic token used for the move. - pub failed_moves: HashMap, - /// Retryable failed executions whose `tasks` row still exists unchanged. - /// Values contain the actual `tasks.updated_at` read from the DB. - pub retry_checks: HashMap, -} - -#[derive(Debug, Clone)] -pub struct FailedScheduling { - pub id: i64, - pub timestamp: i64, - pub task_id: i64, - pub error: String, -} - -#[derive(Debug, Clone)] -pub struct FailedTask { - pub id: i64, - pub timestamp: i64, - pub task_id: i64, - pub error: String, -} - +/// Migration-only store over the legacy scheduler's SQLite database. #[derive(Clone)] pub struct SchedulerDatabase { conn: Arc>, @@ -174,11 +58,12 @@ impl SchedulerDatabase { PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA busy_timeout=5000; - PRAGMA cache_size=-65536; ", )?; - // Create tables + // Mirrors the legacy schema so existing databases can be read. The + // timestamp columns are retained for compatibility but are not used by + // migration. conn.execute( "CREATE TABLE IF NOT EXISTS tasks ( id INTEGER PRIMARY KEY, @@ -186,29 +71,9 @@ impl SchedulerDatabase { authority TEXT NOT NULL, execution_interval_millis INTEGER NOT NULL, executions_left INTEGER NOT NULL, - last_execution_millis INTEGER NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL - )", - [], - )?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS failed_scheduling ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp INTEGER NOT NULL, - task_id INTEGER, - error TEXT NOT NULL - )", - [], - )?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS failed_tasks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp INTEGER NOT NULL, - task_id INTEGER, - error TEXT NOT NULL + last_execution_millis INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT 0 )", [], )?; @@ -218,157 +83,37 @@ impl SchedulerDatabase { }) } - pub async fn insert_task(&self, task: &DbTask) -> TaskSchedulerResult { + /// Inserts (or replaces) a task. Used to seed legacy tasks, including from + /// tests exercising migration. + pub async fn insert_task(&self, task: &DbTask) -> TaskSchedulerResult<()> { let instructions_bin = bincode::serialize(&task.instructions)?; let authority_str = task.authority.to_string(); - let conn = self.conn.lock().await; - let previous_updated_at: Option = conn - .query_row( - "SELECT updated_at FROM tasks WHERE id = ?", - [task.id], - |row| row.get(0), - ) - .optional()?; - let now = previous_updated_at - .map(|updated_at| Utc::now().timestamp_millis().max(updated_at + 1)) - .unwrap_or_else(|| Utc::now().timestamp_millis()); - - conn.execute( - "INSERT OR REPLACE INTO tasks + let now = Utc::now().timestamp_millis(); + self.conn.lock().await.execute( + "INSERT OR REPLACE INTO tasks (id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + VALUES (?, ?, ?, ?, ?, 0, ?, ?)", params![ task.id, instructions_bin, authority_str, task.execution_interval_millis, task.executions_left, - task.last_execution_millis, now, now, ], )?; - - Ok(now) - } - - pub async fn update_task_after_execution( - &self, - task_id: i64, - last_execution: i64, - ) -> TaskSchedulerResult<()> { - let now = Utc::now().timestamp_millis(); - - self.conn.lock().await.execute( - "UPDATE tasks - SET executions_left = executions_left - 1, - last_execution_millis = ?, - updated_at = ? - WHERE id = ?", - params![last_execution, now, task_id], - )?; - - Ok(()) - } - - pub async fn insert_failed_scheduling( - &self, - task_id: i64, - error: String, - ) -> TaskSchedulerResult<()> { - self.conn.lock().await.execute( - "INSERT INTO failed_scheduling (timestamp, task_id, error) VALUES (?, ?, ?)", - params![Utc::now().timestamp_millis(), task_id, error], - )?; - - Ok(()) - } - - pub async fn insert_failed_task( - &self, - task_id: i64, - error: String, - ) -> TaskSchedulerResult<()> { - self.conn.lock().await.execute( - "INSERT INTO failed_tasks (timestamp, task_id, error) VALUES (?, ?, ?)", - params![Utc::now().timestamp_millis(), task_id, error], - )?; - Ok(()) } - pub async fn unschedule_task( - &self, - task_id: i64, - ) -> TaskSchedulerResult<()> { - self.conn.lock().await.execute( - "UPDATE tasks SET executions_left = 0 WHERE id = ?", - [task_id], - )?; - - Ok(()) - } - - pub async fn remove_task(&self, task_id: i64) -> TaskSchedulerResult<()> { - self.conn - .lock() - .await - .execute("DELETE FROM tasks WHERE id = ?", [task_id])?; - - Ok(()) - } - - pub async fn get_task( - &self, - task_id: i64, - ) -> TaskSchedulerResult> { - let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, - updated_at - FROM tasks WHERE id = ?" - )?; - - let mut rows = stmt.query_map([task_id], |row| { - let instructions_bin: Vec = row.get(1)?; - let instructions: Vec = - bincode::deserialize(&instructions_bin).map_err(|e| { - rusqlite::Error::InvalidParameterName(format!( - "instructions: {}", - e - )) - })?; - let authority_str: String = row.get(2)?; - let authority: Pubkey = authority_str.parse().map_err(|e| { - rusqlite::Error::InvalidParameterName(format!( - "authority: {}", - e - )) - })?; - - Ok(DbTask { - id: row.get(0)?, - instructions, - authority, - execution_interval_millis: row.get(3)?, - executions_left: row.get(4)?, - last_execution_millis: row.get(5)?, - updated_at: row.get(6)?, - }) - })?; - - Ok(rows.next().transpose()?) - } - + /// Returns all persisted tasks awaiting migration. pub async fn get_tasks(&self) -> TaskSchedulerResult> { let db = self.conn.lock().await; let mut stmt = db.prepare( - "SELECT id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, - updated_at - FROM tasks" + "SELECT id, instructions, authority, execution_interval_millis, executions_left + FROM tasks", )?; - let mut tasks = Vec::new(); let rows = stmt.query_map([], |row| { let instructions_bin: Vec = row.get(1)?; let instructions: Vec = @@ -392,195 +137,27 @@ impl SchedulerDatabase { authority, execution_interval_millis: row.get(3)?, executions_left: row.get(4)?, - last_execution_millis: row.get(5)?, - updated_at: row.get(6)?, }) })?; - for row in rows { - tasks.push(row?); - } - - Ok(tasks) + Ok(rows.collect::, rusqlite::Error>>()?) } + /// Returns the ids of all persisted tasks. Used by tests to assert the + /// database empties after migration. pub async fn get_task_ids(&self) -> TaskSchedulerResult> { let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT id - FROM tasks", - )?; - + let mut stmt = db.prepare("SELECT id FROM tasks")?; let rows = stmt.query_map([], |row| row.get(0))?; - Ok(rows.collect::, rusqlite::Error>>()?) } - pub async fn get_failed_schedulings( - &self, - ) -> TaskSchedulerResult> { - let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT * - FROM failed_scheduling", - )?; - - let rows = stmt.query_map([], |row| { - Ok(FailedScheduling { - id: row.get(0)?, - timestamp: row.get(1)?, - task_id: row.get(2)?, - error: row.get(3)?, - }) - })?; - - Ok(rows.collect::, rusqlite::Error>>()?) - } - - pub async fn get_failed_tasks( - &self, - ) -> TaskSchedulerResult> { - let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT * - FROM failed_tasks", - )?; - - let rows = stmt.query_map([], |row| { - Ok(FailedTask { - id: row.get(0)?, - timestamp: row.get(1)?, - task_id: row.get(2)?, - error: row.get(3)?, - }) - })?; - - Ok(rows.collect::, rusqlite::Error>>()?) - } - - /// One transaction for a crank completion batch — replaces N separate commits from - /// per-task `update_task_after_execution` / `remove_task` / `move_task_to_failed` calls. - pub async fn apply_crank_batch_completion( - &self, - success_updates: &[CrankSuccessUpdate], - success_removals: &[CrankSuccessRemoval], - failed_moves: &[CrankFailedMove], - retry_checks: &[CrankRetryCheck], - ) -> TaskSchedulerResult { - if success_updates.is_empty() - && success_removals.is_empty() - && failed_moves.is_empty() - && retry_checks.is_empty() - { - return Ok(CrankBatchCompletion::default()); - } - - let mut conn = self.conn.lock().await; - let tx = conn.transaction()?; - let mut completion = CrankBatchCompletion::default(); - - // Continued executions — decrement executions_left via existing UPDATE semantics - for update in success_updates { - let now = Utc::now() - .timestamp_millis() - .max(update.expected_updated_at + 1); - let affected = tx.execute( - "UPDATE tasks SET executions_left = executions_left - 1, - last_execution_millis = ?, updated_at = ? - WHERE id = ? AND updated_at = ?", - params![ - update.last_execution_millis, - now, - update.task_id, - update.expected_updated_at - ], - )?; - if affected == 1 { - completion.success_updates.insert( - update.task_id, - CrankSuccessUpdateCompletion { - expected_updated_at: update.expected_updated_at, - new_updated_at: now, - }, - ); - } - } - - for removal in success_removals { - let affected = tx.execute( - "DELETE FROM tasks WHERE id = ? AND updated_at = ?", - params![removal.task_id, removal.expected_updated_at], - )?; - if affected == 1 { - completion.success_removals.insert( - removal.task_id, - CrankSuccessRemovalCompletion { - expected_updated_at: removal.expected_updated_at, - }, - ); - } - } - - for failed in failed_moves { - let affected = tx.execute( - "DELETE FROM tasks WHERE id = ? AND updated_at = ?", - params![failed.task_id, failed.expected_updated_at], - )?; - if affected == 1 { - let now = Utc::now().timestamp_millis(); - tx.execute( - "INSERT INTO failed_tasks (timestamp, task_id, error) - VALUES (?, ?, ?)", - params![now, failed.task_id, failed.error], - )?; - completion.failed_moves.insert( - failed.task_id, - CrankFailedMoveCompletion { - expected_updated_at: failed.expected_updated_at, - }, - ); - } - } - - for check in retry_checks { - let matched: Option = tx - .query_row( - "SELECT updated_at FROM tasks - WHERE id = ? AND updated_at = ?", - params![check.task_id, check.expected_updated_at], - |row| row.get(0), - ) - .optional()?; - if let Some(updated_at) = matched { - completion.retry_checks.insert( - check.task_id, - CrankRetryCheckCompletion { - current_updated_at: updated_at, - }, - ); - } - } - - tx.commit()?; - Ok(completion) - } - - pub async fn delete_failed_records_older_than( - &self, - cutoff_timestamp_millis: i64, - ) -> TaskSchedulerResult { - let mut conn = self.conn.lock().await; - let tx = conn.transaction()?; - let failed_scheduling_deleted = tx.execute( - "DELETE FROM failed_scheduling WHERE timestamp < ?", - [cutoff_timestamp_millis], - )?; - let failed_tasks_deleted = tx.execute( - "DELETE FROM failed_tasks WHERE timestamp < ?", - [cutoff_timestamp_millis], - )?; - tx.commit()?; - - Ok(failed_scheduling_deleted + failed_tasks_deleted) + /// Removes a task once it has been migrated onto hydra. + pub async fn remove_task(&self, task_id: i64) -> TaskSchedulerResult<()> { + self.conn + .lock() + .await + .execute("DELETE FROM tasks WHERE id = ?", [task_id])?; + Ok(()) } } diff --git a/magicblock-task-scheduler/src/hydra.rs b/magicblock-task-scheduler/src/hydra.rs new file mode 100644 index 000000000..b9aceb5c8 --- /dev/null +++ b/magicblock-task-scheduler/src/hydra.rs @@ -0,0 +1,202 @@ +//! Minimal, self-contained client builders for the hydra ephemeral crank +//! program. +//! +//! The validator's RPC client stack pins `solana-rpc-client-types 4.0.0`, which +//! caps `solana-address` below the `^2.6` that the upstream `hydra-api` crate +//! requires; the two cannot be unified in a single build. Rather than move the +//! whole validator onto pre-release RPC crates, we vendor the small, stable +//! slice of hydra's wire format that the scheduler needs. The integration is +//! pinned to a specific hydra revision, so this layout will not drift +//! unexpectedly. +//! +//! Mirrors `hydra-api` rev `1fc9086` (`crates/hydra-api/src/instruction.rs`, +//! the `ephemeral` builders). See that file for the authoritative wire format. + +use magicblock_program::EPHEMERAL_VAULT_PUBKEY; +use solana_instruction::{AccountMeta, Instruction}; +use solana_pubkey::Pubkey; + +/// Ephemeral-rollup hydra program id (`eHyd5BU8QffvHi4GnXwxrK4WpS7pM2x9UGKHBWii7mf`). +pub const EPHEMERAL_PROGRAM_ID: Pubkey = Pubkey::new_from_array([ + 9, 141, 175, 94, 62, 38, 45, 106, 31, 231, 193, 37, 229, 238, 178, 89, 200, + 202, 82, 70, 56, 177, 52, 125, 239, 164, 240, 139, 173, 185, 238, 142, +]); + +/// Seed prefix for the crank PDA: `[b"crank", seed]`. +const CRANK_SEED_PREFIX: &[u8] = b"crank"; + +/// Instruction discriminators (one byte each). +const IX_CREATE: u8 = 0; +const IX_CANCEL: u8 = 2; + +/// Per-meta writable flag in the scheduled-ix wire format. +const META_FLAG_WRITABLE: u8 = 0b0000_0010; + +/// Flat per-trigger reward (lamports) paid to the cranker. Matches +/// `hydra_api::consts::CRANKER_REWARD` for the deployed program, which is built +/// without the `hydra-api/ephemeral` feature, so `BASE_FEE_LAMPORTS = 5_000` +/// and `CRANKER_REWARD = 2 * BASE_FEE_LAMPORTS`. +pub const CRANKER_REWARD: u64 = 10_000; + +/// One scheduled-ix account meta as stored on-chain. Scheduled instructions may +/// not carry signer flags, so only the writable bit is represented. +pub struct SchedMeta { + pub pubkey: [u8; 32], + pub is_writable: bool, +} + +/// One scheduled instruction template. +pub struct ScheduledIx<'a> { + pub program_id: [u8; 32], + pub metas: &'a [SchedMeta], + pub data: &'a [u8], +} + +/// Scheduling knobs for a hydra `Create` instruction. +pub struct CreateArgs<'a> { + pub seed: [u8; 32], + /// All-zeros = unkillable (no cancel authority). + pub authority: [u8; 32], + pub start_slot: u64, + pub interval_slots: u64, + /// `0` on the wire means "infinite"; the scheduler always sets a finite + /// iteration count. + pub remaining: u64, + pub priority_tip: u64, + pub cu_limit: u32, + /// Scheduled instructions in execution order. Must be non-empty. + pub scheduled: &'a [ScheduledIx<'a>], +} + +impl CreateArgs<'_> { + /// Serializes the `Create` instruction data exactly as hydra's on-chain + /// entrypoint parses it. + fn serialize(&self) -> Vec { + // 1 discriminator + 100-byte fixed prefix + variable body. + let body_len: usize = self + .scheduled + .iter() + .map(|s| 1 + 2 + 32 + 33 * s.metas.len() + s.data.len()) + .sum(); + let mut data = Vec::with_capacity(1 + 100 + body_len); + + data.push(IX_CREATE); + data.extend_from_slice(&self.seed); + data.extend_from_slice(&self.authority); + data.extend_from_slice(&self.start_slot.to_le_bytes()); + data.extend_from_slice(&self.interval_slots.to_le_bytes()); + data.extend_from_slice(&self.remaining.to_le_bytes()); + data.extend_from_slice(&self.priority_tip.to_le_bytes()); + data.extend_from_slice(&self.cu_limit.to_le_bytes()); + + for s in self.scheduled { + data.push(s.metas.len() as u8); + data.extend_from_slice(&(s.data.len() as u16).to_le_bytes()); + data.extend_from_slice(&s.program_id); + for m in s.metas { + let flag = if m.is_writable { META_FLAG_WRITABLE } else { 0 }; + data.push(flag); + data.extend_from_slice(&m.pubkey); + } + data.extend_from_slice(s.data); + } + + data + } +} + +/// Builders targeting the ephemeral-rollup hydra program. +pub mod ephemeral { + use super::*; + + /// Derives `(crank_pda, bump)` under the ephemeral hydra program. + pub fn find_crank_pda(seed: &[u8; 32]) -> (Pubkey, u8) { + Pubkey::find_program_address( + &[CRANK_SEED_PREFIX, seed], + &EPHEMERAL_PROGRAM_ID, + ) + } + + /// Builds a hydra `Create` instruction for the ephemeral program. + pub fn create( + sponsor: Pubkey, + crank: Pubkey, + args: &CreateArgs<'_>, + ) -> Instruction { + Instruction { + program_id: EPHEMERAL_PROGRAM_ID, + accounts: vec![ + AccountMeta::new(sponsor, true), + AccountMeta::new(crank, false), + AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), + AccountMeta::new_readonly(magicblock_program::id(), false), + ], + data: args.serialize(), + } + } + + /// Builds a hydra `Cancel` instruction for the ephemeral program. The + /// remaining crank balance is refunded to the ephemeral vault. + pub fn cancel(authority: Pubkey, crank: Pubkey) -> Instruction { + Instruction { + program_id: EPHEMERAL_PROGRAM_ID, + accounts: vec![ + AccountMeta::new(authority, true), + AccountMeta::new(crank, false), + AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), + AccountMeta::new_readonly(magicblock_program::id(), false), + ], + data: vec![IX_CANCEL], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ephemeral_program_id_matches_base58() { + assert_eq!( + EPHEMERAL_PROGRAM_ID.to_string(), + "eHyd5BU8QffvHi4GnXwxrK4WpS7pM2x9UGKHBWii7mf" + ); + } + + #[test] + fn create_data_layout_matches_wire_format() { + let program_id = [7u8; 32]; + let acct = [9u8; 32]; + let metas = [SchedMeta { + pubkey: acct, + is_writable: true, + }]; + let scheduled = [ScheduledIx { + program_id, + metas: &metas, + data: &[1, 2, 3], + }]; + let args = CreateArgs { + seed: [1u8; 32], + authority: [2u8; 32], + start_slot: 5, + interval_slots: 7, + remaining: 9, + priority_tip: 11, + cu_limit: 0, + scheduled: &scheduled, + }; + let data = args.serialize(); + + // 1 disc + 100 fixed prefix + (1 + 2 + 32 + 33*1 + 3) body. + assert_eq!(data.len(), 1 + 100 + (1 + 2 + 32 + 33 + 3)); + assert_eq!(data[0], IX_CREATE); + // seed starts right after the discriminator. + assert_eq!(&data[1..33], &[1u8; 32]); + // authority follows the seed. + assert_eq!(&data[33..65], &[2u8; 32]); + // The writable meta flag is encoded for the single scheduled account. + let meta_flag_off = 1 + 100 + 1 + 2 + 32; + assert_eq!(data[meta_flag_off], META_FLAG_WRITABLE); + } +} diff --git a/magicblock-task-scheduler/src/lib.rs b/magicblock-task-scheduler/src/lib.rs index 1951c2004..d8bb21fe5 100644 --- a/magicblock-task-scheduler/src/lib.rs +++ b/magicblock-task-scheduler/src/lib.rs @@ -1,7 +1,17 @@ pub mod db; pub mod errors; +mod hydra; pub mod service; pub use db::SchedulerDatabase; pub use errors::TaskSchedulerError; +/// Flat per-execution reward (lamports) that hydra pays the external cranker. +pub use hydra::CRANKER_REWARD; +/// Program id of the ephemeral hydra crank program that the scheduler targets. +pub use hydra::EPHEMERAL_PROGRAM_ID as HYDRA_EPHEMERAL_PROGRAM_ID; +/// Derives the on-chain hydra crank account address for a task from its +/// `(authority, task_id)` — the deterministic, per-authority namespace the +/// scheduler uses. Tests and tooling can use this to locate the crank account a +/// schedule request created. +pub use service::crank_pubkey; pub use service::TaskSchedulerService; diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index 5d5816e4c..99b8e61cb 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -1,266 +1,102 @@ -use std::{ - collections::HashMap, - path::Path, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, -}; +use std::{path::Path, sync::Arc, time::Duration as StdDuration}; -use futures_util::{future::poll_fn, FutureExt, StreamExt}; -use magicblock_config::config::TaskSchedulerConfig; use magicblock_core::link::transactions::ScheduledTasksRx; use magicblock_ledger::LatestBlock; use magicblock_program::{ args::{CancelTaskRequest, ScheduleTaskRequest, TaskRequest}, - instruction_utils::InstructionUtils, validator::{validator_authority, validator_authority_id}, }; +use solana_instruction::Instruction; use solana_message::Message; +use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_signature::Signature; use solana_transaction::Transaction; use tokio::{ select, - sync::mpsc, - task::{JoinHandle, JoinSet}, - time::{interval, Duration, MissedTickBehavior}, -}; -use tokio_util::{ - sync::CancellationToken, - time::{delay_queue::Key, DelayQueue}, + task::JoinHandle, + time::{Duration, Instant}, }; +use tokio_util::sync::CancellationToken; use tracing::*; use crate::{ - db::{ - CrankFailedMove, CrankRetryCheck, CrankSuccessRemoval, - CrankSuccessUpdate, DbTask, SchedulerDatabase, - }, + db::{DbTask, SchedulerDatabase}, errors::{TaskSchedulerError, TaskSchedulerResult}, + hydra::{ + ephemeral, CreateArgs, SchedMeta, ScheduledIx, CRANKER_REWARD, + EPHEMERAL_PROGRAM_ID, + }, }; -const MAX_TASK_EXECUTION_RETRIES: u32 = 10; -const TASK_EXECUTION_RETRY_BASE_DELAY: Duration = Duration::from_millis(100); -const TASK_EXECUTION_RETRY_MAX_DELAY: Duration = Duration::from_secs(5); +/// How long migration waits for the validator to produce a usable blockhash +/// before giving up. +const BLOCK_READY_TIMEOUT: Duration = Duration::from_secs(60); +/// The task scheduler migrates any tasks persisted by the legacy +/// (validator-funded) scheduler onto the hydra crank program at startup, then +/// serves runtime schedule/cancel requests by sending hydra transactions. The +/// SQLite database is used solely for that one-time migration; the runtime path +/// is stateless and derives each task's crank PDA deterministically from +/// `(authority, task_id)`. pub struct TaskSchedulerService { - /// Database for persisting tasks + /// Migration-only database of legacy tasks. db: SchedulerDatabase, - /// RPC client used to send transactions + /// RPC client used to send hydra transactions. rpc_client: Arc, - /// Used to receive scheduled tasks from the transaction executor + /// Used to receive scheduled tasks from the transaction executor. scheduled_tasks: ScheduledTasksRx, - /// Provides latest blockhash for signing transactions + /// Provides latest blockhash and slot for building transactions. block: LatestBlock, - /// Queue of tasks to execute - task_queue: DelayQueue, - /// Map of task IDs to their corresponding keys in the task queue - task_queue_keys: HashMap, - /// Latest in-memory instance version for queued or in-flight tasks - task_versions: HashMap, - /// Number of consecutive failed execution attempts for each task - task_execution_retries: HashMap, - /// Token used to cancel the task scheduler + /// Token used to cancel the task scheduler. token: CancellationToken, - /// Minimum interval between task executions - min_interval: Duration, - /// How long failed task and scheduling records are retained. - failed_task_retention: Duration, - /// How often failed task and scheduling records are cleaned up. - failed_task_cleanup_interval: Duration, - /// Slot interval of the validator + /// Slot interval of the validator, used to convert millisecond intervals + /// into the slot-based cadence hydra expects. slot_interval: Duration, - /// Shared counter for noop instructions (unique crank transactions). - tx_counter: Arc, } -enum ProcessingOutcome { - Success, - Recoverable(Box), -} - -// SAFETY: TaskSchedulerService is moved into a single Tokio task in `start()` and never cloned. -// It runs exclusively on that task's thread. All fields (SchedulerDatabase, TransactionSchedulerHandle, -// ScheduledTasksRx, LatestBlock, DelayQueue, HashMap, AtomicU64, CancellationToken) are Send+Sync, -// and the service maintains exclusive ownership throughout its lifetime. +// SAFETY: TaskSchedulerService is moved into a single Tokio task in `start()` +// and never cloned. It runs exclusively on that task. All fields are Send+Sync. unsafe impl Send for TaskSchedulerService {} unsafe impl Sync for TaskSchedulerService {} impl TaskSchedulerService { - /// Creates a new `TaskSchedulerService` with the given configuration. + /// Creates a new `TaskSchedulerService`. pub fn new( path: &Path, - config: &TaskSchedulerConfig, rpc_url: String, scheduled_tasks: ScheduledTasksRx, block: LatestBlock, slot_interval: Duration, token: CancellationToken, ) -> TaskSchedulerResult { - if config.min_interval.as_millis() > u32::MAX as u128 { - return Err(TaskSchedulerError::InvalidConfiguration(format!( - "min_interval must be less than or equal to u32::MAX: {}", - config.min_interval.as_millis() - ))); - } - if config.reset { - match std::fs::remove_file(path) { - Ok(_) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debug!("Database file not found, skip resetting"); - } - Err(e) => { - warn!("Failed to remove database file: {}", e); - return Err(TaskSchedulerError::Io(e)); - } - } - } - - // Reschedule all persisted tasks let db = SchedulerDatabase::new(path)?; Ok(Self { db, rpc_client: Arc::new(RpcClient::new(rpc_url)), scheduled_tasks, block, - task_queue: DelayQueue::new(), - task_queue_keys: HashMap::new(), - task_versions: HashMap::new(), - task_execution_retries: HashMap::new(), - tx_counter: Arc::new(AtomicU64::default()), token, - min_interval: config.min_interval, - failed_task_retention: config.failed_task_retention, - failed_task_cleanup_interval: config.failed_task_cleanup_interval, slot_interval, }) } /// Starts the `TaskSchedulerService` and returns a handle to the task. pub async fn start( - mut self, + self, ) -> TaskSchedulerResult>> { - self.load_persisted_tasks().await?; Ok(tokio::spawn(self.run())) } - async fn load_persisted_tasks(&mut self) -> TaskSchedulerResult<()> { - self.task_queue.clear(); - self.task_queue_keys.clear(); - self.task_execution_retries.clear(); - - // Reschedule all tasks that are due - let tasks = self.db.get_tasks().await?; - let now = chrono::Utc::now().timestamp_millis(); - debug!( - "Task scheduler starting at {} with {} tasks", - now, - tasks.len() - ); - for task in tasks { - if !is_valid_task_interval(task.execution_interval_millis) - || task.executions_left <= 0 - { - warn!( - "Task {} has an invalid parameters: (interval={}, executions_left={}). Skipping.", - task.id, task.execution_interval_millis, task.executions_left - ); - self.db.remove_task(task.id).await?; - continue; - } - - let next_execution = - task.last_execution_millis + task.execution_interval_millis; - // Earliest reschedule time is 2 slot interval. - // This avoids, scheduling before the first blockhash is produced on restart. - let timeout = Duration::from_millis( - next_execution - .saturating_sub(now) - .max(2 * self.slot_interval.as_millis() as i64) - as u64, - ); - let task_id = task.id; - self.task_versions.insert(task_id, task.updated_at); - let key = self.task_queue.insert(task, timeout); - self.task_queue_keys.insert(task_id, key); - } - - Ok(()) - } - - /// Main loop of the task scheduler. + /// Main loop: migrate persisted tasks once, then serve runtime requests. async fn run(mut self) -> TaskSchedulerResult<()> { - let mut failed_task_cleanup = interval( - self.failed_task_cleanup_interval - .max(Duration::from_millis(1)), - ); - failed_task_cleanup.set_missed_tick_behavior(MissedTickBehavior::Delay); - - let (crank_tx, mut crank_rx) = mpsc::unbounded_channel(); + if let Err(e) = self.migrate_persisted_tasks().await { + error!("Task migration failed: {}", e); + } loop { select! { - Some(expired) = self.task_queue.next() => { - // A task expired, batch all expired tasks - let first = expired.into_inner(); - self.task_queue_keys.remove(&first.id); - let mut batch = vec![first]; - while let Some(expired) = poll_fn(|cx| self.task_queue.poll_expired(cx)) - .now_or_never() - .flatten() - { - let task = expired.into_inner(); - self.task_queue_keys.remove(&task.id); - batch.push(task); - } - - let rpc_client = self.rpc_client.clone(); - let block = self.block.clone(); - let tx_counter = self.tx_counter.clone(); - let crank_tx = crank_tx.clone(); - - tokio::spawn(async move { - let result = - Self::send_crank_batch(rpc_client, &block, tx_counter, &batch).await; - let _ = crank_tx.send((batch, result)); - }); - } - Some((batch, result)) = crank_rx.recv() => { - // The batch has been sent, updates queue and db - self.on_crank_batch_completed(batch, result).await?; - } - Some(task) = self.scheduled_tasks.recv() => { - // Received a new request from the transaction executor - let id = task.id(); - match self.process_request(task).await { - Ok(ProcessingOutcome::Success) => {} - Ok(ProcessingOutcome::Recoverable(e)) => { - warn!("Failed to process request ID={}: {e:?}", id); - } - Err(e) => { - error!("Failed to process request: {}", e); - return Err(e); - } - } - } - _ = failed_task_cleanup.tick() => { - let cutoff = chrono::Utc::now().timestamp_millis().saturating_sub( - self.failed_task_retention.as_millis().min(i64::MAX as u128) as i64, - ); - let deleted = match self.db.delete_failed_records_older_than(cutoff).await { - Ok(deleted) => deleted, - Err(e) => { - error!("Failed to cleanup old failed task records: {}", e); - continue; - } - }; - if deleted > 0 { - debug!( - deleted, - cutoff_timestamp_millis = cutoff, - "Cleaned up old failed task records" - ); - } + Some(request) = self.scheduled_tasks.recv() => { + self.process_request(request).await; } _ = self.token.cancelled() => { break; @@ -269,515 +105,333 @@ impl TaskSchedulerService { } info!("TaskSchedulerService shutdown!"); - drop(crank_tx); - while let Some((batch, result)) = crank_rx.recv().await { - self.on_crank_batch_completed(batch, result).await?; - } - Ok(()) } - /// Processes [TaskRequest] provided by the transaction executor. - async fn process_request( - &mut self, - request: TaskRequest, - ) -> TaskSchedulerResult { - let task_id = request.id(); - match request { - TaskRequest::Schedule(schedule_request) => { - if let Err(e) = - self.process_schedule_request(schedule_request).await - { - self.db - .insert_failed_scheduling(task_id, format!("{:?}", e)) - .await?; - error!( - "Failed to process schedule request {}: {}", - task_id, e - ); - - return Ok(ProcessingOutcome::Recoverable(Box::new(e))); - } - } - TaskRequest::Cancel(cancel_request) => { - if let Err(e) = - self.process_cancel_request(&cancel_request).await - { - self.db - .insert_failed_scheduling(task_id, format!("{:?}", e)) - .await?; - error!( - "Failed to process cancel request for task {}: {}", - task_id, e - ); - - return Ok(ProcessingOutcome::Recoverable(Box::new(e))); - } - } - }; - - Ok(ProcessingOutcome::Success) - } - - /// Processes [ScheduleTaskRequest] provided by the transaction executor. - async fn process_schedule_request( - &mut self, - mut task: ScheduleTaskRequest, - ) -> TaskSchedulerResult<()> { - if !is_valid_task_interval(task.execution_interval_millis) { - // If the interval is too large or zero, we don't schedule the task + /// Migrates every task persisted by the legacy scheduler onto hydra, then + /// empties the database. Invalid tasks are dropped without rescheduling. + /// Migration is best-effort: a task is removed from the database whether or + /// not its hydra crank could be created, so the database always empties. + async fn migrate_persisted_tasks(&self) -> TaskSchedulerResult<()> { + let tasks = self.db.get_tasks().await?; + if tasks.is_empty() { return Ok(()); } - - task.execution_interval_millis = task - .execution_interval_millis - .clamp(self.min_interval.as_millis() as i64, u32::MAX as i64); - - let mut task = DbTask::from(task); - let task_id = task.id; - - // Check if the task already exists in the database - if let Some(db_task) = self.db.get_task(task_id).await? { - if db_task.authority != task.authority { - return Err(TaskSchedulerError::UnauthorizedReplacing( - task_id, - db_task.authority.to_string(), - task.authority.to_string(), - )); - } + info!("Migrating {} persisted task(s) onto hydra", tasks.len()); + + // Drop tasks that can no longer correspond to a live crank without + // touching the network. + let (valid, invalid): (Vec, Vec) = + tasks.into_iter().partition(|task| { + is_valid_task_interval(task.execution_interval_millis) + && task.executions_left > 0 + }); + for task in invalid { + warn!( + "Dropping invalid task {} during migration (interval={}, executions_left={})", + task.id, task.execution_interval_millis, task.executions_left + ); + self.db.remove_task(task.id).await?; } - task.updated_at = self.db.insert_task(&task).await?; - self.remove_task_from_queue(task_id); - self.task_execution_retries.remove(&task_id); - self.task_versions.insert(task_id, task.updated_at); - let key = self.task_queue.insert(task, Duration::from_millis(0)); - self.task_queue_keys.insert(task_id, key); - debug!("Registered task {} from context", task_id); - - Ok(()) - } - - /// Processes [CancelTaskRequest] provided by the transaction executor. - async fn process_cancel_request( - &mut self, - cancel_request: &CancelTaskRequest, - ) -> TaskSchedulerResult<()> { - let Some(task) = self.db.get_task(cancel_request.task_id).await? else { - // Task not found in the database, cleanup the queue - self.remove_task_from_queue(cancel_request.task_id); - self.task_execution_retries.remove(&cancel_request.task_id); - return Ok(()); - }; - - // Check if the task authority is the same as the cancel request authority - if task.authority != cancel_request.authority { - error!( - "Task authority {} does not match cancel request authority {}", - task.authority, cancel_request.authority - ); + if valid.is_empty() { return Ok(()); } - // Remove task from queue and database - self.remove_task_runtime_state(task.id); - self.task_execution_retries.remove(&task.id); - self.db.remove_task(task.id).await?; - debug!("Removed task {} from database", task.id); + // Sending a crank create needs a usable blockhash, which is only + // available once the validator has produced a block. + self.wait_for_block_ready().await; + + for task in valid { + if let Err(e) = self + .schedule_crank( + &task.authority, + task.id, + task.execution_interval_millis, + task.executions_left, + &task.instructions, + ) + .await + { + warn!("Failed to migrate task {} onto hydra: {}", task.id, e); + } + self.db.remove_task(task.id).await?; + } + info!("Task migration complete; database emptied"); Ok(()) } - /// Sends a batch of crank transactions to the RPC client. - async fn send_crank_batch( - rpc_client: Arc, - block: &LatestBlock, - tx_counter: Arc, - tasks: &[DbTask], - ) -> TaskSchedulerResult)>> - { - let mut join_set: JoinSet<(DbTask, TaskSchedulerResult)> = - JoinSet::new(); - let blockhash = block.load().blockhash; - for task in tasks { - let rpc_client = rpc_client.clone(); - let tx_counter = tx_counter.clone(); - let task = task.clone(); - join_set.spawn(async move { - let ixs = vec![ - InstructionUtils::noop_instruction( - tx_counter.fetch_add(1, Ordering::Relaxed), - ), - InstructionUtils::execute_task_instruction( - task.authority, - task.instructions.clone(), - ), - ]; - let tx = Transaction::new( - &[validator_authority()], - Message::new(&ixs, Some(&validator_authority_id())), - blockhash, - ); - let res = rpc_client - .send_transaction(&tx) - .await - .map_err(Box::new) - .map_err(TaskSchedulerError::from); - (task, res) - }); - } - Ok(join_set.join_all().await) - } - - /// Called when a crank batch is completed. - async fn on_crank_batch_completed( - &mut self, - batch: Vec, - result: TaskSchedulerResult< - Vec<(DbTask, TaskSchedulerResult)>, - >, - ) -> TaskSchedulerResult<()> { - let now_millis = chrono::Utc::now().timestamp_millis(); - let mut success_updates: Vec = Vec::new(); - let mut success_removals: Vec = Vec::new(); - let mut failed_records: Vec = Vec::new(); - let mut retry_checks: Vec = Vec::new(); - - // Decide what must happen to cranks - match result { - Ok(ref result) => { - // Batch completed, update individual crank based on tx status - for (task, res) in result { - if let Err(e) = res { - self.prepare_crank_failure_outcome( - task, - e, - &mut failed_records, - &mut retry_checks, - )?; - } else { - self.prepare_crank_success_outcome( - task, - now_millis, - &mut success_updates, - &mut success_removals, - )?; - } - } - } - Err(ref e) => { - // Whole batch failed, fail all cranks - for task in &batch { - self.prepare_crank_failure_outcome( - task, - e, - &mut failed_records, - &mut retry_checks, - )?; - } + /// Waits until the validator has a usable blockhash, or the scheduler is + /// cancelled, or [`BLOCK_READY_TIMEOUT`] elapses. + async fn wait_for_block_ready(&self) { + let start = Instant::now(); + while start.elapsed() < BLOCK_READY_TIMEOUT { + if self.token.is_cancelled() + || self.block.load().blockhash != Default::default() + { + return; } + tokio::time::sleep(Duration::from_millis(100)).await; } + warn!("Timed out waiting for a usable blockhash before migration"); + } - // Apply db updates for the whole batch - let completion = self - .db - .apply_crank_batch_completion( - &success_updates, - &success_removals, - &failed_records, - &retry_checks, - ) - .await?; - - // Update queue, retries and versions based on decisions - match result { - Ok(result) => { - for (task, res) in &result { - if let Err(e) = res { - if completion.failed_moves.get(&task.id).is_some_and( - |m| m.expected_updated_at == task.updated_at, - ) || completion - .retry_checks - .get(&task.id) - .is_some_and(|c| { - c.current_updated_at == task.updated_at - }) - { - self.apply_crank_failure_outcome(task, e)?; - } - } else if let Some(update) = - completion.success_updates.get(&task.id) - { - if update.expected_updated_at == task.updated_at { - self.apply_crank_success_outcome( - task, - now_millis, - Some(update.new_updated_at), - )?; - } - } else if completion - .success_removals - .get(&task.id) - .is_some_and(|r| { - r.expected_updated_at == task.updated_at - }) - { - self.apply_crank_success_outcome( - task, now_millis, None, - )?; - } - } + /// Processes a [TaskRequest] from the transaction executor. + async fn process_request(&self, request: TaskRequest) { + let task_id = request.id(); + let result = match request { + TaskRequest::Schedule(schedule_request) => { + self.process_schedule_request(schedule_request).await } - Err(ref e) => { - for task in &batch { - if completion.failed_moves.get(&task.id).is_some_and(|m| { - m.expected_updated_at == task.updated_at - }) || completion.retry_checks.get(&task.id).is_some_and( - |c| c.current_updated_at == task.updated_at, - ) { - self.apply_crank_failure_outcome(task, e)?; - } - } + TaskRequest::Cancel(cancel_request) => { + self.process_cancel_request(&cancel_request).await } + }; + if let Err(e) = result { + error!("Failed to process task request {}: {}", task_id, e); } - - Ok(()) } - /// Either: - /// - reschedules crank with remaining iterations - /// - remove exhausted cranks - fn prepare_crank_success_outcome( + /// Schedules a task: creates and funds its hydra crank. + async fn process_schedule_request( &self, - task: &DbTask, - now_millis: i64, - success_updates: &mut Vec, - success_removals: &mut Vec, + task: ScheduleTaskRequest, ) -> TaskSchedulerResult<()> { - let executed_at = next_execution_millis( - task.last_execution_millis, - task.execution_interval_millis, - now_millis, - ); - - if task.executions_left > 1 { - success_updates.push(CrankSuccessUpdate { - task_id: task.id, - last_execution_millis: executed_at, - expected_updated_at: task.updated_at, - }); - } else { - success_removals.push(CrankSuccessRemoval { - task_id: task.id, - expected_updated_at: task.updated_at, - }); + if !is_valid_task_interval(task.execution_interval_millis) { + // Too large or zero: ignore. + return Ok(()); } - + let interval_millis = + task.execution_interval_millis.clamp(1, u32::MAX as i64); + + self.schedule_crank( + &task.authority, + task.id, + interval_millis, + task.iterations, + &task.instructions, + ) + .await?; + debug!("Created hydra crank for task {}", task.id); Ok(()) } - /// Either: - /// - re-queues for retry - /// - queues a permanent failure for SQLite along with clearing retry counters and stale queue keys. - fn prepare_crank_failure_outcome( + /// Cancels a task's hydra crank, if one exists for `(authority, task_id)`. + async fn process_cancel_request( &self, - task: &DbTask, - error: &TaskSchedulerError, - failed_records: &mut Vec, - retry_checks: &mut Vec, + cancel_request: &CancelTaskRequest, ) -> TaskSchedulerResult<()> { - debug!("Failed to execute task {}: {}", task.id, error); + let crank = + crank_pubkey(&cancel_request.authority, cancel_request.task_id); - if !is_retryable_task_execution_error(error) { - // Unretryable crank are moved to failed cranks - failed_records.push(CrankFailedMove { - task_id: task.id, - expected_updated_at: task.updated_at, - error: error.to_string(), - }); - return Ok(()); - } - - let retries = *self.task_execution_retries.get(&task.id).unwrap_or(&0); - if retries >= MAX_TASK_EXECUTION_RETRIES { - // Crank exhausted retries, fail it - failed_records.push(CrankFailedMove { - task_id: task.id, - expected_updated_at: task.updated_at, - error: error.to_string(), - }); - return Ok(()); - } - - // Schedule for retry - retry_checks.push(CrankRetryCheck { - task_id: task.id, - expected_updated_at: task.updated_at, - }); + // Does not check if the crank exists, so it will fail if it does not exist + self.send_cancel(crank).await?; + debug!("Cancelled hydra crank for task {}", cancel_request.task_id); Ok(()) } - /// Updates the delay queue for the next firing after SQLite confirms the same task instance. - fn apply_crank_success_outcome( - &mut self, - task: &DbTask, - now_millis: i64, - updated_at: Option, + /// Creates and funds the hydra crank for a task. If a crank already exists + /// at the deterministic PDA (a reschedule), it is closed first so the new + /// schedule can recreate it. + async fn schedule_crank( + &self, + authority: &Pubkey, + task_id: i64, + interval_millis: i64, + iterations: i64, + instructions: &[Instruction], ) -> TaskSchedulerResult<()> { - if !self.task_version_matches(task) { - debug!( - task_id = task.id, - expected_updated_at = task.updated_at, - "Skipping stale successful crank completion" - ); - return Ok(()); - } - - let executed_at = next_execution_millis( - task.last_execution_millis, - task.execution_interval_millis, - now_millis, - ); + let crank = crank_pubkey(authority, task_id); - if let Some(updated_at) = updated_at { - let next_execution = executed_at + task.execution_interval_millis; - let delay = delay_until_millis(next_execution, now_millis); - let new_task = DbTask { - executions_left: task.executions_left - 1, - last_execution_millis: executed_at, - updated_at, - ..task.clone() - }; - let key = self.task_queue.insert(new_task, delay); - self.task_queue_keys.insert(task.id, key); - self.task_versions.insert(task.id, updated_at); - } else { - self.remove_task_runtime_state(task.id); - } - - self.task_execution_retries.remove(&task.id); + self.send_create_and_fund( + authority, + task_id, + interval_millis, + iterations, + instructions, + crank, + ) + .await?; Ok(()) } - /// Either: - /// - re-queues for retry - /// - records a permanent failure in runtime state after SQLite confirms the same task instance. - fn apply_crank_failure_outcome( - &mut self, - task: &DbTask, - error: &TaskSchedulerError, - ) -> TaskSchedulerResult<()> { - if !self.task_version_matches(task) { - debug!( - task_id = task.id, - expected_updated_at = task.updated_at, - "Skipping stale failed crank completion" - ); - return Ok(()); - } + /// Returns whether a hydra-owned crank account currently exists at `crank`. + async fn crank_exists(&self, crank: &Pubkey) -> bool { + matches!( + self.rpc_client.get_account(crank).await, + Ok(account) if account.owner == EPHEMERAL_PROGRAM_ID + ) + } - if !is_retryable_task_execution_error(error) { - self.task_execution_retries.remove(&task.id); - self.remove_task_runtime_state(task.id); - return Ok(()); - } + /// Builds and sends the transaction that creates and funds a hydra crank. + /// It cancels the crank if it already exists. + async fn send_create_and_fund( + &self, + authority: &Pubkey, + task_id: i64, + interval_millis: i64, + iterations: i64, + instructions: &[Instruction], + crank: Pubkey, + ) -> TaskSchedulerResult { + let crank_exists = self.crank_exists(&crank).await; + + let snapshot = self.block.load(); + let start_slot = snapshot.slot; + let blockhash = snapshot.blockhash; + + let interval_slots = + interval_slots(interval_millis, self.slot_interval); + let iterations = iterations.max(0) as u64; + + let create_ix = build_create_ix( + authority, + task_id, + crank, + start_slot, + interval_slots, + iterations, + instructions, + ); - let Some(retries) = ({ - let retries = - self.task_execution_retries.entry(task.id).or_default(); - if *retries >= MAX_TASK_EXECUTION_RETRIES { - None - } else { - *retries += 1; - Some(*retries) - } - }) else { - self.task_execution_retries.remove(&task.id); - self.remove_task_runtime_state(task.id); - return Ok(()); + // Fund the crank so the external cranker can execute every iteration. + let funding = iterations.saturating_mul(CRANKER_REWARD); + let transfer_ix = solana_system_interface::instruction::transfer( + &validator_authority_id(), + &crank, + funding, + ); + + let ixs = if crank_exists { + let cancel_ix = ephemeral::cancel(validator_authority_id(), crank); + vec![cancel_ix, create_ix, transfer_ix] + } else { + vec![create_ix, transfer_ix] }; - let delay = self.task_execution_retry_delay(retries); - debug!( - task_id = task.id, - retry = retries, - max_retries = MAX_TASK_EXECUTION_RETRIES, - delay_ms = delay.as_millis(), - error = %error, - "Retrying failed task execution" + let tx = Transaction::new( + &[validator_authority()], + Message::new(&ixs, Some(&validator_authority_id())), + blockhash, ); - let key = self.task_queue.insert(task.clone(), delay); - self.task_queue_keys.insert(task.id, key); - self.task_versions.insert(task.id, task.updated_at); - - Ok(()) + self.rpc_client + .send_transaction(&tx) + .await + .map_err(Box::new) + .map_err(TaskSchedulerError::from) } - fn task_version_matches(&self, task: &DbTask) -> bool { - self.task_versions.get(&task.id) == Some(&task.updated_at) + /// Sends a crank cancellation (fire-and-forget). + async fn send_cancel( + &self, + crank: Pubkey, + ) -> TaskSchedulerResult { + let blockhash = self.block.load().blockhash; + let cancel_ix = ephemeral::cancel(validator_authority_id(), crank); + let tx = Transaction::new( + &[validator_authority()], + Message::new(&[cancel_ix], Some(&validator_authority_id())), + blockhash, + ); + self.rpc_client + .send_transaction(&tx) + .await + .map_err(Box::new) + .map_err(TaskSchedulerError::from) } +} - /// Removes a task from the queue. - fn remove_task_from_queue(&mut self, task_id: i64) { - if let Some(key) = self.task_queue_keys.remove(&task_id) { - self.task_queue.remove(&key); - } - } +/// Derives the deterministic hydra crank account address for a task. +/// +/// The seed is `hash(authority, task_id)`, so each authority gets its own crank +/// namespace: a different authority scheduling the same `task_id` gets an +/// independent crank, and cancel/reschedule need no database lookup. +pub fn crank_pubkey(authority: &Pubkey, task_id: i64) -> Pubkey { + let seed = solana_sha256_hasher::hashv(&[ + authority.as_ref(), + &task_id.to_le_bytes(), + ]) + .to_bytes(); + ephemeral::find_crank_pda(&seed).0 +} - fn remove_task_runtime_state(&mut self, task_id: i64) { - self.remove_task_from_queue(task_id); - self.task_versions.remove(&task_id); - } +/// Builds the hydra `Create` instruction embedding the task's instructions as +/// the scheduled crank payload. Account signer flags are intentionally dropped: +/// hydra rejects scheduled instructions that declare signers. +fn build_create_ix( + authority: &Pubkey, + task_id: i64, + crank: Pubkey, + start_slot: u64, + interval_slots: u64, + iterations: u64, + instructions: &[Instruction], +) -> Instruction { + let seed = solana_sha256_hasher::hashv(&[ + authority.as_ref(), + &task_id.to_le_bytes(), + ]) + .to_bytes(); + + let metas_per_ix: Vec> = instructions + .iter() + .map(|ix| { + ix.accounts + .iter() + .map(|acc| SchedMeta { + pubkey: acc.pubkey.to_bytes(), + is_writable: acc.is_writable, + }) + .collect() + }) + .collect(); + + let scheduled: Vec = instructions + .iter() + .zip(metas_per_ix.iter()) + .map(|(ix, metas)| ScheduledIx { + program_id: ix.program_id.to_bytes(), + metas: metas.as_slice(), + data: ix.data.as_slice(), + }) + .collect(); + + let args = CreateArgs { + seed, + // The validator authority is the cancel authority for the crank. + authority: validator_authority_id().to_bytes(), + start_slot, + interval_slots, + remaining: iterations, + priority_tip: 0, + cu_limit: 0, + scheduled: scheduled.as_slice(), + }; - /// Calculates the retry delay for the next task execution. - fn task_execution_retry_delay(&self, retry: u32) -> Duration { - let multiplier = 1u32 - .checked_shl(retry.saturating_sub(1)) - .unwrap_or(u32::MAX); - self.slot_interval - .max(TASK_EXECUTION_RETRY_BASE_DELAY) - .checked_mul(multiplier) - .unwrap_or(TASK_EXECUTION_RETRY_MAX_DELAY) - .min(TASK_EXECUTION_RETRY_MAX_DELAY) - } + ephemeral::create(validator_authority_id(), crank, &args) } fn is_valid_task_interval(interval: i64) -> bool { interval > 0 && interval < u32::MAX as i64 } -fn next_execution_millis( - last_execution_millis: i64, - execution_interval_millis: i64, - now: i64, -) -> i64 { - if last_execution_millis == 0 { - now - } else { - last_execution_millis + execution_interval_millis - } -} - -fn delay_until_millis(execution_millis: i64, now: i64) -> Duration { - Duration::from_millis(execution_millis.saturating_sub(now).max(0) as u64) -} - -fn is_retryable_task_execution_error(error: &TaskSchedulerError) -> bool { - // `send_crank_batch` maps Solana send and verification failures to Rpc. - matches!(error, TaskSchedulerError::Rpc(_)) +/// Converts a millisecond execution interval into a slot count (rounding up, +/// with a one-slot minimum) for hydra's slot-based cadence. +fn interval_slots(interval_millis: i64, slot_interval: StdDuration) -> u64 { + let slot_millis = (slot_interval.as_millis() as i64).max(1); + let interval_millis = interval_millis.max(0); + // Ceiling division without the unstable `i64::div_ceil`. + let slots = (interval_millis + slot_millis - 1) / slot_millis; + slots.max(1) as u64 } #[cfg(test)] mod tests { use magicblock_core::coordination_mode::switch_to_primary_mode; - use magicblock_program::{ - args::ScheduleTaskRequest, - validator::generate_validator_authority_if_needed, - }; use serial_test::serial; - use solana_pubkey::Pubkey; use tokio::{sync::mpsc, time::timeout}; use super::*; @@ -792,15 +446,7 @@ mod tests { "http://localhost:8899".to_string(), )), block: LatestBlock::default(), - task_queue: DelayQueue::new(), - task_queue_keys: HashMap::new(), - task_versions: HashMap::new(), - task_execution_retries: HashMap::new(), - tx_counter: Arc::new(AtomicU64::default()), token: CancellationToken::new(), - min_interval: Duration::from_millis(1000), - failed_task_retention: Duration::from_secs(60), - failed_task_cleanup_interval: Duration::from_secs(60), slot_interval: Duration::from_millis(1000), scheduled_tasks, } @@ -808,270 +454,72 @@ mod tests { #[serial] #[test] - fn test_first_execution_anchors_cadence_at_now() { - assert_eq!(next_execution_millis(0, 50, 1_000), 1_000); + fn test_interval_millis_rounds_up_to_slots() { + let slot = StdDuration::from_millis(50); + assert_eq!(interval_slots(1, slot), 1); + assert_eq!(interval_slots(50, slot), 1); + assert_eq!(interval_slots(51, slot), 2); + assert_eq!(interval_slots(100, slot), 2); } #[serial] #[test] - fn test_recurring_execution_preserves_fixed_rate_cadence() { - let executed_at = next_execution_millis(1_000, 50, 1_090); - assert_eq!(executed_at, 1_050); - - let delay = delay_until_millis(executed_at + 50, 1_090); - assert_eq!(delay, Duration::from_millis(10)); - } - - #[serial] - #[test] - fn test_overdue_execution_is_rescheduled_immediately() { - assert_eq!(delay_until_millis(1_100, 1_150), Duration::from_millis(0)); + fn test_crank_pubkey_namespaced_by_authority_and_id() { + let a = Pubkey::new_unique(); + let b = Pubkey::new_unique(); + // Deterministic. + assert_eq!(crank_pubkey(&a, 1), crank_pubkey(&a, 1)); + // Different task id -> different crank. + assert_ne!(crank_pubkey(&a, 1), crank_pubkey(&a, 2)); + // Different authority, same id -> different crank (per-authority namespace). + assert_ne!(crank_pubkey(&a, 1), crank_pubkey(&b, 1)); } #[serial] #[tokio::test] - async fn test_schedule_invalid_tasks() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - generate_validator_authority_if_needed(); - - let (tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - - let service = test_service(db.clone(), rx); - - let handle = service.start().await.unwrap(); - - // Invalid task interval - tx.send(TaskRequest::Schedule(ScheduleTaskRequest { - id: 1, - authority: Pubkey::new_unique(), - execution_interval_millis: u32::MAX as i64, - iterations: 1, - instructions: vec![], - })) - .unwrap(); - // Valid task interval - tx.send(TaskRequest::Schedule(ScheduleTaskRequest { - id: 1, - authority: Pubkey::new_unique(), - execution_interval_millis: u32::MAX as i64 - 1, - iterations: 1, - instructions: vec![], - })) - .unwrap(); - - // After processing the requests, only one task stays in the DB - timeout(Duration::from_secs(1), async move { - loop { - let tasks = db.get_tasks().await.unwrap(); - if tasks.len() > 1 { - return Err::<(), String>(format!( - "Tasks should be 1, got {}", - tasks.len() - )); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .unwrap_err(); - - handle.abort(); - } - - #[serial] - #[tokio::test] - async fn test_remove_invalid_tasks_on_startup() { + async fn test_migration_drops_invalid_tasks_and_empties_db() { magicblock_core::logger::init_for_tests(); switch_to_primary_mode(); let (_tx, rx) = mpsc::unbounded_channel(); let db = SchedulerDatabase::new(":memory:").unwrap(); - // Invalid task interval + // Invalid interval. db.insert_task(&DbTask { id: 1, authority: Pubkey::new_unique(), execution_interval_millis: u32::MAX as i64, executions_left: 1, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, instructions: vec![], }) .await .unwrap(); - // Valid task interval + // Exhausted (no executions left). db.insert_task(&DbTask { id: 2, authority: Pubkey::new_unique(), - execution_interval_millis: u32::MAX as i64 - 1, - executions_left: 1, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, - instructions: vec![], - }) - .await - .unwrap(); - let service = test_service(db.clone(), rx); - - let handle = service.start().await.unwrap(); - - // After starting, only one task should be in the database - timeout(Duration::from_secs(1), async move { - loop { - let tasks = db.get_tasks().await?; - if tasks.len() == 1 { - return Ok::<_, TaskSchedulerError>(()); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .unwrap() - .unwrap(); - handle.abort(); - } - - #[serial] - #[tokio::test] - async fn test_completed_tasks_are_removed_on_startup() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - - let (_tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - db.insert_task(&DbTask { - id: 1, - authority: Pubkey::new_unique(), execution_interval_millis: 50, executions_left: 0, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, - instructions: vec![], - }) - .await - .unwrap(); - db.insert_task(&DbTask { - id: 2, - authority: Pubkey::new_unique(), - execution_interval_millis: 50, - executions_left: 1, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, instructions: vec![], }) .await .unwrap(); - let mut service = test_service(db.clone(), rx); - service.min_interval = Duration::from_millis(10); - + let service = test_service(db.clone(), rx); let handle = service.start().await.unwrap(); - timeout(Duration::from_secs(1), async move { + // Migration drops both invalid tasks without any network access, so the + // database empties promptly. + timeout(Duration::from_secs(2), async move { loop { - let tasks = db.get_task_ids().await?; - if tasks == vec![2] { - return Ok::<_, TaskSchedulerError>(()); + if db.get_task_ids().await.unwrap().is_empty() { + return; } - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(50)).await; } }) .await - .unwrap() - .unwrap(); - handle.abort(); - } - - #[serial] - #[tokio::test] - async fn test_stale_crank_completion_does_not_mutate_replaced_task() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - - let (_tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - let authority = Pubkey::new_unique(); - let mut old_task = DbTask { - id: 1, - authority, - execution_interval_millis: 50, - executions_left: 2, - last_execution_millis: 0, - updated_at: 0, - instructions: vec![], - }; - old_task.updated_at = db.insert_task(&old_task).await.unwrap(); - - let mut service = test_service(db.clone(), rx); - - let mut replacement = DbTask { - executions_left: 5, - ..old_task.clone() - }; - replacement.updated_at = db.insert_task(&replacement).await.unwrap(); - let key = service - .task_queue - .insert(replacement.clone(), Duration::from_secs(10)); - service.task_queue_keys.insert(replacement.id, key); - service - .task_versions - .insert(replacement.id, replacement.updated_at); - - service - .on_crank_batch_completed( - vec![old_task.clone()], - Ok(vec![(old_task, Ok(Signature::new_unique()))]), - ) - .await - .unwrap(); - - let persisted = db.get_task(replacement.id).await.unwrap().unwrap(); - assert_eq!(persisted.executions_left, replacement.executions_left); - assert_eq!(persisted.updated_at, replacement.updated_at); - - let key = service.task_queue_keys.remove(&replacement.id).unwrap(); - let queued = service.task_queue.remove(&key).into_inner(); - assert_eq!(queued.updated_at, replacement.updated_at); - assert_eq!(queued.executions_left, replacement.executions_left); - } - - #[serial] - #[tokio::test] - async fn test_failed_records_are_cleaned_up_periodically() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - - let (_tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - - db.insert_failed_scheduling(1, "schedule failed".to_string()) - .await - .unwrap(); - db.insert_failed_task(2, "task failed".to_string()) - .await - .unwrap(); - tokio::time::sleep(Duration::from_millis(2)).await; - - let mut service = test_service(db.clone(), rx); - service.failed_task_retention = Duration::from_millis(1); - service.failed_task_cleanup_interval = Duration::from_millis(5); + .expect("database should empty after migration"); - let handle = service.start().await.unwrap(); - - timeout(Duration::from_secs(1), async move { - loop { - if db.get_failed_schedulings().await?.is_empty() - && db.get_failed_tasks().await?.is_empty() - { - return Ok::<_, TaskSchedulerError>(()); - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - }) - .await - .unwrap() - .unwrap(); handle.abort(); } } diff --git a/programs/magicblock/src/magicblock_processor.rs b/programs/magicblock/src/magicblock_processor.rs index d39ed2486..0ce806c32 100644 --- a/programs/magicblock/src/magicblock_processor.rs +++ b/programs/magicblock/src/magicblock_processor.rs @@ -21,9 +21,7 @@ use crate::{ }, mutate_accounts::process_mutate_accounts, process_scheduled_commit_sent, - schedule_task::{ - process_cancel_task, process_execute_crank, process_schedule_task, - }, + schedule_task::{process_cancel_task, process_schedule_task}, schedule_transactions::{ process_accept_scheduled_commits, process_add_action_callback, process_execute_callback, process_schedule_cloned_account_undelegation, @@ -226,40 +224,6 @@ declare_process_instruction!( remote_slot, authority, ), - ExecuteCrank { - authority, - instructions, - } => process_execute_crank( - signers, - invoke_context, - &authority, - instructions, - ), - } - } -); - -declare_process_instruction!( - CrankEntrypoint, - DEFAULT_COMPUTE_UNITS, - |invoke_context| { - let instruction = deserialize_instruction(invoke_context)?; - let transaction_context = &invoke_context.transaction_context; - let instruction_context = - transaction_context.get_current_instruction_context()?; - let signers = instruction_context.get_signers()?; - - match instruction { - MagicBlockInstruction::ExecuteCrank { - authority, - instructions, - } => process_execute_crank( - signers, - invoke_context, - &authority, - instructions, - ), - _ => Err(InstructionError::InvalidInstructionData), } } ); @@ -317,37 +281,11 @@ declare_process_instruction!( #[cfg(test)] mod test { - use magicblock_magic_program_api::args::ScheduleTaskArgs; use solana_instruction::AccountMeta; use solana_program_runtime::invoke_context::mock_process_instruction; use super::*; - #[test] - fn crank_entrypoint_rejects_non_execute_crank_instructions() { - let data = bincode::serialize(&MagicBlockInstruction::ScheduleTask( - ScheduleTaskArgs { - task_id: 1, - execution_interval_millis: 10, - iterations: 1, - instructions: vec![], - }, - )) - .unwrap(); - - mock_process_instruction( - &crate::CRANK_PROGRAM_ID, - None, - &data, - Vec::new(), - vec![AccountMeta::new_readonly(crate::CRANK_PROGRAM_ID, false)], - Err(InstructionError::InvalidInstructionData), - CrankEntrypoint::vm, - |_invoke_context| {}, - |_invoke_context| {}, - ); - } - #[test] fn callback_entrypoint_rejects_invalid_instruction_data() { let data = vec![0xFF, 0xFF, 0xFF, 0xFF]; diff --git a/programs/magicblock/src/schedule_task/mod.rs b/programs/magicblock/src/schedule_task/mod.rs index f21836594..b91df93d9 100644 --- a/programs/magicblock/src/schedule_task/mod.rs +++ b/programs/magicblock/src/schedule_task/mod.rs @@ -1,44 +1,31 @@ mod process_cancel_task; -mod process_execute_task; mod process_schedule_task; -use magicblock_magic_program_api::{ - instruction::MagicBlockInstruction, pda::crank_signer_pda, -}; +use magicblock_magic_program_api::instruction::MagicBlockInstruction; pub(crate) use process_cancel_task::*; -pub(crate) use process_execute_task::*; pub(crate) use process_schedule_task::*; use solana_instruction::{error::InstructionError, Instruction}; use solana_log_collector::ic_msg; use solana_program_runtime::invoke_context::InvokeContext; -use solana_pubkey::Pubkey; use crate::validator::effective_validator_authority_id; -// Assert that the task instructions do not have signers aside from the crank signer +// Assert that the task instructions do not have signers // Assert they don't use the validator either // Assert they are not a privileged instruction pub(crate) fn validate_cranks_instructions( invoke_context: &mut InvokeContext, - authority: &Pubkey, instructions: &[Instruction], ) -> Result<(), InstructionError> { - let crank_signer = crank_signer_pda(authority); for instruction in instructions { for account in &instruction.accounts { - if account.is_signer && account.pubkey.ne(&crank_signer) { + if account.is_signer { ic_msg!( invoke_context, "Crank ERR: only the crank signer PDA can be a signer in cranks (invalid signer: '{}')", account.pubkey, ); return Err(InstructionError::MissingRequiredSignature); - } else if account.is_writable && account.pubkey.eq(&crank_signer) { - ic_msg!( - invoke_context, - "Crank ERR: the crank signer PDA cannot be a writable account in cranks", - ); - return Err(InstructionError::Immutable); } else if account.pubkey.eq(&effective_validator_authority_id()) { ic_msg!( invoke_context, diff --git a/programs/magicblock/src/schedule_task/process_execute_task.rs b/programs/magicblock/src/schedule_task/process_execute_task.rs deleted file mode 100644 index 542bcbc79..000000000 --- a/programs/magicblock/src/schedule_task/process_execute_task.rs +++ /dev/null @@ -1,381 +0,0 @@ -use std::collections::HashSet; - -use magicblock_magic_program_api::{pda::crank_signer_pda, CRANK_PROGRAM_ID}; -use solana_instruction::{error::InstructionError, Instruction}; -use solana_log_collector::ic_msg; -use solana_program_runtime::invoke_context::InvokeContext; -use solana_pubkey::Pubkey; - -use crate::{ - schedule_task::validate_cranks_instructions, - utils::accounts::get_instruction_pubkey_with_idx, - validator::effective_validator_authority_id, -}; - -pub(crate) fn process_execute_crank( - signers: HashSet, - invoke_context: &mut InvokeContext, - authority: &Pubkey, - instructions: Vec, -) -> Result<(), InstructionError> { - const VALIDATOR_IDX: u16 = 0; - const CRANK_SIGNER_IDX: u16 = 1; - let crank_signer = crank_signer_pda(authority); - - const ACCOUNTS_START: usize = CRANK_SIGNER_IDX as usize + 1; - - { - let transaction_context = &*invoke_context.transaction_context; - let ix_ctx = transaction_context.get_current_instruction_context()?; - let ix_accs_len = ix_ctx.get_number_of_instruction_accounts() as usize; - - // Assert crank executor program. - let program_key = ix_ctx.get_program_key()?; - if program_key != &crate::id() && program_key != &CRANK_PROGRAM_ID { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: crank executor program account not found" - ); - return Err(InstructionError::UnsupportedProgramId); - } - - // Assert enough accounts - if ix_accs_len < ACCOUNTS_START { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: not enough accounts to execute crank ({}), need crank signer and instructions", - ix_accs_len - ); - return Err(InstructionError::MissingAccount); - } - - // Assert Validator is signer - // Only the validator can execute a crank - let validator_pubkey = get_instruction_pubkey_with_idx( - transaction_context, - VALIDATOR_IDX, - )?; - let validator_authority = effective_validator_authority_id(); - if validator_pubkey != &validator_authority { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: validator pubkey {} is not the expected validator", - validator_pubkey - ); - return Err(InstructionError::IncorrectAuthority); - } - if !signers.contains(validator_pubkey) { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: validator pubkey {} is not in signers", - validator_pubkey - ); - return Err(InstructionError::MissingRequiredSignature); - } - - // Assert Crank signer is provided - let crank_signer_pubkey = get_instruction_pubkey_with_idx( - transaction_context, - CRANK_SIGNER_IDX, - )?; - if crank_signer_pubkey != &crank_signer { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: crank signer pubkey {} is not the expected Crank signer", - crank_signer_pubkey - ); - return Err(InstructionError::InvalidSeeds); - } - } - - // Already validated when scheduling the task. - // This check prevents the validator from manually sending transactions disguised as cranks. - validate_cranks_instructions(invoke_context, authority, &instructions)?; - - let len = instructions.len(); - for ix in instructions { - invoke_context.native_invoke(ix, &[crank_signer])?; - } - - ic_msg!(invoke_context, "Executed crank with {} instructions", len); - - Ok(()) -} - -#[cfg(test)] -mod test { - use magicblock_magic_program_api::args::ScheduleTaskArgs; - use solana_account::AccountSharedData; - use solana_instruction::AccountMeta; - use solana_keypair::Keypair; - use solana_sdk_ids::system_program; - - use super::*; - use crate::{ - test_utils::process_instruction, - utils::instruction_utils::InstructionUtils, - validator::{init_validator_authority, validator_authority_id}, - }; - - pub fn complex_ix(payer: Pubkey) -> Instruction { - let mut ix = InstructionUtils::noop_instruction(0); - ix.accounts.push(AccountMeta::new(payer, false)); - ix - } - - #[test] - fn test_execute_task_simple() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Ok(()), - ); - } - - #[test] - fn test_execute_task_complex() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let payer = Pubkey::new_unique(); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![complex_ix(payer)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - payer, - AccountSharedData::new(1000000, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Ok(()), - ); - } - - #[test] - fn fail_execute_task_without_crank_signer() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - vec![], - Err(InstructionError::MissingAccount), - ); - } - - #[test] - fn fail_execute_task_wrong_validator() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let wrong_validator = Pubkey::new_unique(); - let transaction_accounts = vec![ - ( - wrong_validator, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::IncorrectAuthority), - ); - } - - #[test] - fn fail_execute_task_validator_not_in_signers() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let mut ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - ix.accounts[0].is_signer = false; - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::MissingRequiredSignature), - ); - } - - #[test] - fn fail_execute_task_wrong_crank_signer() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let wrong_crank_signer = Pubkey::new_unique(); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - wrong_crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::InvalidSeeds), - ); - } - - #[test] - fn fail_execute_task_missing_accounts() { - init_validator_authority(Keypair::new()); - let payer = Pubkey::new_unique(); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![complex_ix(payer)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::MissingAccount), - ); - } - - #[test] - fn fail_execute_task_with_invalid_instructions() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let payer = Pubkey::new_unique(); - let mut inner_ix = InstructionUtils::schedule_task_instruction( - &payer, - ScheduleTaskArgs { - task_id: 0, - execution_interval_millis: 1, - iterations: 1, - instructions: vec![InstructionUtils::noop_instruction(0)], - }, - ); - - for (signer, writable, pubkey, expected) in [ - ( - true, - false, - Pubkey::new_unique(), - InstructionError::MissingRequiredSignature, - ), - (false, true, crank_signer, InstructionError::Immutable), - ( - false, - false, - validator_authority_id(), - InstructionError::IncorrectAuthority, - ), - ] { - inner_ix.accounts[0].is_signer = signer; - inner_ix.accounts[0].is_writable = writable; - inner_ix.accounts[0].pubkey = pubkey; - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![inner_ix.clone()], - ); - - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(expected), - ); - } - } -} diff --git a/programs/magicblock/src/schedule_task/process_schedule_task.rs b/programs/magicblock/src/schedule_task/process_schedule_task.rs index dfe20c63b..e478c59e7 100644 --- a/programs/magicblock/src/schedule_task/process_schedule_task.rs +++ b/programs/magicblock/src/schedule_task/process_schedule_task.rs @@ -93,11 +93,7 @@ pub(crate) fn process_schedule_task( return Err(InstructionError::InvalidInstructionData); } - validate_cranks_instructions( - invoke_context, - &payer_pubkey, - &args.instructions, - )?; + validate_cranks_instructions(invoke_context, &args.instructions)?; let schedule_request = ScheduleTaskRequest { id: args.task_id, diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index 8a9702b86..53445fdbb 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -9,9 +9,7 @@ use magicblock_magic_program_api::{ AccountModification, AccountModificationForInstruction, MagicBlockInstruction, PostDelegationActionExecutorInstruction, }, - pda::crank_signer_pda, - CRANK_PROGRAM_ID, MAGIC_CONTEXT_PUBKEY, - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, + MAGIC_CONTEXT_PUBKEY, POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, }; use solana_hash::Hash; use solana_instruction::{AccountMeta, Instruction}; @@ -253,47 +251,6 @@ impl InstructionUtils { ) } - // ----------------- - // Execute Crank - // ----------------- - pub fn execute_task_instruction( - authority: Pubkey, - instructions: Vec, - ) -> Instruction { - let mut account_metas = vec![ - AccountMeta::new_readonly(validator_authority_id(), true), - AccountMeta::new_readonly(crank_signer_pda(&authority), false), - ]; - for instruction in &instructions { - account_metas - .push(AccountMeta::new_readonly(instruction.program_id, false)); - account_metas.extend(instruction.accounts.iter().map(|account| { - AccountMeta { - pubkey: account.pubkey, - is_signer: false, - is_writable: account.is_writable, - } - })); - } - Instruction::new_with_bincode( - CRANK_PROGRAM_ID, - &MagicBlockInstruction::ExecuteCrank { - authority, - instructions, - }, - account_metas, - ) - } - - pub fn execute_task( - authority: Pubkey, - instructions: Vec, - recent_blockhash: Hash, - ) -> Transaction { - let ix = Self::execute_task_instruction(authority, instructions); - Self::into_transaction(&validator_authority(), ix, recent_blockhash) - } - // ----------------- // Executable Check // ----------------- diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 8021b20a7..f09f718bb 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -4195,7 +4195,9 @@ dependencies = [ "solana-pubkey 4.1.0", "solana-rpc-client", "solana-rpc-client-api", + "solana-sha256-hasher 3.1.0", "solana-signature", + "solana-system-interface 3.1.0", "solana-transaction", "solana-transaction-error 3.1.0", "thiserror 2.0.18", diff --git a/test-integration/configs/schedule-task.devnet.toml b/test-integration/configs/schedule-task.devnet.toml index babad3bb1..0ec185729 100644 --- a/test-integration/configs/schedule-task.devnet.toml +++ b/test-integration/configs/schedule-task.devnet.toml @@ -30,5 +30,9 @@ path = "../../target/deploy/magicblock_committor_program.so" id = "9hgprgZiRWmy8KkfvUuaVkDGrqo9GzeXMohwq6BazgUY" path = "../target/deploy/program_schedulecommit.so" +[[programs]] +id = "eHyd5BU8QffvHi4GnXwxrK4WpS7pM2x9UGKHBWii7mf" +path = "../programs/hydra/hydra.so" + [metrics] address = "0.0.0.0:9000" diff --git a/test-integration/programs/hydra/hydra.so b/test-integration/programs/hydra/hydra.so new file mode 100755 index 0000000000000000000000000000000000000000..431201c42016701fec2b461dd27ffd68b741a08b GIT binary patch literal 24536 zcmdUXe{|H>b>|yNUkhuK3S&>LXiFF{GBRCkY4j7Fk`=~a!uHfMW-RQDf(A*JgaK(Z zvf=5wX|RK1LQYb^cB0VsR04$MbCwgjZ4KErJ0xvt;$+W~?K)+%yN$XhIpL%^3EQS8 zG_j+-pZjAp^Oe-tF8!m=9^JY3-S^(T_uY5jeee4nKDznWx0J+U&Pu5(XKd-e6-v8S zjCUD_bM3CxmC@f@-4chE0euqMMHDW1NZJ{6O9cn}E%d4F5IhP8&?yyzh3(3S-u@_8 z&x1?Lg>KN=l@J-@9ZN}l6rLBdGq^Ay;J<1Z&0qQ*b$c-*l1s@UZjB2tlY4!Llf z=pC*VJyWqex&DH5OLi8!t}XQp}C#m&iicAp{u|ILY6$UdLuim*-kZz@~r5=6P6 z^2EzTZa=B|+|TUP4!t;6*&;an5OS20ooV`J<0bFqii|XO{lwl%l-CqqC>8#`vY*W6 zh#xpg;Qn6hT`zb}#|z1CK!M&b63nT5enRM#5j$^K^A`)un~Q~e#LrW)3Lap^2m==$ z6@NB=IzY-Srz*@hMC*&yuznYGVa8*%+%PAA4g9powc?<(xB2a) z?q^9N^B3p~e8V#vrQHh&fs-;m))C3eegwOe(>y@rxcdp~6}ShO-|Ui#l{#wT499ps zC(Z)|(JpTq8)4M_V|K447Hgk!cX0zhs_QX7{#CY(QT*`ANB7 z;dU#YmGR8qOfR!nRDT=y+)7wg>vQGSqqKDUcvbYk} zBc*Y{Ps+OVr!I0o;Y#su$cfUacF%P)s?m$=t@~>3L7|)9*#6K@^P8Zm0}A6MnCIJ^ zKMX3he;)&9{AS<$JW@N93-p1WDcx7%<+5+&+9d!t} zm0<2pVZi!8sOqFYHz>~KvO-^RyTA>}U*a#22PIV^as=-88Lz}*qcT6fo@4sEj;}S` z+^uL5I2HV87M2X#WPRke3mtw3`At5H(_MG=GJ`)O{FNJpKVEVhf5~+S-bsM<)_tH7 zMx<@b?pCcAQ~$Gpt#BN*Me%!3SEKcT#jC^WzCE7yHQ{mb@|17uFDbF*U=_IE1i z4KQ%O%pF_&t9+PIXK^AN(|vD^&@qqbFQ;}}p?1t|k@_k!;w=4#0=VnrLO(!y!M?i= z3trhPu!`3Ki$i+H+ja?z7N? zx^WL3mik@Tys4P}{%%->`Ws-t`vSM8Kir-mxBs}*@A{0ux%RZ+sJ#338lT_g`0&#N z#@_-doa86X5yPCgms|0))Z2Uo>_ub;&CBxp=#rN>zt4;c|Gd13%mr_`pn2*W zWu-1hhAJmJfgZm_@+a#bd?<7E|BvOeeI;-|V%$%QpJTnhN&JYj@bkPM5ps#}oX$h# z-GorNKzt~jR1f}sg~)pf*O~7I7qut9L&3Tvk=@Jm;dv&=^Q>`WmrSdxOi%?C$PZ5` zr~Wyf=(wrUAG7=zuW;SvwBGWp1JoAnqy4g+Y&qDE)}M|G{yO?pw$mRfl3kPhvYk4# zJEAcX^%1-HOZeAhn+r5=^By)lR;$-LB_q{D?TAG zr|TSf0`j>Qg1uB_>m2QX8-Ks9hfadXFZ+of`XL^_+97@7Z{@gr$39R*9~647SzzU( z0y8h77w58DAO-D18o#nzFbd&^tzKY8<6L$NYH7a6&)+0_)RcdF_M@K}cYoyWqhEm{q5d= zOmKg)4&TlF$CutBc^1&)>c22Orb!RW11WC&oad|h21CD6&l7jZ{P~p%H9zq>&;FOK zo&pmUulaqp;(aoYx}I@vfoFtA6GHC>H?2SVFuh`_(A96S-!grSwhbzKDrz}ik5p7K z+*7e+7Fl9O^TAYX6*GoMuZSKeX9RA$Ca~=)!_33tzqUVHe2vH6k6~P9hipv9%fJ~g zTcLDWz^Ssi_NefyYq>t$cA5M4wF$=kb_r@ZUEj!?!5MDr5Iqa%JDI+f>EYUThFE`F zL4Uke=wo#ZvnO{8tWFDT>t;Asv6EpY&BMER>^*D`e^%xb`w8};p1Y;q)}x&lqI##w z{+j!P9Z)hmME`i%4ATQAF!!9+r`0aRhHtii#bY02hD>j9EJCSY{!#_m>zCB$Pjr6+i}mZ&?^QQX4_5*tp1$9wc`xODvmMq z)8apwXGFiK9aFJya-hz9O!X#Eq<5(V>dax)TkYZ}#6RO@62Q&RGlwJkOSg1xR{cLA za?I&}iRH`{&_(}?Ob@r6XNYyj`X8MZdc_38Z1sN^*!B&9dM}3iUyMZeiL!3=o`>=m z#jkUHVSL6$5mt*ogkx8^9(ffsD&*rl>6Ge0W~rlg@IU6$_28#u9U;%5>RI?vH8lyG z$PXN4F@H*xy~^X~{Z{=dUMBf|-~{ITS!8}vWnY(mWLo$hjZ8`Tgb?CoU*r0~3AB7Y z9=yR4X4)Yo-AhuIim8k{Njy^S8@wZ@N>8)i{x0-N{Wi#i$4gH!?(bpX{0p`GrHbOm zt)Ke*hk<*HoceW{)M`H{fh5>sRqb*fB2ss>)J#Mg^GWnl_A<+etr8EAg#aO z_>rzn6kw>>`u8ukGhRjHADy}WNg(IG@WW3p-yw2YoJ^G-Wj?=#CWwk35&4nt-171p zcY;y1gbaj;ajRqHcq>b!@7t5{xCOpelw&eJJToF{D-UYeda4w>8?QK z-K_IO^y~Nh`Y`J6U$x^mx&oDVv(6LIU;pTH{M7Et&mTG46{x%doPQvJraVsgbfyE} zKQQxK7nj??NFR(8Zj(6aAK>Z*}sTEf&4%C z!Z-g4v_C!Ym$h91G7Qwu$$$LS@9{&G6W?F<&8`4akUSCj<0S95|G2FkbhdABocY3| zmpBgEde2B0cT;7@-`C-6VXl1F1}Y`VjkRsx-kk z=pZx&tK1QS7!NZMxNXY!uf=cGZW&z%@lr%fvgch4_&ko8_f3NPj#~bLd#xhB?N&eN zsLR(E`E9rQK}T!8zQ}L8#ef%N74qVVmq1UgKogCi|Y(xEHp`e(yJ}=k{Tn z#CyM~UhzNU`0h8|qxhdFzEbf&WE^s$q(7KuSaa*^vzI;;cYdoRrvAYXGClV(2JWG( zz-!r&u9wiJR<8fa9?ExiLewWC|Y1)fDaKda9`nin{VX7 z9FO{_l8S${gJIJx)9xUB%aWWZXb(#T?u*<$9GfWeBiyG#vvBRW*mo6LQGW~Ai{tYS zjn4)4DvHlL)DH^mRTQ7~+>!SOwHNL+I6kW%q{?0veNTuy_I$#hk@bUp1q`9CC#)09 zaM*T?_2{JUTwig7am$Nfk6x;Sf7}NCwc<|}kHa=u4>`3f&TV&rpZZ6C=lWQze4f>* z^i=8JamBit?-LjKaSsEI*QnTV*WYq|4ueyH{FvVjtEHVkC3@x7aJ8%K7nsp|I>?Pu zbL-()f?%HSK|+A97FenKSLOl1Ga8@zNlw`3p-n=sDGg(A40~T<4Ncohs4EF1| zdxGojeFuL==A}~g#W@Qq=jY`?9f$NpnVXldbA>$z3{OftJa_k(IsWABTK-rJ2S9)J zn5=_A>8CI*-=Mf0Rv+YcxS#sicHzO_uzG;$V-<&H6NHad49}9PZiL<+LPcRe1GI2+ zXNX7zet`IH_r>rxYQg6u_y_WAJu)-Doug9jGpv`Nlzr0PfAN#DPulxSep2>HJI`gN zZ?K$xa-5;PZ{tr$eC%4L@%^~OL$CMST`Ms&WS`>_2mPsQ+`eliDL}=}XZ}Qjaew+c z*Dp`%yb!>*c%Y`V->Pb3sCkb05>EH+NSPZ|h+beQYIJxj%2j|zXdXNlt7TJIF!skq2x@z+lY zo(gCnDt=1%Q-RpcPf35N0JDvH8805Z#kl3Gra$5oTE>G?wwIj){Dh3t73lnJ*7=)| z@uGRNpOA68RE}uA>?cHyXr7-6e#~-Nyn+8>B)^%-&Ue7de%hs)K=GH_UI!-Olrrt09C6))_|f9gI4A-v!Sv zOm5!qp4a`ZT;-5+hqR2wKBhB6$#d3jP6Zg>D?9Mvv{MSFrHjP{uo&o(R|XLljb(^u$-U4ugnjy zPMUF_oR<3Z^ba^Lrq5nu7`Q*+3fLDGvgb5YWi`C7(eo+hv;Ofu1?MYzpX^VF;cea3 zlqB2%>L)AdbwS>8Ha`wlh@Y1D9k|m>x4hWmhOZwMy1sYm>jxB{mUUX+t@woEx^Al_ z9@;t1*NFL{^QGp|feWR-DX}NLXC(8rjLCG4ko|T=Qt*8IAt!-c>z2{P zQ?Wb_`!I^>6EBnXYH=^%Cj-PE;Y*eMOxERH(mt+n5OEGYOqD+2p85Mk(f-N%<%saZ zPrpfDEiX2I?I$4VXx>q`zk%KOi%xq$4Q^P&M<@Zdo-tN&Nynd&`{c}= z{SMMsT3560A=p#><~;poBKBoJkT8&cPr>$i+c)7KzYF`Gong6Cx3sVygkx3LF|waL zqWUalJUlAT@eAb_yXtfOf_bXscVWf4pkFuPDFI?t|+*49txmh5_!%jJ< z&*O4Ruhe@itRFJx>=)x2R|MqaN#MTA{imnD6S3d_6+f|lC}_Zs%s%UC{)+4U@rx`- zSgrBy_=MsT=j>cz^N92272>e>u`rKskgnEmKh-7ZgFOq-?4EL0WIR3Be0$3-`x^N&?b&wGyt>mKhpEO}(EAI#6?e~|pYJ5@|S z@H3%TMd+hM&&l;5pR?*B^?yeFS^i?@RGi1QM)-0%?&XL)13K=(2!9{Zy{@-ZsW`g7 zT^u7-D(lwYP6|;;mCE||w^uP9FTI1IoqKVQ0XxS_h2Q$M`sUs}jt=d>Lkk=2!hG?>|hPdvD9yn|!I#AMktzPWJmuLK?&= zJ7<5P{@~a4{#@XG$_&0!N-L)t&^1@~tM2|^BNMDN{! z?833jTp#Wcf3v)Ns_cvOY>(_QA?;zEBk*hOFP9(!70|EHC)D5Bz|VULhn^@Qy@$r| z@o|AiWq%CE#Gi8r98?g8?;>dXRJdn``OhJ7Q958J#WPS+!7p*ex(B}nGUdU-Z_n#- zo&m##&F^8p+#zs@yfD#*c4?n~-|FLB?J`dY4EJ2;@oR3m;u1v4^ND-9T%GV$ikV%8 z#zV;r2<#{Kg$F;hQ}CK4ud(6ny|gvNzmlqQ!%`nUevQYo{>%>NhFSdlIb=@%zu|_K z=T)lw`F6J~*6uspZrzqYVCbi0e&WHad~OZ*XdXj(2^h?d;p4(*^Xgv_dGXv3dRV;2 zJuL9$gYsjo{Jy95L3>Dc?kSazzN2)~2b7e%43y^Wb%u-C+3aBN@5cj~ufR#-VZT$V ze2%+u{uS}J%9jL(UaRSAUY?g0mWSSpU^}GTf8~bj`d(z1E9eLQ^o#m2z1H>dBVbxr zS7g4Aad#x2$p56`Ie8pHeCijvf91z{Y+-p*?!SnfgU>Sz*B;|>EI+pQCj6w#@4Ai3 z*CylGK3J*e2|6E$obtHGac)AB@9kq(9op&S@ zXW;xJI#0=COz@9LyD)h~acLJO4=K*Tg~@}8w=)crfs^;q4M#mj2%Nlk zpR;wmn}}49zb)M%-|-jz^fdB3^S1)N_6T2w_$-e2Q{q?XZ)b#WFZwI!XE;(nfcljp zgZ-`p{Y*sodJE*T-^YOOVubI*#D~24T7+KpQyRzgzhA~fd)jpJ1^V~jSU7K9O6}2a zG;em^{tKQ`%|SKiO_$d z^eYkikBOeULrj4<4nOsq1do@>dkAzdbORK>QPxeR@|!-0?*p|IC&=lB2wGf>p6BMy zRX<~kbP_+QJsU5*P4a-!wY+jN8)f`d>~k?JgmBwZ9lwJs78<`>$M1hzT!&Kt2mrnL$9 zKe_+D*aQ5_;8g617~*jieXQW*V(b2m*q=CTNpHmdlHM`F?}In;FThi=-7&mpQrPZJ zZg-vINTK|Y*Y=yK(opU{CW)B}&woLEX0y;!!B&o2;YL`4{9v=rgZh&FPOO~jispfo zgrU!A97djsVtHIDke$-}U~arkIvy6l%^dHQqIsVi?*lrXSn%fQ_uGq%w_e9f++_Ve zRWu&vla_;<)3Z^>D+i5XwWG!< z#9^`f5w`mqCBMe;G+xrpc2AXjg!6)o?t}EZUx}}_uJ6}%y;}OU`$?ReiqS z4?#Fer_xg;zaqHC_qp+$j;HJTrj3WEe>DEQ@qSjkt~aWF3+!J-`jK+1U*ta0_OE!X zmE$V%f`Q1sTtWM3U#wQ_9{ViIRT!6Hs(5Sz-|rRDu`U89_nY{4A8?Y6$6n(115&Zi zJ7`$QkNUthvK;Y}d$_+jJ4fdArs*xee}UfP5xxJ0)^R+R<^3cTd!Fxz3gyL$4&3`i z-dM)nPIxNz9RHrMkPrJ>;AEV5?4bB<>;(I5As<3T;O@}=p5}XuRO}eP&s4~V{W5Ts z%r|GB&!}JMe)iAqhsSk1_4}JPAK&BW0GN+Mk@@%n&BvU+lmATm$~kfVy8E4CeQ%n5 z7mF*O)$wHJ{Y@KhvAA+V$4iiRAuqmJaphcoK06}w`B9qB6vY+X4;T73@=1CRf!n2G zYq+kk9 z=cI-5P>3e_@6O}Djp--n(UA`a?z8jg$ae$x;ygO+6S(8^=#YW_-JyAOJkbl>C+E@S z952sB3+0C?0{7fJeq0d;?%8>C#JRwIlBHWCjEG#!b?r_Z^PEm>z@gupiN5zCHH8!+sL%5EM7wl|3kKa69Zf z=Oc0I`#+KUibRC|J*Brs=-(l_Unc&NUM-1P`Yy4Ttp~qK{fYb!_j@>R1KRsrdAys| z%AdD)SLpi_XM_&B;dk-IpPrEVeyUE;=QV|PUQdsX_U#)UO^gmD(!ITj1B0UjiJ_$E_+44mYolMSeY9tMaQTN{{p5!)p8oBBx9>yQ)W7E_kl0OKqdysldaSFf ztFKGeHPkiMHPtoOwbZrNt*@`Eudh$mH`F)QH`O=Sx74@RuTRz`>yyc3L$Wd1lx$A6 zBwLf~8|oVB8jm?cMjjfIAo9deCo03fp zO^r=WP0dX$O|4Dqo9mkEo0H8A&5g}X&CSg%&8^MrTk2ZsTaqmeEsZTrEzK=0Ev+r< zTkBfuTa&E~t&Ocst<9}1t*x!=*OQ3rY54Wj^?GWyo=9f2k@WulzQhwlBZ=(Z!Ja>EyG`}%sbBR%)^?H?WS!$X7nNAKG=)O+tp)+u}Tz5(AmlD-d2MCzfI zLnHU~4DH)Dv>$AsXMMIOy?1Z+7dGn~9_ZWGH-NB;3A)Ra)gc_eN%$&bCU~B3?2T6boN#$-MJRtkc)e8;+_-(N zzdLC?C$z4N=++N@K8Sr7rI7w3NbhB9@9h>k%{Gcz!zaE`aDeMP_%Z2zsZxHr* z{K*CA_?gc_{ouN7Avyn+`dG+rpqoz>%6SL%BaydQe#HIS1>^y}Z2|gnqFetD(|nmO zTZtDt+WWN)azhmJsfkX(mkDs^ap=ueMLq=jBltr_AJ^yqomAw6D6)GZACpq0 z{o{@m<*@E$JyEL+LWq;r3E(*{+PwpD%8|)t# zog@0})9DdLhtvB9dv@{HeSQ0ShW*_4eZ#v*GP-MmA*q<}O-JyHaxBZ&cdK!Y|6l}f zRfn?rnF!vk4rcY!5xiC%&gzFF_*Tuot^RxjpSinOpYC5R#;>g_#t$Wn@yii>CW7CH z;8hL9?b{>x&Impn!5tisis@4u!8;=Oa0DNZ;Flu!)d=oBP&{5$1h0$WJ0tj^2tFRc z&qwgf5qu_s--zH_w-n3Q{$MeFWot1$v8@>2`cN_69l;MqaQipD%>Lusi`!2}@QV@r zas%YxIL3L+@3ufZqHN=zoyT{47X>^hTF4b!|j=| z;r49U@a|Nx{`M}V)gRQ24f_}!k=!MXKfrNcFs<+&kKLaMaz~>-T3zM+l}z)Ax?_?Y(zo$T6e}srCC9ruPl@xO=k$ z#66ncO*}(rBAxyd{$iWJ2Qs51bl)(T|M%FXIv?K@b?x*8O_EB8{`>6RFm6H7?(>vN z`UVUra18-VP_w%EVdZu;5$TW*1AN3LUHM*qF|t5k=A_K?%sTV8~@ z#n{%~@*{(m9}x?esQx&|Apf@ZmcJRa{0%;bV&j|q%|tWZnL+<_bjg>kg+%%u9smCW Dtd4-c literal 0 HcmV?d00001 diff --git a/test-integration/test-ledger-restore/src/lib.rs b/test-integration/test-ledger-restore/src/lib.rs index e12c674fb..0bd527696 100644 --- a/test-integration/test-ledger-restore/src/lib.rs +++ b/test-integration/test-ledger-restore/src/lib.rs @@ -19,8 +19,7 @@ use integration_test_tools::{ use magicblock_config::{ config::{ accounts::AccountsDbConfig, ledger::LedgerConfig, - scheduler::TaskSchedulerConfig, validator::ValidatorConfig, - LifecycleMode, LoadableProgram, + validator::ValidatorConfig, LifecycleMode, LoadableProgram, }, consts::DEFAULT_LEDGER_BLOCK_TIME_MS, types::{crypto::SerdePubkey, network::Remote, StorageDirectory}, @@ -146,10 +145,6 @@ pub fn setup_validator_with_local_remote_and_resume_strategy( }, accountsdb: accountsdb_config.clone(), programs, - task_scheduler: TaskSchedulerConfig { - reset: reset_ledger, - ..Default::default() - }, lifecycle: LifecycleMode::Ephemeral, remotes: vec![ Remote::from_str(IntegrationTestContext::url_chain()).unwrap(), diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index e3422bb40..9527346f8 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -1,4 +1,5 @@ use std::{ + path::PathBuf, process::Child, str::FromStr, time::{Duration, Instant}, @@ -18,13 +19,15 @@ use integration_test_tools::{ use magicblock_config::{ config::{ accounts::AccountsDbConfig, ledger::LedgerConfig, - scheduler::TaskSchedulerConfig, validator::ValidatorConfig, - LifecycleMode, + validator::ValidatorConfig, LifecycleMode, }, types::{network::Remote, StorageDirectory}, ValidatorParams, }; use magicblock_program::Pubkey; +use magicblock_task_scheduler::{ + db::DbTask, SchedulerDatabase, HYDRA_EPHEMERAL_PROGRAM_ID, +}; use program_flexi_counter::{ instruction::{ create_delegate_ix_with_commit_frequency_ms, create_init_ix, @@ -36,23 +39,18 @@ use solana_sdk::{ signature::Keypair, signer::Signer, transaction::Transaction, }; use tempfile::TempDir; +use tokio::runtime::Runtime; pub const TASK_SCHEDULER_TICK_MILLIS: u64 = 50; -pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { - let (default_tmpdir, temp_dir) = resolve_tmp_dir(TMP_DIR_CONFIG); - - let config = ValidatorParams { +fn validator_config(temp_dir: PathBuf) -> ValidatorParams { + ValidatorParams { lifecycle: LifecycleMode::Ephemeral, remotes: vec![ Remote::from_str(IntegrationTestContext::url_chain()).unwrap(), Remote::from_str(IntegrationTestContext::ws_url_chain()).unwrap(), ], accountsdb: AccountsDbConfig::default(), - task_scheduler: TaskSchedulerConfig { - reset: true, - ..Default::default() - }, validator: ValidatorConfig { ..Default::default() }, @@ -61,9 +59,16 @@ pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { block_time: Duration::from_millis(TASK_SCHEDULER_TICK_MILLIS), ..Default::default() }, - storage: StorageDirectory(temp_dir.clone()), + storage: StorageDirectory(temp_dir), ..Default::default() - }; + } +} + +fn start_validator( + config: ValidatorParams, + default_tmpdir: TempDir, + temp_dir: PathBuf, +) -> (TempDir, Child, IntegrationTestContext) { let (default_tmpdir_config, Some(mut validator), port) = start_magicblock_validator_with_config_struct_and_temp_dir( config, @@ -82,6 +87,33 @@ pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { (default_tmpdir_config, validator, ctx) } +pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { + setup_validator_with_migration_tasks(&[]) +} + +pub fn setup_validator_with_migration_tasks( + tasks: &[DbTask], +) -> (TempDir, Child, IntegrationTestContext) { + let (default_tmpdir, temp_dir) = resolve_tmp_dir(TMP_DIR_CONFIG); + + // Seed the migration database before the validator opens it. The handle is + // dropped (flushing the WAL) before startup. + { + let db_path = SchedulerDatabase::path(&temp_dir); + let db = SchedulerDatabase::new(&db_path) + .expect("failed to open seed database"); + let runtime = Runtime::new().expect("failed to create runtime"); + for task in tasks { + runtime + .block_on(db.insert_task(task)) + .expect("failed to seed task"); + } + } + + let config = validator_config(temp_dir.clone()); + start_validator(config, default_tmpdir, temp_dir) +} + pub fn create_delegated_counter( ctx: &IntegrationTestContext, payer: &Keypair, @@ -133,6 +165,98 @@ pub fn create_delegated_counter( expect!(ctx.wait_for_delta_slot_ephem(10), validator); } +pub fn wait_for_hydra_crank( + ctx: &IntegrationTestContext, + crank_pda: &Pubkey, + expected_lamports: u64, + max_timeout: Duration, + validator: &mut Child, +) { + let now = Instant::now(); + while now.elapsed() < max_timeout { + let maybe_account = ctx + .try_ephem_client() + .ok() + .and_then(|client| client.get_account(crank_pda).ok()); + if let Some(account) = maybe_account { + assert!( + account.owner.to_bytes() + == HYDRA_EPHEMERAL_PROGRAM_ID.to_bytes(), + cleanup(validator), + "crank account {} not owned by hydra program (owner: {})", + crank_pda, + account.owner + ); + assert!( + account.lamports >= expected_lamports, + cleanup(validator), + "crank account {} underfunded: {} < {}", + crank_pda, + account.lamports, + expected_lamports + ); + return; + } + expect!(ctx.wait_for_next_slot_ephem(), validator); + } + assert!( + false, + cleanup(validator), + "hydra crank account {} was not created before timeout", crank_pda + ); +} + +pub fn wait_for_hydra_crank_closed( + ctx: &IntegrationTestContext, + crank_pda: &Pubkey, + max_timeout: Duration, + validator: &mut Child, +) { + let now = Instant::now(); + while now.elapsed() < max_timeout { + let closed = match ctx + .try_ephem_client() + .ok() + .and_then(|client| client.get_account(crank_pda).ok()) + { + // Account fully removed. + None => true, + // Closed ephemeral accounts are drained to zero lamports. + Some(account) => account.lamports == 0, + }; + if closed { + return; + } + expect!(ctx.wait_for_next_slot_ephem(), validator); + } + assert!( + false, + cleanup(validator), + "hydra crank account {} was not closed before timeout", crank_pda + ); +} + +pub fn wait_for_empty_db( + runtime: &Runtime, + db: &SchedulerDatabase, + max_timeout: Duration, + validator: &mut Child, +) { + let now = Instant::now(); + while now.elapsed() < max_timeout { + let ids = expect!(runtime.block_on(db.get_task_ids()), validator); + if ids.is_empty() { + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + assert!( + false, + cleanup(validator), + "migration database was not emptied before timeout" + ); +} + pub fn wait_for_incremented_counter( ctx: &IntegrationTestContext, counter_pda: &Pubkey, diff --git a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs index 951f98239..a9347c0ad 100644 --- a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs +++ b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs @@ -1,6 +1,7 @@ -use cleanass::{assert, assert_eq}; +use std::time::Duration; + use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; +use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, @@ -9,16 +10,17 @@ use solana_sdk::{ native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, transaction::Transaction, }; -use test_task_scheduler::{create_delegated_counter, setup_validator}; -use tokio::runtime::Runtime; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, +}; #[test] fn test_cancel_ongoing_task() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); + let (_temp_dir, mut validator, ctx) = setup_validator(); let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); expect!( ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), @@ -67,10 +69,18 @@ fn test_cancel_ongoing_task() { validator ); - // Wait for the task to be scheduled - expect!(ctx.wait_for_delta_slot_ephem(2), validator); + // The crank is created and funded for all scheduled iterations. + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + let expected_lamports = iterations as u64 * CRANKER_REWARD; + wait_for_hydra_crank( + &ctx, + &crank_pda, + expected_lamports, + Duration::from_secs(10), + &mut validator, + ); - // Cancel the task + // Cancel the task while it is still ongoing. let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -97,65 +107,12 @@ fn test_cancel_ongoing_task() { validator ); - // Wait for the task to be cancelled - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Check that the task was cancelled - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - - let task = expect!(runtime.block_on(db.get_task(task_id)), validator); - assert!(task.is_none(), cleanup(&mut validator)); - - // Check that the counter was incremented but not as much as the number of executions - let counter_account = expect!( - ctx.try_ephem_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let counter = - expect!(FlexiCounter::try_decode(&counter_account.data), validator); - assert!( - counter.count < iterations as u64, - cleanup(&mut validator), - "counter.count: {}", - counter.count, - ); - assert!( - counter.count > 0, - cleanup(&mut validator), - "counter.count: {}", - counter.count, + // Cancelling closes the crank. + wait_for_hydra_crank_closed( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, ); cleanup(&mut validator); diff --git a/test-integration/test-task-scheduler/tests/test_independent_authority.rs b/test-integration/test-task-scheduler/tests/test_independent_authority.rs new file mode 100644 index 000000000..b5179f41f --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_independent_authority.rs @@ -0,0 +1,135 @@ +use std::time::Duration; + +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; +use program_flexi_counter::instruction::{ + create_cancel_task_ix, create_schedule_task_ix, +}; +use solana_sdk::{ + native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, + transaction::Transaction, +}; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, +}; + +#[test] +fn test_independent_cranks_per_authority() { + let (_temp_dir, mut validator, ctx) = setup_validator(); + + let payer = Keypair::new(); + let other = Keypair::new(); + + expect!( + ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + expect!( + ctx.airdrop_chain(&other.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + + create_delegated_counter(&ctx, &payer, &mut validator, 0); + create_delegated_counter(&ctx, &other, &mut validator, 0); + + let ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + + // Both authorities schedule the same task_id with different iteration counts. + let task_id = 1; + let payer_iterations = 3; + let other_iterations = 6; + for (signer, iterations) in + [(&payer, payer_iterations), (&other, other_iterations)] + { + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_schedule_task_ix( + signer.pubkey(), + task_id, + 100, + iterations, + false, + false, + )], + Some(&signer.pubkey()), + &[signer], + ephem_blockhash, + ), + &[signer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + } + + // Each authority owns its own independent crank at a distinct PDA, funded + // for its own iteration count. + let payer_crank = crank_pubkey(&payer.pubkey(), task_id); + let other_crank = crank_pubkey(&other.pubkey(), task_id); + assert_ne!(payer_crank, other_crank); + wait_for_hydra_crank( + &ctx, + &payer_crank, + payer_iterations as u64 * CRANKER_REWARD, + Duration::from_secs(10), + &mut validator, + ); + wait_for_hydra_crank( + &ctx, + &other_crank, + other_iterations as u64 * CRANKER_REWARD, + Duration::from_secs(10), + &mut validator, + ); + + // Cancelling one authority's task closes only its crank; the other remains. + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_cancel_task_ix(payer.pubkey(), task_id,)], + Some(&payer.pubkey()), + &[&payer], + ephem_blockhash, + ), + &[&payer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + wait_for_hydra_crank_closed( + &ctx, + &payer_crank, + Duration::from_secs(10), + &mut validator, + ); + // The other authority's crank is unaffected. + wait_for_hydra_crank( + &ctx, + &other_crank, + other_iterations as u64 * CRANKER_REWARD, + Duration::from_secs(10), + &mut validator, + ); + + cleanup(&mut validator); +} diff --git a/test-integration/test-task-scheduler/tests/test_migration.rs b/test-integration/test-task-scheduler/tests/test_migration.rs new file mode 100644 index 000000000..fd75f8730 --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_migration.rs @@ -0,0 +1,65 @@ +use std::time::Duration; + +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::{ + crank_pubkey, db::DbTask, SchedulerDatabase, CRANKER_REWARD, +}; +use solana_sdk::{ + instruction::{AccountMeta, Instruction}, + pubkey::Pubkey, + signature::Keypair, + signer::Signer, +}; +use test_task_scheduler::{ + setup_validator_with_migration_tasks, wait_for_empty_db, + wait_for_hydra_crank, +}; +use tokio::runtime::Runtime; + +/// Tasks persisted by the legacy (validator-funded) scheduler are migrated onto +/// hydra at startup: a funded hydra crank is created for each, and the +/// migration database is emptied. The database is used only for this one-time +/// migration. +#[test] +fn test_migration_reschedules_tasks_and_empties_db() { + let authority = Keypair::new().pubkey(); + let task_id = 42; + let iterations = 2; + // A simple, signer-free instruction is enough for the crank to be created; + // migration does not execute it. + let instructions = vec![Instruction { + program_id: program_flexi_counter::ID, + accounts: vec![AccountMeta::new(Pubkey::new_unique(), false)], + data: vec![0], + }]; + let task = DbTask { + id: task_id, + authority, + execution_interval_millis: 100, + executions_left: iterations, + instructions, + }; + + let (temp_dir, mut validator, ctx) = + setup_validator_with_migration_tasks(&[task]); + + // Migration creates a funded hydra crank for the persisted task... + let crank_pda = crank_pubkey(&authority, task_id); + wait_for_hydra_crank( + &ctx, + &crank_pda, + iterations as u64 * CRANKER_REWARD, + Duration::from_secs(15), + &mut validator, + ); + + // ...and empties the migration database. + let db = expect!( + SchedulerDatabase::new(SchedulerDatabase::path(temp_dir.path())), + validator + ); + let runtime = expect!(Runtime::new(), validator); + wait_for_empty_db(&runtime, &db, Duration::from_secs(15), &mut validator); + + cleanup(&mut validator); +} diff --git a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs index f85928981..8bd175381 100644 --- a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs @@ -1,8 +1,7 @@ use std::time::Duration; -use cleanass::assert_eq; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; +use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, @@ -12,17 +11,16 @@ use solana_sdk::{ transaction::Transaction, }; use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_incremented_counter, + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, }; -use tokio::runtime::Runtime; #[test] fn test_reschedule_task() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); + let (_temp_dir, mut validator, ctx) = setup_validator(); let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); expect!( ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), @@ -34,9 +32,9 @@ fn test_reschedule_task() { let ephem_blockhash = expect!(ctx.try_get_latest_blockhash_ephem(), validator); - // Schedule a task let task_id = 1; let execution_interval_millis = 100; + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); let iterations = 2; let sig = expect!( ctx.send_transaction_ephem_with_preflight( @@ -66,12 +64,17 @@ fn test_reschedule_task() { .ok_or_else(|| anyhow::anyhow!("Transaction failed")), validator ); - - // Wait for the task to be scheduled - expect!(ctx.wait_for_delta_slot_ephem(5), validator); + wait_for_hydra_crank( + &ctx, + &crank_pda, + iterations as u64 * CRANKER_REWARD, + Duration::from_secs(10), + &mut validator, + ); // Reschedule the task let new_execution_interval_millis = 200; + let new_iterations = 5; let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -79,7 +82,7 @@ fn test_reschedule_task() { payer.pubkey(), task_id, new_execution_interval_millis, - iterations, + new_iterations, false, false, )], @@ -101,51 +104,17 @@ fn test_reschedule_task() { validator ); - // Wait for the rescheduled task to finish all remaining executions. - wait_for_incremented_counter( + // Funding now covers the larger iteration count, which the original crank + // could not have held, proving the crank was recreated. + wait_for_hydra_crank( &ctx, - &counter_pda, - 2 * iterations as u64, + &crank_pda, + new_iterations as u64 * CRANKER_REWARD, Duration::from_secs(10), &mut validator, ); - // Check that the completed task was removed from the database - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - let task = expect!(runtime.block_on(db.get_task(task_id)), validator); - assert_eq!(task, None, cleanup(&mut validator)); - - // Cancel the task + // Cancel and confirm the active crank is closed. let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -167,12 +136,12 @@ fn test_reschedule_task() { .ok_or_else(|| anyhow::anyhow!("Transaction failed")), validator ); - - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Check that the task was cancelled - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!(tasks.len(), 0, cleanup(&mut validator)); + wait_for_hydra_crank_closed( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); cleanup(&mut validator); } diff --git a/test-integration/test-task-scheduler/tests/test_schedule_error.rs b/test-integration/test-task-scheduler/tests/test_schedule_error.rs deleted file mode 100644 index 0a624e678..000000000 --- a/test-integration/test-task-scheduler/tests/test_schedule_error.rs +++ /dev/null @@ -1,173 +0,0 @@ -use std::{ - thread::sleep, - time::{Duration, Instant}, -}; - -use cleanass::{assert, assert_eq}; -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; -use program_flexi_counter::{ - instruction::{create_cancel_task_ix, create_schedule_task_ix}, - state::FlexiCounter, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{create_delegated_counter, setup_validator}; -use tokio::runtime::Runtime; - -// Test that a task with an error is unscheduled -#[test] -fn test_schedule_error() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); - - let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 2; - let execution_interval_millis = 100; - let iterations = 3; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - true, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that the task eventually exhausts retries - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - let started = Instant::now(); - let failed_tasks = loop { - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - if failed_tasks.len() == 1 - || started.elapsed() >= Duration::from_secs(45) - { - break failed_tasks; - } - sleep(Duration::from_millis(200)); - }; - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - assert_eq!( - failed_tasks.len(), - 1, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks, - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - - assert!( - expect!(runtime.block_on(db.get_task(task_id)), validator).is_none(), - cleanup(&mut validator) - ); - - // Check that the counter was not incremented - let counter_account = expect!( - ctx.try_ephem_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let counter = - expect!(FlexiCounter::try_decode(&counter_account.data), validator); - assert!( - counter.count == 0, - cleanup(&mut validator), - "counter.count: {}", - counter.count, - ); - - // Cancel the task - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_cancel_task_ix(payer.pubkey(), task_id,)], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - expect!(ctx.wait_for_delta_slot_ephem(2), validator); - - // Check that the task was cancelled - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - - cleanup(&mut validator); -} diff --git a/test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs b/test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs deleted file mode 100644 index 554f49e3a..000000000 --- a/test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::time::Duration; - -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_program::{ - args::ScheduleTaskArgs, instruction_utils::InstructionUtils, - pda::crank_signer_pda, MAGIC_CONTEXT_PUBKEY, -}; -use program_schedulecommit::{ - api::{ - delegate_account_cpi_instruction, init_account_instruction, - schedule_commit_cpi_instruction, UserSeeds, - }, - ScheduleCommitType, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, signature::Keypair, - signer::Signer, transaction::Transaction, -}; -use test_task_scheduler::{setup_validator, wait_for_committed_count}; - -#[test] -fn test_crank_can_execute_program_that_cpis_into_magic() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let player = Keypair::new(); - let (committee, _) = Pubkey::find_program_address( - &[ - UserSeeds::MagicScheduleCommit.bytes(), - player.pubkey().as_ref(), - ], - &program_schedulecommit::ID, - ); - - expect!( - ctx.airdrop_chain(&player.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - let chain_blockhash = expect!( - ctx.try_chain_client().and_then(|client| client - .get_latest_blockhash() - .map_err(|e| anyhow::anyhow!( - "Failed to get latest chain blockhash: {}", - e - ))), - validator - ); - - expect!( - ctx.send_transaction_chain( - &mut Transaction::new_signed_with_payer( - &[init_account_instruction( - player.pubkey(), - player.pubkey(), - committee, - )], - Some(&player.pubkey()), - &[&player], - chain_blockhash, - ), - &[&player] - ), - validator - ); - - expect!( - ctx.send_transaction_chain( - &mut Transaction::new_signed_with_payer( - &[delegate_account_cpi_instruction( - player.pubkey(), - None, - player.pubkey(), - UserSeeds::MagicScheduleCommit, - )], - Some(&player.pubkey()), - &[&player], - chain_blockhash, - ), - &[&player] - ), - validator - ); - - expect!(ctx.wait_for_delta_slot_ephem(10), validator); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - let task_id = 17; - let crank_signer = crank_signer_pda(&player.pubkey()); - let mut crank_ix = schedule_commit_cpi_instruction( - crank_signer, - magicblock_program::ID, - MAGIC_CONTEXT_PUBKEY, - None, - &[player.pubkey()], - &[committee], - ScheduleCommitType::Commit, - ); - crank_ix.accounts[0].is_writable = false; - - let schedule_sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[InstructionUtils::schedule_task_instruction( - &player.pubkey(), - ScheduleTaskArgs { - task_id, - execution_interval_millis: 10, - iterations: 1, - instructions: vec![crank_ix], - }, - )], - Some(&player.pubkey()), - &[&player], - ephem_blockhash, - ), - &[&player] - ), - validator - ); - let schedule_status = - expect!(ctx.get_transaction_ephem(&schedule_sig), validator); - expect!( - schedule_status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Scheduling transaction failed")), - validator - ); - - wait_for_committed_count( - &ctx, - &committee, - 1, - Duration::from_secs(15), - &mut validator, - ); - - cleanup(&mut validator); -} diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task.rs b/test-integration/test-task-scheduler/tests/test_schedule_task.rs index 2d974906e..084622591 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task.rs @@ -1,8 +1,7 @@ use std::time::Duration; -use cleanass::assert_eq; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; +use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, @@ -12,17 +11,16 @@ use solana_sdk::{ transaction::Transaction, }; use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_incremented_counter, + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, }; -use tokio::runtime::Runtime; #[test] fn test_schedule_task() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); + let (_temp_dir, mut validator, ctx) = setup_validator(); let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); expect!( ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), @@ -67,51 +65,17 @@ fn test_schedule_task() { validator ); - // Wait for the task to be scheduled and executed - // Check that the counter was incremented - wait_for_incremented_counter( + // The crank is created by hydra and funded for every iteration. + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + let expected_lamports = iterations as u64 * CRANKER_REWARD; + wait_for_hydra_crank( &ctx, - &counter_pda, - iterations as u64, + &crank_pda, + expected_lamports, Duration::from_secs(10), &mut validator, ); - // Check that the completed task was removed from the database - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - let task = expect!(runtime.block_on(db.get_task(task_id)), validator); - assert_eq!(task, None, cleanup(&mut validator)); - // Cancel the task let sig = expect!( ctx.send_transaction_ephem_with_preflight( @@ -135,11 +99,13 @@ fn test_schedule_task() { validator ); - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Check that the task was cancelled - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!(tasks.len(), 0, cleanup(&mut validator)); + // Cancelling closes the hydra crank. + wait_for_hydra_crank_closed( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); cleanup(&mut validator); } diff --git a/test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs b/test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs deleted file mode 100644 index 4390422d8..000000000 --- a/test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs +++ /dev/null @@ -1,200 +0,0 @@ -use std::time::{Duration, Instant}; - -use cleanass::{assert, assert_eq}; -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; -use program_flexi_counter::instruction::create_schedule_task_ix; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{create_delegated_counter, setup_validator}; -use tokio::runtime::Runtime; - -#[test] -fn test_unauthorized_reschedule() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); - - let payer = Keypair::new(); - let different_payer = Keypair::new(); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - expect!( - ctx.airdrop_chain(&different_payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - create_delegated_counter(&ctx, &different_payer, &mut validator, 0); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 1; - let execution_interval_millis = 100; - let iterations = 1000; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - false, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Wait for the task to be scheduled - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Reschedule the same task with a different payer - let new_execution_interval_millis = 200; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - different_payer.pubkey(), - task_id, - new_execution_interval_millis, - iterations, - false, - false, - )], - Some(&different_payer.pubkey()), - &[&different_payer], - ephem_blockhash, - ), - &[&different_payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that one task is scheduled but another one is failed to schedule - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = wait_for_failed_schedulings( - &db, - &runtime, - 1, - Duration::from_secs(10), - &ctx, - &mut validator, - ); - assert_eq!( - failed_scheduling.len(), - 1, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - assert_eq!( - failed_scheduling[0].task_id, task_id, - cleanup(&mut validator), - "failed_scheduling: {:?}", failed_scheduling, - ); - assert!( - failed_scheduling[0].error.contains("UnauthorizedReplacing"), - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!(tasks.len(), 1, cleanup(&mut validator)); - - let task = expect!( - runtime - .block_on(db.get_task(task_id)) - .ok() - .flatten() - .ok_or(anyhow::anyhow!("Task not found")), - validator - ); - assert_eq!(task.authority, payer.pubkey(), cleanup(&mut validator)); - assert_eq!( - task.execution_interval_millis, execution_interval_millis, - cleanup(&mut validator) - ); - assert!( - task.executions_left > 0, - cleanup(&mut validator), - "task.executions_left: {}", - task.executions_left - ); - - cleanup(&mut validator); -} - -fn wait_for_failed_schedulings( - db: &SchedulerDatabase, - runtime: &Runtime, - expected_len: usize, - timeout: Duration, - ctx: &integration_test_tools::IntegrationTestContext, - validator: &mut std::process::Child, -) -> Vec { - let start = Instant::now(); - while start.elapsed() < timeout { - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - if failed_scheduling.len() == expected_len { - return failed_scheduling; - } - - expect!(ctx.wait_for_next_slot_ephem(), validator); - } - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - expected_len, - cleanup(validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - failed_scheduling -} diff --git a/test-integration/test-task-scheduler/tests/test_use_crank_signer.rs b/test-integration/test-task-scheduler/tests/test_use_crank_signer.rs deleted file mode 100644 index e35c8e477..000000000 --- a/test-integration/test-task-scheduler/tests/test_use_crank_signer.rs +++ /dev/null @@ -1,91 +0,0 @@ -use std::time::Duration; - -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_program::{ - args::ScheduleTaskArgs, instruction_utils::InstructionUtils, - pda::crank_signer_pda, -}; -use program_flexi_counter::{ - instruction::FlexiCounterInstruction, state::FlexiCounter, -}; -use solana_sdk::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, - signature::Keypair, - signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_incremented_counter, -}; - -#[test] -fn test_use_crank_signer() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 9; - let execution_interval_millis = 10; - let iterations = 5; - let crank_signer = crank_signer_pda(&payer.pubkey()); - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[InstructionUtils::schedule_task_instruction( - &payer.pubkey(), - ScheduleTaskArgs { - task_id, - execution_interval_millis, - iterations, - instructions: vec![Instruction::new_with_borsh( - program_flexi_counter::ID, - &FlexiCounterInstruction::AddUnsigned { count: 1 }, - vec![ - AccountMeta::new(counter_pda, false), - AccountMeta::new_readonly(crank_signer, true) - ], - )], - } - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that the counter was incremented - wait_for_incremented_counter( - &ctx, - &counter_pda, - iterations as u64, - Duration::from_secs(10), - &mut validator, - ); - - cleanup(&mut validator); -} From 51cc38f888209f573085eed791083c2d2f2817a4 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Tue, 30 Jun 2026 09:51:50 +0200 Subject: [PATCH 02/13] feat: use hydra api --- Cargo.lock | 157 +++++++++----- Cargo.toml | 1 + magicblock-task-scheduler/Cargo.toml | 3 +- magicblock-task-scheduler/src/hydra.rs | 202 ------------------ magicblock-task-scheduler/src/lib.rs | 5 - magicblock-task-scheduler/src/service.rs | 9 +- test-integration/Cargo.lock | 168 ++++++++++----- test-integration/Cargo.toml | 1 + .../test-task-scheduler/Cargo.toml | 5 +- .../test-task-scheduler/src/lib.rs | 5 +- .../tests/test_cancel_ongoing_task.rs | 3 +- .../tests/test_independent_authority.rs | 3 +- .../tests/test_migration.rs | 5 +- .../tests/test_reschedule_task.rs | 3 +- .../tests/test_schedule_task.rs | 3 +- 15 files changed, 245 insertions(+), 328 deletions(-) delete mode 100644 magicblock-task-scheduler/src/hydra.rs diff --git a/Cargo.lock b/Cargo.lock index 353748056..903370e19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a48ac39333d364f102b03ef00b59cd5b1f878ccb3c5f0f67df20f44824d51f" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "digest 0.10.7", "ed25519-dalek 1.0.1", "libsecp256k1", @@ -144,7 +144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84debd4abe0cbab5a6aac2ee50e3969ef0e0961f7dff7e8f96bda0be7998bca2" dependencies = [ "agave-bls12-381", - "bincode", + "bincode 1.3.3", "libsecp256k1", "num-traits", "solana-account 3.4.0", @@ -768,6 +768,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.69.5" @@ -1912,6 +1931,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ephemeral-rollups-pinocchio" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a09b61c47e5b4eeb7add34e9d43d836784a66d78fa704606bebaa453bef73e21" +dependencies = [ + "bincode 2.0.1", + "pinocchio 0.10.2", + "pinocchio-pubkey", + "pinocchio-system", + "solana-address 2.5.0", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2391,7 +2423,7 @@ dependencies = [ name = "guinea" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "magicblock-magic-program-api", "serde", "solana-program 3.0.0", @@ -2642,6 +2674,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "hydra-api" +version = "0.1.1" +source = "git+https://github.com/magicblock-labs/hydra.git?rev=4f9e3c44e9e2e4feab66e6e7903099e095c74db3#4f9e3c44e9e2e4feab66e6e7903099e095c74db3" +dependencies = [ + "ephemeral-rollups-pinocchio", + "pinocchio 0.10.2", + "solana-address 2.5.0", + "solana-instruction 3.3.0", + "solana-pubkey 4.1.0", +] + [[package]] name = "hyper" version = "1.10.1" @@ -3433,7 +3477,7 @@ name = "magicblock-account-cloner" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "magicblock-accounts-db", "magicblock-chainlink", "magicblock-committor-service", @@ -3531,7 +3575,7 @@ dependencies = [ "agave-geyser-plugin-interface", "arc-swap", "base64 0.21.7", - "bincode", + "bincode 1.3.3", "bs58", "fastwebsockets", "futures", @@ -3645,7 +3689,7 @@ dependencies = [ "arc-swap", "assert_matches", "async-trait", - "bincode", + "bincode 1.3.3", "borsh", "futures-util", "helius-laserstream", @@ -3715,7 +3759,7 @@ name = "magicblock-committor-service" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "borsh", "futures-util", "lazy_static", @@ -3785,7 +3829,7 @@ dependencies = [ name = "magicblock-core" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bytes", "console-subscriber", "flume", @@ -3816,7 +3860,7 @@ name = "magicblock-delegation-program-api" version = "0.3.0" source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b#25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh", "bytemuck", "const-crypto", @@ -3846,7 +3890,7 @@ name = "magicblock-ledger" version = "0.13.2" dependencies = [ "arc-swap", - "bincode", + "bincode 1.3.3", "byteorder", "fs_extra", "libc", @@ -3887,7 +3931,7 @@ dependencies = [ name = "magicblock-magic-program-api" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "const-crypto", "serde", "solana-program 2.3.0", @@ -3918,7 +3962,7 @@ dependencies = [ "agave-feature-set", "agave-precompiles", "agave-syscalls", - "bincode", + "bincode 1.3.3", "blake3", "guinea", "magicblock-accounts-db", @@ -3969,7 +4013,7 @@ name = "magicblock-program" version = "0.13.2" dependencies = [ "assert_matches", - "bincode", + "bincode 1.3.3", "lazy_static", "magicblock-chainlink", "magicblock-core", @@ -4010,7 +4054,7 @@ name = "magicblock-replicator" version = "0.13.2" dependencies = [ "async-nats", - "bincode", + "bincode 1.3.3", "bytes", "futures", "magicblock-accounts-db", @@ -4062,7 +4106,7 @@ dependencies = [ name = "magicblock-services" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "futures-util", "magicblock-core", "magicblock-magic-program-api", @@ -4110,9 +4154,10 @@ dependencies = [ name = "magicblock-task-scheduler" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "futures-util", + "hydra-api", "magicblock-config", "magicblock-core", "magicblock-ledger", @@ -6289,7 +6334,7 @@ name = "solana-account" version = "3.4.0" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "serde_bytes", @@ -6309,7 +6354,7 @@ checksum = "ff1acff600a621d4445e0610fa71a53bef5390a5e349cfbd30651917593fb57d" dependencies = [ "Inflector", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "bv", "serde", @@ -6364,7 +6409,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8f5152a288ef1912300fc6efa6c2d1f9bb55d9398eb6c72326360b8063987da" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-program-error 2.2.2", "solana-program-memory 2.3.1", @@ -6377,7 +6422,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9cf16495d9eb53e3d04e72366a33bb1c20c24e78c171d8b8f5978357b63ae95" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-address 2.5.0", "solana-program-error 3.0.1", @@ -6434,7 +6479,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673f67efe870b64a65cb39e6194be5b26527691ce5922909939961a6e6b395" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6451,7 +6496,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e8df0b083c10ce32490410f3795016b1b5d9b4d094658c0a5e496753645b7cd" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6509,7 +6554,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19a3787b8cf9c9fe3dd360800e8b70982b9e5a8af9e11c354b6665dd4a003adc" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-instruction 2.3.3", ] @@ -6520,7 +6565,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "278a1a5bad62cd9da89ac8d4b7ec444e83caa8ae96aa656dfc27684b28d49a5d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-instruction-error", ] @@ -6599,7 +6644,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219bfba64973ac9e64aa181f03fd56ac319e2d50d8a23d16c54bbd7fa9807a47" dependencies = [ "agave-syscalls", - "bincode", + "bincode 1.3.3", "qualifier_attr", "solana-account 3.4.0", "solana-bincode 3.1.0", @@ -6741,7 +6786,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e401ae56aed512821cc7a0adaa412ff97fecd2dff4602be7b1330d2daec0c4" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -6997,7 +7042,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43f5c5382b449e8e4e3016fb05e418c53d57782d8b5c30aa372fc265654b956d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 2.2.1", @@ -7016,7 +7061,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75ca9b5cbb6f500f7fd73db5bd95640f71a83f04d6121a0e59a43b202dca2731" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -7078,7 +7123,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "749eccc960e85c9b33608450093d256006253e1cb436b8380e71777840a3f675" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "memmap2 0.5.10", "solana-account 3.4.0", @@ -7164,7 +7209,7 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab5682934bd1f65f8d2c16f21cb532526fcc1a09f796e2cacdb091eee5774ad" dependencies = [ - "bincode", + "bincode 1.3.3", "getrandom 0.2.17", "js-sys", "num-traits", @@ -7182,7 +7227,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97881335fc698deb46c6571945969aae6d93a14e2fff792a368b4fac872f116" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh", "serde", "serde_derive", @@ -7435,7 +7480,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1796aabce376ff74bf89b78d268fa5e683d7d7a96a0a4e4813ec34de49d5314b" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7458,7 +7503,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0448b1fd891c5f46491e5dc7d9986385ba3c852c340db2911dd29faa01d2b08d" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7648,7 +7693,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98eca145bd3545e2fbb07166e895370576e47a00a7d824e325390d33bf467210" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "bs58", "bytemuck", @@ -7870,7 +7915,7 @@ version = "4.0.0" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "cfg-if", "itertools 0.12.1", "log", @@ -8031,7 +8076,7 @@ checksum = "10dd50b329ce569340c1deab3667d21e26a41e65cc6460e8a5bb8b57aff8420d" dependencies = [ "async-trait", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "futures", "indicatif", @@ -8146,7 +8191,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f03df7969f5e723ad31b6c9eadccc209037ac4caa34d8dc259316b05c11e82b" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "serde", "solana-account 3.4.0", @@ -8542,7 +8587,7 @@ dependencies = [ name = "solana-storage-proto" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "enum-iterator 1.5.0", "prost", @@ -8722,7 +8767,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "450479004fee3396c88cc4aa2f9b2b8db9c77be42ee7c1c53e6fac9eaec5fd51" dependencies = [ - "bincode", + "bincode 1.3.3", "log", "serde", "solana-account 3.4.0", @@ -8764,7 +8809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c3595f95069f3d90f275bb9bd235a1973c4d059028b0a7f81baca2703815db" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -8801,7 +8846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6690d3dd88f15c21edff68eb391ef8800df7a1f5cec84ee3e8d1abf05affdf74" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -8860,7 +8905,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96697cff5075a028265324255efed226099f6d761ca67342b230d09f72cc48d2" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-address 2.5.0", @@ -8881,7 +8926,7 @@ name = "solana-transaction-context" version = "4.0.0" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "solana-account 3.4.0", @@ -8924,7 +8969,7 @@ dependencies = [ "Inflector", "agave-reserved-account-keys", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "borsh", "bs58", "log", @@ -8965,7 +9010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6886b7bb8fbba5937b4a38fa67ed442d7971629244b8fbd95c7963b2126bc9" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "serde", "serde_json", @@ -9002,7 +9047,7 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b80d57478d6599d30acc31cc5ae7f93ec2361a06aefe8ea79bc81739a08af4c3" dependencies = [ - "bincode", + "bincode 1.3.3", "num-derive", "num-traits", "serde", @@ -9026,7 +9071,7 @@ version = "5.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d444ce30b6b4f9c281ba06061ea96638d063b53c2171b1e41ba02ebff79ed28f" dependencies = [ - "bincode", + "bincode 1.3.3", "cfg_eval", "num-derive", "num-traits", @@ -9053,7 +9098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4537fd6efe65f53ccd28d54d2ad43275b024834a4a8ca4dfa4babfa01e6d11ab" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "log", "num-derive", "num-traits", @@ -9116,7 +9161,7 @@ checksum = "9602bcb1f7af15caef92b91132ec2347e1c51a72ecdbefdaefa3eac4b8711475" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -9153,7 +9198,7 @@ checksum = "09670ff59f60e6ddc2209c2e4353992a9b9f01d4e244f3e9d67bd5146e33d388" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -10302,6 +10347,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "3.3.0" @@ -10403,6 +10454,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "void" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 010b6ccd0..06cce05a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ guinea = { path = "./programs/guinea" } helius-laserstream = { git = "https://github.com/magicblock-labs/laserstream-sdk.git", branch = "mbv-4.0+conn-fix" } http-body-util = "0.1.3" humantime = { version = "1.1", package = "humantime-serde" } +hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "4f9e3c44e9e2e4feab66e6e7903099e095c74db3" } hyper = "1.6.0" hyper-util = "0.1.15" isocountry = "0.3.2" diff --git a/magicblock-task-scheduler/Cargo.toml b/magicblock-task-scheduler/Cargo.toml index 842a932d7..c188b4669 100644 --- a/magicblock-task-scheduler/Cargo.toml +++ b/magicblock-task-scheduler/Cargo.toml @@ -11,7 +11,7 @@ edition.workspace = true bincode = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } -tracing = { workspace = true } +hydra-api = { workspace = true, features = ["client"] } magicblock-core = { workspace = true } magicblock-config = { workspace = true } magicblock-ledger = { workspace = true } @@ -30,6 +30,7 @@ solana-transaction-error = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true, features = ["time"] } +tracing = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/magicblock-task-scheduler/src/hydra.rs b/magicblock-task-scheduler/src/hydra.rs deleted file mode 100644 index b9aceb5c8..000000000 --- a/magicblock-task-scheduler/src/hydra.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! Minimal, self-contained client builders for the hydra ephemeral crank -//! program. -//! -//! The validator's RPC client stack pins `solana-rpc-client-types 4.0.0`, which -//! caps `solana-address` below the `^2.6` that the upstream `hydra-api` crate -//! requires; the two cannot be unified in a single build. Rather than move the -//! whole validator onto pre-release RPC crates, we vendor the small, stable -//! slice of hydra's wire format that the scheduler needs. The integration is -//! pinned to a specific hydra revision, so this layout will not drift -//! unexpectedly. -//! -//! Mirrors `hydra-api` rev `1fc9086` (`crates/hydra-api/src/instruction.rs`, -//! the `ephemeral` builders). See that file for the authoritative wire format. - -use magicblock_program::EPHEMERAL_VAULT_PUBKEY; -use solana_instruction::{AccountMeta, Instruction}; -use solana_pubkey::Pubkey; - -/// Ephemeral-rollup hydra program id (`eHyd5BU8QffvHi4GnXwxrK4WpS7pM2x9UGKHBWii7mf`). -pub const EPHEMERAL_PROGRAM_ID: Pubkey = Pubkey::new_from_array([ - 9, 141, 175, 94, 62, 38, 45, 106, 31, 231, 193, 37, 229, 238, 178, 89, 200, - 202, 82, 70, 56, 177, 52, 125, 239, 164, 240, 139, 173, 185, 238, 142, -]); - -/// Seed prefix for the crank PDA: `[b"crank", seed]`. -const CRANK_SEED_PREFIX: &[u8] = b"crank"; - -/// Instruction discriminators (one byte each). -const IX_CREATE: u8 = 0; -const IX_CANCEL: u8 = 2; - -/// Per-meta writable flag in the scheduled-ix wire format. -const META_FLAG_WRITABLE: u8 = 0b0000_0010; - -/// Flat per-trigger reward (lamports) paid to the cranker. Matches -/// `hydra_api::consts::CRANKER_REWARD` for the deployed program, which is built -/// without the `hydra-api/ephemeral` feature, so `BASE_FEE_LAMPORTS = 5_000` -/// and `CRANKER_REWARD = 2 * BASE_FEE_LAMPORTS`. -pub const CRANKER_REWARD: u64 = 10_000; - -/// One scheduled-ix account meta as stored on-chain. Scheduled instructions may -/// not carry signer flags, so only the writable bit is represented. -pub struct SchedMeta { - pub pubkey: [u8; 32], - pub is_writable: bool, -} - -/// One scheduled instruction template. -pub struct ScheduledIx<'a> { - pub program_id: [u8; 32], - pub metas: &'a [SchedMeta], - pub data: &'a [u8], -} - -/// Scheduling knobs for a hydra `Create` instruction. -pub struct CreateArgs<'a> { - pub seed: [u8; 32], - /// All-zeros = unkillable (no cancel authority). - pub authority: [u8; 32], - pub start_slot: u64, - pub interval_slots: u64, - /// `0` on the wire means "infinite"; the scheduler always sets a finite - /// iteration count. - pub remaining: u64, - pub priority_tip: u64, - pub cu_limit: u32, - /// Scheduled instructions in execution order. Must be non-empty. - pub scheduled: &'a [ScheduledIx<'a>], -} - -impl CreateArgs<'_> { - /// Serializes the `Create` instruction data exactly as hydra's on-chain - /// entrypoint parses it. - fn serialize(&self) -> Vec { - // 1 discriminator + 100-byte fixed prefix + variable body. - let body_len: usize = self - .scheduled - .iter() - .map(|s| 1 + 2 + 32 + 33 * s.metas.len() + s.data.len()) - .sum(); - let mut data = Vec::with_capacity(1 + 100 + body_len); - - data.push(IX_CREATE); - data.extend_from_slice(&self.seed); - data.extend_from_slice(&self.authority); - data.extend_from_slice(&self.start_slot.to_le_bytes()); - data.extend_from_slice(&self.interval_slots.to_le_bytes()); - data.extend_from_slice(&self.remaining.to_le_bytes()); - data.extend_from_slice(&self.priority_tip.to_le_bytes()); - data.extend_from_slice(&self.cu_limit.to_le_bytes()); - - for s in self.scheduled { - data.push(s.metas.len() as u8); - data.extend_from_slice(&(s.data.len() as u16).to_le_bytes()); - data.extend_from_slice(&s.program_id); - for m in s.metas { - let flag = if m.is_writable { META_FLAG_WRITABLE } else { 0 }; - data.push(flag); - data.extend_from_slice(&m.pubkey); - } - data.extend_from_slice(s.data); - } - - data - } -} - -/// Builders targeting the ephemeral-rollup hydra program. -pub mod ephemeral { - use super::*; - - /// Derives `(crank_pda, bump)` under the ephemeral hydra program. - pub fn find_crank_pda(seed: &[u8; 32]) -> (Pubkey, u8) { - Pubkey::find_program_address( - &[CRANK_SEED_PREFIX, seed], - &EPHEMERAL_PROGRAM_ID, - ) - } - - /// Builds a hydra `Create` instruction for the ephemeral program. - pub fn create( - sponsor: Pubkey, - crank: Pubkey, - args: &CreateArgs<'_>, - ) -> Instruction { - Instruction { - program_id: EPHEMERAL_PROGRAM_ID, - accounts: vec![ - AccountMeta::new(sponsor, true), - AccountMeta::new(crank, false), - AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), - AccountMeta::new_readonly(magicblock_program::id(), false), - ], - data: args.serialize(), - } - } - - /// Builds a hydra `Cancel` instruction for the ephemeral program. The - /// remaining crank balance is refunded to the ephemeral vault. - pub fn cancel(authority: Pubkey, crank: Pubkey) -> Instruction { - Instruction { - program_id: EPHEMERAL_PROGRAM_ID, - accounts: vec![ - AccountMeta::new(authority, true), - AccountMeta::new(crank, false), - AccountMeta::new(EPHEMERAL_VAULT_PUBKEY, false), - AccountMeta::new_readonly(magicblock_program::id(), false), - ], - data: vec![IX_CANCEL], - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ephemeral_program_id_matches_base58() { - assert_eq!( - EPHEMERAL_PROGRAM_ID.to_string(), - "eHyd5BU8QffvHi4GnXwxrK4WpS7pM2x9UGKHBWii7mf" - ); - } - - #[test] - fn create_data_layout_matches_wire_format() { - let program_id = [7u8; 32]; - let acct = [9u8; 32]; - let metas = [SchedMeta { - pubkey: acct, - is_writable: true, - }]; - let scheduled = [ScheduledIx { - program_id, - metas: &metas, - data: &[1, 2, 3], - }]; - let args = CreateArgs { - seed: [1u8; 32], - authority: [2u8; 32], - start_slot: 5, - interval_slots: 7, - remaining: 9, - priority_tip: 11, - cu_limit: 0, - scheduled: &scheduled, - }; - let data = args.serialize(); - - // 1 disc + 100 fixed prefix + (1 + 2 + 32 + 33*1 + 3) body. - assert_eq!(data.len(), 1 + 100 + (1 + 2 + 32 + 33 + 3)); - assert_eq!(data[0], IX_CREATE); - // seed starts right after the discriminator. - assert_eq!(&data[1..33], &[1u8; 32]); - // authority follows the seed. - assert_eq!(&data[33..65], &[2u8; 32]); - // The writable meta flag is encoded for the single scheduled account. - let meta_flag_off = 1 + 100 + 1 + 2 + 32; - assert_eq!(data[meta_flag_off], META_FLAG_WRITABLE); - } -} diff --git a/magicblock-task-scheduler/src/lib.rs b/magicblock-task-scheduler/src/lib.rs index d8bb21fe5..854f34be6 100644 --- a/magicblock-task-scheduler/src/lib.rs +++ b/magicblock-task-scheduler/src/lib.rs @@ -1,14 +1,9 @@ pub mod db; pub mod errors; -mod hydra; pub mod service; pub use db::SchedulerDatabase; pub use errors::TaskSchedulerError; -/// Flat per-execution reward (lamports) that hydra pays the external cranker. -pub use hydra::CRANKER_REWARD; -/// Program id of the ephemeral hydra crank program that the scheduler targets. -pub use hydra::EPHEMERAL_PROGRAM_ID as HYDRA_EPHEMERAL_PROGRAM_ID; /// Derives the on-chain hydra crank account address for a task from its /// `(authority, task_id)` — the deterministic, per-authority namespace the /// scheduler uses. Tests and tooling can use this to locate the crank account a diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index 99b8e61cb..aefedc255 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -1,5 +1,10 @@ use std::{path::Path, sync::Arc, time::Duration as StdDuration}; +use hydra_api::{ + consts::CRANKER_REWARD, + ephemeral::ID as EPHEMERAL_PROGRAM_ID, + instruction::{ephemeral, CreateArgs, SchedMeta, ScheduledIx}, +}; use magicblock_core::link::transactions::ScheduledTasksRx; use magicblock_ledger::LatestBlock; use magicblock_program::{ @@ -23,10 +28,6 @@ use tracing::*; use crate::{ db::{DbTask, SchedulerDatabase}, errors::{TaskSchedulerError, TaskSchedulerResult}, - hydra::{ - ephemeral, CreateArgs, SchedMeta, ScheduledIx, CRANKER_REWARD, - EPHEMERAL_PROGRAM_ID, - }, }; /// How long migration waits for the validator to produce a usable blockhash diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index c7267c889..62578541e 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -120,7 +120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a48ac39333d364f102b03ef00b59cd5b1f878ccb3c5f0f67df20f44824d51f" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "digest 0.10.7", "ed25519-dalek 1.0.1", "libsecp256k1", @@ -153,7 +153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84debd4abe0cbab5a6aac2ee50e3969ef0e0961f7dff7e8f96bda0be7998bca2" dependencies = [ "agave-bls12-381", - "bincode", + "bincode 1.3.3", "libsecp256k1", "num-traits", "solana-account 3.4.0", @@ -786,6 +786,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.69.5" @@ -1939,6 +1958,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ephemeral-rollups-pinocchio" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a09b61c47e5b4eeb7add34e9d43d836784a66d78fa704606bebaa453bef73e21" +dependencies = [ + "bincode 2.0.1", + "pinocchio 0.10.2", + "pinocchio-pubkey", + "pinocchio-system", + "solana-address 2.5.0", +] + [[package]] name = "ephemeral-rollups-sdk" version = "0.14.4" @@ -1946,7 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "838564736408ada011e9986c819b26cf0195bb2dd238e10be258731979213d75" dependencies = [ "base64ct", - "bincode", + "bincode 1.3.3", "bytemuck", "ephemeral-rollups-sdk-attribute-action", "ephemeral-rollups-sdk-attribute-commit", @@ -2509,7 +2541,7 @@ dependencies = [ name = "guinea" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "magicblock-magic-program-api 0.13.2", "serde", "solana-program 3.0.0", @@ -2746,6 +2778,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "hydra-api" +version = "0.1.1" +source = "git+https://github.com/magicblock-labs/hydra.git?rev=4f9e3c44e9e2e4feab66e6e7903099e095c74db3#4f9e3c44e9e2e4feab66e6e7903099e095c74db3" +dependencies = [ + "ephemeral-rollups-pinocchio", + "pinocchio 0.10.2", + "solana-address 2.5.0", + "solana-instruction 3.3.0", + "solana-pubkey 4.1.0", +] + [[package]] name = "hyper" version = "1.10.1" @@ -3500,7 +3544,7 @@ name = "magicblock-account-cloner" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "magicblock-accounts-db", "magicblock-chainlink", "magicblock-committor-service", @@ -3594,7 +3638,7 @@ dependencies = [ "agave-geyser-plugin-interface", "arc-swap", "base64 0.21.7", - "bincode", + "bincode 1.3.3", "bs58", "fastwebsockets", "futures", @@ -3702,7 +3746,7 @@ dependencies = [ "agave-feature-set", "arc-swap", "async-trait", - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "futures-util", "helius-laserstream", @@ -3771,7 +3815,7 @@ name = "magicblock-committor-service" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "futures-util", "lru", @@ -3832,7 +3876,7 @@ dependencies = [ name = "magicblock-core" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bytes", "flume", "magicblock-magic-program-api 0.13.2", @@ -3862,7 +3906,7 @@ name = "magicblock-delegation-program-api" version = "0.3.0" source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b#25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "bytemuck", "const-crypto", @@ -3893,7 +3937,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "288904a9950bd20f27f0ef934f320ab1410bd35a6d5c9cf138eca276442b6b2e" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 0.10.4", "borsh 1.7.0", "bytemuck", @@ -3925,7 +3969,7 @@ name = "magicblock-ledger" version = "0.13.2" dependencies = [ "arc-swap", - "bincode", + "bincode 1.3.3", "byteorder", "fs_extra", "libc", @@ -3964,7 +4008,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dc8fba0307c90b91b70c9ed06d4242d6c4159f331b2f05bf8f875c2a94e0e98" dependencies = [ - "bincode", + "bincode 1.3.3", "const-crypto", "serde", "solana-program 3.0.0", @@ -3975,7 +4019,7 @@ dependencies = [ name = "magicblock-magic-program-api" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "const-crypto", "serde", "solana-program 3.0.0", @@ -4004,7 +4048,7 @@ dependencies = [ "agave-feature-set", "agave-precompiles", "agave-syscalls", - "bincode", + "bincode 1.3.3", "blake3", "magicblock-accounts-db", "magicblock-core", @@ -4047,7 +4091,7 @@ dependencies = [ name = "magicblock-program" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "lazy_static", "magicblock-core", "magicblock-delegation-program-api 0.3.0", @@ -4084,7 +4128,7 @@ name = "magicblock-replicator" version = "0.13.2" dependencies = [ "async-nats", - "bincode", + "bincode 1.3.3", "bytes", "futures", "magicblock-accounts-db", @@ -4134,7 +4178,7 @@ dependencies = [ name = "magicblock-services" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "futures-util", "magicblock-core", "magicblock-magic-program-api 0.13.2", @@ -4182,9 +4226,10 @@ dependencies = [ name = "magicblock-task-scheduler" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "futures-util", + "hydra-api", "magicblock-config", "magicblock-core", "magicblock-ledger", @@ -5035,7 +5080,7 @@ dependencies = [ name = "program-flexi-counter" version = "0.0.0" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "ephemeral-rollups-sdk", "magicblock-magic-program-api 0.13.2", @@ -6461,7 +6506,7 @@ name = "solana-account" version = "3.4.0" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "serde_bytes", @@ -6481,7 +6526,7 @@ checksum = "ff1acff600a621d4445e0610fa71a53bef5390a5e349cfbd30651917593fb57d" dependencies = [ "Inflector", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "bv", "serde", @@ -6536,7 +6581,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8f5152a288ef1912300fc6efa6c2d1f9bb55d9398eb6c72326360b8063987da" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-program-error 2.2.2", "solana-program-memory 2.3.1", @@ -6549,7 +6594,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9cf16495d9eb53e3d04e72366a33bb1c20c24e78c171d8b8f5978357b63ae95" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-address 2.5.0", "solana-program-error 3.0.1", @@ -6606,7 +6651,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673f67efe870b64a65cb39e6194be5b26527691ce5922909939961a6e6b395" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6623,7 +6668,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e8df0b083c10ce32490410f3795016b1b5d9b4d094658c0a5e496753645b7cd" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6681,7 +6726,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19a3787b8cf9c9fe3dd360800e8b70982b9e5a8af9e11c354b6665dd4a003adc" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-instruction 2.3.3", ] @@ -6692,7 +6737,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "278a1a5bad62cd9da89ac8d4b7ec444e83caa8ae96aa656dfc27684b28d49a5d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-instruction-error", ] @@ -6771,7 +6816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219bfba64973ac9e64aa181f03fd56ac319e2d50d8a23d16c54bbd7fa9807a47" dependencies = [ "agave-syscalls", - "bincode", + "bincode 1.3.3", "qualifier_attr", "solana-account 3.4.0", "solana-bincode 3.1.0", @@ -6913,7 +6958,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e401ae56aed512821cc7a0adaa412ff97fecd2dff4602be7b1330d2daec0c4" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -7169,7 +7214,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43f5c5382b449e8e4e3016fb05e418c53d57782d8b5c30aa372fc265654b956d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 2.2.1", @@ -7188,7 +7233,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75ca9b5cbb6f500f7fd73db5bd95640f71a83f04d6121a0e59a43b202dca2731" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -7250,7 +7295,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "749eccc960e85c9b33608450093d256006253e1cb436b8380e71777840a3f675" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "memmap2 0.5.10", "solana-account 3.4.0", @@ -7336,7 +7381,7 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab5682934bd1f65f8d2c16f21cb532526fcc1a09f796e2cacdb091eee5774ad" dependencies = [ - "bincode", + "bincode 1.3.3", "getrandom 0.2.17", "js-sys", "num-traits", @@ -7354,7 +7399,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97881335fc698deb46c6571945969aae6d93a14e2fff792a368b4fac872f116" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "serde", "serde_derive", @@ -7607,7 +7652,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1796aabce376ff74bf89b78d268fa5e683d7d7a96a0a4e4813ec34de49d5314b" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7630,7 +7675,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0448b1fd891c5f46491e5dc7d9986385ba3c852c340db2911dd29faa01d2b08d" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7820,7 +7865,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98eca145bd3545e2fbb07166e895370576e47a00a7d824e325390d33bf467210" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "bs58", "bytemuck", @@ -8042,7 +8087,7 @@ version = "4.0.0" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "cfg-if", "itertools 0.12.1", "log", @@ -8205,7 +8250,7 @@ checksum = "10dd50b329ce569340c1deab3667d21e26a41e65cc6460e8a5bb8b57aff8420d" dependencies = [ "async-trait", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "futures", "indicatif", @@ -8320,7 +8365,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f03df7969f5e723ad31b6c9eadccc209037ac4caa34d8dc259316b05c11e82b" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "serde", "solana-account 3.4.0", @@ -8703,7 +8748,7 @@ dependencies = [ name = "solana-storage-proto" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "prost", "protobuf-src", @@ -8882,7 +8927,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "450479004fee3396c88cc4aa2f9b2b8db9c77be42ee7c1c53e6fac9eaec5fd51" dependencies = [ - "bincode", + "bincode 1.3.3", "log", "serde", "solana-account 3.4.0", @@ -8924,7 +8969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c3595f95069f3d90f275bb9bd235a1973c4d059028b0a7f81baca2703815db" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -8961,7 +9006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6690d3dd88f15c21edff68eb391ef8800df7a1f5cec84ee3e8d1abf05affdf74" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -9020,7 +9065,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96697cff5075a028265324255efed226099f6d761ca67342b230d09f72cc48d2" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-address 2.5.0", @@ -9041,7 +9086,7 @@ name = "solana-transaction-context" version = "4.0.0" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "solana-account 3.4.0", @@ -9084,7 +9129,7 @@ dependencies = [ "Inflector", "agave-reserved-account-keys", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "bs58", "log", @@ -9125,7 +9170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6886b7bb8fbba5937b4a38fa67ed442d7971629244b8fbd95c7963b2126bc9" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "serde", "serde_json", @@ -9162,7 +9207,7 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b80d57478d6599d30acc31cc5ae7f93ec2361a06aefe8ea79bc81739a08af4c3" dependencies = [ - "bincode", + "bincode 1.3.3", "num-derive", "num-traits", "serde", @@ -9186,7 +9231,7 @@ version = "5.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d444ce30b6b4f9c281ba06061ea96638d063b53c2171b1e41ba02ebff79ed28f" dependencies = [ - "bincode", + "bincode 1.3.3", "cfg_eval", "num-derive", "num-traits", @@ -9213,7 +9258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4537fd6efe65f53ccd28d54d2ad43275b024834a4a8ca4dfa4babfa01e6d11ab" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "log", "num-derive", "num-traits", @@ -9276,7 +9321,7 @@ checksum = "9602bcb1f7af15caef92b91132ec2347e1c51a72ecdbefdaefa3eac4b8711475" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -9313,7 +9358,7 @@ checksum = "09670ff59f60e6ddc2209c2e4353992a9b9f01d4e244f3e9d67bd5146e33d388" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -9817,7 +9862,7 @@ dependencies = [ name = "test-chainlink" version = "0.0.0" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "futures", "integration-test-tools", @@ -10022,6 +10067,7 @@ dependencies = [ "anyhow", "chrono", "cleanass", + "hydra-api", "integration-test-tools", "magicblock-config", "magicblock-program", @@ -10683,6 +10729,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "2.12.1" @@ -10800,6 +10852,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "void" version = "1.0.2" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index b27717add..d6951509c 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -41,6 +41,7 @@ ephemeral-rollups-sdk = { version = "0.14", default-features = false, features = "modular-sdk", ] } futures = "0.3.31" +hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "4f9e3c44e9e2e4feab66e6e7903099e095c74db3" } integration-test-tools = { path = "test-tools" } isocountry = "0.3.2" lazy_static = "1.4.0" diff --git a/test-integration/test-task-scheduler/Cargo.toml b/test-integration/test-task-scheduler/Cargo.toml index 4ac807154..892730a92 100644 --- a/test-integration/test-task-scheduler/Cargo.toml +++ b/test-integration/test-task-scheduler/Cargo.toml @@ -7,13 +7,16 @@ edition = "2021" anyhow = { workspace = true } chrono = { workspace = true } cleanass = { workspace = true } +hydra-api = { workspace = true } integration-test-tools = { workspace = true } tracing = { workspace = true } magicblock-task-scheduler = { path = "../../magicblock-task-scheduler" } magicblock-program = { path = "../../programs/magicblock" } magicblock-config = { path = "../../magicblock-config" } program-flexi-counter = { path = "../programs/flexi-counter" } -program-schedulecommit = { path = "../programs/schedulecommit", features = ["no-entrypoint"] } +program-schedulecommit = { path = "../programs/schedulecommit", features = [ + "no-entrypoint", +] } solana-sdk = { workspace = true } solana-rpc-client = { workspace = true } tempfile = { workspace = true } diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index 9527346f8..2e1d1f1b0 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -6,6 +6,7 @@ use std::{ }; use cleanass::assert; +use hydra_api::ephemeral::ID as HYDRA_EPHEMERAL_PROGRAM_ID; use integration_test_tools::{ expect, loaded_accounts::LoadedAccounts, @@ -25,9 +26,7 @@ use magicblock_config::{ ValidatorParams, }; use magicblock_program::Pubkey; -use magicblock_task_scheduler::{ - db::DbTask, SchedulerDatabase, HYDRA_EPHEMERAL_PROGRAM_ID, -}; +use magicblock_task_scheduler::{db::DbTask, SchedulerDatabase}; use program_flexi_counter::{ instruction::{ create_delegate_ix_with_commit_frequency_ms, create_init_ix, diff --git a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs index a9347c0ad..b4ac60248 100644 --- a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs +++ b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs @@ -1,7 +1,8 @@ use std::time::Duration; +use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, diff --git a/test-integration/test-task-scheduler/tests/test_independent_authority.rs b/test-integration/test-task-scheduler/tests/test_independent_authority.rs index b5179f41f..82bf1deac 100644 --- a/test-integration/test-task-scheduler/tests/test_independent_authority.rs +++ b/test-integration/test-task-scheduler/tests/test_independent_authority.rs @@ -1,7 +1,8 @@ use std::time::Duration; +use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::instruction::{ create_cancel_task_ix, create_schedule_task_ix, }; diff --git a/test-integration/test-task-scheduler/tests/test_migration.rs b/test-integration/test-task-scheduler/tests/test_migration.rs index fd75f8730..a2582cfe7 100644 --- a/test-integration/test-task-scheduler/tests/test_migration.rs +++ b/test-integration/test-task-scheduler/tests/test_migration.rs @@ -1,9 +1,8 @@ use std::time::Duration; +use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::{ - crank_pubkey, db::DbTask, SchedulerDatabase, CRANKER_REWARD, -}; +use magicblock_task_scheduler::{crank_pubkey, db::DbTask, SchedulerDatabase}; use solana_sdk::{ instruction::{AccountMeta, Instruction}, pubkey::Pubkey, diff --git a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs index 8bd175381..2e882239b 100644 --- a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs @@ -1,7 +1,8 @@ use std::time::Duration; +use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task.rs b/test-integration/test-task-scheduler/tests/test_schedule_task.rs index 084622591..335af65cf 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task.rs @@ -1,7 +1,8 @@ use std::time::Duration; +use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::{crank_pubkey, CRANKER_REWARD}; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, From 7850f1696cc8622a9f2360ff2857ecd4b874d662 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 00:47:53 +0200 Subject: [PATCH 03/13] feat: add crank faucet --- Cargo.lock | 14 +- Cargo.toml | 28 ++-- config.example.toml | 11 ++ magicblock-api/Cargo.toml | 1 + magicblock-api/src/errors.rs | 12 +- magicblock-api/src/fund_account.rs | 72 ++++++++- magicblock-api/src/magic_validator.rs | 35 ++++- magicblock-config/src/config/mod.rs | 2 + magicblock-config/src/config/scheduler.rs | 28 ++++ magicblock-config/src/consts.rs | 7 + magicblock-config/src/lib.rs | 4 +- magicblock-config/src/types/crypto.rs | 6 + magicblock-task-scheduler/Cargo.toml | 3 + magicblock-task-scheduler/src/errors.rs | 3 + magicblock-task-scheduler/src/service.rs | 146 ++++++++++++++---- test-integration/Cargo.lock | 14 +- test-integration/Cargo.toml | 17 +- test-integration/programs/hydra/hydra.so | Bin 24536 -> 26624 bytes .../test-task-scheduler/src/lib.rs | 49 ++++-- .../tests/test_cancel_ongoing_task.rs | 5 +- .../tests/test_independent_authority.rs | 4 - .../tests/test_migration.rs | 2 - .../tests/test_reschedule_task.rs | 3 - .../tests/test_schedule_task.rs | 3 - .../tests/test_undrained_validator.rs | 100 ++++++++++++ 25 files changed, 469 insertions(+), 100 deletions(-) create mode 100644 magicblock-config/src/config/scheduler.rs create mode 100644 test-integration/test-task-scheduler/tests/test_undrained_validator.rs diff --git a/Cargo.lock b/Cargo.lock index 903370e19..3ef4173f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2677,7 +2677,7 @@ dependencies = [ [[package]] name = "hydra-api" version = "0.1.1" -source = "git+https://github.com/magicblock-labs/hydra.git?rev=4f9e3c44e9e2e4feab66e6e7903099e095c74db3#4f9e3c44e9e2e4feab66e6e7903099e095c74db3" +source = "git+https://github.com/magicblock-labs/hydra.git?rev=449532ee0ee792e9a3809b16fd9d289c2024341a#449532ee0ee792e9a3809b16fd9d289c2024341a" dependencies = [ "ephemeral-rollups-pinocchio", "pinocchio 0.10.2", @@ -3670,6 +3670,7 @@ dependencies = [ "solana-sha256-hasher 3.1.0", "solana-signature 3.3.0", "solana-signer", + "solana-system-interface 3.1.0", "solana-system-program", "solana-sysvar 3.1.1", "solana-transaction", @@ -4164,13 +4165,16 @@ dependencies = [ "magicblock-program", "rusqlite", "serial_test", + "solana-account 3.4.0", "solana-instruction 3.3.0", + "solana-keypair", "solana-message 3.1.0", "solana-pubkey 4.1.0", "solana-rpc-client", "solana-rpc-client-api", "solana-sha256-hasher 3.1.0", "solana-signature 3.3.0", + "solana-signer", "solana-system-interface 3.1.0", "solana-transaction", "solana-transaction-error 3.1.0", @@ -6332,7 +6336,7 @@ dependencies = [ [[package]] name = "solana-account" version = "3.4.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "bincode 1.3.3", "qualifier_attr", @@ -7912,7 +7916,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "base64 0.22.1", "bincode 1.3.3", @@ -8608,7 +8612,7 @@ dependencies = [ [[package]] name = "solana-svm" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "ahash 0.8.12", "log", @@ -8924,7 +8928,7 @@ dependencies = [ [[package]] name = "solana-transaction-context" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "bincode 1.3.3", "qualifier_attr", diff --git a/Cargo.toml b/Cargo.toml index 06cce05a0..d14f0192f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,7 +84,7 @@ guinea = { path = "./programs/guinea" } helius-laserstream = { git = "https://github.com/magicblock-labs/laserstream-sdk.git", branch = "mbv-4.0+conn-fix" } http-body-util = "0.1.3" humantime = { version = "1.1", package = "humantime-serde" } -hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "4f9e3c44e9e2e4feab66e6e7903099e095c74db3" } +hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "449532ee0ee792e9a3809b16fd9d289c2024341a" } hyper = "1.6.0" hyper-util = "0.1.15" isocountry = "0.3.2" @@ -140,9 +140,8 @@ parking_lot = "0.12" paste = "1.0" prometheus = "0.13.4" # Keep in sync with `solana-storage-proto` codegen. - - -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82", features = [ +# TODO: Update to latest version is merged +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a", features = [ "dev-context-only-utils", ] } solana-transaction-error = { version = "3.0" } @@ -195,7 +194,8 @@ serde_json = "1.0" serde_with = "3.16" serial_test = "3.2" sha3 = "0.10.8" -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-account-decoder = { version = "4.0" } solana-account-decoder-client-types = { version = "4.0" } solana-account-info = { version = "3.1" } @@ -247,7 +247,8 @@ solana-program = "3.0" solana-program-error = { version = "3.0" } solana-program-option = { version = "3.0" } solana-program-pack = { version = "3.0" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-pubkey = { version = "4.1" } solana-pubsub-client = { version = "4.0" } solana-rent = { version = "3.0" } @@ -279,14 +280,19 @@ solana-transaction = { version = "3.0" } [workspace.dependencies.solana-svm] features = ["dev-context-only-utils"] git = "https://github.com/magicblock-labs/magicblock-svm.git" -rev = "b275f82" +# TODO: Update to latest version is merged +rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" [patch.crates-io] -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-storage-proto = { path = "storage-proto" } -solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } # Fork is used to enable `disable_manual_compaction` usage # Fork is based on commit d4e9e16 of rocksdb (parent commit of 0.23.0 release) # without patching update isn't possible due to conflict with solana deps diff --git a/config.example.toml b/config.example.toml index 1b05477a7..33a52553e 100644 --- a/config.example.toml +++ b/config.example.toml @@ -311,6 +311,17 @@ fqdn = "https://validator.example.com" # Env: MBV_CHAIN_OPERATION__CLAIM_FEES_FREQUENCY claim-fees-frequency = "24h" +# ============================================================================== +# Internal Task Scheduler +# ============================================================================== +[task-scheduler] +# Keypair (Base58) the task scheduler uses to pay for hydra cranks. It is +# delegated to this validator on startup and must be funded separately (the +# validator does not fund it). Use a dedicated account, not the validator +# identity. +# Env: MBV_TASK_SCHEDULER__FAUCET_KEYPAIR +faucet-keypair = "4SkhwWzaUdkXS5RQZHNFtmaTjvPRNKEZjLRiYPm3y7MGQTWBYYzrwAcvK1vMDwrTHrAy2aGqMJn2s7qbjfxqDLU9" + # ============================================================================== # Pre-loaded Programs # ============================================================================== diff --git a/magicblock-api/Cargo.toml b/magicblock-api/Cargo.toml index 1ddbc66de..d18b437a9 100644 --- a/magicblock-api/Cargo.toml +++ b/magicblock-api/Cargo.toml @@ -53,6 +53,7 @@ solana-sdk-ids = { workspace = true } solana-sha256-hasher = { workspace = true } solana-signature = { workspace = true } solana-signer = { workspace = true } +solana-system-interface = { workspace = true } solana-system-program = { workspace = true } solana-sysvar = { workspace = true } solana-transaction = { workspace = true } diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 86c2c8834..502dd6f4d 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -36,6 +36,9 @@ pub enum ApiError { #[error("Failed to delegate magic fee vault for validator '{0}': {1}")] FailedToDelegateMagicFeeVault(Pubkey, String), + #[error("Failed to fund and delegate task scheduler faucet '{0}': {1}")] + FailedToDelegateFaucet(Pubkey, String), + #[error("CommittorServiceError")] CommittorServiceError( Box, @@ -68,15 +71,6 @@ pub enum ApiError { #[error("Ledger Path is missing a parent directory: {0}")] LedgerPathIsMissingParent(String), - #[error("Ledger Path has an invalid faucet keypair file: {0} ({1})")] - LedgerInvalidFaucetKeypair(String, String), - - #[error("Ledger Path is missing a faucet keypair file: {0}")] - LedgerIsMissingFaucetKeypair(String), - - #[error("Ledger could not write faucet keypair file: {0} ({1})")] - LedgerCouldNotWriteFaucetKeypair(String, String), - #[error("Ledger Path has an invalid validator keypair file: {0} ({1})")] LedgerInvalidValidatorKeypair(String, String), diff --git a/magicblock-api/src/fund_account.rs b/magicblock-api/src/fund_account.rs index 2c5171f95..6399e569b 100644 --- a/magicblock-api/src/fund_account.rs +++ b/magicblock-api/src/fund_account.rs @@ -1,9 +1,16 @@ use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_magic_program_api as magic_program; -use magicblock_program::MagicContext; +use magicblock_program::{validator::validator_authority, MagicContext}; use solana_account::{AccountSharedData, WritableAccount}; +use solana_commitment_config::CommitmentConfig; +use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rent::Rent; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_signer::Signer; +use tracing::*; + +use crate::errors::{ApiError, ApiResult}; pub(crate) fn fund_account( accountsdb: &AccountsDb, @@ -66,3 +73,66 @@ pub(crate) fn fund_ephemeral_vault(accountsdb: &AccountsDb) { let _ = accountsdb .insert_account(&magic_program::EPHEMERAL_VAULT_PUBKEY, &vault); } + +/// Delegates the task scheduler faucet to this validator on the base chain if +/// it is not already delegated. Delegating gives the faucet real (base-chain +/// backed) lamports usable inside the ephemeral rollup, unlike the validator +/// identity. The faucet must already be funded — the validator does not fund +/// it (operators and integration tests airdrop to it). +pub(crate) async fn ensure_faucet_delegated_on_chain( + rpc_url: String, + faucet: &Keypair, +) -> ApiResult<()> { + let validator_keypair = validator_authority(); + let validator_pubkey = validator_keypair.pubkey(); + let faucet_pubkey = faucet.pubkey(); + + let rpc = + RpcClient::new_with_commitment(rpc_url, CommitmentConfig::confirmed()); + + // A faucet already owned by the delegation program is delegated. + let already_delegated = matches!( + rpc.get_account(&faucet_pubkey).await, + Ok(account) if account.owner == dlp_api::id() + ); + if already_delegated { + info!(%faucet_pubkey, "Crank faucet already delegated, skipping"); + return Ok(()); + } + + info!(%faucet_pubkey, "Delegating crank faucet"); + // Hand the on-curve faucet to the delegation program and delegate it to + // this validator. This makes it a writable, base-chain-backed account + // inside the ephemeral rollup that can sponsor crank creation (mirrors the + // on-curve delegation flow used in tests). The faucet must already be + // funded; the validator does not fund it. + let assign_ix = solana_system_interface::instruction::assign( + &faucet_pubkey, + &dlp_api::id(), + ); + let delegate_ix = dlp_api::instruction_builder::delegate( + validator_pubkey, + faucet_pubkey, + None, + dlp_api::args::DelegateArgs { + commit_frequency_ms: u32::MAX, + seeds: vec![], + validator: Some(validator_pubkey), + }, + ); + + let blockhash = rpc.get_latest_blockhash().await.map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + let tx = solana_transaction::Transaction::new_signed_with_payer( + &[assign_ix, delegate_ix], + Some(&validator_pubkey), + &[&validator_keypair, faucet], + blockhash, + ); + rpc.send_and_confirm_transaction(&tx).await.map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + info!(%faucet_pubkey, "Crank faucet delegated"); + Ok(()) +} diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index ac1230f9e..0f0128a40 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -85,7 +85,8 @@ use crate::{ domain_registry_manager::DomainRegistryManager, errors::{ApiError, ApiResult}, fund_account::{ - fund_ephemeral_vault, fund_magic_context, init_validator_identity, + ensure_faucet_delegated_on_chain, fund_ephemeral_vault, + fund_magic_context, init_validator_identity, }, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, ledger::{ @@ -116,6 +117,9 @@ pub struct MagicValidator { scheduled_commits_processor: Option>, rpc_handle: thread::JoinHandle<()>, identity: Pubkey, + /// Faucet keypair used by the task scheduler to pay for hydra cranks. + /// Delegated on startup so its lamports are real (base-chain backed). + faucet_keypair: Keypair, transaction_scheduler: TransactionSchedulerHandle, _metrics: (MetricsService, tokio::task::JoinHandle<()>), claim_fees_task: ClaimFeesTask, @@ -166,6 +170,12 @@ impl MagicValidator { )?; log_timing("startup", "sync_validator_keypair", step_start); + // The task scheduler pays for hydra cranks from a configured faucet + // account (delegated on startup) rather than the validator identity, + // which is not a delegated account. + let faucet_keypair = + config.task_scheduler.faucet_keypair.insecure_clone(); + let latest_block = ledger.latest_block().load(); let mut accountsdb = AccountsDb::new(&config.accountsdb, &config.storage, last_slot)?; @@ -425,6 +435,7 @@ impl MagicValidator { let task_scheduler = TaskSchedulerService::new( &task_scheduler_db_path, config.aperture.listen.http(), + faucet_keypair.insecure_clone(), dispatch .tasks_service .take() @@ -451,6 +462,7 @@ impl MagicValidator { claim_fees_task: ClaimFeesTask::new(), rpc_handle, identity: validator_pubkey, + faucet_keypair, transaction_scheduler: dispatch.transaction_scheduler, task_scheduler: Some(task_scheduler), transaction_execution, @@ -869,6 +881,7 @@ impl MagicValidator { fn spawn_primary_onchain_setup(&self) { let rpc_url = self.config.rpc_url().to_owned(); let identity = self.identity; + let faucet_keypair = self.faucet_keypair.insecure_clone(); let chain_operation_config = self.config.chain_operation.clone(); let block_time_ms = self.config.ledger.block_time_ms(); let base_fee = self.config.validator.basefee; @@ -911,6 +924,26 @@ impl MagicValidator { error!("Exiting process"); std::process::exit(1); } + + let step_start = Instant::now(); + let result = ensure_faucet_delegated_on_chain( + rpc_url.clone(), + &faucet_keypair, + ) + .await; + log_timing( + "startup_background", + "ensure_faucet_delegated_on_chain", + step_start, + ); + // Without the faucet being funded and delegated the task scheduler + // cannot pay for hydra cranks. + if let Err(err) = result { + error!(error = ?err, "Task scheduler faucet setup failed"); + error!("Exiting process"); + std::process::exit(1); + } + if let Some(ref config) = chain_operation_config { if !config.claim_fees_frequency.is_zero() { let step_start = Instant::now(); diff --git a/magicblock-config/src/config/mod.rs b/magicblock-config/src/config/mod.rs index 86e6fe3e2..5d9377d10 100644 --- a/magicblock-config/src/config/mod.rs +++ b/magicblock-config/src/config/mod.rs @@ -7,6 +7,7 @@ pub mod ledger; pub mod lifecycle; pub mod metrics; pub mod program; +pub mod scheduler; pub mod validator; // Re-export types for backward compatibility and easier access @@ -20,4 +21,5 @@ pub use grpc::GrpcConfig; pub use ledger::LedgerConfig; pub use lifecycle::LifecycleMode; pub use program::LoadableProgram; +pub use scheduler::TaskSchedulerConfig; pub use validator::ValidatorConfig; diff --git a/magicblock-config/src/config/scheduler.rs b/magicblock-config/src/config/scheduler.rs new file mode 100644 index 000000000..b6b30c66f --- /dev/null +++ b/magicblock-config/src/config/scheduler.rs @@ -0,0 +1,28 @@ +use serde::{Deserialize, Serialize}; +use solana_keypair::Keypair; + +use crate::{consts, types::SerdeKeypair}; + +/// Configuration for the internal task scheduler. +/// +/// Task execution is performed by the external hydra cranker service, so the +/// validator only needs the keypair used to pay for (sponsor, fund, and cancel) +/// hydra cranks. This faucet keypair is delegated on startup; it must be funded +/// separately (the validator does not fund it). +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(rename_all = "kebab-case", deny_unknown_fields, default)] +pub struct TaskSchedulerConfig { + /// Keypair the task scheduler uses to pay for hydra cranks, encoded in + /// Base58. + pub faucet_keypair: SerdeKeypair, +} + +impl Default for TaskSchedulerConfig { + fn default() -> Self { + Self { + faucet_keypair: SerdeKeypair::from(Keypair::from_base58_string( + consts::DEFAULT_TASK_SCHEDULER_FAUCET_KEYPAIR, + )), + } + } +} diff --git a/magicblock-config/src/consts.rs b/magicblock-config/src/consts.rs index 7e8d07836..2fd2ba59b 100644 --- a/magicblock-config/src/consts.rs +++ b/magicblock-config/src/consts.rs @@ -14,6 +14,13 @@ pub const DEFAULT_STORAGE_DIRECTORY: &str = "magicblock-test-storage/"; pub const DEFAULT_VALIDATOR_KEYPAIR: &str = "9Vo7TbA5YfC5a33JhAi9Fb41usA6JwecHNRw3f9MzzHAM8hFnXTzL5DcEHwsAFjuUZ8vNQcJ4XziRFpMc3gTgBQ"; +/// Keypair the task scheduler uses to pay for hydra cranks. It is delegated on +/// startup and must be funded (the validator does not fund it). +/// WARNING: This keypair is for development/testing only. +/// Production deployments MUST provide their own faucet keypair via config. +pub const DEFAULT_TASK_SCHEDULER_FAUCET_KEYPAIR: &str = + "4SkhwWzaUdkXS5RQZHNFtmaTjvPRNKEZjLRiYPm3y7MGQTWBYYzrwAcvK1vMDwrTHrAy2aGqMJn2s7qbjfxqDLU9"; + /// Default base fee in lamports for transactions pub const DEFAULT_BASE_FEE: u64 = 0; diff --git a/magicblock-config/src/lib.rs b/magicblock-config/src/lib.rs index 79a394d9c..acf5b31ba 100644 --- a/magicblock-config/src/lib.rs +++ b/magicblock-config/src/lib.rs @@ -22,7 +22,8 @@ pub mod types; use crate::{ config::{ AccountsDbConfig, ChainLinkConfig, ChainOperationConfig, - CommittorConfig, LedgerConfig, LoadableProgram, ValidatorConfig, + CommittorConfig, LedgerConfig, LoadableProgram, TaskSchedulerConfig, + ValidatorConfig, }, types::Remote, }; @@ -66,6 +67,7 @@ pub struct ValidatorParams { pub chainlink: ChainLinkConfig, pub chain_operation: Option, pub programs: Vec, + pub task_scheduler: TaskSchedulerConfig, } impl ValidatorParams { diff --git a/magicblock-config/src/types/crypto.rs b/magicblock-config/src/types/crypto.rs index e5d24721e..f509e179f 100644 --- a/magicblock-config/src/types/crypto.rs +++ b/magicblock-config/src/types/crypto.rs @@ -32,6 +32,12 @@ impl Clone for SerdeKeypair { } } +impl From for SerdeKeypair { + fn from(keypair: Keypair) -> Self { + Self(keypair) + } +} + impl FromStr for SerdeKeypair { type Err = String; fn from_str(s: &str) -> Result { diff --git a/magicblock-task-scheduler/Cargo.toml b/magicblock-task-scheduler/Cargo.toml index c188b4669..73ef17938 100644 --- a/magicblock-task-scheduler/Cargo.toml +++ b/magicblock-task-scheduler/Cargo.toml @@ -17,13 +17,16 @@ magicblock-config = { workspace = true } magicblock-ledger = { workspace = true } magicblock-program = { workspace = true } rusqlite = { workspace = true } +solana-account = { workspace = true } solana-instruction = { workspace = true } +solana-keypair = { workspace = true } solana-message = { workspace = true } solana-pubkey = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-sha256-hasher = { workspace = true } solana-signature = { workspace = true } +solana-signer = { workspace = true } solana-system-interface = { workspace = true, features = ["bincode"] } solana-transaction = { workspace = true } solana-transaction-error = { workspace = true } diff --git a/magicblock-task-scheduler/src/errors.rs b/magicblock-task-scheduler/src/errors.rs index c5604aeff..4fdcaf56f 100644 --- a/magicblock-task-scheduler/src/errors.rs +++ b/magicblock-task-scheduler/src/errors.rs @@ -24,4 +24,7 @@ pub enum TaskSchedulerError { #[error("Batch size mismatch: expected {0}, got {1}")] SizeMismatch(usize, usize), + + #[error("Faucet not ready")] + FaucetNotReady, } diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index aefedc255..c2fe9f71f 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -1,7 +1,7 @@ use std::{path::Path, sync::Arc, time::Duration as StdDuration}; use hydra_api::{ - consts::CRANKER_REWARD, + consts::{CRANKER_REWARD, CRANK_HEADER_SIZE, SERIALIZED_META_SIZE}, ephemeral::ID as EPHEMERAL_PROGRAM_ID, instruction::{ephemeral, CreateArgs, SchedMeta, ScheduledIx}, }; @@ -10,12 +10,15 @@ use magicblock_ledger::LatestBlock; use magicblock_program::{ args::{CancelTaskRequest, ScheduleTaskRequest, TaskRequest}, validator::{validator_authority, validator_authority_id}, + EPHEMERAL_RENT_PER_BYTE, }; +use solana_account::AccountSharedData; use solana_instruction::Instruction; -use solana_message::Message; +use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_signature::Signature; +use solana_signer::Signer; use solana_transaction::Transaction; use tokio::{ select, @@ -34,6 +37,10 @@ use crate::{ /// before giving up. const BLOCK_READY_TIMEOUT: Duration = Duration::from_secs(60); +/// How long the scheduler waits for the faucet to be delegated and funded in +/// the ephemeral rollup before it starts paying for cranks. +const FAUCET_READY_TIMEOUT: Duration = Duration::from_secs(60); + /// The task scheduler migrates any tasks persisted by the legacy /// (validator-funded) scheduler onto the hydra crank program at startup, then /// serves runtime schedule/cancel requests by sending hydra transactions. The @@ -45,6 +52,10 @@ pub struct TaskSchedulerService { db: SchedulerDatabase, /// RPC client used to send hydra transactions. rpc_client: Arc, + /// Delegated faucet keypair used to pay for (sponsor, fund, sign, and + /// cancel) hydra cranks. Used instead of the validator identity, which is + /// not a delegated account. + faucet: Keypair, /// Used to receive scheduled tasks from the transaction executor. scheduled_tasks: ScheduledTasksRx, /// Provides latest blockhash and slot for building transactions. @@ -65,6 +76,7 @@ impl TaskSchedulerService { pub fn new( path: &Path, rpc_url: String, + faucet: Keypair, scheduled_tasks: ScheduledTasksRx, block: LatestBlock, slot_interval: Duration, @@ -74,6 +86,7 @@ impl TaskSchedulerService { Ok(Self { db, rpc_client: Arc::new(RpcClient::new(rpc_url)), + faucet, scheduled_tasks, block, token, @@ -94,6 +107,10 @@ impl TaskSchedulerService { error!("Task migration failed: {}", e); } + // Ensure the faucet is funded before serving runtime requests so the + // first scheduled task is not dropped for lack of a payer. + self.wait_for_faucet_ready().await?; + loop { select! { Some(request) = self.scheduled_tasks.recv() => { @@ -139,12 +156,16 @@ impl TaskSchedulerService { return Ok(()); } - // Sending a crank create needs a usable blockhash, which is only - // available once the validator has produced a block. + // Sending a crank create needs a usable blockhash and a funded faucet + // (delegated and cloned into the ephemeral rollup) to pay with. self.wait_for_block_ready().await; + info!("Migration: block ready"); + self.wait_for_faucet_ready().await?; + info!("Migration: faucet ready"); for task in valid { - if let Err(e) = self + info!("Migration: creating crank for task {}", task.id); + match self .schedule_crank( &task.authority, task.id, @@ -154,7 +175,15 @@ impl TaskSchedulerService { ) .await { - warn!("Failed to migrate task {} onto hydra: {}", task.id, e); + Ok(()) => { + info!("Migration: created crank for task {}", task.id) + } + Err(e) => { + warn!( + "Failed to migrate task {} onto hydra: {}", + task.id, e + ) + } } self.db.remove_task(task.id).await?; } @@ -163,6 +192,28 @@ impl TaskSchedulerService { Ok(()) } + /// Waits until the faucet has a non-zero balance in the ephemeral rollup + /// (i.e. it has been delegated on the base chain and cloned in), or the + /// scheduler is cancelled, or [`FAUCET_READY_TIMEOUT`] elapses. + async fn wait_for_faucet_ready(&self) -> TaskSchedulerResult<()> { + let faucet = self.faucet.pubkey(); + let start = Instant::now(); + while start.elapsed() < FAUCET_READY_TIMEOUT { + if self.token.is_cancelled() { + return Ok(()); + } + if matches!( + self.rpc_client.get_account(&faucet).await, + Ok(account) if account.lamports > 0 + ) { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + warn!(%faucet, "Timed out waiting for faucet to be delegated before paying cranks"); + Err(TaskSchedulerError::FaucetNotReady) + } + /// Waits until the validator has a usable blockhash, or the scheduler is /// cancelled, or [`BLOCK_READY_TIMEOUT`] elapses. async fn wait_for_block_ready(&self) { @@ -246,7 +297,7 @@ impl TaskSchedulerService { ) -> TaskSchedulerResult<()> { let crank = crank_pubkey(authority, task_id); - self.send_create_and_fund( + self.send_create( authority, task_id, interval_millis, @@ -267,8 +318,8 @@ impl TaskSchedulerService { } /// Builds and sends the transaction that creates and funds a hydra crank. - /// It cancels the crank if it already exists. - async fn send_create_and_fund( + /// It cancels the crank first if it already exists. + async fn send_create( &self, authority: &Pubkey, task_id: i64, @@ -287,7 +338,9 @@ impl TaskSchedulerService { interval_slots(interval_millis, self.slot_interval); let iterations = iterations.max(0) as u64; + let faucet_pubkey = self.faucet.pubkey(); let create_ix = build_create_ix( + &faucet_pubkey, authority, task_id, crank, @@ -296,49 +349,57 @@ impl TaskSchedulerService { iterations, instructions, ); - - // Fund the crank so the external cranker can execute every iteration. - let funding = iterations.saturating_mul(CRANKER_REWARD); - let transfer_ix = solana_system_interface::instruction::transfer( - &validator_authority_id(), + let reward_pool = iterations.saturating_mul(CRANKER_REWARD); + let funding = + reward_pool.saturating_add(crank_rent_floor(instructions)); + let fund_ix = solana_system_interface::instruction::transfer( + &faucet_pubkey, &crank, funding, ); + let validator = validator_authority(); let ixs = if crank_exists { - let cancel_ix = ephemeral::cancel(validator_authority_id(), crank); - vec![cancel_ix, create_ix, transfer_ix] + let cancel_ix = + ephemeral::cancel(faucet_pubkey, crank, faucet_pubkey); + vec![cancel_ix, create_ix, fund_ix] } else { - vec![create_ix, transfer_ix] + vec![create_ix, fund_ix] }; - - let tx = Transaction::new( - &[validator_authority()], - Message::new(&ixs, Some(&validator_authority_id())), + let tx = Transaction::new_signed_with_payer( + &ixs, + Some(&validator_authority_id()), + &[&validator, &self.faucet], blockhash, ); + // Confirm before returning so successive cranks (all sponsored by the + // same faucet) don't race on the faucet's delegated account state. self.rpc_client - .send_transaction(&tx) + .send_and_confirm_transaction(&tx) .await .map_err(Box::new) .map_err(TaskSchedulerError::from) } - /// Sends a crank cancellation (fire-and-forget). + /// Sends and confirms a crank cancellation. The crank's remaining lamports + /// are returned to the faucet (the cancel recipient). async fn send_cancel( &self, crank: Pubkey, ) -> TaskSchedulerResult { + let faucet_pubkey = self.faucet.pubkey(); let blockhash = self.block.load().blockhash; - let cancel_ix = ephemeral::cancel(validator_authority_id(), crank); - let tx = Transaction::new( - &[validator_authority()], - Message::new(&[cancel_ix], Some(&validator_authority_id())), + let cancel_ix = ephemeral::cancel(faucet_pubkey, crank, faucet_pubkey); + let validator = validator_authority(); + let tx = Transaction::new_signed_with_payer( + &[cancel_ix], + Some(&validator_authority_id()), + &[&validator, &self.faucet], blockhash, ); self.rpc_client - .send_transaction(&tx) + .send_and_confirm_transaction(&tx) .await .map_err(Box::new) .map_err(TaskSchedulerError::from) @@ -362,7 +423,29 @@ pub fn crank_pubkey(authority: &Pubkey, task_id: i64) -> Pubkey { /// Builds the hydra `Create` instruction embedding the task's instructions as /// the scheduled crank payload. Account signer flags are intentionally dropped: /// hydra rejects scheduled instructions that declare signers. +/// Rent-exempt minimum for the crank account hydra will allocate for these +/// scheduled instructions. Mirrors hydra's on-chain size accounting +/// (`CRANK_HEADER_SIZE + region_len`, where each scheduled ix serializes to +/// `2 + metas + program_id + 2 + data`). +fn crank_rent_floor(instructions: &[Instruction]) -> u64 { + let region_len: usize = instructions + .iter() + .map(|ix| { + 2 + ix.accounts.len() * SERIALIZED_META_SIZE + + 32 + + 2 + + ix.data.len() + }) + .sum::() + + CRANK_HEADER_SIZE; + let total_size = + (region_len + AccountSharedData::ACCOUNT_STATIC_SIZE as usize) as u64; + total_size * EPHEMERAL_RENT_PER_BYTE +} + +#[allow(clippy::too_many_arguments)] fn build_create_ix( + faucet: &Pubkey, authority: &Pubkey, task_id: i64, crank: Pubkey, @@ -402,8 +485,8 @@ fn build_create_ix( let args = CreateArgs { seed, - // The validator authority is the cancel authority for the crank. - authority: validator_authority_id().to_bytes(), + // The faucet is the sponsor and the cancel authority for the crank. + authority: faucet.to_bytes(), start_slot, interval_slots, remaining: iterations, @@ -412,7 +495,7 @@ fn build_create_ix( scheduled: scheduled.as_slice(), }; - ephemeral::create(validator_authority_id(), crank, &args) + ephemeral::create(*faucet, crank, &args) } fn is_valid_task_interval(interval: i64) -> bool { @@ -446,6 +529,7 @@ mod tests { rpc_client: Arc::new(RpcClient::new( "http://localhost:8899".to_string(), )), + faucet: Keypair::new(), block: LatestBlock::default(), token: CancellationToken::new(), slot_interval: Duration::from_millis(1000), diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 62578541e..83c49b35d 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -2781,7 +2781,7 @@ dependencies = [ [[package]] name = "hydra-api" version = "0.1.1" -source = "git+https://github.com/magicblock-labs/hydra.git?rev=4f9e3c44e9e2e4feab66e6e7903099e095c74db3#4f9e3c44e9e2e4feab66e6e7903099e095c74db3" +source = "git+https://github.com/magicblock-labs/hydra.git?rev=449532ee0ee792e9a3809b16fd9d289c2024341a#449532ee0ee792e9a3809b16fd9d289c2024341a" dependencies = [ "ephemeral-rollups-pinocchio", "pinocchio 0.10.2", @@ -3728,6 +3728,7 @@ dependencies = [ "solana-sha256-hasher 3.1.0", "solana-signature", "solana-signer", + "solana-system-interface 3.1.0", "solana-system-program", "solana-sysvar 3.1.1", "solana-transaction", @@ -4235,13 +4236,16 @@ dependencies = [ "magicblock-ledger", "magicblock-program", "rusqlite", + "solana-account 3.4.0", "solana-instruction 3.3.0", + "solana-keypair", "solana-message 3.1.0", "solana-pubkey 4.1.0", "solana-rpc-client", "solana-rpc-client-api", "solana-sha256-hasher 3.1.0", "solana-signature", + "solana-signer", "solana-system-interface 3.1.0", "solana-transaction", "solana-transaction-error 3.1.0", @@ -6504,7 +6508,7 @@ dependencies = [ [[package]] name = "solana-account" version = "3.4.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "bincode 1.3.3", "qualifier_attr", @@ -8084,7 +8088,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "base64 0.22.1", "bincode 1.3.3", @@ -8768,7 +8772,7 @@ dependencies = [ [[package]] name = "solana-svm" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "ahash 0.8.12", "log", @@ -9084,7 +9088,7 @@ dependencies = [ [[package]] name = "solana-transaction-context" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "bincode 1.3.3", "qualifier_attr", diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index d6951509c..60f061186 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -41,7 +41,7 @@ ephemeral-rollups-sdk = { version = "0.14", default-features = false, features = "modular-sdk", ] } futures = "0.3.31" -hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "4f9e3c44e9e2e4feab66e6e7903099e095c74db3" } +hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "449532ee0ee792e9a3809b16fd9d289c2024341a" } integration-test-tools = { path = "test-tools" } isocountry = "0.3.2" lazy_static = "1.4.0" @@ -81,7 +81,8 @@ schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" shlex = "1.3.0" -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-address-lookup-table-interface = "3.0" solana-commitment-config = "3.1.1" solana-compute-budget-interface = "3.0" @@ -122,9 +123,13 @@ url = "2.5.0" solana-storage-proto = { path = "../storage-proto" } # same reason as above rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } # Fix libsodium for ARM builds - upstream crates.io version breaks Linux ARM binaries libsodium-rs = { git = "https://github.com/jedisct1/libsodium-rs.git", rev = "0397a6c5785233f9f2ac91f3eedc3cceb74e0060" } diff --git a/test-integration/programs/hydra/hydra.so b/test-integration/programs/hydra/hydra.so index 431201c42016701fec2b461dd27ffd68b741a08b..7ca74bf5757d49d42ee33a4b05e8b8dcdb2b1d68 100755 GIT binary patch literal 26624 zcmdUY4RqAkb>|yN1G1V{_+x^s4Q6DJkm+KNzJ*Tl3S$$6CN;*4MO-UrB#nhI9~zAU zJpE4^KJ0{$CdH1OVA9=40@=8YPDtAXx=nU)x~GMw#l2L<{QBHy1Bd45Xjg`JKQ z%Nc4Os!X~UxPX7UilNEzldAtGnCw1A0REd3vygp0&KW_6$ZsiIi3+K`-l9Zn^bkoqx>NMH6N*kvL015}Q=A91^)?m^}^yCfr}j*2+LQQyyqb1y-Z z%iBxkOZ`h}{AhjZr+%T>!PEFevVJmsbA3YZrpj3V+mt^W=ZxI5lJ6%)F2(`tuj41C zerI-zIz$DkN6Y0#7#onZ3g8w|b{*U{%ora81o^0M7p-cCh+kX6etr(f*Wi;-8mxcPE8%t|H7@uG z8JB+YEXxUM#J`~@Qn%VY*UP9zFS57J*SU8J-TcPphe7J!IA!fs7%M?P-|GC+pi=oe z893uN`{w(R$|0Sh589d3c_p?`=8b&()c(%(gIoL^5idV)FxBPtMqEmeK)^17xy8bO z@qtj)O<&h2&gHT~UwXU1HI2W-KcEj%vPATVy3aFSiNQu`ZhW0)`df~#wOrgSZ4o#X z{oxEO8Fa|_$ZZlj{0{n?eio;D77s9kKPmi`YlS~nax4GHZ4|tl0JmG`fl3&WlF7@@ zBfU&^3n?G{GCPl;cIPthWZ(t~<#|M*`F(n!&LiqyHjnHjb7GvE9jh?EQ-U{f7j&^+Q76OYMSvw>~X+<$%B{ z9tW%rwL9YOQ#$z3^=7YJr|>5iZr~reM+G-OitwKZ>-8fN48hF|%3xLoMDE`f;a)Fk&Z2_tu(K|4r{d-!R| z--^kblC|I0!73zg8v~vfxIBGvd4gR2Lz2JslLBYU(}1J&*2`;rev9?PPvfY6J+yGH z=eGZ@aq|ldbK+iZ=^n|q{)(~}ksWkhmY+wom+teePO@Rl>WPQ6%G>T+bL zg=8ntV+&;cWb(m>G@Jj|te4FzQTIc}{VDNtjQ5`sKjJL>Jnu(@oFhD^{ZLs)2&FT` zhty5^;2%_obx+|q^S$7r^5l0&7?-57dzn7m&jh)jHEwK`ZgrJ$%Af@O;VBDAKF1TS zH(C0(tUu~29Cta*w{_NDDvR=AxvVE!4)&w*r}ctAL7&P_`l2MOYtmn)Q-^X_iuP49 ze~(p&9mgszaX*e#$a;OO;xfnA!hO#LYB=VLbw8tdLG9^Pej(o_;>(>7J<$F)2*0|B9&j)cS7i`dY z_>kaJCAIDf>EutyJOKTu_>7-YJ}qbK^)@QDXa(63ok3{@LB_q{OFtqor{f&!1gz)U z2o6w|jdPR(Zv2Bf9=Zu)eK|<{XdmM7>)j#~-<8MYH|BvN`jF6btpY0_6PS4(Qk=`K zhZZysY5dBrM=gXuZTSK-8t1a>kxTwce*ROkNA1Gz&V2Bb$J`g(Vmh~=c94EZbK2K0 z^c7@BZXcYG(ja|M1Db-Gt0=!nf5MUc%Q}B+{HWY4^b8g!l(J>g4(Nq`%7yuwgg@IX zFj-Q@KWv^hy{(>DiO7vDkaeQj9qqb{N>-7d73tY0a#253pI;AE>U@^DTlHKfe11~= zXsND8f*Nt0O6}7iBYb&(z0UbZ)#Jj0?;AP)kaG;FALISao_~x}yMk&Nhi~QlNl8gnLb9-2Bqy4^&GE9DykT6uPC3vDlwz$!DQra zW(!oWX{T(#`_<7N&PGJy_Yv5aX{4^vBzUK9*pZJ>Dm< zIwi29m*H5&7KWJ=SLR*FNy zQFt!Mc09hH>A{A546!a@J8mBqdc`(|*^c7^t6vnj@)*OhilYquDe<4o0cl^@j>*XD z9H=vgwA}=X+FdGvI`g!)TkYb<#Xnpy_tW4JXQ714jXP*8^mi?K?k#6C0G%_WvCxj3y`x@s*oj|{w)pE%)SvdIR;HL9} zGMS(Ka%vFc(g44_it$+4VuojEytDrhJK7tI-pX`e%fNY^_kBX-_H5L1pIX6vLge>s z#44TouU2rMka~JHiXNzk^Y0eAZ;CU%WSKae=@E-6F2!8`T4r;7kup9(%Eu3thxIA+ zPDcNy+U3U#?fHn=+t#sDrN8RZ#BY9MdLb?k5FPs;SRi+lz9^wp%jjcqdHSg8CxGJQ zHLkClaFCJ_nHW|jIte2=&ojuMU#w}~D0FQI;w^f{)|YEqq`W_&{@x(vE1wmZxmBR; z-*8?_s=L}qvx^h7^Q}r<7uk74ML>pWeld!#c&1K;$WS{_W4Fdc)EbjyLd zRjXyfM_j@9jFqF;Nss*am3}ZB^Bzj(`)#wz7r^KDEI(4ua&p@KVC)L_=k)3vLq8$& zjrq|R%1WtzS-)~Y`hxwVJj;<^t%Wsbao7X>OY=^Vy#9>v^*>9BGQU8qi!DK})gRPI zKJLCJ|bIy*-PUn0ae zN_?ni|Kr8=FF`%JPP6A5;QOZbJ-CBviLYZ7=Vs!Bk5!zWA(wO`6<_Cht}u>c-JLs0 znUv7K*eBcm72}9XaNLFoEsiJoX$|Zs_7AFM9?ILb`k3~M%pXCu%)^<9uM7Re1%^S# zVZr;jdRl);|MfI$JY1%6@1&IPS*P)EnZ~=5Qcur1?MDgc&fgtpf&QfEwd4VG82K&b z;}^yU$7MRVLjwMb^cf^HPVix=Hh&HQ`)(=dhoD;YztQst_~Et!Ik0B~{GpR7rIc$H z=yhF}QM<%SXnar>{1odxTjxccthao3{3=TN_f`qqv`FBidM?qS<7A7RPmIX4=6wG~ z6mu=5C2k)<$cH7P?tYM6a9sOOpSxO|F~7xm+y&ypImBW3%>)LLJI|S*ZZ(|ihe<-0iRZPhr zl6Vv}ihub-ai#~244gkG<$}7SOz;P#Tu^sdaVZzn?N^+E3+nbM-pMeiJEnM-;^Pcq zS2D+pKeUnYnEIW?iJ_)~k%D_#4e^&I0t-D!qn3(hi39qeUj z&&{%}UuV3(?rRL82mC(j{+P!da3uH>;v)P48T}11p2G3ao)g*g>zs5){#^2m`qfnQ zRbCHZTn*4CSZQ(K9M>Bhm+_NBrIfG^UPci5z%Qn6!MEUmc8(U%xgPLtQ?fjW)kpPQgvP&GIr{Zdr9X}`_d z;XkuLJ1?sIkjT%MTTpDz*>Zo%jH@1do}oQwjFlYYt2Dv(v(jI>kHC0`BZ;M-) zKcGMQ9SPAI_>wZY%#wQ!R9av32}F6wn|==8RE}_@Zlxy|{0`%5Hhq?1ZVv-z=iwF~ ztv%NMSjkn^JL*QHoc2TIK`noqK(uE${lq$!^|(#-ARMV%>B+LsGj8`nRz0wnaa+fs z-6U$(fA3H^AW|~@k|n>(xSg}DTAoo{#zUp*<;TU(p3?I6yrZ&}*=OrnfBky=iSyk4 zWXX7OeRbOyU-jq?hQZ^fna`dtRX)dLn%7mIwSwEYS-MVe#Pi#!k8)|ju}+B7&-}VD zt$GP4SQkFa1?+yys#ve!dhaIJz$nijG!$mr*`V^~*z*V%n7&gMPyQ9r+x7?c+{&Kc zE$tWi`Sz;4^Xs<9Bp>bj5w$5FN2smbzoZ5iSuIU@mB?UnE#rcbIXN)y6^h>jU?~A^n^&bn_dra#; z9is15`sEPa_E+_x_GOfRONhQp>1RXqr<8s;MBheq9~XOOO32`pVkHuAyv7-?8u09W3GOTu|eEdo- zkoAwR?GnBR893i5xIM46=WN!W){bdhNTM`%ee^b_kClj=O6d+4lnIXZR})8mJ+E=b z&Nob6*iOj>f5G;Q($jygm*gy_A5r&zr61KFGCJ?%{dyCtObZL^=ZVj`;O#K5FRCh`jKA2K?;j zqK3;HV)1kjRF^UUihM{-CVw;`-!wu&rzqn*fXZ_3VK7* zRM}@|4*Xrb=(!4?zZKXA&spr=C+zSB<`uoqYI;E){4QnpT1zHoFz(07r5|kFFh%=n z_zBt#=m#{vlb9*l{U5U{;sX0gr}~LB-*qcJ8T~zOXXSD!AFLO7OYadp|D2;)>v=%v zw$I4Rf8hG^p(p$}8U3)-BimFLtP^?2M?0hL5a-)Dc`W)J7MNKt^`#fQgWC~XAp4%= zf^YH3db0G({JhH7mouY1|BIF0&*eSY2+8ZfuOcJg^0Cjo548L1R&DQDhf_o3Cm14s z>Gxl)1@b?B@O$s=S%+08@+CkZ|NK`su~zM$Z~j2fIx(pEyC3aj{wgB>@Z{Bx0-659 zAAEd?owJi0L9tlrzh?d5w-WEBz)JI5Qs!sIFz@^ax)DyWbe#Ro+qr48v={Y6ANm9C zy~}@6mFn57?P(@QqlEmc_q=-;zaO}J^RM-6*7h_@JVyRkKlseU$p6ZR55Ls2nFy5f z^~O2fcmJrP6Le^t zEPI|6&q?#0?Z2@9hsEq(ncbf&JP$}?d=LnOE(>N-JekT;C z&Nxdr+D-pETOUBb->3aU>Xo1V44kQPxsPD??3Zt-U&?)yS-)Ju_CcJ%N;K;J72_#AKTEB{+efrc?6|<=#;=Pp9(3sX zM&o?YA?qHmaXaXcb&X#q{t8~Zs*GnA7Yf|ffu-Z@VMNwpyyKDe!q4tGthHD#%aAKaQkV=UoQ^} z3**7tt~37W`6Bb%a|+B;*fAmwe1d-aDar3YzD@Ml-Yalyi@+lr8Ttv;qq<9R(WAdw z;+5BRl3y$HlAo++zMdN0H>^c;rTQi%uJzPtJYOq;*iW`AKcW@I8U7~zwQ>ICCp$TR zNey>wY0wzLKUf7 zG|vv}JliSlLVnRaJB%nv`giI)OZi3f?6A(WVg~0Qj59ya4TbZp-ESj4=n1<}x4GoP z&#*m$BhnAKZZ6LEVj#nh3hp0L{Rx=Sm2*9f+K+u|2g-kS`Jet4%GW&h{;8fu?MF+jSne=Bh%Pak3iKOyl1>l3uwpOXFx>t}Hd=b+*B)3>SL z?YZJ!GJl-p*!`M8k_&vV!enHv%wq?LND1}sBZxRhKi%{Jd)9gc9s? zIW#|G{1xhTjc}|R(SNqTgFo(|bKBJTWw9GQ?Zvv*Pe}afCb_7e?h{`nK=)VV{z7nE z?k^OMcebCtXE9bG_Znd35ohMgy* zUir(AJ`yML{N&fLmw*2;7O-YBhn)PpXZN4_!9>Z<`+W8kNe z+iTBLryG_s-Dl1TJR$i@n;E5h*Wx$RcfgCt&*;PQQex+pND1#hX8TFr6T>{e1^iS$ z{mid-E495t&GMWM`#G3ysGq(lt!b7xKdAKFI-z4;g5_7;F3**QvgbL&o{QT3K-7nM zWk~O%+r7BT7L`Mlk=xFEIfhn9djMzk{}UO9#!v2mIA!^(YG0LcpE-+SG=9d#e=_5W zOFXDt&uIRALd>TZh-A;RFixK!n#r|!V~}WZ;@hjx`ZHM~=RY=%gT>OH{(wO9Ba4@I zF9P;JviWTKc5zhTOsB8K^ywxXImwdWl=f-dwdbcnmDV>YORF}iPhknhQj$TEifRQ#%zo-ght7SK@uc0m7|y`t_Y%L&?KT>2Bz z5AYLInb$|pmF*r7=tvd^lM(T^V1@K2J+I{cjL~Th;)>yj$H0Tdr?`BfUJ_qzKK=mf zg?YWahgtax{eAn7<@_}sqAw!4jYrf|P9?nheXLaWOF?z{H3;%+x3Ij*I~X|oI}-4( zZS~~)WTf2$t^M=&!%x@}-NcW1BvyK>^nd9}K1ax`75T}?ixD&^*sw_Jm;Sk4{k>ZM z;G3%d2SxSU`{!l{|B%#=_LS4l!g&V@7tXKcgu}i&>Af5qFR2N~`X(u^qJJzdrzxHy zB43RB5yeQr|Hksukq7xNXCxy}MlfWm=wnItvGR8yE=IoOFcV#je3Sk*1pa=;TK)?u z$w*%W=YxgiFs`ERD*HpB{?OOIZ5h3_Wf86EvDqx6+eQ84>0Ov-Pdh`pB@9ZeD$_6!rUTefMj9V!@kN->2uP zZ?)DJzsc?USW$iGPZ|zxww=vdFIRgL?fM(V^-?^bW!Fq?$r)bH+C5L)m&eSP8<+aw zFYrfup6S&;f=4jwB%a`x=^u-Jnd9}Fmm~gT<9JriS8h;__{Cf~uic=WIGV+A67352 zPb~UpHz<$A&C1K${V?18V#%*?JdKs~vfYyrHF7ydR47#idZY7gD=pku1+A$;gL! z9bBj{{5|UK5`7~XhoyKj@(jOkU&x1fE$XD6SY)60ZR9!j+d@8siYPrrAwYK8!{;N( z$Wea(sE`lyWziK;seU1G#{b;?@L8=-{r;x)$M<=?i2gVi>W_;*nwgnx@5Ilk zy(?k<>x{eKE^hBlv+sOy-MNSJckZbMBOuU=s0JHx`T7*h;vc*L8cFqv3UJ6NkmH6H}U#Mj+dOT z5jvfVvE6OG5p_GIJn?h+DdEGs%;jqZ_nWkQjl?x8Z*d@2vRl>{Z(5FYeANAb<0sAs z{%Q5oHkMR;^KzvAx$TJFpd9hQsJnQR+A(m0_0QLip&OJV?U-wiT{kEv&W=KH<1N{P z;s%$)ymK`arxsA00=+jxFID=!5S^}Y&-mNKUsKCvVU}7Z_OkKd?^b^%|HC;5)>(k| zeqg%9l@#Oa35b$LaH$LffyWMn}^-hezY1yW^>Wf%x8` z(QWbFd(tCM?%2I|XxBh`UwrrbN8`JnjQ8z+|E_^-M|^Z!I=&;lD?XHsdq&!$zSJDj{8NpTZDT=xI={dLbxl0`#`u06z?O^kS7(9 z#6%*|kZ4RaC7Kg0iPl70qCK&?A<@v#(Adz_(A?0{(Av<}(B81RG11u2*x1)Y#P2)ZEn4)Y{b6)ZVnZInmtE+}PaI+}zyK+}hmM+}^ypCDGE* z(%90}(%jP0(%RD2(%!PVHPPD8+SuCE+T7aG+S=OI+TOalEz#D{*4Wn6*4)<8*4ozA z*504mB1w#c(YWBGT4-)#uN3xs2Cwe(kn z%PFN${)TXU{K13)dafm-mCkZ$+{d+G2hnA9=-g+Brbof9kd7cCPbgdwovyv|{s5v1 z`9U{b3h6jr(xd$%e$eOR*GCye{CIutdU8O&o*dAdpd_V2eL!y%auL0J9{HeOPkzNb z{MSe}_DM*E^@47FQAkI>NCYkB$L*MUmNJ*#aUFV`_)Y$D>RC+DH#HAAz{DCuebini+3-%SzS`e-ZBtLSr=>V!2alIh+B z21+)L5?Jg}GG8&9(4$X&Aa1XSC|myTnagd8+4Vd2ut?|LCs1y|&>d+(^Y&HrO#^B^ z=^AnN_ulubf*5P2YoW0CJoGj5(BC}|Jv9$~$2|0X^U&RFV0PKvJGKt(+Ozwq^w#Xq z;I8z*)_$f@ZO2K(gYGf82P4z#ubz6F9aN8&ohf}+T`nU4W zo#~zZ!+!Sn&SB~BUKo;+jh#~={Hz>1<@0;hxW>OPgtx0hS^nh^-m4B~`KLm7y*ixb z4~OuCJg&*>(-p#pL-?@}emaC-3gPWbi|w&LgeUGP&fgNk_lNLPA^dy@zZ$~JbwaWB zCqj5v2=5KyheP=35Pm*{Uk>5dLinOzE^cpq2=5HxTSEA;L~;3}EyZ|!dkDY37{3t0 zkN%tDe5cQW&0bX@eA|19`Fnq@7{B^(F@ElmVtm_U#rXaZek_FB-}W?p?B7r@y!`QE zc~v32K7_Z2@U9TPC4_Ga;r2IItbTjf&~SUl(D3u2^7gKw zw=*om?X28zJF_(0&Xf(ey`|yyOxtjKmTmYxOyZO(=tEmY*QpaaaGYD5)$s-#b9|SM zAH%P#AtEK~;((LvsC2hN+!&_2k~{HsT&l^%2i*6k(bFlrKzE_}Ka*uJe|`{J-0xm6-jk&QAJ) zBDOvX|2r-(!ng%V#pfw_HUS!Jj%4HB;6}@k?EHMCDF2%x(B0VP`*&TeeWt%ow+_c) zzd#>TGyLz$T+)hA9(r1Nbsv6qfx4}{tJ4Eqg;{n#z^vGTTlG-&HbTmM=8 ZrhhAywYJmGeA<_mpIv#1{0Q6s{{z7$pzHtu literal 24536 zcmdUXe{|H>b>|yNUkhuK3S&>LXiFF{GBRCkY4j7Fk`=~a!uHfMW-RQDf(A*JgaK(Z zvf=5wX|RK1LQYb^cB0VsR04$MbCwgjZ4KErJ0xvt;$+W~?K)+%yN$XhIpL%^3EQS8 zG_j+-pZjAp^Oe-tF8!m=9^JY3-S^(T_uY5jeee4nKDznWx0J+U&Pu5(XKd-e6-v8S zjCUD_bM3CxmC@f@-4chE0euqMMHDW1NZJ{6O9cn}E%d4F5IhP8&?yyzh3(3S-u@_8 z&x1?Lg>KN=l@J-@9ZN}l6rLBdGq^Ay;J<1Z&0qQ*b$c-*l1s@UZjB2tlY4!Llf z=pC*VJyWqex&DH5OLi8!t}XQp}C#m&iicAp{u|ILY6$UdLuim*-kZz@~r5=6P6 z^2EzTZa=B|+|TUP4!t;6*&;an5OS20ooV`J<0bFqii|XO{lwl%l-CqqC>8#`vY*W6 zh#xpg;Qn6hT`zb}#|z1CK!M&b63nT5enRM#5j$^K^A`)un~Q~e#LrW)3Lap^2m==$ z6@NB=IzY-Srz*@hMC*&yuznYGVa8*%+%PAA4g9powc?<(xB2a) z?q^9N^B3p~e8V#vrQHh&fs-;m))C3eegwOe(>y@rxcdp~6}ShO-|Ui#l{#wT499ps zC(Z)|(JpTq8)4M_V|K447Hgk!cX0zhs_QX7{#CY(QT*`ANB7 z;dU#YmGR8qOfR!nRDT=y+)7wg>vQGSqqKDUcvbYk} zBc*Y{Ps+OVr!I0o;Y#su$cfUacF%P)s?m$=t@~>3L7|)9*#6K@^P8Zm0}A6MnCIJ^ zKMX3he;)&9{AS<$JW@N93-p1WDcx7%<+5+&+9d!t} zm0<2pVZi!8sOqFYHz>~KvO-^RyTA>}U*a#22PIV^as=-88Lz}*qcT6fo@4sEj;}S` z+^uL5I2HV87M2X#WPRke3mtw3`At5H(_MG=GJ`)O{FNJpKVEVhf5~+S-bsM<)_tH7 zMx<@b?pCcAQ~$Gpt#BN*Me%!3SEKcT#jC^WzCE7yHQ{mb@|17uFDbF*U=_IE1i z4KQ%O%pF_&t9+PIXK^AN(|vD^&@qqbFQ;}}p?1t|k@_k!;w=4#0=VnrLO(!y!M?i= z3trhPu!`3Ki$i+H+ja?z7N? zx^WL3mik@Tys4P}{%%->`Ws-t`vSM8Kir-mxBs}*@A{0ux%RZ+sJ#338lT_g`0&#N z#@_-doa86X5yPCgms|0))Z2Uo>_ub;&CBxp=#rN>zt4;c|Gd13%mr_`pn2*W zWu-1hhAJmJfgZm_@+a#bd?<7E|BvOeeI;-|V%$%QpJTnhN&JYj@bkPM5ps#}oX$h# z-GorNKzt~jR1f}sg~)pf*O~7I7qut9L&3Tvk=@Jm;dv&=^Q>`WmrSdxOi%?C$PZ5` zr~Wyf=(wrUAG7=zuW;SvwBGWp1JoAnqy4g+Y&qDE)}M|G{yO?pw$mRfl3kPhvYk4# zJEAcX^%1-HOZeAhn+r5=^By)lR;$-LB_q{D?TAG zr|TSf0`j>Qg1uB_>m2QX8-Ks9hfadXFZ+of`XL^_+97@7Z{@gr$39R*9~647SzzU( z0y8h77w58DAO-D18o#nzFbd&^tzKY8<6L$NYH7a6&)+0_)RcdF_M@K}cYoyWqhEm{q5d= zOmKg)4&TlF$CutBc^1&)>c22Orb!RW11WC&oad|h21CD6&l7jZ{P~p%H9zq>&;FOK zo&pmUulaqp;(aoYx}I@vfoFtA6GHC>H?2SVFuh`_(A96S-!grSwhbzKDrz}ik5p7K z+*7e+7Fl9O^TAYX6*GoMuZSKeX9RA$Ca~=)!_33tzqUVHe2vH6k6~P9hipv9%fJ~g zTcLDWz^Ssi_NefyYq>t$cA5M4wF$=kb_r@ZUEj!?!5MDr5Iqa%JDI+f>EYUThFE`F zL4Uke=wo#ZvnO{8tWFDT>t;Asv6EpY&BMER>^*D`e^%xb`w8};p1Y;q)}x&lqI##w z{+j!P9Z)hmME`i%4ATQAF!!9+r`0aRhHtii#bY02hD>j9EJCSY{!#_m>zCB$Pjr6+i}mZ&?^QQX4_5*tp1$9wc`xODvmMq z)8apwXGFiK9aFJya-hz9O!X#Eq<5(V>dax)TkYZ}#6RO@62Q&RGlwJkOSg1xR{cLA za?I&}iRH`{&_(}?Ob@r6XNYyj`X8MZdc_38Z1sN^*!B&9dM}3iUyMZeiL!3=o`>=m z#jkUHVSL6$5mt*ogkx8^9(ffsD&*rl>6Ge0W~rlg@IU6$_28#u9U;%5>RI?vH8lyG z$PXN4F@H*xy~^X~{Z{=dUMBf|-~{ITS!8}vWnY(mWLo$hjZ8`Tgb?CoU*r0~3AB7Y z9=yR4X4)Yo-AhuIim8k{Njy^S8@wZ@N>8)i{x0-N{Wi#i$4gH!?(bpX{0p`GrHbOm zt)Ke*hk<*HoceW{)M`H{fh5>sRqb*fB2ss>)J#Mg^GWnl_A<+etr8EAg#aO z_>rzn6kw>>`u8ukGhRjHADy}WNg(IG@WW3p-yw2YoJ^G-Wj?=#CWwk35&4nt-171p zcY;y1gbaj;ajRqHcq>b!@7t5{xCOpelw&eJJToF{D-UYeda4w>8?QK z-K_IO^y~Nh`Y`J6U$x^mx&oDVv(6LIU;pTH{M7Et&mTG46{x%doPQvJraVsgbfyE} zKQQxK7nj??NFR(8Zj(6aAK>Z*}sTEf&4%C z!Z-g4v_C!Ym$h91G7Qwu$$$LS@9{&G6W?F<&8`4akUSCj<0S95|G2FkbhdABocY3| zmpBgEde2B0cT;7@-`C-6VXl1F1}Y`VjkRsx-kk z=pZx&tK1QS7!NZMxNXY!uf=cGZW&z%@lr%fvgch4_&ko8_f3NPj#~bLd#xhB?N&eN zsLR(E`E9rQK}T!8zQ}L8#ef%N74qVVmq1UgKogCi|Y(xEHp`e(yJ}=k{Tn z#CyM~UhzNU`0h8|qxhdFzEbf&WE^s$q(7KuSaa*^vzI;;cYdoRrvAYXGClV(2JWG( zz-!r&u9wiJR<8fa9?ExiLewWC|Y1)fDaKda9`nin{VX7 z9FO{_l8S${gJIJx)9xUB%aWWZXb(#T?u*<$9GfWeBiyG#vvBRW*mo6LQGW~Ai{tYS zjn4)4DvHlL)DH^mRTQ7~+>!SOwHNL+I6kW%q{?0veNTuy_I$#hk@bUp1q`9CC#)09 zaM*T?_2{JUTwig7am$Nfk6x;Sf7}NCwc<|}kHa=u4>`3f&TV&rpZZ6C=lWQze4f>* z^i=8JamBit?-LjKaSsEI*QnTV*WYq|4ueyH{FvVjtEHVkC3@x7aJ8%K7nsp|I>?Pu zbL-()f?%HSK|+A97FenKSLOl1Ga8@zNlw`3p-n=sDGg(A40~T<4Ncohs4EF1| zdxGojeFuL==A}~g#W@Qq=jY`?9f$NpnVXldbA>$z3{OftJa_k(IsWABTK-rJ2S9)J zn5=_A>8CI*-=Mf0Rv+YcxS#sicHzO_uzG;$V-<&H6NHad49}9PZiL<+LPcRe1GI2+ zXNX7zet`IH_r>rxYQg6u_y_WAJu)-Doug9jGpv`Nlzr0PfAN#DPulxSep2>HJI`gN zZ?K$xa-5;PZ{tr$eC%4L@%^~OL$CMST`Ms&WS`>_2mPsQ+`eliDL}=}XZ}Qjaew+c z*Dp`%yb!>*c%Y`V->Pb3sCkb05>EH+NSPZ|h+beQYIJxj%2j|zXdXNlt7TJIF!skq2x@z+lY zo(gCnDt=1%Q-RpcPf35N0JDvH8805Z#kl3Gra$5oTE>G?wwIj){Dh3t73lnJ*7=)| z@uGRNpOA68RE}uA>?cHyXr7-6e#~-Nyn+8>B)^%-&Ue7de%hs)K=GH_UI!-Olrrt09C6))_|f9gI4A-v!Sv zOm5!qp4a`ZT;-5+hqR2wKBhB6$#d3jP6Zg>D?9Mvv{MSFrHjP{uo&o(R|XLljb(^u$-U4ugnjy zPMUF_oR<3Z^ba^Lrq5nu7`Q*+3fLDGvgb5YWi`C7(eo+hv;Ofu1?MYzpX^VF;cea3 zlqB2%>L)AdbwS>8Ha`wlh@Y1D9k|m>x4hWmhOZwMy1sYm>jxB{mUUX+t@woEx^Al_ z9@;t1*NFL{^QGp|feWR-DX}NLXC(8rjLCG4ko|T=Qt*8IAt!-c>z2{P zQ?Wb_`!I^>6EBnXYH=^%Cj-PE;Y*eMOxERH(mt+n5OEGYOqD+2p85Mk(f-N%<%saZ zPrpfDEiX2I?I$4VXx>q`zk%KOi%xq$4Q^P&M<@Zdo-tN&Nynd&`{c}= z{SMMsT3560A=p#><~;poBKBoJkT8&cPr>$i+c)7KzYF`Gong6Cx3sVygkx3LF|waL zqWUalJUlAT@eAb_yXtfOf_bXscVWf4pkFuPDFI?t|+*49txmh5_!%jJ< z&*O4Ruhe@itRFJx>=)x2R|MqaN#MTA{imnD6S3d_6+f|lC}_Zs%s%UC{)+4U@rx`- zSgrBy_=MsT=j>cz^N92272>e>u`rKskgnEmKh-7ZgFOq-?4EL0WIR3Be0$3-`x^N&?b&wGyt>mKhpEO}(EAI#6?e~|pYJ5@|S z@H3%TMd+hM&&l;5pR?*B^?yeFS^i?@RGi1QM)-0%?&XL)13K=(2!9{Zy{@-ZsW`g7 zT^u7-D(lwYP6|;;mCE||w^uP9FTI1IoqKVQ0XxS_h2Q$M`sUs}jt=d>Lkk=2!hG?>|hPdvD9yn|!I#AMktzPWJmuLK?&= zJ7<5P{@~a4{#@XG$_&0!N-L)t&^1@~tM2|^BNMDN{! z?833jTp#Wcf3v)Ns_cvOY>(_QA?;zEBk*hOFP9(!70|EHC)D5Bz|VULhn^@Qy@$r| z@o|AiWq%CE#Gi8r98?g8?;>dXRJdn``OhJ7Q958J#WPS+!7p*ex(B}nGUdU-Z_n#- zo&m##&F^8p+#zs@yfD#*c4?n~-|FLB?J`dY4EJ2;@oR3m;u1v4^ND-9T%GV$ikV%8 z#zV;r2<#{Kg$F;hQ}CK4ud(6ny|gvNzmlqQ!%`nUevQYo{>%>NhFSdlIb=@%zu|_K z=T)lw`F6J~*6uspZrzqYVCbi0e&WHad~OZ*XdXj(2^h?d;p4(*^Xgv_dGXv3dRV;2 zJuL9$gYsjo{Jy95L3>Dc?kSazzN2)~2b7e%43y^Wb%u-C+3aBN@5cj~ufR#-VZT$V ze2%+u{uS}J%9jL(UaRSAUY?g0mWSSpU^}GTf8~bj`d(z1E9eLQ^o#m2z1H>dBVbxr zS7g4Aad#x2$p56`Ie8pHeCijvf91z{Y+-p*?!SnfgU>Sz*B;|>EI+pQCj6w#@4Ai3 z*CylGK3J*e2|6E$obtHGac)AB@9kq(9op&S@ zXW;xJI#0=COz@9LyD)h~acLJO4=K*Tg~@}8w=)crfs^;q4M#mj2%Nlk zpR;wmn}}49zb)M%-|-jz^fdB3^S1)N_6T2w_$-e2Q{q?XZ)b#WFZwI!XE;(nfcljp zgZ-`p{Y*sodJE*T-^YOOVubI*#D~24T7+KpQyRzgzhA~fd)jpJ1^V~jSU7K9O6}2a zG;em^{tKQ`%|SKiO_$d z^eYkikBOeULrj4<4nOsq1do@>dkAzdbORK>QPxeR@|!-0?*p|IC&=lB2wGf>p6BMy zRX<~kbP_+QJsU5*P4a-!wY+jN8)f`d>~k?JgmBwZ9lwJs78<`>$M1hzT!&Kt2mrnL$9 zKe_+D*aQ5_;8g617~*jieXQW*V(b2m*q=CTNpHmdlHM`F?}In;FThi=-7&mpQrPZJ zZg-vINTK|Y*Y=yK(opU{CW)B}&woLEX0y;!!B&o2;YL`4{9v=rgZh&FPOO~jispfo zgrU!A97djsVtHIDke$-}U~arkIvy6l%^dHQqIsVi?*lrXSn%fQ_uGq%w_e9f++_Ve zRWu&vla_;<)3Z^>D+i5XwWG!< z#9^`f5w`mqCBMe;G+xrpc2AXjg!6)o?t}EZUx}}_uJ6}%y;}OU`$?ReiqS z4?#Fer_xg;zaqHC_qp+$j;HJTrj3WEe>DEQ@qSjkt~aWF3+!J-`jK+1U*ta0_OE!X zmE$V%f`Q1sTtWM3U#wQ_9{ViIRT!6Hs(5Sz-|rRDu`U89_nY{4A8?Y6$6n(115&Zi zJ7`$QkNUthvK;Y}d$_+jJ4fdArs*xee}UfP5xxJ0)^R+R<^3cTd!Fxz3gyL$4&3`i z-dM)nPIxNz9RHrMkPrJ>;AEV5?4bB<>;(I5As<3T;O@}=p5}XuRO}eP&s4~V{W5Ts z%r|GB&!}JMe)iAqhsSk1_4}JPAK&BW0GN+Mk@@%n&BvU+lmATm$~kfVy8E4CeQ%n5 z7mF*O)$wHJ{Y@KhvAA+V$4iiRAuqmJaphcoK06}w`B9qB6vY+X4;T73@=1CRf!n2G zYq+kk9 z=cI-5P>3e_@6O}Djp--n(UA`a?z8jg$ae$x;ygO+6S(8^=#YW_-JyAOJkbl>C+E@S z952sB3+0C?0{7fJeq0d;?%8>C#JRwIlBHWCjEG#!b?r_Z^PEm>z@gupiN5zCHH8!+sL%5EM7wl|3kKa69Zf z=Oc0I`#+KUibRC|J*Brs=-(l_Unc&NUM-1P`Yy4Ttp~qK{fYb!_j@>R1KRsrdAys| z%AdD)SLpi_XM_&B;dk-IpPrEVeyUE;=QV|PUQdsX_U#)UO^gmD(!ITj1B0UjiJ_$E_+44mYolMSeY9tMaQTN{{p5!)p8oBBx9>yQ)W7E_kl0OKqdysldaSFf ztFKGeHPkiMHPtoOwbZrNt*@`Eudh$mH`F)QH`O=Sx74@RuTRz`>yyc3L$Wd1lx$A6 zBwLf~8|oVB8jm?cMjjfIAo9deCo03fp zO^r=WP0dX$O|4Dqo9mkEo0H8A&5g}X&CSg%&8^MrTk2ZsTaqmeEsZTrEzK=0Ev+r< zTkBfuTa&E~t&Ocst<9}1t*x!=*OQ3rY54Wj^?GWyo=9f2k@WulzQhwlBZ=(Z!Ja>EyG`}%sbBR%)^?H?WS!$X7nNAKG=)O+tp)+u}Tz5(AmlD-d2MCzfI zLnHU~4DH)Dv>$AsXMMIOy?1Z+7dGn~9_ZWGH-NB;3A)Ra)gc_eN%$&bCU~B3?2T6boN#$-MJRtkc)e8;+_-(N zzdLC?C$z4N=++N@K8Sr7rI7w3NbhB9@9h>k%{Gcz!zaE`aDeMP_%Z2zsZxHr* z{K*CA_?gc_{ouN7Avyn+`dG+rpqoz>%6SL%BaydQe#HIS1>^y}Z2|gnqFetD(|nmO zTZtDt+WWN)azhmJsfkX(mkDs^ap=ueMLq=jBltr_AJ^yqomAw6D6)GZACpq0 z{o{@m<*@E$JyEL+LWq;r3E(*{+PwpD%8|)t# zog@0})9DdLhtvB9dv@{HeSQ0ShW*_4eZ#v*GP-MmA*q<}O-JyHaxBZ&cdK!Y|6l}f zRfn?rnF!vk4rcY!5xiC%&gzFF_*Tuot^RxjpSinOpYC5R#;>g_#t$Wn@yii>CW7CH z;8hL9?b{>x&Impn!5tisis@4u!8;=Oa0DNZ;Flu!)d=oBP&{5$1h0$WJ0tj^2tFRc z&qwgf5qu_s--zH_w-n3Q{$MeFWot1$v8@>2`cN_69l;MqaQipD%>Lusi`!2}@QV@r zas%YxIL3L+@3ufZqHN=zoyT{47X>^hTF4b!|j=| z;r49U@a|Nx{`M}V)gRQ24f_}!k=!MXKfrNcFs<+&kKLaMaz~>-T3zM+l}z)Ax?_?Y(zo$T6e}srCC9ruPl@xO=k$ z#66ncO*}(rBAxyd{$iWJ2Qs51bl)(T|M%FXIv?K@b?x*8O_EB8{`>6RFm6H7?(>vN z`UVUra18-VP_w%EVdZu;5$TW*1AN3LUHM*qF|t5k=A_K?%sTV8~@ z#n{%~@*{(m9}x?esQx&|Apf@ZmcJRa{0%;bV&j|q%|tWZnL+<_bjg>kg+%%u9smCW Dtd4-c diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index 2e1d1f1b0..6fa89a6a1 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -20,9 +20,10 @@ use integration_test_tools::{ use magicblock_config::{ config::{ accounts::AccountsDbConfig, ledger::LedgerConfig, - validator::ValidatorConfig, LifecycleMode, + scheduler::TaskSchedulerConfig, validator::ValidatorConfig, + LifecycleMode, }, - types::{network::Remote, StorageDirectory}, + types::{crypto::SerdeKeypair, network::Remote, StorageDirectory}, ValidatorParams, }; use magicblock_program::Pubkey; @@ -35,14 +36,15 @@ use program_flexi_counter::{ }; use program_schedulecommit::MainAccount; use solana_sdk::{ - signature::Keypair, signer::Signer, transaction::Transaction, + native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, + transaction::Transaction, }; use tempfile::TempDir; use tokio::runtime::Runtime; pub const TASK_SCHEDULER_TICK_MILLIS: u64 = 50; -fn validator_config(temp_dir: PathBuf) -> ValidatorParams { +fn validator_config(temp_dir: PathBuf, faucet: &Keypair) -> ValidatorParams { ValidatorParams { lifecycle: LifecycleMode::Ephemeral, remotes: vec![ @@ -58,11 +60,34 @@ fn validator_config(temp_dir: PathBuf) -> ValidatorParams { block_time: Duration::from_millis(TASK_SCHEDULER_TICK_MILLIS), ..Default::default() }, + task_scheduler: TaskSchedulerConfig { + faucet_keypair: SerdeKeypair::from(faucet.insecure_clone()), + }, storage: StorageDirectory(temp_dir), ..Default::default() } } +fn airdrop_faucet(faucet: &Keypair) { + let chain_ctx = IntegrationTestContext::try_new_chain_only() + .expect("failed to connect to base chain to fund faucet"); + let validator = LoadedAccounts::with_delegation_program_test_authority() + .validator_authority(); + let payer = Keypair::new(); + chain_ctx + .airdrop_chain(&payer.pubkey(), LAMPORTS_PER_SOL) + .expect("failed to airdrop faucet delegation payer"); + chain_ctx + .airdrop_chain(&faucet.pubkey(), 100 * LAMPORTS_PER_SOL) + .expect("failed to airdrop to task scheduler faucet"); + let (_, confirmed) = chain_ctx + .delegate_account_to_validator(&payer, faucet, Some(validator)) + .expect("failed to delegate task scheduler faucet"); + if !confirmed { + panic!("task scheduler faucet delegation not confirmed"); + } +} + fn start_validator( config: ValidatorParams, default_tmpdir: TempDir, @@ -109,7 +134,12 @@ pub fn setup_validator_with_migration_tasks( } } - let config = validator_config(temp_dir.clone()); + // The faucet that pays for cranks must be funded before the validator + // delegates it on startup. + let faucet = Keypair::new(); + airdrop_faucet(&faucet); + + let config = validator_config(temp_dir.clone(), &faucet); start_validator(config, default_tmpdir, temp_dir) } @@ -167,7 +197,6 @@ pub fn create_delegated_counter( pub fn wait_for_hydra_crank( ctx: &IntegrationTestContext, crank_pda: &Pubkey, - expected_lamports: u64, max_timeout: Duration, validator: &mut Child, ) { @@ -186,14 +215,6 @@ pub fn wait_for_hydra_crank( crank_pda, account.owner ); - assert!( - account.lamports >= expected_lamports, - cleanup(validator), - "crank account {} underfunded: {} < {}", - crank_pda, - account.lamports, - expected_lamports - ); return; } expect!(ctx.wait_for_next_slot_ephem(), validator); diff --git a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs index b4ac60248..3a727411d 100644 --- a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs +++ b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ @@ -36,7 +35,7 @@ fn test_cancel_ongoing_task() { // Schedule a task let task_id = 3; let execution_interval_millis = 100; - let iterations = 1000000; + let iterations = 1000; let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -72,11 +71,9 @@ fn test_cancel_ongoing_task() { // The crank is created and funded for all scheduled iterations. let crank_pda = crank_pubkey(&payer.pubkey(), task_id); - let expected_lamports = iterations as u64 * CRANKER_REWARD; wait_for_hydra_crank( &ctx, &crank_pda, - expected_lamports, Duration::from_secs(10), &mut validator, ); diff --git a/test-integration/test-task-scheduler/tests/test_independent_authority.rs b/test-integration/test-task-scheduler/tests/test_independent_authority.rs index 82bf1deac..d33cc82f1 100644 --- a/test-integration/test-task-scheduler/tests/test_independent_authority.rs +++ b/test-integration/test-task-scheduler/tests/test_independent_authority.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::instruction::{ @@ -82,14 +81,12 @@ fn test_independent_cranks_per_authority() { wait_for_hydra_crank( &ctx, &payer_crank, - payer_iterations as u64 * CRANKER_REWARD, Duration::from_secs(10), &mut validator, ); wait_for_hydra_crank( &ctx, &other_crank, - other_iterations as u64 * CRANKER_REWARD, Duration::from_secs(10), &mut validator, ); @@ -127,7 +124,6 @@ fn test_independent_cranks_per_authority() { wait_for_hydra_crank( &ctx, &other_crank, - other_iterations as u64 * CRANKER_REWARD, Duration::from_secs(10), &mut validator, ); diff --git a/test-integration/test-task-scheduler/tests/test_migration.rs b/test-integration/test-task-scheduler/tests/test_migration.rs index a2582cfe7..3a7eaedc7 100644 --- a/test-integration/test-task-scheduler/tests/test_migration.rs +++ b/test-integration/test-task-scheduler/tests/test_migration.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; use magicblock_task_scheduler::{crank_pubkey, db::DbTask, SchedulerDatabase}; use solana_sdk::{ @@ -47,7 +46,6 @@ fn test_migration_reschedules_tasks_and_empties_db() { wait_for_hydra_crank( &ctx, &crank_pda, - iterations as u64 * CRANKER_REWARD, Duration::from_secs(15), &mut validator, ); diff --git a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs index 2e882239b..2dc43c22a 100644 --- a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ @@ -68,7 +67,6 @@ fn test_reschedule_task() { wait_for_hydra_crank( &ctx, &crank_pda, - iterations as u64 * CRANKER_REWARD, Duration::from_secs(10), &mut validator, ); @@ -110,7 +108,6 @@ fn test_reschedule_task() { wait_for_hydra_crank( &ctx, &crank_pda, - new_iterations as u64 * CRANKER_REWARD, Duration::from_secs(10), &mut validator, ); diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task.rs b/test-integration/test-task-scheduler/tests/test_schedule_task.rs index 335af65cf..12a8211fd 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task.rs @@ -1,6 +1,5 @@ use std::time::Duration; -use hydra_api::CRANKER_REWARD; use integration_test_tools::{expect, validator::cleanup}; use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ @@ -68,11 +67,9 @@ fn test_schedule_task() { // The crank is created by hydra and funded for every iteration. let crank_pda = crank_pubkey(&payer.pubkey(), task_id); - let expected_lamports = iterations as u64 * CRANKER_REWARD; wait_for_hydra_crank( &ctx, &crank_pda, - expected_lamports, Duration::from_secs(10), &mut validator, ); diff --git a/test-integration/test-task-scheduler/tests/test_undrained_validator.rs b/test-integration/test-task-scheduler/tests/test_undrained_validator.rs new file mode 100644 index 000000000..f4c5328b5 --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_undrained_validator.rs @@ -0,0 +1,100 @@ +use std::time::Duration; + +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::crank_pubkey; +use program_flexi_counter::{ + instruction::create_schedule_task_ix, state::FlexiCounter, +}; +use solana_sdk::{ + native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, + transaction::Transaction, +}; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, +}; + +/// The validator identity must not be drained to pay for hydra cranks: a +/// dedicated, delegated faucet account pays instead. Scheduling a task creates +/// and funds a crank, and the validator identity's balance is left untouched. +#[test] +fn test_undrained_validator() { + let (_temp_dir, mut validator, ctx) = setup_validator(); + + let payer = Keypair::new(); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + + expect!( + ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + + create_delegated_counter(&ctx, &payer, &mut validator, 0); + + // The default validator identity keypair (see consts::DEFAULT_VALIDATOR_KEYPAIR). + let validator_keypair = Keypair::from_base58_string("9Vo7TbA5YfC5a33JhAi9Fb41usA6JwecHNRw3f9MzzHAM8hFnXTzL5DcEHwsAFjuUZ8vNQcJ4XziRFpMc3gTgBQ"); + let validator_identity = validator_keypair.pubkey(); + let validator_balance_before = expect!( + ctx.fetch_ephem_account_balance(&validator_identity), + validator + ); + + let ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + + // Schedule a task + let task_id = 1; + let execution_interval_millis = 100; + let iterations = 3; + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_schedule_task_ix( + payer.pubkey(), + task_id, + execution_interval_millis, + iterations, + false, + false, + )], + Some(&payer.pubkey()), + &[&payer], + ephem_blockhash, + ), + &[&payer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + // The crank is created by hydra and funded for every iteration (paid for by + // the faucet, not the validator identity). + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + wait_for_hydra_crank( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); + + // The validator identity must not have been drained to pay for the crank. + let validator_balance_after = expect!( + ctx.fetch_ephem_account_balance(&validator_identity), + validator + ); + assert!( + validator_balance_after >= validator_balance_before, + "validator identity was drained paying for cranks: {} < {}", + validator_balance_after, + validator_balance_before + ); + + cleanup(&mut validator); +} From c5608ea067defb5b4195ad2f06910d90befec4f3 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 11:15:44 +0200 Subject: [PATCH 04/13] feat: optional crank faucet --- magicblock-api/src/magic_validator.rs | 188 +++++++++--------- magicblock-config/src/config/scheduler.rs | 6 +- magicblock-config/src/types/crypto.rs | 6 - magicblock-task-scheduler/src/service.rs | 6 +- .../test-task-scheduler/src/lib.rs | 4 +- 5 files changed, 108 insertions(+), 102 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 0f0128a40..6e753f30a 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -117,9 +117,7 @@ pub struct MagicValidator { scheduled_commits_processor: Option>, rpc_handle: thread::JoinHandle<()>, identity: Pubkey, - /// Faucet keypair used by the task scheduler to pay for hydra cranks. - /// Delegated on startup so its lamports are real (base-chain backed). - faucet_keypair: Keypair, + faucet_keypair: Option, transaction_scheduler: TransactionSchedulerHandle, _metrics: (MetricsService, tokio::task::JoinHandle<()>), claim_fees_task: ClaimFeesTask, @@ -173,8 +171,7 @@ impl MagicValidator { // The task scheduler pays for hydra cranks from a configured faucet // account (delegated on startup) rather than the validator identity, // which is not a delegated account. - let faucet_keypair = - config.task_scheduler.faucet_keypair.insecure_clone(); + let faucet_keypair = config.task_scheduler.faucet_keypair.clone(); let latest_block = ledger.latest_block().load(); let mut accountsdb = @@ -435,7 +432,7 @@ impl MagicValidator { let task_scheduler = TaskSchedulerService::new( &task_scheduler_db_path, config.aperture.listen.http(), - faucet_keypair.insecure_clone(), + faucet_keypair.clone().map(|k| k.insecure_clone()), dispatch .tasks_service .take() @@ -443,7 +440,11 @@ impl MagicValidator { ledger.latest_block().clone(), Duration::from_millis(config.ledger.block_time_ms()), token.clone(), - )?; + ) + .inspect_err( + |e| error!(error = ?e, "Failed to initialize task scheduler"), + ) + .ok(); log_timing("startup", "task_scheduler_init", step_start); Ok(Self { @@ -462,9 +463,9 @@ impl MagicValidator { claim_fees_task: ClaimFeesTask::new(), rpc_handle, identity: validator_pubkey, - faucet_keypair, + faucet_keypair: faucet_keypair.map(|k| k.insecure_clone()), transaction_scheduler: dispatch.transaction_scheduler, - task_scheduler: Some(task_scheduler), + task_scheduler, transaction_execution, replication_handle: None, mode_tx, @@ -881,105 +882,110 @@ impl MagicValidator { fn spawn_primary_onchain_setup(&self) { let rpc_url = self.config.rpc_url().to_owned(); let identity = self.identity; - let faucet_keypair = self.faucet_keypair.insecure_clone(); let chain_operation_config = self.config.chain_operation.clone(); let block_time_ms = self.config.ledger.block_time_ms(); let base_fee = self.config.validator.basefee; // Ephemeral mode does a non-blocking startup balance check. // Intentionally fire-and-forget: the task itself exits the process on failure. - tokio::spawn(async move { - let step_start = Instant::now(); - let result = MagicValidator::ensure_validator_funded_on_chain( - rpc_url.clone(), - identity, - ) - .await; - log_timing( - "startup_background", - "ensure_funded_on_chain", - step_start, - ); - if let Err(err) = result { - error!(error = ?err, "Validator balance check failed"); - error!("Exiting process"); - std::process::exit(1); - } + // Skipped if no faucet keypair is configured. + if let Some(faucet_keypair) = + self.faucet_keypair.as_ref().map(|k| k.insecure_clone()) + { + tokio::spawn(async move { + let step_start = Instant::now(); + let result = MagicValidator::ensure_validator_funded_on_chain( + rpc_url.clone(), + identity, + ) + .await; + log_timing( + "startup_background", + "ensure_funded_on_chain", + step_start, + ); + if let Err(err) = result { + error!(error = ?err, "Validator balance check failed"); + error!("Exiting process"); + std::process::exit(1); + } - let step_start = Instant::now(); - let result = MagicValidator::ensure_magic_fee_vault_on_chain( - rpc_url.clone(), - ) - .await; - log_timing( - "startup_background", - "ensure_magic_fee_vault_on_chain", - step_start, - ); + let step_start = Instant::now(); + let result = MagicValidator::ensure_magic_fee_vault_on_chain( + rpc_url.clone(), + ) + .await; + log_timing( + "startup_background", + "ensure_magic_fee_vault_on_chain", + step_start, + ); - // Without magic fee vault being properly set up - // transactions scheduling commits will fail - if let Err(err) = result { - error!(error = ?err, "Magic fee vault setup failed"); - error!("Exiting process"); - std::process::exit(1); - } + // Without magic fee vault being properly set up + // transactions scheduling commits will fail + if let Err(err) = result { + error!(error = ?err, "Magic fee vault setup failed"); + error!("Exiting process"); + std::process::exit(1); + } - let step_start = Instant::now(); - let result = ensure_faucet_delegated_on_chain( - rpc_url.clone(), - &faucet_keypair, - ) - .await; - log_timing( - "startup_background", - "ensure_faucet_delegated_on_chain", - step_start, - ); - // Without the faucet being funded and delegated the task scheduler - // cannot pay for hydra cranks. - if let Err(err) = result { - error!(error = ?err, "Task scheduler faucet setup failed"); - error!("Exiting process"); - std::process::exit(1); - } + let step_start = Instant::now(); + let result = ensure_faucet_delegated_on_chain( + rpc_url.clone(), + &faucet_keypair, + ) + .await; + log_timing( + "startup_background", + "ensure_faucet_delegated_on_chain", + step_start, + ); + // Without the faucet being funded and delegated the task scheduler + // cannot pay for hydra cranks. + if let Err(err) = result { + error!(error = ?err, "Task scheduler faucet setup failed"); + error!("Exiting process"); + std::process::exit(1); + } - if let Some(ref config) = chain_operation_config { - if !config.claim_fees_frequency.is_zero() { - let step_start = Instant::now(); - if let Err(err) = claim_fees(rpc_url.clone()).await { - error!( - error = ?err, - "Failed to claim validator fees on startup" + if let Some(ref config) = chain_operation_config { + if !config.claim_fees_frequency.is_zero() { + let step_start = Instant::now(); + if let Err(err) = claim_fees(rpc_url.clone()).await { + error!( + error = ?err, + "Failed to claim validator fees on startup" + ); + } + log_timing( + "startup_background", + "claim_fees_on_startup", + step_start, ); } + } + if let Some(ref config) = chain_operation_config { + let step_start = Instant::now(); + if let Err(error) = + MagicValidator::register_validator_on_chain( + &rpc_url, + config, + block_time_ms, + base_fee, + ) + .await + { + error!(%error, "Validator registration failed, exitting"); + std::process::exit(1); + } log_timing( "startup_background", - "claim_fees_on_startup", + "register_validator_on_chain", step_start, ); } - } - if let Some(ref config) = chain_operation_config { - let step_start = Instant::now(); - if let Err(error) = MagicValidator::register_validator_on_chain( - &rpc_url, - config, - block_time_ms, - base_fee, - ) - .await - { - error!(%error, "Validator registration failed, exitting"); - std::process::exit(1); - } - log_timing( - "startup_background", - "register_validator_on_chain", - step_start, - ); - } - }); + }); + } } #[instrument(skip(self))] diff --git a/magicblock-config/src/config/scheduler.rs b/magicblock-config/src/config/scheduler.rs index b6b30c66f..c16f678b5 100644 --- a/magicblock-config/src/config/scheduler.rs +++ b/magicblock-config/src/config/scheduler.rs @@ -14,15 +14,15 @@ use crate::{consts, types::SerdeKeypair}; pub struct TaskSchedulerConfig { /// Keypair the task scheduler uses to pay for hydra cranks, encoded in /// Base58. - pub faucet_keypair: SerdeKeypair, + pub faucet_keypair: Option, } impl Default for TaskSchedulerConfig { fn default() -> Self { Self { - faucet_keypair: SerdeKeypair::from(Keypair::from_base58_string( + faucet_keypair: Some(SerdeKeypair(Keypair::from_base58_string( consts::DEFAULT_TASK_SCHEDULER_FAUCET_KEYPAIR, - )), + ))), } } } diff --git a/magicblock-config/src/types/crypto.rs b/magicblock-config/src/types/crypto.rs index f509e179f..e5d24721e 100644 --- a/magicblock-config/src/types/crypto.rs +++ b/magicblock-config/src/types/crypto.rs @@ -32,12 +32,6 @@ impl Clone for SerdeKeypair { } } -impl From for SerdeKeypair { - fn from(keypair: Keypair) -> Self { - Self(keypair) - } -} - impl FromStr for SerdeKeypair { type Err = String; fn from_str(s: &str) -> Result { diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index c2fe9f71f..85d41c689 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -76,12 +76,16 @@ impl TaskSchedulerService { pub fn new( path: &Path, rpc_url: String, - faucet: Keypair, + faucet: Option, scheduled_tasks: ScheduledTasksRx, block: LatestBlock, slot_interval: Duration, token: CancellationToken, ) -> TaskSchedulerResult { + let Some(faucet) = faucet else { + warn!("No faucet keypair configured, skipping task scheduler"); + return Err(TaskSchedulerError::FaucetNotReady); + }; let db = SchedulerDatabase::new(path)?; Ok(Self { db, diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index 6fa89a6a1..e804913f9 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -61,7 +61,9 @@ fn validator_config(temp_dir: PathBuf, faucet: &Keypair) -> ValidatorParams { ..Default::default() }, task_scheduler: TaskSchedulerConfig { - faucet_keypair: SerdeKeypair::from(faucet.insecure_clone()), + faucet_keypair: Some( + SerdeKeypair::from_str(&faucet.to_base58_string()).unwrap(), + ), }, storage: StorageDirectory(temp_dir), ..Default::default() From 24dd7fb2eca6b4321dbd9b4110d6fa6f24240b2a Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 12:18:07 +0200 Subject: [PATCH 05/13] fix: check faucet funds --- .../test-task-scheduler/src/lib.rs | 14 +- .../tests/test_cancel_ongoing_task.rs | 2 +- .../tests/test_independent_authority.rs | 2 +- .../tests/test_migration.rs | 2 +- .../tests/test_reschedule_task.rs | 2 +- .../tests/test_schedule_task.rs | 2 +- .../tests/test_schedule_task_signed.rs | 2 +- .../tests/test_scheduled_commits.rs | 126 -------------- .../tests/test_undrained_faucet.rs | 163 ++++++++++++++++++ .../tests/test_undrained_validator.rs | 100 ----------- 10 files changed, 179 insertions(+), 236 deletions(-) delete mode 100644 test-integration/test-task-scheduler/tests/test_scheduled_commits.rs create mode 100644 test-integration/test-task-scheduler/tests/test_undrained_faucet.rs delete mode 100644 test-integration/test-task-scheduler/tests/test_undrained_validator.rs diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index e804913f9..43a5e1a04 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -94,7 +94,12 @@ fn start_validator( config: ValidatorParams, default_tmpdir: TempDir, temp_dir: PathBuf, -) -> (TempDir, Child, IntegrationTestContext) { +) -> (TempDir, Child, IntegrationTestContext, Option) { + let faucet_keypair = config + .task_scheduler + .faucet_keypair + .clone() + .map(|k| k.insecure_clone()); let (default_tmpdir_config, Some(mut validator), port) = start_magicblock_validator_with_config_struct_and_temp_dir( config, @@ -110,16 +115,17 @@ fn start_validator( IntegrationTestContext::try_new_with_ephem_port(port), validator ); - (default_tmpdir_config, validator, ctx) + (default_tmpdir_config, validator, ctx, faucet_keypair) } -pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { +pub fn setup_validator( +) -> (TempDir, Child, IntegrationTestContext, Option) { setup_validator_with_migration_tasks(&[]) } pub fn setup_validator_with_migration_tasks( tasks: &[DbTask], -) -> (TempDir, Child, IntegrationTestContext) { +) -> (TempDir, Child, IntegrationTestContext, Option) { let (default_tmpdir, temp_dir) = resolve_tmp_dir(TMP_DIR_CONFIG); // Seed the migration database before the validator opens it. The handle is diff --git a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs index 3a727411d..39dbcd8d3 100644 --- a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs +++ b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs @@ -17,7 +17,7 @@ use test_task_scheduler::{ #[test] fn test_cancel_ongoing_task() { - let (_temp_dir, mut validator, ctx) = setup_validator(); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); diff --git a/test-integration/test-task-scheduler/tests/test_independent_authority.rs b/test-integration/test-task-scheduler/tests/test_independent_authority.rs index d33cc82f1..0ac09cd74 100644 --- a/test-integration/test-task-scheduler/tests/test_independent_authority.rs +++ b/test-integration/test-task-scheduler/tests/test_independent_authority.rs @@ -16,7 +16,7 @@ use test_task_scheduler::{ #[test] fn test_independent_cranks_per_authority() { - let (_temp_dir, mut validator, ctx) = setup_validator(); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); let other = Keypair::new(); diff --git a/test-integration/test-task-scheduler/tests/test_migration.rs b/test-integration/test-task-scheduler/tests/test_migration.rs index 3a7eaedc7..a71441c1f 100644 --- a/test-integration/test-task-scheduler/tests/test_migration.rs +++ b/test-integration/test-task-scheduler/tests/test_migration.rs @@ -38,7 +38,7 @@ fn test_migration_reschedules_tasks_and_empties_db() { instructions, }; - let (temp_dir, mut validator, ctx) = + let (temp_dir, mut validator, ctx, _) = setup_validator_with_migration_tasks(&[task]); // Migration creates a funded hydra crank for the persisted task... diff --git a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs index 2dc43c22a..57510f310 100644 --- a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs @@ -17,7 +17,7 @@ use test_task_scheduler::{ #[test] fn test_reschedule_task() { - let (_temp_dir, mut validator, ctx) = setup_validator(); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task.rs b/test-integration/test-task-scheduler/tests/test_schedule_task.rs index 12a8211fd..8b5cdd9af 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task.rs @@ -17,7 +17,7 @@ use test_task_scheduler::{ #[test] fn test_schedule_task() { - let (_temp_dir, mut validator, ctx) = setup_validator(); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs b/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs index 55bb2f3f5..2f84a1763 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs @@ -12,7 +12,7 @@ use test_task_scheduler::{create_delegated_counter, setup_validator}; /// Test that a task can be scheduled and executed when it has multiple signers #[test] fn test_schedule_task_signed() { - let (_temp_dir, mut validator, ctx) = setup_validator(); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); expect!( diff --git a/test-integration/test-task-scheduler/tests/test_scheduled_commits.rs b/test-integration/test-task-scheduler/tests/test_scheduled_commits.rs deleted file mode 100644 index 0722724ae..000000000 --- a/test-integration/test-task-scheduler/tests/test_scheduled_commits.rs +++ /dev/null @@ -1,126 +0,0 @@ -use cleanass::assert; -use integration_test_tools::{expect, validator::cleanup}; -use program_flexi_counter::{ - instruction::create_schedule_task_ix, state::FlexiCounter, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{create_delegated_counter, setup_validator}; - -#[test] -#[ignore] -fn test_scheduled_commits() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - let commit_frequency_ms = 400; - create_delegated_counter(&ctx, &payer, &mut validator, commit_frequency_ms); - - eprintln!("Delegated counter: {:?}", counter_pda); - - // Schedule a task - let task_id = 5; - // Interval matching mainnet block time - let execution_interval_millis = 400; - let iterations = 3; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - false, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that the counter value is properly incremented on mainnet - const MAX_TRIES: u32 = 30; - let mut tries = 0; - let mut chain_values = vec![1, 2]; - let mut ephem_values = vec![1, 2]; - loop { - let chain_counter_account = expect!( - ctx.try_chain_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let chain_counter = expect!( - FlexiCounter::try_decode(&chain_counter_account.data), - validator - ); - - let ephem_counter_account = expect!( - ctx.try_ephem_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let ephem_counter = expect!( - FlexiCounter::try_decode(&ephem_counter_account.data), - validator - ); - - let chain_value_index = - chain_values.iter().position(|x| x == &chain_counter.count); - if let Some(index) = chain_value_index { - chain_values.remove(index); - } - - let ephem_value_index = - ephem_values.iter().position(|x| x == &ephem_counter.count); - if let Some(index) = ephem_value_index { - ephem_values.remove(index); - } - - if chain_values.is_empty() && ephem_values.is_empty() { - break; - } - - tries += 1; - if tries > MAX_TRIES { - assert!( - false, - cleanup(&mut validator), - "Missed some values: ephem_values: {:?}, chain_values: {:?}", - ephem_values, - chain_values - ); - } - - std::thread::sleep(std::time::Duration::from_millis(50)); - } - - cleanup(&mut validator); -} diff --git a/test-integration/test-task-scheduler/tests/test_undrained_faucet.rs b/test-integration/test-task-scheduler/tests/test_undrained_faucet.rs new file mode 100644 index 000000000..d50972641 --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_undrained_faucet.rs @@ -0,0 +1,163 @@ +use std::time::Duration; + +use hydra_api::instruction::ephemeral; +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::crank_pubkey; +use program_flexi_counter::{ + instruction::{create_schedule_task_ix, FlexiCounterInstruction}, + state::FlexiCounter, +}; +use solana_sdk::{ + message::{AccountMeta, Instruction}, + native_token::LAMPORTS_PER_SOL, + signature::Keypair, + signer::Signer, + transaction::Transaction, +}; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, +}; + +/// The validator identity must not be drained to pay for hydra cranks: a +/// dedicated, delegated faucet account pays instead. Scheduling a task creates +/// and funds a crank, and the validator identity's balance is left untouched. +#[test] +fn test_undrained_faucet() { + let (_temp_dir, mut validator, ctx, faucet_keypair) = setup_validator(); + + let payer = Keypair::new(); + let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + + expect!( + ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + + create_delegated_counter(&ctx, &payer, &mut validator, 0); + + // The default validator identity keypair (see consts::DEFAULT_VALIDATOR_KEYPAIR). + let faucet_keypair = expect!( + faucet_keypair + .ok_or_else(|| anyhow::anyhow!("Faucet keypair not found")), + validator + ); + let faucet_pk = faucet_keypair.pubkey(); + let faucet_balance_before = + expect!(ctx.fetch_ephem_account_balance(&faucet_pk), validator); + + let mut ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + + // Schedule a task + let task_id = 1; + let execution_interval_millis = 100; + let iterations = 3; + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_schedule_task_ix( + payer.pubkey(), + task_id, + execution_interval_millis, + iterations, + false, + false, + )], + Some(&payer.pubkey()), + &[&payer], + ephem_blockhash, + ), + &[&payer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + // The crank is created by hydra and funded for every iteration (paid for by + // the faucet, not the validator identity). + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + wait_for_hydra_crank( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); + + // The faucet executes the crank + for _ in 0..iterations { + ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[ + ephemeral::trigger(crank_pda, faucet_pk,), + Instruction::new_with_borsh( + program_flexi_counter::ID, + &FlexiCounterInstruction::AddUnsigned { count: 1 }, + vec![AccountMeta::new(counter_pda, false)], + ) + ], + Some(&faucet_pk), + &[&faucet_keypair], + ephem_blockhash, + ), + &[&faucet_keypair] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + expect!(ctx.wait_for_next_slot_ephem(), validator); + } + + // Close the crank + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[ephemeral::close(faucet_pk, crank_pda, faucet_pk)], + Some(&faucet_pk), + &[&faucet_keypair], + ephem_blockhash, + ), + &[&faucet_keypair] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + // The validator identity must not have been drained to pay for the crank. + let faucet_balance_after = + expect!(ctx.fetch_ephem_account_balance(&faucet_pk), validator); + assert!( + faucet_balance_after >= faucet_balance_before, + "faucet was drained paying for cranks: {} < {}", + faucet_balance_after, + faucet_balance_before + ); + + cleanup(&mut validator); +} diff --git a/test-integration/test-task-scheduler/tests/test_undrained_validator.rs b/test-integration/test-task-scheduler/tests/test_undrained_validator.rs deleted file mode 100644 index f4c5328b5..000000000 --- a/test-integration/test-task-scheduler/tests/test_undrained_validator.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::time::Duration; - -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::crank_pubkey; -use program_flexi_counter::{ - instruction::create_schedule_task_ix, state::FlexiCounter, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_hydra_crank, -}; - -/// The validator identity must not be drained to pay for hydra cranks: a -/// dedicated, delegated faucet account pays instead. Scheduling a task creates -/// and funds a crank, and the validator identity's balance is left untouched. -#[test] -fn test_undrained_validator() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let payer = Keypair::new(); - let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - - // The default validator identity keypair (see consts::DEFAULT_VALIDATOR_KEYPAIR). - let validator_keypair = Keypair::from_base58_string("9Vo7TbA5YfC5a33JhAi9Fb41usA6JwecHNRw3f9MzzHAM8hFnXTzL5DcEHwsAFjuUZ8vNQcJ4XziRFpMc3gTgBQ"); - let validator_identity = validator_keypair.pubkey(); - let validator_balance_before = expect!( - ctx.fetch_ephem_account_balance(&validator_identity), - validator - ); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 1; - let execution_interval_millis = 100; - let iterations = 3; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - false, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // The crank is created by hydra and funded for every iteration (paid for by - // the faucet, not the validator identity). - let crank_pda = crank_pubkey(&payer.pubkey(), task_id); - wait_for_hydra_crank( - &ctx, - &crank_pda, - Duration::from_secs(10), - &mut validator, - ); - - // The validator identity must not have been drained to pay for the crank. - let validator_balance_after = expect!( - ctx.fetch_ephem_account_balance(&validator_identity), - validator - ); - assert!( - validator_balance_after >= validator_balance_before, - "validator identity was drained paying for cranks: {} < {}", - validator_balance_after, - validator_balance_before - ); - - cleanup(&mut validator); -} From e6848ecbf254f86ba5c68b424f2001788ab80ef8 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 12:38:52 +0200 Subject: [PATCH 06/13] fix: do not pre delegate --- magicblock-api/src/magic_validator.rs | 27 +++++++++++++++---- .../test-task-scheduler/src/lib.rs | 12 --------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 6e753f30a..7b8cf8204 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -879,7 +879,9 @@ impl MagicValidator { Ok(()) } - fn spawn_primary_onchain_setup(&self) { + fn spawn_primary_onchain_setup( + &self, + ) -> Option> { let rpc_url = self.config.rpc_url().to_owned(); let identity = self.identity; let chain_operation_config = self.config.chain_operation.clone(); @@ -892,7 +894,7 @@ impl MagicValidator { if let Some(faucet_keypair) = self.faucet_keypair.as_ref().map(|k| k.insecure_clone()) { - tokio::spawn(async move { + Some(tokio::spawn(async move { let step_start = Instant::now(); let result = MagicValidator::ensure_validator_funded_on_chain( rpc_url.clone(), @@ -984,7 +986,9 @@ impl MagicValidator { step_start, ); } - }); + })) + } else { + None } } @@ -1036,6 +1040,8 @@ impl MagicValidator { } } + let mut primary_onchain_setup = None; + // Notify the scheduler that ledger replay and bank cleanup is complete. if self.is_standalone { self.mode_tx @@ -1047,7 +1053,7 @@ impl MagicValidator { )) })?; if matches!(self.config.lifecycle, LifecycleMode::Ephemeral) { - self.spawn_primary_onchain_setup(); + primary_onchain_setup = self.spawn_primary_onchain_setup(); } } else if let Some(replicator) = self.replication_service.take() { self.replication_handle.replace(replicator.spawn()); @@ -1055,7 +1061,7 @@ impl MagicValidator { &self.config.validator.replication_mode, ) && matches!(self.config.lifecycle, LifecycleMode::Ephemeral) { - self.spawn_primary_onchain_setup(); + primary_onchain_setup = self.spawn_primary_onchain_setup(); } } @@ -1105,6 +1111,17 @@ impl MagicValidator { }; if is_primary_mode { tokio::spawn(async move { + if let Some(primary_onchain_setup) = primary_onchain_setup { + if let Err(err) = primary_onchain_setup.await { + error!( + error = ?err, + "Primary on-chain setup task failed before task scheduler start" + ); + error!("Exiting process"); + std::process::exit(1); + } + } + let step_start = Instant::now(); let join_handle = match task_scheduler.start().await { Ok(join_handle) => join_handle, diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index 43a5e1a04..641bdce95 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -73,21 +73,9 @@ fn validator_config(temp_dir: PathBuf, faucet: &Keypair) -> ValidatorParams { fn airdrop_faucet(faucet: &Keypair) { let chain_ctx = IntegrationTestContext::try_new_chain_only() .expect("failed to connect to base chain to fund faucet"); - let validator = LoadedAccounts::with_delegation_program_test_authority() - .validator_authority(); - let payer = Keypair::new(); - chain_ctx - .airdrop_chain(&payer.pubkey(), LAMPORTS_PER_SOL) - .expect("failed to airdrop faucet delegation payer"); chain_ctx .airdrop_chain(&faucet.pubkey(), 100 * LAMPORTS_PER_SOL) .expect("failed to airdrop to task scheduler faucet"); - let (_, confirmed) = chain_ctx - .delegate_account_to_validator(&payer, faucet, Some(validator)) - .expect("failed to delegate task scheduler faucet"); - if !confirmed { - panic!("task scheduler faucet delegation not confirmed"); - } } fn start_validator( From 4a740cdd4f7b002d11ddad6ccd5f76b4b8751f8a Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 14:20:00 +0200 Subject: [PATCH 07/13] feat: ensure delegated to us --- magicblock-api/src/crank_faucet.rs | 126 ++++++++++++++++++++ magicblock-api/src/fund_account.rs | 72 +----------- magicblock-api/src/lib.rs | 1 + magicblock-api/src/magic_validator.rs | 163 ++++++++++++-------------- 4 files changed, 200 insertions(+), 162 deletions(-) create mode 100644 magicblock-api/src/crank_faucet.rs diff --git a/magicblock-api/src/crank_faucet.rs b/magicblock-api/src/crank_faucet.rs new file mode 100644 index 000000000..e4f87eee6 --- /dev/null +++ b/magicblock-api/src/crank_faucet.rs @@ -0,0 +1,126 @@ +use dlp_api::state::DelegationRecord; +use magicblock_program::validator::validator_authority; +use solana_commitment_config::CommitmentConfig; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_signer::Signer; +use tracing::*; + +use crate::errors::{ApiError, ApiResult}; + +pub(crate) fn delegation_record_authority( + data: &[u8], + delegation_record_pubkey: Pubkey, +) -> Result { + let delegation_record_size = DelegationRecord::size_with_discriminator(); + if data.len() < delegation_record_size { + return Err(format!( + "delegation record {delegation_record_pubkey} is too small" + )); + } + + DelegationRecord::try_from_bytes_with_discriminator( + &data[..delegation_record_size], + ) + .copied() + .map(|record| record.authority) + .map_err(|err| { + format!( + "failed to decode delegation record {delegation_record_pubkey}: {err:?}" + ) + }) +} + +/// Delegates the task scheduler faucet to this validator on the base chain if +/// it is not already delegated. Delegating gives the faucet real (base-chain +/// backed) lamports usable inside the ephemeral rollup, unlike the validator +/// identity. The faucet must already be funded — the validator does not fund +/// it (operators and integration tests airdrop to it). +pub(crate) async fn ensure_faucet_delegated_on_chain( + rpc_url: String, + faucet: &Keypair, +) -> ApiResult<()> { + let validator_keypair = validator_authority(); + let validator_pubkey = validator_keypair.pubkey(); + let faucet_pubkey = faucet.pubkey(); + let delegation_record_pubkey = + dlp_api::pda::delegation_record_pda_from_delegated_account( + &faucet_pubkey, + ); + + let rpc = + RpcClient::new_with_commitment(rpc_url, CommitmentConfig::confirmed()); + + let accounts = rpc + .get_multiple_accounts(&[faucet_pubkey, delegation_record_pubkey]) + .await + .map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + + if matches!( + accounts[0].as_ref(), + Some(account) if account.owner == dlp_api::id() + ) { + let Some(delegation_record) = accounts[1].as_ref() else { + return Err(ApiError::FailedToDelegateFaucet( + faucet_pubkey, + format!( + "faucet is owned by the delegation program but missing delegation record {delegation_record_pubkey}" + ), + )); + }; + let authority = delegation_record_authority( + &delegation_record.data, + delegation_record_pubkey, + ) + .map_err(|err| ApiError::FailedToDelegateFaucet(faucet_pubkey, err))?; + if authority == validator_pubkey { + info!(%faucet_pubkey, %validator_pubkey, "Crank faucet already delegated, skipping"); + return Ok(()); + } + return Err(ApiError::FailedToDelegateFaucet( + faucet_pubkey, + format!( + "faucet already delegated to validator {authority}, expected {validator_pubkey}" + ), + )); + } + + info!(%faucet_pubkey, "Delegating crank faucet"); + // Hand the on-curve faucet to the delegation program and delegate it to + // this validator. This makes it a writable, base-chain-backed account + // inside the ephemeral rollup that can sponsor crank creation (mirrors the + // on-curve delegation flow used in tests). The faucet must already be + // funded; the validator does not fund it. + let assign_ix = solana_system_interface::instruction::assign( + &faucet_pubkey, + &dlp_api::id(), + ); + let delegate_ix = dlp_api::instruction_builder::delegate( + validator_pubkey, + faucet_pubkey, + None, + dlp_api::args::DelegateArgs { + commit_frequency_ms: u32::MAX, + seeds: vec![], + validator: Some(validator_pubkey), + }, + ); + + let blockhash = rpc.get_latest_blockhash().await.map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + let tx = solana_transaction::Transaction::new_signed_with_payer( + &[assign_ix, delegate_ix], + Some(&validator_pubkey), + &[&validator_keypair, faucet], + blockhash, + ); + rpc.send_and_confirm_transaction(&tx).await.map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + info!(%faucet_pubkey, "Crank faucet delegated"); + Ok(()) +} diff --git a/magicblock-api/src/fund_account.rs b/magicblock-api/src/fund_account.rs index 6399e569b..2c5171f95 100644 --- a/magicblock-api/src/fund_account.rs +++ b/magicblock-api/src/fund_account.rs @@ -1,16 +1,9 @@ use magicblock_accounts_db::{traits::AccountsBank, AccountsDb}; use magicblock_magic_program_api as magic_program; -use magicblock_program::{validator::validator_authority, MagicContext}; +use magicblock_program::MagicContext; use solana_account::{AccountSharedData, WritableAccount}; -use solana_commitment_config::CommitmentConfig; -use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_rent::Rent; -use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_signer::Signer; -use tracing::*; - -use crate::errors::{ApiError, ApiResult}; pub(crate) fn fund_account( accountsdb: &AccountsDb, @@ -73,66 +66,3 @@ pub(crate) fn fund_ephemeral_vault(accountsdb: &AccountsDb) { let _ = accountsdb .insert_account(&magic_program::EPHEMERAL_VAULT_PUBKEY, &vault); } - -/// Delegates the task scheduler faucet to this validator on the base chain if -/// it is not already delegated. Delegating gives the faucet real (base-chain -/// backed) lamports usable inside the ephemeral rollup, unlike the validator -/// identity. The faucet must already be funded — the validator does not fund -/// it (operators and integration tests airdrop to it). -pub(crate) async fn ensure_faucet_delegated_on_chain( - rpc_url: String, - faucet: &Keypair, -) -> ApiResult<()> { - let validator_keypair = validator_authority(); - let validator_pubkey = validator_keypair.pubkey(); - let faucet_pubkey = faucet.pubkey(); - - let rpc = - RpcClient::new_with_commitment(rpc_url, CommitmentConfig::confirmed()); - - // A faucet already owned by the delegation program is delegated. - let already_delegated = matches!( - rpc.get_account(&faucet_pubkey).await, - Ok(account) if account.owner == dlp_api::id() - ); - if already_delegated { - info!(%faucet_pubkey, "Crank faucet already delegated, skipping"); - return Ok(()); - } - - info!(%faucet_pubkey, "Delegating crank faucet"); - // Hand the on-curve faucet to the delegation program and delegate it to - // this validator. This makes it a writable, base-chain-backed account - // inside the ephemeral rollup that can sponsor crank creation (mirrors the - // on-curve delegation flow used in tests). The faucet must already be - // funded; the validator does not fund it. - let assign_ix = solana_system_interface::instruction::assign( - &faucet_pubkey, - &dlp_api::id(), - ); - let delegate_ix = dlp_api::instruction_builder::delegate( - validator_pubkey, - faucet_pubkey, - None, - dlp_api::args::DelegateArgs { - commit_frequency_ms: u32::MAX, - seeds: vec![], - validator: Some(validator_pubkey), - }, - ); - - let blockhash = rpc.get_latest_blockhash().await.map_err(|err| { - ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) - })?; - let tx = solana_transaction::Transaction::new_signed_with_payer( - &[assign_ix, delegate_ix], - Some(&validator_pubkey), - &[&validator_keypair, faucet], - blockhash, - ); - rpc.send_and_confirm_transaction(&tx).await.map_err(|err| { - ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) - })?; - info!(%faucet_pubkey, "Crank faucet delegated"); - Ok(()) -} diff --git a/magicblock-api/src/lib.rs b/magicblock-api/src/lib.rs index 6d0db7546..4b17fe0fa 100644 --- a/magicblock-api/src/lib.rs +++ b/magicblock-api/src/lib.rs @@ -1,3 +1,4 @@ +mod crank_faucet; pub mod domain_registry_manager; pub mod errors; mod fund_account; diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 7b8cf8204..7251e877e 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -82,11 +82,11 @@ use tokio_util::sync::CancellationToken; use tracing::*; use crate::{ + crank_faucet::ensure_faucet_delegated_on_chain, domain_registry_manager::DomainRegistryManager, errors::{ApiError, ApiResult}, fund_account::{ - ensure_faucet_delegated_on_chain, fund_ephemeral_vault, - fund_magic_context, init_validator_identity, + fund_ephemeral_vault, fund_magic_context, init_validator_identity, }, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, ledger::{ @@ -879,58 +879,55 @@ impl MagicValidator { Ok(()) } - fn spawn_primary_onchain_setup( - &self, - ) -> Option> { + fn spawn_primary_onchain_setup(&self) { let rpc_url = self.config.rpc_url().to_owned(); let identity = self.identity; let chain_operation_config = self.config.chain_operation.clone(); let block_time_ms = self.config.ledger.block_time_ms(); let base_fee = self.config.validator.basefee; + let faucet_keypair = + self.faucet_keypair.as_ref().map(|k| k.insecure_clone()); // Ephemeral mode does a non-blocking startup balance check. // Intentionally fire-and-forget: the task itself exits the process on failure. - // Skipped if no faucet keypair is configured. - if let Some(faucet_keypair) = - self.faucet_keypair.as_ref().map(|k| k.insecure_clone()) - { - Some(tokio::spawn(async move { - let step_start = Instant::now(); - let result = MagicValidator::ensure_validator_funded_on_chain( - rpc_url.clone(), - identity, - ) - .await; - log_timing( - "startup_background", - "ensure_funded_on_chain", - step_start, - ); - if let Err(err) = result { - error!(error = ?err, "Validator balance check failed"); - error!("Exiting process"); - std::process::exit(1); - } + tokio::spawn(async move { + let step_start = Instant::now(); + let result = MagicValidator::ensure_validator_funded_on_chain( + rpc_url.clone(), + identity, + ) + .await; + log_timing( + "startup_background", + "ensure_funded_on_chain", + step_start, + ); + if let Err(err) = result { + error!(error = ?err, "Validator balance check failed"); + error!("Exiting process"); + std::process::exit(1); + } - let step_start = Instant::now(); - let result = MagicValidator::ensure_magic_fee_vault_on_chain( - rpc_url.clone(), - ) - .await; - log_timing( - "startup_background", - "ensure_magic_fee_vault_on_chain", - step_start, - ); + let step_start = Instant::now(); + let result = MagicValidator::ensure_magic_fee_vault_on_chain( + rpc_url.clone(), + ) + .await; + log_timing( + "startup_background", + "ensure_magic_fee_vault_on_chain", + step_start, + ); - // Without magic fee vault being properly set up - // transactions scheduling commits will fail - if let Err(err) = result { - error!(error = ?err, "Magic fee vault setup failed"); - error!("Exiting process"); - std::process::exit(1); - } + // Without magic fee vault being properly set up + // transactions scheduling commits will fail + if let Err(err) = result { + error!(error = ?err, "Magic fee vault setup failed"); + error!("Exiting process"); + std::process::exit(1); + } + if let Some(faucet_keypair) = faucet_keypair { let step_start = Instant::now(); let result = ensure_faucet_delegated_on_chain( rpc_url.clone(), @@ -949,47 +946,44 @@ impl MagicValidator { error!("Exiting process"); std::process::exit(1); } + } - if let Some(ref config) = chain_operation_config { - if !config.claim_fees_frequency.is_zero() { - let step_start = Instant::now(); - if let Err(err) = claim_fees(rpc_url.clone()).await { - error!( - error = ?err, - "Failed to claim validator fees on startup" - ); - } - log_timing( - "startup_background", - "claim_fees_on_startup", - step_start, - ); - } - } - if let Some(ref config) = chain_operation_config { + if let Some(ref config) = chain_operation_config { + if !config.claim_fees_frequency.is_zero() { let step_start = Instant::now(); - if let Err(error) = - MagicValidator::register_validator_on_chain( - &rpc_url, - config, - block_time_ms, - base_fee, - ) - .await - { - error!(%error, "Validator registration failed, exitting"); - std::process::exit(1); + if let Err(err) = claim_fees(rpc_url.clone()).await { + error!( + error = ?err, + "Failed to claim validator fees on startup" + ); } log_timing( "startup_background", - "register_validator_on_chain", + "claim_fees_on_startup", step_start, ); } - })) - } else { - None - } + } + if let Some(ref config) = chain_operation_config { + let step_start = Instant::now(); + if let Err(error) = MagicValidator::register_validator_on_chain( + &rpc_url, + config, + block_time_ms, + base_fee, + ) + .await + { + error!(%error, "Validator registration failed, exitting"); + std::process::exit(1); + } + log_timing( + "startup_background", + "register_validator_on_chain", + step_start, + ); + } + }); } #[instrument(skip(self))] @@ -1040,8 +1034,6 @@ impl MagicValidator { } } - let mut primary_onchain_setup = None; - // Notify the scheduler that ledger replay and bank cleanup is complete. if self.is_standalone { self.mode_tx @@ -1053,7 +1045,7 @@ impl MagicValidator { )) })?; if matches!(self.config.lifecycle, LifecycleMode::Ephemeral) { - primary_onchain_setup = self.spawn_primary_onchain_setup(); + self.spawn_primary_onchain_setup(); } } else if let Some(replicator) = self.replication_service.take() { self.replication_handle.replace(replicator.spawn()); @@ -1061,7 +1053,7 @@ impl MagicValidator { &self.config.validator.replication_mode, ) && matches!(self.config.lifecycle, LifecycleMode::Ephemeral) { - primary_onchain_setup = self.spawn_primary_onchain_setup(); + self.spawn_primary_onchain_setup(); } } @@ -1111,17 +1103,6 @@ impl MagicValidator { }; if is_primary_mode { tokio::spawn(async move { - if let Some(primary_onchain_setup) = primary_onchain_setup { - if let Err(err) = primary_onchain_setup.await { - error!( - error = ?err, - "Primary on-chain setup task failed before task scheduler start" - ); - error!("Exiting process"); - std::process::exit(1); - } - } - let step_start = Instant::now(); let join_handle = match task_scheduler.start().await { Ok(join_handle) => join_handle, From 22a74652214acb79862c41f795291488e02d7c0b Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 14:53:27 +0200 Subject: [PATCH 08/13] feat: use last execution timestamp --- .../crates/magicblock-task-scheduler.md | 11 ++- magicblock-task-scheduler/src/db.rs | 13 ++-- magicblock-task-scheduler/src/service.rs | 67 +++++++++++++++++-- .../tests/test_migration.rs | 1 + 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/.agents/context/crates/magicblock-task-scheduler.md b/.agents/context/crates/magicblock-task-scheduler.md index 1b9f51a96..7738cf123 100644 --- a/.agents/context/crates/magicblock-task-scheduler.md +++ b/.agents/context/crates/magicblock-task-scheduler.md @@ -74,13 +74,10 @@ Main consumers: Important API: - `SchedulerDatabase::path(path)` returns `path.join("task_scheduler.sqlite")`; -- `new(path)` opens SQLite, enables WAL, `synchronous=NORMAL`, `busy_timeout=5000`, and a larger page cache, then creates `tasks`, `failed_scheduling`, and `failed_tasks` tables if missing; -- `insert_task`, `get_task`, `get_tasks`, `get_task_ids`, `remove_task`, and `unschedule_task` manage scheduled task rows; -- `insert_failed_scheduling`, `insert_failed_task`, `get_failed_schedulings`, and `get_failed_tasks` manage diagnostic failure records; -- `apply_crank_batch_completion(...)` atomically applies one batch of success updates, success removals, failed moves, and retry checks using optimistic `tasks.updated_at` tokens; -- `delete_failed_records_older_than(cutoff)` removes old rows from both failure tables in one transaction. +- `new(path)` opens SQLite, enables WAL, `synchronous=NORMAL`, and `busy_timeout=5000`, then creates the legacy-compatible `tasks` table if missing; +- `insert_task`, `get_tasks`, `get_task_ids`, and `remove_task` manage the migration-only task rows used to seed Hydra cranks. -`DbTask` is the persisted runtime task shape. It stores task IDs and timestamps as `i64`, serializes `Vec` with `bincode`, stores authority as a stringified `Pubkey`, and uses `executions_left`, `last_execution_millis`, and `updated_at` to drive future scheduling. +`DbTask` is the legacy persisted task shape used during Hydra migration. It stores task IDs and timestamps as `i64`, serializes `Vec` with `bincode`, stores authority as a stringified `Pubkey`, and carries `last_execution_millis` so migration can preserve the legacy cadence when choosing the Hydra `start_slot`. ### `TaskSchedulerService` @@ -119,7 +116,7 @@ MagicValidator::start -> if Replica: do not start task scheduler ``` -On `start()`, `load_persisted_tasks` reads all rows from `tasks`, removes invalid rows (`execution_interval_millis <= 0`, `>= u32::MAX`, or `executions_left <= 0`), and inserts valid rows into the delay queue. Restarted tasks are delayed until the later of their next scheduled time and two slot intervals. That two-slot minimum avoids cranking before the validator has produced a fresh blockhash after restart. +On `start()`, `migrate_persisted_tasks` reads all legacy rows from `tasks`, removes invalid rows (`execution_interval_millis <= 0`, `>= u32::MAX`, or `executions_left <= 0`), waits for a usable blockhash and delegated/funded faucet, creates each valid Hydra crank, and removes each row whether migration succeeds or fails so the legacy database empties. If a legacy row has `last_execution_millis > 0`, migration preserves its cadence by converting the remaining wall-clock delay until `last_execution_millis + execution_interval_millis` into slots and adding those slots to the current block snapshot slot for Hydra `start_slot`; overdue or never-run tasks start at the current slot. ### Schedule request flow diff --git a/magicblock-task-scheduler/src/db.rs b/magicblock-task-scheduler/src/db.rs index e45abc6da..26575ba00 100644 --- a/magicblock-task-scheduler/src/db.rs +++ b/magicblock-task-scheduler/src/db.rs @@ -26,6 +26,8 @@ pub struct DbTask { pub execution_interval_millis: i64, /// Number of times this task still needs to be executed. pub executions_left: i64, + /// Legacy scheduler timestamp of the last successful execution. + pub last_execution_millis: i64, } impl From for DbTask { @@ -36,6 +38,7 @@ impl From for DbTask { authority: task.authority, execution_interval_millis: task.execution_interval_millis, executions_left: task.iterations, + last_execution_millis: 0, } } } @@ -62,8 +65,8 @@ impl SchedulerDatabase { )?; // Mirrors the legacy schema so existing databases can be read. The - // timestamp columns are retained for compatibility but are not used by - // migration. + // last execution timestamp is used to preserve cadence when migrating + // legacy rows onto hydra cranks. conn.execute( "CREATE TABLE IF NOT EXISTS tasks ( id INTEGER PRIMARY KEY, @@ -92,13 +95,14 @@ impl SchedulerDatabase { self.conn.lock().await.execute( "INSERT OR REPLACE INTO tasks (id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, 0, ?, ?)", + VALUES (?, ?, ?, ?, ?, ?, ?, ?)", params![ task.id, instructions_bin, authority_str, task.execution_interval_millis, task.executions_left, + task.last_execution_millis, now, now, ], @@ -110,7 +114,7 @@ impl SchedulerDatabase { pub async fn get_tasks(&self) -> TaskSchedulerResult> { let db = self.conn.lock().await; let mut stmt = db.prepare( - "SELECT id, instructions, authority, execution_interval_millis, executions_left + "SELECT id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis FROM tasks", )?; @@ -137,6 +141,7 @@ impl SchedulerDatabase { authority, execution_interval_millis: row.get(3)?, executions_left: row.get(4)?, + last_execution_millis: row.get(5)?, }) })?; diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index 85d41c689..d3295e770 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -1,5 +1,6 @@ use std::{path::Path, sync::Arc, time::Duration as StdDuration}; +use chrono::Utc; use hydra_api::{ consts::{CRANKER_REWARD, CRANK_HEADER_SIZE, SERIALIZED_META_SIZE}, ephemeral::ID as EPHEMERAL_PROGRAM_ID, @@ -176,6 +177,7 @@ impl TaskSchedulerService { task.execution_interval_millis, task.executions_left, &task.instructions, + Some(task.last_execution_millis), ) .await { @@ -267,6 +269,7 @@ impl TaskSchedulerService { interval_millis, task.iterations, &task.instructions, + None, ) .await?; debug!("Created hydra crank for task {}", task.id); @@ -298,16 +301,15 @@ impl TaskSchedulerService { interval_millis: i64, iterations: i64, instructions: &[Instruction], + last_execution_millis: Option, ) -> TaskSchedulerResult<()> { - let crank = crank_pubkey(authority, task_id); - self.send_create( authority, task_id, interval_millis, iterations, instructions, - crank, + last_execution_millis, ) .await?; Ok(()) @@ -330,12 +332,23 @@ impl TaskSchedulerService { interval_millis: i64, iterations: i64, instructions: &[Instruction], - crank: Pubkey, + last_execution_millis: Option, ) -> TaskSchedulerResult { + let crank = crank_pubkey(authority, task_id); let crank_exists = self.crank_exists(&crank).await; let snapshot = self.block.load(); - let start_slot = snapshot.slot; + let start_slot = last_execution_millis + .map(|last_execution_millis| { + legacy_start_slot( + last_execution_millis, + interval_millis, + Utc::now().timestamp_millis(), + snapshot.slot, + self.slot_interval, + ) + }) + .unwrap_or(snapshot.slot); let blockhash = snapshot.blockhash; let interval_slots = @@ -506,6 +519,29 @@ fn is_valid_task_interval(interval: i64) -> bool { interval > 0 && interval < u32::MAX as i64 } +/// Computes the hydra start slot for a task migrated from the legacy scheduler. +/// If the legacy task is overdue or has never run, it remains due immediately. +fn legacy_start_slot( + last_execution_millis: i64, + interval_millis: i64, + current_millis: i64, + current_slot: u64, + slot_interval: StdDuration, +) -> u64 { + if last_execution_millis <= 0 { + return current_slot; + } + + let next_execution_millis = + last_execution_millis.saturating_add(interval_millis.max(0)); + let remaining_millis = next_execution_millis.saturating_sub(current_millis); + if remaining_millis == 0 { + return current_slot; + } + + current_slot.saturating_add(interval_slots(remaining_millis, slot_interval)) +} + /// Converts a millisecond execution interval into a slot count (rounding up, /// with a one-slot minimum) for hydra's slot-based cadence. fn interval_slots(interval_millis: i64, slot_interval: StdDuration) -> u64 { @@ -551,6 +587,25 @@ mod tests { assert_eq!(interval_slots(100, slot), 2); } + #[serial] + #[test] + fn test_legacy_start_slot_preserves_remaining_delay() { + let slot_interval = StdDuration::from_millis(1_000); + + assert_eq!( + legacy_start_slot(10_000, 30_000, 25_000, 100, slot_interval), + 115 + ); + assert_eq!( + legacy_start_slot(10_000, 30_000, 40_000, 100, slot_interval), + 100 + ); + assert_eq!( + legacy_start_slot(0, 30_000, 25_000, 100, slot_interval), + 100 + ); + } + #[serial] #[test] fn test_crank_pubkey_namespaced_by_authority_and_id() { @@ -578,6 +633,7 @@ mod tests { authority: Pubkey::new_unique(), execution_interval_millis: u32::MAX as i64, executions_left: 1, + last_execution_millis: 0, instructions: vec![], }) .await @@ -588,6 +644,7 @@ mod tests { authority: Pubkey::new_unique(), execution_interval_millis: 50, executions_left: 0, + last_execution_millis: 0, instructions: vec![], }) .await diff --git a/test-integration/test-task-scheduler/tests/test_migration.rs b/test-integration/test-task-scheduler/tests/test_migration.rs index a71441c1f..0dab1b88a 100644 --- a/test-integration/test-task-scheduler/tests/test_migration.rs +++ b/test-integration/test-task-scheduler/tests/test_migration.rs @@ -35,6 +35,7 @@ fn test_migration_reschedules_tasks_and_empties_db() { authority, execution_interval_millis: 100, executions_left: iterations, + last_execution_millis: 0, instructions, }; From e8398af5dfbd3f91d3b2617d510ef0b12197d4eb Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 14:54:49 +0200 Subject: [PATCH 09/13] feat: keep invalid tasks in db --- magicblock-task-scheduler/src/service.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index d3295e770..f3b6f5181 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -182,7 +182,8 @@ impl TaskSchedulerService { .await { Ok(()) => { - info!("Migration: created crank for task {}", task.id) + self.db.remove_task(task.id).await?; + debug!("Migration: created crank for task {}", task.id) } Err(e) => { warn!( @@ -191,10 +192,9 @@ impl TaskSchedulerService { ) } } - self.db.remove_task(task.id).await?; } - info!("Task migration complete; database emptied"); + info!("Task migration complete"); Ok(()) } From 1c4640389304347bdf204f495e10924ac73dec50 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 15:00:43 +0200 Subject: [PATCH 10/13] fix: prevent 0 iteration --- magicblock-task-scheduler/src/service.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index f3b6f5181..90328d456 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -303,6 +303,9 @@ impl TaskSchedulerService { instructions: &[Instruction], last_execution_millis: Option, ) -> TaskSchedulerResult<()> { + if iterations <= 0 { + return Ok(()); + } self.send_create( authority, task_id, @@ -353,7 +356,7 @@ impl TaskSchedulerService { let interval_slots = interval_slots(interval_millis, self.slot_interval); - let iterations = iterations.max(0) as u64; + let iterations = iterations as u64; let faucet_pubkey = self.faucet.pubkey(); let create_ix = build_create_ix( From b7a862e0a22f5542d314c5963c1daa9f1fc94aec Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 15:09:33 +0200 Subject: [PATCH 11/13] feat: activate bincode feature --- magicblock-api/Cargo.toml | 2 +- programs/magicblock/src/schedule_task/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/magicblock-api/Cargo.toml b/magicblock-api/Cargo.toml index d18b437a9..bf5cd5a2d 100644 --- a/magicblock-api/Cargo.toml +++ b/magicblock-api/Cargo.toml @@ -53,7 +53,7 @@ solana-sdk-ids = { workspace = true } solana-sha256-hasher = { workspace = true } solana-signature = { workspace = true } solana-signer = { workspace = true } -solana-system-interface = { workspace = true } +solana-system-interface = { workspace = true, features = ["bincode"] } solana-system-program = { workspace = true } solana-sysvar = { workspace = true } solana-transaction = { workspace = true } diff --git a/programs/magicblock/src/schedule_task/mod.rs b/programs/magicblock/src/schedule_task/mod.rs index b91df93d9..75f85bcc5 100644 --- a/programs/magicblock/src/schedule_task/mod.rs +++ b/programs/magicblock/src/schedule_task/mod.rs @@ -22,7 +22,7 @@ pub(crate) fn validate_cranks_instructions( if account.is_signer { ic_msg!( invoke_context, - "Crank ERR: only the crank signer PDA can be a signer in cranks (invalid signer: '{}')", + "Crank ERR: scheduled crank instructions cannot include signer accounts (invalid signer: '{}')", account.pubkey, ); return Err(InstructionError::MissingRequiredSignature); From 9cc46711eae1b898eb6b7457920df3c92ae41d96 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 15:10:08 +0200 Subject: [PATCH 12/13] fix: clarify error --- magicblock-api/src/errors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 502dd6f4d..c7a36c5bf 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -36,7 +36,7 @@ pub enum ApiError { #[error("Failed to delegate magic fee vault for validator '{0}': {1}")] FailedToDelegateMagicFeeVault(Pubkey, String), - #[error("Failed to fund and delegate task scheduler faucet '{0}': {1}")] + #[error("Failed to delegate task scheduler faucet '{0}': {1}")] FailedToDelegateFaucet(Pubkey, String), #[error("CommittorServiceError")] From 12bcb81d3269a45bc7ede19aa6cf30f7298dfe44 Mon Sep 17 00:00:00 2001 From: Dodecahedr0x Date: Wed, 1 Jul 2026 15:19:40 +0200 Subject: [PATCH 13/13] fix: update lockfile --- Cargo.lock | 2 +- test-integration/Cargo.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a4d46da98..118f993ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3477,7 +3477,7 @@ name = "magicblock-account-cloner" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "magicblock-chainlink", "magicblock-core", "magicblock-ledger", diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 1a1545040..c2a650c15 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -3544,7 +3544,7 @@ name = "magicblock-account-cloner" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "magicblock-chainlink", "magicblock-core", "magicblock-ledger",