diff --git a/.agents/context/crates/magicblock-task-scheduler.md b/.agents/context/crates/magicblock-task-scheduler.md index 1b9f51a96..7738cf123 100644 --- a/.agents/context/crates/magicblock-task-scheduler.md +++ b/.agents/context/crates/magicblock-task-scheduler.md @@ -74,13 +74,10 @@ Main consumers: Important API: - `SchedulerDatabase::path(path)` returns `path.join("task_scheduler.sqlite")`; -- `new(path)` opens SQLite, enables WAL, `synchronous=NORMAL`, `busy_timeout=5000`, and a larger page cache, then creates `tasks`, `failed_scheduling`, and `failed_tasks` tables if missing; -- `insert_task`, `get_task`, `get_tasks`, `get_task_ids`, `remove_task`, and `unschedule_task` manage scheduled task rows; -- `insert_failed_scheduling`, `insert_failed_task`, `get_failed_schedulings`, and `get_failed_tasks` manage diagnostic failure records; -- `apply_crank_batch_completion(...)` atomically applies one batch of success updates, success removals, failed moves, and retry checks using optimistic `tasks.updated_at` tokens; -- `delete_failed_records_older_than(cutoff)` removes old rows from both failure tables in one transaction. +- `new(path)` opens SQLite, enables WAL, `synchronous=NORMAL`, and `busy_timeout=5000`, then creates the legacy-compatible `tasks` table if missing; +- `insert_task`, `get_tasks`, `get_task_ids`, and `remove_task` manage the migration-only task rows used to seed Hydra cranks. -`DbTask` is the persisted runtime task shape. It stores task IDs and timestamps as `i64`, serializes `Vec` with `bincode`, stores authority as a stringified `Pubkey`, and uses `executions_left`, `last_execution_millis`, and `updated_at` to drive future scheduling. +`DbTask` is the legacy persisted task shape used during Hydra migration. It stores task IDs and timestamps as `i64`, serializes `Vec` with `bincode`, stores authority as a stringified `Pubkey`, and carries `last_execution_millis` so migration can preserve the legacy cadence when choosing the Hydra `start_slot`. ### `TaskSchedulerService` @@ -119,7 +116,7 @@ MagicValidator::start -> if Replica: do not start task scheduler ``` -On `start()`, `load_persisted_tasks` reads all rows from `tasks`, removes invalid rows (`execution_interval_millis <= 0`, `>= u32::MAX`, or `executions_left <= 0`), and inserts valid rows into the delay queue. Restarted tasks are delayed until the later of their next scheduled time and two slot intervals. That two-slot minimum avoids cranking before the validator has produced a fresh blockhash after restart. +On `start()`, `migrate_persisted_tasks` reads all legacy rows from `tasks`, removes invalid rows (`execution_interval_millis <= 0`, `>= u32::MAX`, or `executions_left <= 0`), waits for a usable blockhash and delegated/funded faucet, creates each valid Hydra crank, and removes each row whether migration succeeds or fails so the legacy database empties. If a legacy row has `last_execution_millis > 0`, migration preserves its cadence by converting the remaining wall-clock delay until `last_execution_millis + execution_interval_millis` into slots and adding those slots to the current block snapshot slot for Hydra `start_slot`; overdue or never-run tasks start at the current slot. ### Schedule request flow diff --git a/Cargo.lock b/Cargo.lock index af6285d2f..118f993ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a48ac39333d364f102b03ef00b59cd5b1f878ccb3c5f0f67df20f44824d51f" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "digest 0.10.7", "ed25519-dalek 1.0.1", "libsecp256k1", @@ -144,7 +144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84debd4abe0cbab5a6aac2ee50e3969ef0e0961f7dff7e8f96bda0be7998bca2" dependencies = [ "agave-bls12-381", - "bincode", + "bincode 1.3.3", "libsecp256k1", "num-traits", "solana-account 3.4.0", @@ -768,6 +768,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.69.5" @@ -1912,6 +1931,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ephemeral-rollups-pinocchio" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a09b61c47e5b4eeb7add34e9d43d836784a66d78fa704606bebaa453bef73e21" +dependencies = [ + "bincode 2.0.1", + "pinocchio 0.10.2", + "pinocchio-pubkey", + "pinocchio-system", + "solana-address 2.5.0", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2391,7 +2423,7 @@ dependencies = [ name = "guinea" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "magicblock-magic-program-api", "serde", "solana-program 3.0.0", @@ -2642,6 +2674,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "hydra-api" +version = "0.1.1" +source = "git+https://github.com/magicblock-labs/hydra.git?rev=449532ee0ee792e9a3809b16fd9d289c2024341a#449532ee0ee792e9a3809b16fd9d289c2024341a" +dependencies = [ + "ephemeral-rollups-pinocchio", + "pinocchio 0.10.2", + "solana-address 2.5.0", + "solana-instruction 3.3.0", + "solana-pubkey 4.1.0", +] + [[package]] name = "hyper" version = "1.10.1" @@ -3433,7 +3477,7 @@ name = "magicblock-account-cloner" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "magicblock-chainlink", "magicblock-core", "magicblock-ledger", @@ -3516,7 +3560,7 @@ dependencies = [ "agave-geyser-plugin-interface", "arc-swap", "base64 0.21.7", - "bincode", + "bincode 1.3.3", "bs58", "fastwebsockets", "futures", @@ -3611,6 +3655,7 @@ dependencies = [ "solana-sha256-hasher 3.1.0", "solana-signature 3.3.0", "solana-signer", + "solana-system-interface 3.1.0", "solana-system-program", "solana-sysvar 3.1.1", "solana-transaction", @@ -3630,7 +3675,7 @@ dependencies = [ "arc-swap", "assert_matches", "async-trait", - "bincode", + "bincode 1.3.3", "borsh", "futures-util", "helius-laserstream", @@ -3700,7 +3745,7 @@ name = "magicblock-committor-service" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "borsh", "futures-util", "lru 0.16.4", @@ -3766,7 +3811,7 @@ dependencies = [ name = "magicblock-core" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bytes", "console-subscriber", "flume", @@ -3797,7 +3842,7 @@ name = "magicblock-delegation-program-api" version = "0.3.0" source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b#25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh", "bytemuck", "const-crypto", @@ -3827,7 +3872,7 @@ name = "magicblock-ledger" version = "0.13.2" dependencies = [ "arc-swap", - "bincode", + "bincode 1.3.3", "byteorder", "fs_extra", "libc", @@ -3868,7 +3913,7 @@ dependencies = [ name = "magicblock-magic-program-api" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "const-crypto", "serde", "solana-program 2.3.0", @@ -3899,7 +3944,7 @@ dependencies = [ "agave-feature-set", "agave-precompiles", "agave-syscalls", - "bincode", + "bincode 1.3.3", "blake3", "guinea", "magicblock-accounts-db", @@ -3950,7 +3995,7 @@ name = "magicblock-program" version = "0.13.2" dependencies = [ "assert_matches", - "bincode", + "bincode 1.3.3", "lazy_static", "magicblock-chainlink", "magicblock-core", @@ -3991,7 +4036,7 @@ name = "magicblock-replicator" version = "0.13.2" dependencies = [ "async-nats", - "bincode", + "bincode 1.3.3", "bytes", "futures", "magicblock-accounts-db", @@ -4043,7 +4088,7 @@ dependencies = [ name = "magicblock-services" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "futures-util", "magicblock-core", "magicblock-magic-program-api", @@ -4091,21 +4136,27 @@ dependencies = [ name = "magicblock-task-scheduler" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "futures-util", + "hydra-api", "magicblock-config", "magicblock-core", "magicblock-ledger", "magicblock-program", "rusqlite", "serial_test", + "solana-account 3.4.0", "solana-instruction 3.3.0", + "solana-keypair", "solana-message 3.1.0", "solana-pubkey 4.1.0", "solana-rpc-client", "solana-rpc-client-api", + "solana-sha256-hasher 3.1.0", "solana-signature 3.3.0", + "solana-signer", + "solana-system-interface 3.1.0", "solana-transaction", "solana-transaction-error 3.1.0", "thiserror 2.0.18", @@ -6266,9 +6317,9 @@ dependencies = [ [[package]] name = "solana-account" version = "3.4.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "serde_bytes", @@ -6288,7 +6339,7 @@ checksum = "ff1acff600a621d4445e0610fa71a53bef5390a5e349cfbd30651917593fb57d" dependencies = [ "Inflector", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "bv", "serde", @@ -6343,7 +6394,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8f5152a288ef1912300fc6efa6c2d1f9bb55d9398eb6c72326360b8063987da" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-program-error 2.2.2", "solana-program-memory 2.3.1", @@ -6356,7 +6407,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9cf16495d9eb53e3d04e72366a33bb1c20c24e78c171d8b8f5978357b63ae95" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-address 2.5.0", "solana-program-error 3.0.1", @@ -6413,7 +6464,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673f67efe870b64a65cb39e6194be5b26527691ce5922909939961a6e6b395" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6430,7 +6481,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e8df0b083c10ce32490410f3795016b1b5d9b4d094658c0a5e496753645b7cd" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6488,7 +6539,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19a3787b8cf9c9fe3dd360800e8b70982b9e5a8af9e11c354b6665dd4a003adc" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-instruction 2.3.3", ] @@ -6499,7 +6550,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "278a1a5bad62cd9da89ac8d4b7ec444e83caa8ae96aa656dfc27684b28d49a5d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-instruction-error", ] @@ -6578,7 +6629,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219bfba64973ac9e64aa181f03fd56ac319e2d50d8a23d16c54bbd7fa9807a47" dependencies = [ "agave-syscalls", - "bincode", + "bincode 1.3.3", "qualifier_attr", "solana-account 3.4.0", "solana-bincode 3.1.0", @@ -6720,7 +6771,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e401ae56aed512821cc7a0adaa412ff97fecd2dff4602be7b1330d2daec0c4" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -6976,7 +7027,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43f5c5382b449e8e4e3016fb05e418c53d57782d8b5c30aa372fc265654b956d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 2.2.1", @@ -6995,7 +7046,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75ca9b5cbb6f500f7fd73db5bd95640f71a83f04d6121a0e59a43b202dca2731" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -7057,7 +7108,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "749eccc960e85c9b33608450093d256006253e1cb436b8380e71777840a3f675" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "memmap2 0.5.10", "solana-account 3.4.0", @@ -7143,7 +7194,7 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab5682934bd1f65f8d2c16f21cb532526fcc1a09f796e2cacdb091eee5774ad" dependencies = [ - "bincode", + "bincode 1.3.3", "getrandom 0.2.17", "js-sys", "num-traits", @@ -7161,7 +7212,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97881335fc698deb46c6571945969aae6d93a14e2fff792a368b4fac872f116" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh", "serde", "serde_derive", @@ -7414,7 +7465,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1796aabce376ff74bf89b78d268fa5e683d7d7a96a0a4e4813ec34de49d5314b" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7437,7 +7488,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0448b1fd891c5f46491e5dc7d9986385ba3c852c340db2911dd29faa01d2b08d" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7627,7 +7678,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98eca145bd3545e2fbb07166e895370576e47a00a7d824e325390d33bf467210" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "bs58", "bytemuck", @@ -7846,10 +7897,10 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "cfg-if", "itertools 0.12.1", "log", @@ -8010,7 +8061,7 @@ checksum = "10dd50b329ce569340c1deab3667d21e26a41e65cc6460e8a5bb8b57aff8420d" dependencies = [ "async-trait", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "futures", "indicatif", @@ -8125,7 +8176,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f03df7969f5e723ad31b6c9eadccc209037ac4caa34d8dc259316b05c11e82b" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "serde", "solana-account 3.4.0", @@ -8521,7 +8572,7 @@ dependencies = [ name = "solana-storage-proto" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "enum-iterator 1.5.0", "prost", @@ -8542,7 +8593,7 @@ dependencies = [ [[package]] name = "solana-svm" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "ahash 0.8.12", "log", @@ -8701,7 +8752,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "450479004fee3396c88cc4aa2f9b2b8db9c77be42ee7c1c53e6fac9eaec5fd51" dependencies = [ - "bincode", + "bincode 1.3.3", "log", "serde", "solana-account 3.4.0", @@ -8743,7 +8794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c3595f95069f3d90f275bb9bd235a1973c4d059028b0a7f81baca2703815db" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -8780,7 +8831,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6690d3dd88f15c21edff68eb391ef8800df7a1f5cec84ee3e8d1abf05affdf74" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -8839,7 +8890,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96697cff5075a028265324255efed226099f6d761ca67342b230d09f72cc48d2" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-address 2.5.0", @@ -8858,9 +8909,9 @@ dependencies = [ [[package]] name = "solana-transaction-context" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "solana-account 3.4.0", @@ -8903,7 +8954,7 @@ dependencies = [ "Inflector", "agave-reserved-account-keys", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "borsh", "bs58", "log", @@ -8944,7 +8995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6886b7bb8fbba5937b4a38fa67ed442d7971629244b8fbd95c7963b2126bc9" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "serde", "serde_json", @@ -8981,7 +9032,7 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b80d57478d6599d30acc31cc5ae7f93ec2361a06aefe8ea79bc81739a08af4c3" dependencies = [ - "bincode", + "bincode 1.3.3", "num-derive", "num-traits", "serde", @@ -9005,7 +9056,7 @@ version = "5.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d444ce30b6b4f9c281ba06061ea96638d063b53c2171b1e41ba02ebff79ed28f" dependencies = [ - "bincode", + "bincode 1.3.3", "cfg_eval", "num-derive", "num-traits", @@ -9032,7 +9083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4537fd6efe65f53ccd28d54d2ad43275b024834a4a8ca4dfa4babfa01e6d11ab" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "log", "num-derive", "num-traits", @@ -9095,7 +9146,7 @@ checksum = "9602bcb1f7af15caef92b91132ec2347e1c51a72ecdbefdaefa3eac4b8711475" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -9132,7 +9183,7 @@ checksum = "09670ff59f60e6ddc2209c2e4353992a9b9f01d4e244f3e9d67bd5146e33d388" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -10281,6 +10332,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "3.3.0" @@ -10382,6 +10439,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "void" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 010b6ccd0..d14f0192f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ guinea = { path = "./programs/guinea" } helius-laserstream = { git = "https://github.com/magicblock-labs/laserstream-sdk.git", branch = "mbv-4.0+conn-fix" } http-body-util = "0.1.3" humantime = { version = "1.1", package = "humantime-serde" } +hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "449532ee0ee792e9a3809b16fd9d289c2024341a" } hyper = "1.6.0" hyper-util = "0.1.15" isocountry = "0.3.2" @@ -139,9 +140,8 @@ parking_lot = "0.12" paste = "1.0" prometheus = "0.13.4" # Keep in sync with `solana-storage-proto` codegen. - - -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82", features = [ +# TODO: Update to latest version is merged +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a", features = [ "dev-context-only-utils", ] } solana-transaction-error = { version = "3.0" } @@ -194,7 +194,8 @@ serde_json = "1.0" serde_with = "3.16" serial_test = "3.2" sha3 = "0.10.8" -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-account-decoder = { version = "4.0" } solana-account-decoder-client-types = { version = "4.0" } solana-account-info = { version = "3.1" } @@ -246,7 +247,8 @@ solana-program = "3.0" solana-program-error = { version = "3.0" } solana-program-option = { version = "3.0" } solana-program-pack = { version = "3.0" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-pubkey = { version = "4.1" } solana-pubsub-client = { version = "4.0" } solana-rent = { version = "3.0" } @@ -278,14 +280,19 @@ solana-transaction = { version = "3.0" } [workspace.dependencies.solana-svm] features = ["dev-context-only-utils"] git = "https://github.com/magicblock-labs/magicblock-svm.git" -rev = "b275f82" +# TODO: Update to latest version is merged +rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" [patch.crates-io] -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-storage-proto = { path = "storage-proto" } -solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } # Fork is used to enable `disable_manual_compaction` usage # Fork is based on commit d4e9e16 of rocksdb (parent commit of 0.23.0 release) # without patching update isn't possible due to conflict with solana deps diff --git a/config.example.toml b/config.example.toml index f9e12417b..33a52553e 100644 --- a/config.example.toml +++ b/config.example.toml @@ -315,28 +315,12 @@ claim-fees-frequency = "24h" # Internal Task Scheduler # ============================================================================== [task-scheduler] -# If true, clears all pending scheduled tasks on startup. -# Default: false -# Env: MBV_TASK_SCHEDULER__RESET -reset = false - -# The minimum interval between task executions. -# Supports humantime (e.g. "10ms", "1s"). -# Default: "10ms" -# Env: MBV_TASK_SCHEDULER__MIN_INTERVAL -min-interval = "10ms" - -# How long failed task executions and failed scheduling records are retained. -# Supports humantime (e.g. "1h", "7d"). -# Default: "14d" -# Env: MBV_TASK_SCHEDULER__FAILED_TASK_RETENTION -failed-task-retention = "14d" - -# How often old failed task executions and failed scheduling records are deleted. -# Supports humantime (e.g. "1m", "1h"). -# Default: "1h" -# Env: MBV_TASK_SCHEDULER__FAILED_TASK_CLEANUP_INTERVAL -failed-task-cleanup-interval = "1h" +# Keypair (Base58) the task scheduler uses to pay for hydra cranks. It is +# delegated to this validator on startup and must be funded separately (the +# validator does not fund it). Use a dedicated account, not the validator +# identity. +# Env: MBV_TASK_SCHEDULER__FAUCET_KEYPAIR +faucet-keypair = "4SkhwWzaUdkXS5RQZHNFtmaTjvPRNKEZjLRiYPm3y7MGQTWBYYzrwAcvK1vMDwrTHrAy2aGqMJn2s7qbjfxqDLU9" # ============================================================================== # Pre-loaded Programs diff --git a/magicblock-api/Cargo.toml b/magicblock-api/Cargo.toml index 1ddbc66de..bf5cd5a2d 100644 --- a/magicblock-api/Cargo.toml +++ b/magicblock-api/Cargo.toml @@ -53,6 +53,7 @@ solana-sdk-ids = { workspace = true } solana-sha256-hasher = { workspace = true } solana-signature = { workspace = true } solana-signer = { workspace = true } +solana-system-interface = { workspace = true, features = ["bincode"] } solana-system-program = { workspace = true } solana-sysvar = { workspace = true } solana-transaction = { workspace = true } diff --git a/magicblock-api/src/crank_faucet.rs b/magicblock-api/src/crank_faucet.rs new file mode 100644 index 000000000..e4f87eee6 --- /dev/null +++ b/magicblock-api/src/crank_faucet.rs @@ -0,0 +1,126 @@ +use dlp_api::state::DelegationRecord; +use magicblock_program::validator::validator_authority; +use solana_commitment_config::CommitmentConfig; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_signer::Signer; +use tracing::*; + +use crate::errors::{ApiError, ApiResult}; + +pub(crate) fn delegation_record_authority( + data: &[u8], + delegation_record_pubkey: Pubkey, +) -> Result { + let delegation_record_size = DelegationRecord::size_with_discriminator(); + if data.len() < delegation_record_size { + return Err(format!( + "delegation record {delegation_record_pubkey} is too small" + )); + } + + DelegationRecord::try_from_bytes_with_discriminator( + &data[..delegation_record_size], + ) + .copied() + .map(|record| record.authority) + .map_err(|err| { + format!( + "failed to decode delegation record {delegation_record_pubkey}: {err:?}" + ) + }) +} + +/// Delegates the task scheduler faucet to this validator on the base chain if +/// it is not already delegated. Delegating gives the faucet real (base-chain +/// backed) lamports usable inside the ephemeral rollup, unlike the validator +/// identity. The faucet must already be funded — the validator does not fund +/// it (operators and integration tests airdrop to it). +pub(crate) async fn ensure_faucet_delegated_on_chain( + rpc_url: String, + faucet: &Keypair, +) -> ApiResult<()> { + let validator_keypair = validator_authority(); + let validator_pubkey = validator_keypair.pubkey(); + let faucet_pubkey = faucet.pubkey(); + let delegation_record_pubkey = + dlp_api::pda::delegation_record_pda_from_delegated_account( + &faucet_pubkey, + ); + + let rpc = + RpcClient::new_with_commitment(rpc_url, CommitmentConfig::confirmed()); + + let accounts = rpc + .get_multiple_accounts(&[faucet_pubkey, delegation_record_pubkey]) + .await + .map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + + if matches!( + accounts[0].as_ref(), + Some(account) if account.owner == dlp_api::id() + ) { + let Some(delegation_record) = accounts[1].as_ref() else { + return Err(ApiError::FailedToDelegateFaucet( + faucet_pubkey, + format!( + "faucet is owned by the delegation program but missing delegation record {delegation_record_pubkey}" + ), + )); + }; + let authority = delegation_record_authority( + &delegation_record.data, + delegation_record_pubkey, + ) + .map_err(|err| ApiError::FailedToDelegateFaucet(faucet_pubkey, err))?; + if authority == validator_pubkey { + info!(%faucet_pubkey, %validator_pubkey, "Crank faucet already delegated, skipping"); + return Ok(()); + } + return Err(ApiError::FailedToDelegateFaucet( + faucet_pubkey, + format!( + "faucet already delegated to validator {authority}, expected {validator_pubkey}" + ), + )); + } + + info!(%faucet_pubkey, "Delegating crank faucet"); + // Hand the on-curve faucet to the delegation program and delegate it to + // this validator. This makes it a writable, base-chain-backed account + // inside the ephemeral rollup that can sponsor crank creation (mirrors the + // on-curve delegation flow used in tests). The faucet must already be + // funded; the validator does not fund it. + let assign_ix = solana_system_interface::instruction::assign( + &faucet_pubkey, + &dlp_api::id(), + ); + let delegate_ix = dlp_api::instruction_builder::delegate( + validator_pubkey, + faucet_pubkey, + None, + dlp_api::args::DelegateArgs { + commit_frequency_ms: u32::MAX, + seeds: vec![], + validator: Some(validator_pubkey), + }, + ); + + let blockhash = rpc.get_latest_blockhash().await.map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + let tx = solana_transaction::Transaction::new_signed_with_payer( + &[assign_ix, delegate_ix], + Some(&validator_pubkey), + &[&validator_keypair, faucet], + blockhash, + ); + rpc.send_and_confirm_transaction(&tx).await.map_err(|err| { + ApiError::FailedToDelegateFaucet(faucet_pubkey, err.to_string()) + })?; + info!(%faucet_pubkey, "Crank faucet delegated"); + Ok(()) +} diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 4f80a6b72..9fbafc630 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -34,6 +34,9 @@ pub enum ApiError { #[error("Failed to delegate magic fee vault for validator '{0}': {1}")] FailedToDelegateMagicFeeVault(Pubkey, String), + #[error("Failed to delegate task scheduler faucet '{0}': {1}")] + FailedToDelegateFaucet(Pubkey, String), + #[error("CommittorServiceError")] CommittorServiceError( Box, @@ -69,15 +72,6 @@ pub enum ApiError { #[error("Ledger Path is missing a parent directory: {0}")] LedgerPathIsMissingParent(String), - #[error("Ledger Path has an invalid faucet keypair file: {0} ({1})")] - LedgerInvalidFaucetKeypair(String, String), - - #[error("Ledger Path is missing a faucet keypair file: {0}")] - LedgerIsMissingFaucetKeypair(String), - - #[error("Ledger could not write faucet keypair file: {0} ({1})")] - LedgerCouldNotWriteFaucetKeypair(String, String), - #[error("Ledger Path has an invalid validator keypair file: {0} ({1})")] LedgerInvalidValidatorKeypair(String, String), diff --git a/magicblock-api/src/lib.rs b/magicblock-api/src/lib.rs index 6d0db7546..4b17fe0fa 100644 --- a/magicblock-api/src/lib.rs +++ b/magicblock-api/src/lib.rs @@ -1,3 +1,4 @@ +mod crank_faucet; pub mod domain_registry_manager; pub mod errors; mod fund_account; diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index b4d569ab8..278be4c4c 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -80,6 +80,7 @@ use tokio_util::sync::CancellationToken; use tracing::*; use crate::{ + crank_faucet::ensure_faucet_delegated_on_chain, domain_registry_manager::DomainRegistryManager, errors::{ApiError, ApiResult}, fund_account::{ @@ -115,6 +116,7 @@ pub struct MagicValidator { replication_service: Option, rpc_handle: thread::JoinHandle<()>, identity: Pubkey, + faucet_keypair: Option, transaction_scheduler: TransactionSchedulerHandle, _metrics: (MetricsService, tokio::task::JoinHandle<()>), claim_fees_task: ClaimFeesTask, @@ -165,6 +167,11 @@ impl MagicValidator { )?; log_timing("startup", "sync_validator_keypair", step_start); + // The task scheduler pays for hydra cranks from a configured faucet + // account (delegated on startup) rather than the validator identity, + // which is not a delegated account. + let faucet_keypair = config.task_scheduler.faucet_keypair.clone(); + let latest_block = ledger.latest_block().load(); let mut accountsdb = AccountsDb::new(&config.accountsdb, &config.storage, last_slot)?; @@ -425,8 +432,8 @@ impl MagicValidator { let step_start = Instant::now(); let task_scheduler = TaskSchedulerService::new( &task_scheduler_db_path, - &config.task_scheduler, config.aperture.listen.http(), + faucet_keypair.clone().map(|k| k.insecure_clone()), dispatch .tasks_service .take() @@ -434,7 +441,11 @@ impl MagicValidator { ledger.latest_block().clone(), Duration::from_millis(config.ledger.block_time_ms()), token.clone(), - )?; + ) + .inspect_err( + |e| error!(error = ?e, "Failed to initialize task scheduler"), + ) + .ok(); log_timing("startup", "task_scheduler_init", step_start); Ok(Self { @@ -450,8 +461,9 @@ impl MagicValidator { claim_fees_task: ClaimFeesTask::new(), rpc_handle, identity: validator_pubkey, + faucet_keypair: faucet_keypair.map(|k| k.insecure_clone()), transaction_scheduler: dispatch.transaction_scheduler, - task_scheduler: Some(task_scheduler), + task_scheduler, transaction_execution, replication_handle: None, mode_tx, @@ -894,6 +906,8 @@ impl MagicValidator { let chain_operation_config = self.config.chain_operation.clone(); let block_time_ms = self.config.ledger.block_time_ms(); let base_fee = self.config.validator.basefee; + let faucet_keypair = + self.faucet_keypair.as_ref().map(|k| k.insecure_clone()); // Ephemeral mode does a non-blocking startup balance check. // Intentionally fire-and-forget: the task itself exits the process on failure. @@ -933,6 +947,28 @@ impl MagicValidator { error!("Exiting process"); std::process::exit(1); } + + if let Some(faucet_keypair) = faucet_keypair { + let step_start = Instant::now(); + let result = ensure_faucet_delegated_on_chain( + rpc_url.clone(), + &faucet_keypair, + ) + .await; + log_timing( + "startup_background", + "ensure_faucet_delegated_on_chain", + step_start, + ); + // Without the faucet being funded and delegated the task scheduler + // cannot pay for hydra cranks. + if let Err(err) = result { + error!(error = ?err, "Task scheduler faucet setup failed"); + error!("Exiting process"); + std::process::exit(1); + } + } + if let Some(ref config) = chain_operation_config { if !config.claim_fees_frequency.is_zero() { let step_start = Instant::now(); diff --git a/magicblock-config/src/config/scheduler.rs b/magicblock-config/src/config/scheduler.rs index e27eadee6..c16f678b5 100644 --- a/magicblock-config/src/config/scheduler.rs +++ b/magicblock-config/src/config/scheduler.rs @@ -1,42 +1,28 @@ -use std::time::Duration; - use serde::{Deserialize, Serialize}; +use solana_keypair::Keypair; -use crate::consts; +use crate::{consts, types::SerdeKeypair}; /// Configuration for the internal task scheduler. +/// +/// Task execution is performed by the external hydra cranker service, so the +/// validator only needs the keypair used to pay for (sponsor, fund, and cancel) +/// hydra cranks. This faucet keypair is delegated on startup; it must be funded +/// separately (the validator does not fund it). #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(rename_all = "kebab-case", deny_unknown_fields, default)] pub struct TaskSchedulerConfig { - /// If true, clears all pending scheduled tasks on startup. - pub reset: bool, - /// The minimum interval between task executions. - /// Supports humantime (e.g. "10ms", "1s"). - #[serde(with = "humantime")] - pub min_interval: Duration, - /// How long failed task and scheduling records are retained. - /// Supports humantime (e.g. "1h", "7d"). - #[serde(with = "humantime")] - pub failed_task_retention: Duration, - /// How often failed task and scheduling records are cleaned up. - /// Supports humantime (e.g. "1m", "1h"). - #[serde(with = "humantime")] - pub failed_task_cleanup_interval: Duration, + /// Keypair the task scheduler uses to pay for hydra cranks, encoded in + /// Base58. + pub faucet_keypair: Option, } impl Default for TaskSchedulerConfig { fn default() -> Self { Self { - reset: false, - min_interval: Duration::from_millis( - consts::DEFAULT_TASK_SCHEDULER_MIN_INTERVAL_MILLIS, - ), - failed_task_retention: Duration::from_secs( - consts::DEFAULT_TASK_SCHEDULER_FAILED_TASK_RETENTION_SECS, - ), - failed_task_cleanup_interval: Duration::from_secs( - consts::DEFAULT_TASK_SCHEDULER_FAILED_TASK_CLEANUP_INTERVAL_SECS, - ), + faucet_keypair: Some(SerdeKeypair(Keypair::from_base58_string( + consts::DEFAULT_TASK_SCHEDULER_FAUCET_KEYPAIR, + ))), } } } diff --git a/magicblock-config/src/consts.rs b/magicblock-config/src/consts.rs index e1452b7b4..2fd2ba59b 100644 --- a/magicblock-config/src/consts.rs +++ b/magicblock-config/src/consts.rs @@ -14,6 +14,13 @@ pub const DEFAULT_STORAGE_DIRECTORY: &str = "magicblock-test-storage/"; pub const DEFAULT_VALIDATOR_KEYPAIR: &str = "9Vo7TbA5YfC5a33JhAi9Fb41usA6JwecHNRw3f9MzzHAM8hFnXTzL5DcEHwsAFjuUZ8vNQcJ4XziRFpMc3gTgBQ"; +/// Keypair the task scheduler uses to pay for hydra cranks. It is delegated on +/// startup and must be funded (the validator does not fund it). +/// WARNING: This keypair is for development/testing only. +/// Production deployments MUST provide their own faucet keypair via config. +pub const DEFAULT_TASK_SCHEDULER_FAUCET_KEYPAIR: &str = + "4SkhwWzaUdkXS5RQZHNFtmaTjvPRNKEZjLRiYPm3y7MGQTWBYYzrwAcvK1vMDwrTHrAy2aGqMJn2s7qbjfxqDLU9"; + /// Default base fee in lamports for transactions pub const DEFAULT_BASE_FEE: u64 = 0; @@ -64,13 +71,6 @@ pub const DEFAULT_METRICS_ADDR: &str = "0.0.0.0:9000"; /// Default frequency of metrics collection in seconds pub const DEFAULT_METRICS_COLLECT_FREQUENCY_SEC: u64 = 30; -// Task Scheduler Defaults -pub const DEFAULT_TASK_SCHEDULER_MIN_INTERVAL_MILLIS: u64 = 10; -pub const DEFAULT_TASK_SCHEDULER_FAILED_TASK_RETENTION_SECS: u64 = - 14 * 24 * 60 * 60; // 14 days -pub const DEFAULT_TASK_SCHEDULER_FAILED_TASK_CLEANUP_INTERVAL_SECS: u64 = - 60 * 60; // 1 hour - // ChainLink Defaults /// Default delay in milliseconds between resubscribing to accounts after a pubsub reconnection pub const DEFAULT_RESUBSCRIPTION_DELAY_MS: u64 = 50; diff --git a/magicblock-config/src/lib.rs b/magicblock-config/src/lib.rs index f0136d376..acf5b31ba 100644 --- a/magicblock-config/src/lib.rs +++ b/magicblock-config/src/lib.rs @@ -66,8 +66,8 @@ pub struct ValidatorParams { pub ledger: LedgerConfig, pub chainlink: ChainLinkConfig, pub chain_operation: Option, - pub task_scheduler: TaskSchedulerConfig, pub programs: Vec, + pub task_scheduler: TaskSchedulerConfig, } impl ValidatorParams { diff --git a/magicblock-config/src/tests.rs b/magicblock-config/src/tests.rs index 1268d9bc8..8be2a6145 100644 --- a/magicblock-config/src/tests.rs +++ b/magicblock-config/src/tests.rs @@ -387,16 +387,6 @@ fn test_cli_ledger_reset_overrides_toml() { assert!(config.ledger.reset); } -#[test] -#[serial] -fn test_task_scheduler_bool_env() { - // Verify standard boolean parsing from Env vars work on nested fields - let _guard = EnvVarGuard::new("MBV_TASK_SCHEDULER__RESET", "true"); - - let config = run_cli(vec![]); - assert!(config.task_scheduler.reset); -} - #[test] #[parallel] fn test_example_config_full_coverage() { @@ -503,21 +493,6 @@ fn test_example_config_full_coverage() { // ======================================================================== // 11. Optional Sections // ======================================================================== - // Task scheduler reset should be false - assert!(!config.task_scheduler.reset); - assert_eq!( - config.task_scheduler.min_interval, - Duration::from_millis(10) - ); - assert_eq!( - config.task_scheduler.failed_task_retention, - Duration::from_secs(14 * 24 * 60 * 60) - ); - assert_eq!( - config.task_scheduler.failed_task_cleanup_interval, - Duration::from_secs(60 * 60) - ); - // The example file has the programs section with 2 entries assert_eq!( config.programs.len(), @@ -597,14 +572,6 @@ fn test_env_vars_full_coverage() { EnvVarGuard::new("MBV_CHAINLINK__RISK__CACHE_TTL", "45m"), EnvVarGuard::new("MBV_CHAINLINK__RISK__REQUEST_TIMEOUT", "3s"), EnvVarGuard::new("MBV_CHAINLINK__RISK__RISK_SCORE_THRESHOLD", "8"), - // --- Task Scheduler --- - EnvVarGuard::new("MBV_TASK_SCHEDULER__RESET", "true"), - EnvVarGuard::new("MBV_TASK_SCHEDULER__MIN_INTERVAL", "99ms"), - EnvVarGuard::new("MBV_TASK_SCHEDULER__FAILED_TASK_RETENTION", "2h"), - EnvVarGuard::new( - "MBV_TASK_SCHEDULER__FAILED_TASK_CLEANUP_INTERVAL", - "3m", - ), // --- Chain Operation (Optional Section) --- // Figment can instantiate optional structs if their fields are present EnvVarGuard::new("MBV_CHAIN_OPERATION__COUNTRY_CODE", "DE"), @@ -681,21 +648,6 @@ fn test_env_vars_full_coverage() { ); assert_eq!(config.chainlink.risk.risk_score_threshold, 8); - // Task Scheduler - assert!(config.task_scheduler.reset); - assert_eq!( - config.task_scheduler.min_interval, - Duration::from_millis(99) - ); - assert_eq!( - config.task_scheduler.failed_task_retention, - Duration::from_secs(2 * 60 * 60) - ); - assert_eq!( - config.task_scheduler.failed_task_cleanup_interval, - Duration::from_secs(3 * 60) - ); - // Chain Operation // Verify the optional struct was created and populated let chain_op = config diff --git a/magicblock-magic-program-api/src/instruction.rs b/magicblock-magic-program-api/src/instruction.rs index 208db91f6..9450f6061 100644 --- a/magicblock-magic-program-api/src/instruction.rs +++ b/magicblock-magic-program-api/src/instruction.rs @@ -300,17 +300,6 @@ pub enum MagicBlockInstruction { /// - **0.** `[SIGNER]` Validator Authority /// - **1.** `[WRITE]` Account to evict EvictAccount { pubkey: Pubkey }, - - /// Executes a crank - /// - /// # Account references - /// - **0.** `[SIGNER]` Validator authority - /// - **1.** `[]` Crank signer PDA - /// - **2..n** `[]` Accounts required by the embedded instructions - ExecuteCrank { - authority: Pubkey, - instructions: Vec, - }, } impl MagicBlockInstruction { diff --git a/magicblock-magic-program-api/src/pda.rs b/magicblock-magic-program-api/src/pda.rs index 234713ca5..21081b58e 100644 --- a/magicblock-magic-program-api/src/pda.rs +++ b/magicblock-magic-program-api/src/pda.rs @@ -1,14 +1,5 @@ use crate::Pubkey; -pub const CRANK_SEED: &[u8] = b"crank-executor"; -pub fn crank_signer_pda(authority: &Pubkey) -> Pubkey { - Pubkey::find_program_address( - &[CRANK_SEED, authority.as_ref()], - &crate::CRANK_PROGRAM_ID, - ) - .0 -} - /// Callback signer PDA info pub const CALLBACK_SEED: &[u8] = b"callback-executor"; const CALLBACK_SIGNER_PDA: ([u8; 32], u8) = diff --git a/magicblock-processor/src/builtins.rs b/magicblock-processor/src/builtins.rs index c7a500e05..2327be257 100644 --- a/magicblock-processor/src/builtins.rs +++ b/magicblock-processor/src/builtins.rs @@ -49,11 +49,6 @@ pub static BUILTINS: &[Builtin] = &[ name: "magicblock_program", entrypoint: magicblock_processor::Entrypoint::vm, }, - Builtin { - program_id: magicblock_program::CRANK_PROGRAM_ID, - name: "magicblock_crank_program", - entrypoint: magicblock_processor::CrankEntrypoint::vm, - }, Builtin { program_id: magicblock_program::CALLBACK_PROGRAM_ID, name: "magicblock_callback_program", diff --git a/magicblock-task-scheduler/Cargo.toml b/magicblock-task-scheduler/Cargo.toml index fa0ede3d6..73ef17938 100644 --- a/magicblock-task-scheduler/Cargo.toml +++ b/magicblock-task-scheduler/Cargo.toml @@ -11,23 +11,29 @@ edition.workspace = true bincode = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } -tracing = { workspace = true } +hydra-api = { workspace = true, features = ["client"] } magicblock-core = { workspace = true } magicblock-config = { workspace = true } magicblock-ledger = { workspace = true } magicblock-program = { workspace = true } rusqlite = { workspace = true } +solana-account = { workspace = true } solana-instruction = { workspace = true } +solana-keypair = { workspace = true } solana-message = { workspace = true } solana-pubkey = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } +solana-sha256-hasher = { workspace = true } solana-signature = { workspace = true } +solana-signer = { workspace = true } +solana-system-interface = { workspace = true, features = ["bincode"] } solana-transaction = { workspace = true } solana-transaction-error = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true, features = ["time"] } +tracing = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/magicblock-task-scheduler/src/db.rs b/magicblock-task-scheduler/src/db.rs index 3fc980dc8..26575ba00 100644 --- a/magicblock-task-scheduler/src/db.rs +++ b/magicblock-task-scheduler/src/db.rs @@ -1,12 +1,11 @@ use std::{ - collections::HashMap, path::{Path, PathBuf}, sync::Arc, }; use chrono::Utc; use magicblock_program::args::ScheduleTaskRequest; -use rusqlite::{params, Connection, OptionalExtension}; +use rusqlite::{params, Connection}; use solana_instruction::Instruction; use solana_pubkey::Pubkey; use tokio::sync::Mutex; @@ -17,20 +16,18 @@ use crate::errors::TaskSchedulerResult; /// Uses i64 for all timestamps and IDs to avoid overflows #[derive(Debug, Clone, PartialEq, Eq)] pub struct DbTask { - /// Unique identifier for this task + /// Unique identifier for this task. pub id: i64, - /// Instructions to execute when triggered + /// Instructions to execute when triggered. pub instructions: Vec, - /// Authority that can modify or cancel this task + /// Authority that owns this task. pub authority: Pubkey, - /// How frequently the task should be executed, in milliseconds + /// How frequently the task should be executed, in milliseconds. pub execution_interval_millis: i64, /// Number of times this task still needs to be executed. pub executions_left: i64, - /// Timestamp of the last execution of this task in milliseconds since UNIX epoch + /// Legacy scheduler timestamp of the last successful execution. pub last_execution_millis: i64, - /// Timestamp of the latest persisted mutation of this task in milliseconds since UNIX epoch - pub updated_at: i64, } impl From for DbTask { @@ -42,121 +39,11 @@ impl From for DbTask { execution_interval_millis: task.execution_interval_millis, executions_left: task.iterations, last_execution_millis: 0, - updated_at: 0, } } } -#[derive(Debug, Clone, Copy)] -pub struct CrankSuccessUpdate { - /// Task whose successful execution should be persisted. - pub task_id: i64, - /// Actual execution timestamp to store in `tasks.last_execution_millis`. - pub last_execution_millis: i64, - /// Optimistic concurrency token. The update is applied only if the current - /// row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, -} - -#[derive(Debug, Clone, Copy)] -pub struct CrankSuccessRemoval { - /// Task whose final successful execution should remove it from `tasks`. - pub task_id: i64, - /// Optimistic concurrency token. The removal is applied only if the current - /// row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, -} - -#[derive(Debug, Clone)] -pub struct CrankFailedMove { - /// Task to remove from `tasks` and append to `failed_tasks`. - pub task_id: i64, - /// Optimistic concurrency token. The move is applied only if the current - /// row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, - /// Error text persisted in `failed_tasks.error` when the move succeeds. - pub error: String, -} - -#[derive(Debug, Clone, Copy)] -pub struct CrankRetryCheck { - /// Task that failed but remains retryable. - pub task_id: i64, - /// Optimistic concurrency token. The retry is considered valid only if the - /// current row still has this `tasks.updated_at` value. - pub expected_updated_at: i64, -} - -/// Applied metadata for a successful task execution that remains scheduled. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankSuccessUpdateCompletion { - /// Optimistic token supplied by the caller and matched against the previous - /// `tasks.updated_at` value. - pub expected_updated_at: i64, - /// New `tasks.updated_at` value written by the database transaction. - pub new_updated_at: i64, -} - -/// Applied metadata for a task removed after its final successful execution. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankSuccessRemovalCompletion { - /// Optimistic token supplied by the caller and matched against the previous - /// `tasks.updated_at` value before deleting the row. - pub expected_updated_at: i64, -} - -/// Applied metadata for a task moved from `tasks` to `failed_tasks`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankFailedMoveCompletion { - /// Optimistic token supplied by the caller and matched against the previous - /// `tasks.updated_at` value before moving the task. - pub expected_updated_at: i64, -} - -/// Applied metadata for a retryable failed task that still matches its DB row. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CrankRetryCheckCompletion { - /// Actual `tasks.updated_at` value read from the database. This equals the - /// requested optimistic token when the retry check succeeds. - pub current_updated_at: i64, -} - -/// Results for a crank batch transaction keyed by task ID. -/// -/// Each map contains only entries whose optimistic `expected_updated_at` token -/// matched the database state and whose corresponding DB operation succeeded. -#[derive(Debug, Default)] -pub struct CrankBatchCompletion { - /// Continued successful executions. Values include both the matched - /// optimistic token and the new `tasks.updated_at` written by the DB. - pub success_updates: HashMap, - /// Final successful executions removed from `tasks`. Values contain the - /// matched optimistic token used for the delete. - pub success_removals: HashMap, - /// Failed executions moved to `failed_tasks`. Values contain the matched - /// optimistic token used for the move. - pub failed_moves: HashMap, - /// Retryable failed executions whose `tasks` row still exists unchanged. - /// Values contain the actual `tasks.updated_at` read from the DB. - pub retry_checks: HashMap, -} - -#[derive(Debug, Clone)] -pub struct FailedScheduling { - pub id: i64, - pub timestamp: i64, - pub task_id: i64, - pub error: String, -} - -#[derive(Debug, Clone)] -pub struct FailedTask { - pub id: i64, - pub timestamp: i64, - pub task_id: i64, - pub error: String, -} - +/// Migration-only store over the legacy scheduler's SQLite database. #[derive(Clone)] pub struct SchedulerDatabase { conn: Arc>, @@ -174,11 +61,12 @@ impl SchedulerDatabase { PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA busy_timeout=5000; - PRAGMA cache_size=-65536; ", )?; - // Create tables + // Mirrors the legacy schema so existing databases can be read. The + // last execution timestamp is used to preserve cadence when migrating + // legacy rows onto hydra cranks. conn.execute( "CREATE TABLE IF NOT EXISTS tasks ( id INTEGER PRIMARY KEY, @@ -186,29 +74,9 @@ impl SchedulerDatabase { authority TEXT NOT NULL, execution_interval_millis INTEGER NOT NULL, executions_left INTEGER NOT NULL, - last_execution_millis INTEGER NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL - )", - [], - )?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS failed_scheduling ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp INTEGER NOT NULL, - task_id INTEGER, - error TEXT NOT NULL - )", - [], - )?; - - conn.execute( - "CREATE TABLE IF NOT EXISTS failed_tasks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp INTEGER NOT NULL, - task_id INTEGER, - error TEXT NOT NULL + last_execution_millis INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT 0 )", [], )?; @@ -218,23 +86,14 @@ impl SchedulerDatabase { }) } - pub async fn insert_task(&self, task: &DbTask) -> TaskSchedulerResult { + /// Inserts (or replaces) a task. Used to seed legacy tasks, including from + /// tests exercising migration. + pub async fn insert_task(&self, task: &DbTask) -> TaskSchedulerResult<()> { let instructions_bin = bincode::serialize(&task.instructions)?; let authority_str = task.authority.to_string(); - let conn = self.conn.lock().await; - let previous_updated_at: Option = conn - .query_row( - "SELECT updated_at FROM tasks WHERE id = ?", - [task.id], - |row| row.get(0), - ) - .optional()?; - let now = previous_updated_at - .map(|updated_at| Utc::now().timestamp_millis().max(updated_at + 1)) - .unwrap_or_else(|| Utc::now().timestamp_millis()); - - conn.execute( - "INSERT OR REPLACE INTO tasks + let now = Utc::now().timestamp_millis(); + self.conn.lock().await.execute( + "INSERT OR REPLACE INTO tasks (id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", params![ @@ -248,127 +107,17 @@ impl SchedulerDatabase { now, ], )?; - - Ok(now) - } - - pub async fn update_task_after_execution( - &self, - task_id: i64, - last_execution: i64, - ) -> TaskSchedulerResult<()> { - let now = Utc::now().timestamp_millis(); - - self.conn.lock().await.execute( - "UPDATE tasks - SET executions_left = executions_left - 1, - last_execution_millis = ?, - updated_at = ? - WHERE id = ?", - params![last_execution, now, task_id], - )?; - - Ok(()) - } - - pub async fn insert_failed_scheduling( - &self, - task_id: i64, - error: String, - ) -> TaskSchedulerResult<()> { - self.conn.lock().await.execute( - "INSERT INTO failed_scheduling (timestamp, task_id, error) VALUES (?, ?, ?)", - params![Utc::now().timestamp_millis(), task_id, error], - )?; - - Ok(()) - } - - pub async fn insert_failed_task( - &self, - task_id: i64, - error: String, - ) -> TaskSchedulerResult<()> { - self.conn.lock().await.execute( - "INSERT INTO failed_tasks (timestamp, task_id, error) VALUES (?, ?, ?)", - params![Utc::now().timestamp_millis(), task_id, error], - )?; - Ok(()) } - pub async fn unschedule_task( - &self, - task_id: i64, - ) -> TaskSchedulerResult<()> { - self.conn.lock().await.execute( - "UPDATE tasks SET executions_left = 0 WHERE id = ?", - [task_id], - )?; - - Ok(()) - } - - pub async fn remove_task(&self, task_id: i64) -> TaskSchedulerResult<()> { - self.conn - .lock() - .await - .execute("DELETE FROM tasks WHERE id = ?", [task_id])?; - - Ok(()) - } - - pub async fn get_task( - &self, - task_id: i64, - ) -> TaskSchedulerResult> { - let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, - updated_at - FROM tasks WHERE id = ?" - )?; - - let mut rows = stmt.query_map([task_id], |row| { - let instructions_bin: Vec = row.get(1)?; - let instructions: Vec = - bincode::deserialize(&instructions_bin).map_err(|e| { - rusqlite::Error::InvalidParameterName(format!( - "instructions: {}", - e - )) - })?; - let authority_str: String = row.get(2)?; - let authority: Pubkey = authority_str.parse().map_err(|e| { - rusqlite::Error::InvalidParameterName(format!( - "authority: {}", - e - )) - })?; - - Ok(DbTask { - id: row.get(0)?, - instructions, - authority, - execution_interval_millis: row.get(3)?, - executions_left: row.get(4)?, - last_execution_millis: row.get(5)?, - updated_at: row.get(6)?, - }) - })?; - - Ok(rows.next().transpose()?) - } - + /// Returns all persisted tasks awaiting migration. pub async fn get_tasks(&self) -> TaskSchedulerResult> { let db = self.conn.lock().await; let mut stmt = db.prepare( - "SELECT id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis, - updated_at - FROM tasks" + "SELECT id, instructions, authority, execution_interval_millis, executions_left, last_execution_millis + FROM tasks", )?; - let mut tasks = Vec::new(); let rows = stmt.query_map([], |row| { let instructions_bin: Vec = row.get(1)?; let instructions: Vec = @@ -393,194 +142,27 @@ impl SchedulerDatabase { execution_interval_millis: row.get(3)?, executions_left: row.get(4)?, last_execution_millis: row.get(5)?, - updated_at: row.get(6)?, }) })?; - for row in rows { - tasks.push(row?); - } - - Ok(tasks) + Ok(rows.collect::, rusqlite::Error>>()?) } + /// Returns the ids of all persisted tasks. Used by tests to assert the + /// database empties after migration. pub async fn get_task_ids(&self) -> TaskSchedulerResult> { let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT id - FROM tasks", - )?; - + let mut stmt = db.prepare("SELECT id FROM tasks")?; let rows = stmt.query_map([], |row| row.get(0))?; - Ok(rows.collect::, rusqlite::Error>>()?) } - pub async fn get_failed_schedulings( - &self, - ) -> TaskSchedulerResult> { - let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT * - FROM failed_scheduling", - )?; - - let rows = stmt.query_map([], |row| { - Ok(FailedScheduling { - id: row.get(0)?, - timestamp: row.get(1)?, - task_id: row.get(2)?, - error: row.get(3)?, - }) - })?; - - Ok(rows.collect::, rusqlite::Error>>()?) - } - - pub async fn get_failed_tasks( - &self, - ) -> TaskSchedulerResult> { - let db = self.conn.lock().await; - let mut stmt = db.prepare( - "SELECT * - FROM failed_tasks", - )?; - - let rows = stmt.query_map([], |row| { - Ok(FailedTask { - id: row.get(0)?, - timestamp: row.get(1)?, - task_id: row.get(2)?, - error: row.get(3)?, - }) - })?; - - Ok(rows.collect::, rusqlite::Error>>()?) - } - - /// One transaction for a crank completion batch — replaces N separate commits from - /// per-task `update_task_after_execution` / `remove_task` / `move_task_to_failed` calls. - pub async fn apply_crank_batch_completion( - &self, - success_updates: &[CrankSuccessUpdate], - success_removals: &[CrankSuccessRemoval], - failed_moves: &[CrankFailedMove], - retry_checks: &[CrankRetryCheck], - ) -> TaskSchedulerResult { - if success_updates.is_empty() - && success_removals.is_empty() - && failed_moves.is_empty() - && retry_checks.is_empty() - { - return Ok(CrankBatchCompletion::default()); - } - - let mut conn = self.conn.lock().await; - let tx = conn.transaction()?; - let mut completion = CrankBatchCompletion::default(); - - // Continued executions — decrement executions_left via existing UPDATE semantics - for update in success_updates { - let now = Utc::now() - .timestamp_millis() - .max(update.expected_updated_at + 1); - let affected = tx.execute( - "UPDATE tasks SET executions_left = executions_left - 1, - last_execution_millis = ?, updated_at = ? - WHERE id = ? AND updated_at = ?", - params![ - update.last_execution_millis, - now, - update.task_id, - update.expected_updated_at - ], - )?; - if affected == 1 { - completion.success_updates.insert( - update.task_id, - CrankSuccessUpdateCompletion { - expected_updated_at: update.expected_updated_at, - new_updated_at: now, - }, - ); - } - } - - for removal in success_removals { - let affected = tx.execute( - "DELETE FROM tasks WHERE id = ? AND updated_at = ?", - params![removal.task_id, removal.expected_updated_at], - )?; - if affected == 1 { - completion.success_removals.insert( - removal.task_id, - CrankSuccessRemovalCompletion { - expected_updated_at: removal.expected_updated_at, - }, - ); - } - } - - for failed in failed_moves { - let affected = tx.execute( - "DELETE FROM tasks WHERE id = ? AND updated_at = ?", - params![failed.task_id, failed.expected_updated_at], - )?; - if affected == 1 { - let now = Utc::now().timestamp_millis(); - tx.execute( - "INSERT INTO failed_tasks (timestamp, task_id, error) - VALUES (?, ?, ?)", - params![now, failed.task_id, failed.error], - )?; - completion.failed_moves.insert( - failed.task_id, - CrankFailedMoveCompletion { - expected_updated_at: failed.expected_updated_at, - }, - ); - } - } - - for check in retry_checks { - let matched: Option = tx - .query_row( - "SELECT updated_at FROM tasks - WHERE id = ? AND updated_at = ?", - params![check.task_id, check.expected_updated_at], - |row| row.get(0), - ) - .optional()?; - if let Some(updated_at) = matched { - completion.retry_checks.insert( - check.task_id, - CrankRetryCheckCompletion { - current_updated_at: updated_at, - }, - ); - } - } - - tx.commit()?; - Ok(completion) - } - - pub async fn delete_failed_records_older_than( - &self, - cutoff_timestamp_millis: i64, - ) -> TaskSchedulerResult { - let mut conn = self.conn.lock().await; - let tx = conn.transaction()?; - let failed_scheduling_deleted = tx.execute( - "DELETE FROM failed_scheduling WHERE timestamp < ?", - [cutoff_timestamp_millis], - )?; - let failed_tasks_deleted = tx.execute( - "DELETE FROM failed_tasks WHERE timestamp < ?", - [cutoff_timestamp_millis], - )?; - tx.commit()?; - - Ok(failed_scheduling_deleted + failed_tasks_deleted) + /// Removes a task once it has been migrated onto hydra. + pub async fn remove_task(&self, task_id: i64) -> TaskSchedulerResult<()> { + self.conn + .lock() + .await + .execute("DELETE FROM tasks WHERE id = ?", [task_id])?; + Ok(()) } } diff --git a/magicblock-task-scheduler/src/errors.rs b/magicblock-task-scheduler/src/errors.rs index c5604aeff..4fdcaf56f 100644 --- a/magicblock-task-scheduler/src/errors.rs +++ b/magicblock-task-scheduler/src/errors.rs @@ -24,4 +24,7 @@ pub enum TaskSchedulerError { #[error("Batch size mismatch: expected {0}, got {1}")] SizeMismatch(usize, usize), + + #[error("Faucet not ready")] + FaucetNotReady, } diff --git a/magicblock-task-scheduler/src/lib.rs b/magicblock-task-scheduler/src/lib.rs index 1951c2004..854f34be6 100644 --- a/magicblock-task-scheduler/src/lib.rs +++ b/magicblock-task-scheduler/src/lib.rs @@ -4,4 +4,9 @@ pub mod service; pub use db::SchedulerDatabase; pub use errors::TaskSchedulerError; +/// Derives the on-chain hydra crank account address for a task from its +/// `(authority, task_id)` — the deterministic, per-authority namespace the +/// scheduler uses. Tests and tooling can use this to locate the crank account a +/// schedule request created. +pub use service::crank_pubkey; pub use service::TaskSchedulerService; diff --git a/magicblock-task-scheduler/src/service.rs b/magicblock-task-scheduler/src/service.rs index 5d5816e4c..90328d456 100644 --- a/magicblock-task-scheduler/src/service.rs +++ b/magicblock-task-scheduler/src/service.rs @@ -1,266 +1,125 @@ -use std::{ - collections::HashMap, - path::Path, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, -}; +use std::{path::Path, sync::Arc, time::Duration as StdDuration}; -use futures_util::{future::poll_fn, FutureExt, StreamExt}; -use magicblock_config::config::TaskSchedulerConfig; +use chrono::Utc; +use hydra_api::{ + consts::{CRANKER_REWARD, CRANK_HEADER_SIZE, SERIALIZED_META_SIZE}, + ephemeral::ID as EPHEMERAL_PROGRAM_ID, + instruction::{ephemeral, CreateArgs, SchedMeta, ScheduledIx}, +}; use magicblock_core::link::transactions::ScheduledTasksRx; use magicblock_ledger::LatestBlock; use magicblock_program::{ args::{CancelTaskRequest, ScheduleTaskRequest, TaskRequest}, - instruction_utils::InstructionUtils, validator::{validator_authority, validator_authority_id}, + EPHEMERAL_RENT_PER_BYTE, }; -use solana_message::Message; +use solana_account::AccountSharedData; +use solana_instruction::Instruction; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_signature::Signature; +use solana_signer::Signer; use solana_transaction::Transaction; use tokio::{ select, - sync::mpsc, - task::{JoinHandle, JoinSet}, - time::{interval, Duration, MissedTickBehavior}, -}; -use tokio_util::{ - sync::CancellationToken, - time::{delay_queue::Key, DelayQueue}, + task::JoinHandle, + time::{Duration, Instant}, }; +use tokio_util::sync::CancellationToken; use tracing::*; use crate::{ - db::{ - CrankFailedMove, CrankRetryCheck, CrankSuccessRemoval, - CrankSuccessUpdate, DbTask, SchedulerDatabase, - }, + db::{DbTask, SchedulerDatabase}, errors::{TaskSchedulerError, TaskSchedulerResult}, }; -const MAX_TASK_EXECUTION_RETRIES: u32 = 10; -const TASK_EXECUTION_RETRY_BASE_DELAY: Duration = Duration::from_millis(100); -const TASK_EXECUTION_RETRY_MAX_DELAY: Duration = Duration::from_secs(5); +/// How long migration waits for the validator to produce a usable blockhash +/// before giving up. +const BLOCK_READY_TIMEOUT: Duration = Duration::from_secs(60); +/// How long the scheduler waits for the faucet to be delegated and funded in +/// the ephemeral rollup before it starts paying for cranks. +const FAUCET_READY_TIMEOUT: Duration = Duration::from_secs(60); + +/// The task scheduler migrates any tasks persisted by the legacy +/// (validator-funded) scheduler onto the hydra crank program at startup, then +/// serves runtime schedule/cancel requests by sending hydra transactions. The +/// SQLite database is used solely for that one-time migration; the runtime path +/// is stateless and derives each task's crank PDA deterministically from +/// `(authority, task_id)`. pub struct TaskSchedulerService { - /// Database for persisting tasks + /// Migration-only database of legacy tasks. db: SchedulerDatabase, - /// RPC client used to send transactions + /// RPC client used to send hydra transactions. rpc_client: Arc, - /// Used to receive scheduled tasks from the transaction executor + /// Delegated faucet keypair used to pay for (sponsor, fund, sign, and + /// cancel) hydra cranks. Used instead of the validator identity, which is + /// not a delegated account. + faucet: Keypair, + /// Used to receive scheduled tasks from the transaction executor. scheduled_tasks: ScheduledTasksRx, - /// Provides latest blockhash for signing transactions + /// Provides latest blockhash and slot for building transactions. block: LatestBlock, - /// Queue of tasks to execute - task_queue: DelayQueue, - /// Map of task IDs to their corresponding keys in the task queue - task_queue_keys: HashMap, - /// Latest in-memory instance version for queued or in-flight tasks - task_versions: HashMap, - /// Number of consecutive failed execution attempts for each task - task_execution_retries: HashMap, - /// Token used to cancel the task scheduler + /// Token used to cancel the task scheduler. token: CancellationToken, - /// Minimum interval between task executions - min_interval: Duration, - /// How long failed task and scheduling records are retained. - failed_task_retention: Duration, - /// How often failed task and scheduling records are cleaned up. - failed_task_cleanup_interval: Duration, - /// Slot interval of the validator + /// Slot interval of the validator, used to convert millisecond intervals + /// into the slot-based cadence hydra expects. slot_interval: Duration, - /// Shared counter for noop instructions (unique crank transactions). - tx_counter: Arc, } -enum ProcessingOutcome { - Success, - Recoverable(Box), -} - -// SAFETY: TaskSchedulerService is moved into a single Tokio task in `start()` and never cloned. -// It runs exclusively on that task's thread. All fields (SchedulerDatabase, TransactionSchedulerHandle, -// ScheduledTasksRx, LatestBlock, DelayQueue, HashMap, AtomicU64, CancellationToken) are Send+Sync, -// and the service maintains exclusive ownership throughout its lifetime. +// SAFETY: TaskSchedulerService is moved into a single Tokio task in `start()` +// and never cloned. It runs exclusively on that task. All fields are Send+Sync. unsafe impl Send for TaskSchedulerService {} unsafe impl Sync for TaskSchedulerService {} impl TaskSchedulerService { - /// Creates a new `TaskSchedulerService` with the given configuration. + /// Creates a new `TaskSchedulerService`. pub fn new( path: &Path, - config: &TaskSchedulerConfig, rpc_url: String, + faucet: Option, scheduled_tasks: ScheduledTasksRx, block: LatestBlock, slot_interval: Duration, token: CancellationToken, ) -> TaskSchedulerResult { - if config.min_interval.as_millis() > u32::MAX as u128 { - return Err(TaskSchedulerError::InvalidConfiguration(format!( - "min_interval must be less than or equal to u32::MAX: {}", - config.min_interval.as_millis() - ))); - } - if config.reset { - match std::fs::remove_file(path) { - Ok(_) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debug!("Database file not found, skip resetting"); - } - Err(e) => { - warn!("Failed to remove database file: {}", e); - return Err(TaskSchedulerError::Io(e)); - } - } - } - - // Reschedule all persisted tasks + let Some(faucet) = faucet else { + warn!("No faucet keypair configured, skipping task scheduler"); + return Err(TaskSchedulerError::FaucetNotReady); + }; let db = SchedulerDatabase::new(path)?; Ok(Self { db, rpc_client: Arc::new(RpcClient::new(rpc_url)), + faucet, scheduled_tasks, block, - task_queue: DelayQueue::new(), - task_queue_keys: HashMap::new(), - task_versions: HashMap::new(), - task_execution_retries: HashMap::new(), - tx_counter: Arc::new(AtomicU64::default()), token, - min_interval: config.min_interval, - failed_task_retention: config.failed_task_retention, - failed_task_cleanup_interval: config.failed_task_cleanup_interval, slot_interval, }) } /// Starts the `TaskSchedulerService` and returns a handle to the task. pub async fn start( - mut self, + self, ) -> TaskSchedulerResult>> { - self.load_persisted_tasks().await?; Ok(tokio::spawn(self.run())) } - async fn load_persisted_tasks(&mut self) -> TaskSchedulerResult<()> { - self.task_queue.clear(); - self.task_queue_keys.clear(); - self.task_execution_retries.clear(); - - // Reschedule all tasks that are due - let tasks = self.db.get_tasks().await?; - let now = chrono::Utc::now().timestamp_millis(); - debug!( - "Task scheduler starting at {} with {} tasks", - now, - tasks.len() - ); - for task in tasks { - if !is_valid_task_interval(task.execution_interval_millis) - || task.executions_left <= 0 - { - warn!( - "Task {} has an invalid parameters: (interval={}, executions_left={}). Skipping.", - task.id, task.execution_interval_millis, task.executions_left - ); - self.db.remove_task(task.id).await?; - continue; - } - - let next_execution = - task.last_execution_millis + task.execution_interval_millis; - // Earliest reschedule time is 2 slot interval. - // This avoids, scheduling before the first blockhash is produced on restart. - let timeout = Duration::from_millis( - next_execution - .saturating_sub(now) - .max(2 * self.slot_interval.as_millis() as i64) - as u64, - ); - let task_id = task.id; - self.task_versions.insert(task_id, task.updated_at); - let key = self.task_queue.insert(task, timeout); - self.task_queue_keys.insert(task_id, key); - } - - Ok(()) - } - - /// Main loop of the task scheduler. + /// Main loop: migrate persisted tasks once, then serve runtime requests. async fn run(mut self) -> TaskSchedulerResult<()> { - let mut failed_task_cleanup = interval( - self.failed_task_cleanup_interval - .max(Duration::from_millis(1)), - ); - failed_task_cleanup.set_missed_tick_behavior(MissedTickBehavior::Delay); + if let Err(e) = self.migrate_persisted_tasks().await { + error!("Task migration failed: {}", e); + } - let (crank_tx, mut crank_rx) = mpsc::unbounded_channel(); + // Ensure the faucet is funded before serving runtime requests so the + // first scheduled task is not dropped for lack of a payer. + self.wait_for_faucet_ready().await?; loop { select! { - Some(expired) = self.task_queue.next() => { - // A task expired, batch all expired tasks - let first = expired.into_inner(); - self.task_queue_keys.remove(&first.id); - let mut batch = vec![first]; - while let Some(expired) = poll_fn(|cx| self.task_queue.poll_expired(cx)) - .now_or_never() - .flatten() - { - let task = expired.into_inner(); - self.task_queue_keys.remove(&task.id); - batch.push(task); - } - - let rpc_client = self.rpc_client.clone(); - let block = self.block.clone(); - let tx_counter = self.tx_counter.clone(); - let crank_tx = crank_tx.clone(); - - tokio::spawn(async move { - let result = - Self::send_crank_batch(rpc_client, &block, tx_counter, &batch).await; - let _ = crank_tx.send((batch, result)); - }); - } - Some((batch, result)) = crank_rx.recv() => { - // The batch has been sent, updates queue and db - self.on_crank_batch_completed(batch, result).await?; - } - Some(task) = self.scheduled_tasks.recv() => { - // Received a new request from the transaction executor - let id = task.id(); - match self.process_request(task).await { - Ok(ProcessingOutcome::Success) => {} - Ok(ProcessingOutcome::Recoverable(e)) => { - warn!("Failed to process request ID={}: {e:?}", id); - } - Err(e) => { - error!("Failed to process request: {}", e); - return Err(e); - } - } - } - _ = failed_task_cleanup.tick() => { - let cutoff = chrono::Utc::now().timestamp_millis().saturating_sub( - self.failed_task_retention.as_millis().min(i64::MAX as u128) as i64, - ); - let deleted = match self.db.delete_failed_records_older_than(cutoff).await { - Ok(deleted) => deleted, - Err(e) => { - error!("Failed to cleanup old failed task records: {}", e); - continue; - } - }; - if deleted > 0 { - debug!( - deleted, - cutoff_timestamp_millis = cutoff, - "Cleaned up old failed task records" - ); - } + Some(request) = self.scheduled_tasks.recv() => { + self.process_request(request).await; } _ = self.token.cancelled() => { break; @@ -269,515 +128,437 @@ impl TaskSchedulerService { } info!("TaskSchedulerService shutdown!"); - drop(crank_tx); - while let Some((batch, result)) = crank_rx.recv().await { - self.on_crank_batch_completed(batch, result).await?; - } - Ok(()) } - /// Processes [TaskRequest] provided by the transaction executor. - async fn process_request( - &mut self, - request: TaskRequest, - ) -> TaskSchedulerResult { - let task_id = request.id(); - match request { - TaskRequest::Schedule(schedule_request) => { - if let Err(e) = - self.process_schedule_request(schedule_request).await - { - self.db - .insert_failed_scheduling(task_id, format!("{:?}", e)) - .await?; - error!( - "Failed to process schedule request {}: {}", - task_id, e - ); - - return Ok(ProcessingOutcome::Recoverable(Box::new(e))); - } - } - TaskRequest::Cancel(cancel_request) => { - if let Err(e) = - self.process_cancel_request(&cancel_request).await - { - self.db - .insert_failed_scheduling(task_id, format!("{:?}", e)) - .await?; - error!( - "Failed to process cancel request for task {}: {}", - task_id, e - ); - - return Ok(ProcessingOutcome::Recoverable(Box::new(e))); - } - } - }; - - Ok(ProcessingOutcome::Success) - } - - /// Processes [ScheduleTaskRequest] provided by the transaction executor. - async fn process_schedule_request( - &mut self, - mut task: ScheduleTaskRequest, - ) -> TaskSchedulerResult<()> { - if !is_valid_task_interval(task.execution_interval_millis) { - // If the interval is too large or zero, we don't schedule the task + /// Migrates every task persisted by the legacy scheduler onto hydra, then + /// empties the database. Invalid tasks are dropped without rescheduling. + /// Migration is best-effort: a task is removed from the database whether or + /// not its hydra crank could be created, so the database always empties. + async fn migrate_persisted_tasks(&self) -> TaskSchedulerResult<()> { + let tasks = self.db.get_tasks().await?; + if tasks.is_empty() { return Ok(()); } - - task.execution_interval_millis = task - .execution_interval_millis - .clamp(self.min_interval.as_millis() as i64, u32::MAX as i64); - - let mut task = DbTask::from(task); - let task_id = task.id; - - // Check if the task already exists in the database - if let Some(db_task) = self.db.get_task(task_id).await? { - if db_task.authority != task.authority { - return Err(TaskSchedulerError::UnauthorizedReplacing( - task_id, - db_task.authority.to_string(), - task.authority.to_string(), - )); - } + info!("Migrating {} persisted task(s) onto hydra", tasks.len()); + + // Drop tasks that can no longer correspond to a live crank without + // touching the network. + let (valid, invalid): (Vec, Vec) = + tasks.into_iter().partition(|task| { + is_valid_task_interval(task.execution_interval_millis) + && task.executions_left > 0 + }); + for task in invalid { + warn!( + "Dropping invalid task {} during migration (interval={}, executions_left={})", + task.id, task.execution_interval_millis, task.executions_left + ); + self.db.remove_task(task.id).await?; } - task.updated_at = self.db.insert_task(&task).await?; - self.remove_task_from_queue(task_id); - self.task_execution_retries.remove(&task_id); - self.task_versions.insert(task_id, task.updated_at); - let key = self.task_queue.insert(task, Duration::from_millis(0)); - self.task_queue_keys.insert(task_id, key); - debug!("Registered task {} from context", task_id); - - Ok(()) - } - - /// Processes [CancelTaskRequest] provided by the transaction executor. - async fn process_cancel_request( - &mut self, - cancel_request: &CancelTaskRequest, - ) -> TaskSchedulerResult<()> { - let Some(task) = self.db.get_task(cancel_request.task_id).await? else { - // Task not found in the database, cleanup the queue - self.remove_task_from_queue(cancel_request.task_id); - self.task_execution_retries.remove(&cancel_request.task_id); - return Ok(()); - }; - - // Check if the task authority is the same as the cancel request authority - if task.authority != cancel_request.authority { - error!( - "Task authority {} does not match cancel request authority {}", - task.authority, cancel_request.authority - ); + if valid.is_empty() { return Ok(()); } - // Remove task from queue and database - self.remove_task_runtime_state(task.id); - self.task_execution_retries.remove(&task.id); - self.db.remove_task(task.id).await?; - debug!("Removed task {} from database", task.id); + // Sending a crank create needs a usable blockhash and a funded faucet + // (delegated and cloned into the ephemeral rollup) to pay with. + self.wait_for_block_ready().await; + info!("Migration: block ready"); + self.wait_for_faucet_ready().await?; + info!("Migration: faucet ready"); + + for task in valid { + info!("Migration: creating crank for task {}", task.id); + match self + .schedule_crank( + &task.authority, + task.id, + task.execution_interval_millis, + task.executions_left, + &task.instructions, + Some(task.last_execution_millis), + ) + .await + { + Ok(()) => { + self.db.remove_task(task.id).await?; + debug!("Migration: created crank for task {}", task.id) + } + Err(e) => { + warn!( + "Failed to migrate task {} onto hydra: {}", + task.id, e + ) + } + } + } + info!("Task migration complete"); Ok(()) } - /// Sends a batch of crank transactions to the RPC client. - async fn send_crank_batch( - rpc_client: Arc, - block: &LatestBlock, - tx_counter: Arc, - tasks: &[DbTask], - ) -> TaskSchedulerResult)>> - { - let mut join_set: JoinSet<(DbTask, TaskSchedulerResult)> = - JoinSet::new(); - let blockhash = block.load().blockhash; - for task in tasks { - let rpc_client = rpc_client.clone(); - let tx_counter = tx_counter.clone(); - let task = task.clone(); - join_set.spawn(async move { - let ixs = vec![ - InstructionUtils::noop_instruction( - tx_counter.fetch_add(1, Ordering::Relaxed), - ), - InstructionUtils::execute_task_instruction( - task.authority, - task.instructions.clone(), - ), - ]; - let tx = Transaction::new( - &[validator_authority()], - Message::new(&ixs, Some(&validator_authority_id())), - blockhash, - ); - let res = rpc_client - .send_transaction(&tx) - .await - .map_err(Box::new) - .map_err(TaskSchedulerError::from); - (task, res) - }); + /// Waits until the faucet has a non-zero balance in the ephemeral rollup + /// (i.e. it has been delegated on the base chain and cloned in), or the + /// scheduler is cancelled, or [`FAUCET_READY_TIMEOUT`] elapses. + async fn wait_for_faucet_ready(&self) -> TaskSchedulerResult<()> { + let faucet = self.faucet.pubkey(); + let start = Instant::now(); + while start.elapsed() < FAUCET_READY_TIMEOUT { + if self.token.is_cancelled() { + return Ok(()); + } + if matches!( + self.rpc_client.get_account(&faucet).await, + Ok(account) if account.lamports > 0 + ) { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(200)).await; } - Ok(join_set.join_all().await) + warn!(%faucet, "Timed out waiting for faucet to be delegated before paying cranks"); + Err(TaskSchedulerError::FaucetNotReady) } - /// Called when a crank batch is completed. - async fn on_crank_batch_completed( - &mut self, - batch: Vec, - result: TaskSchedulerResult< - Vec<(DbTask, TaskSchedulerResult)>, - >, - ) -> TaskSchedulerResult<()> { - let now_millis = chrono::Utc::now().timestamp_millis(); - let mut success_updates: Vec = Vec::new(); - let mut success_removals: Vec = Vec::new(); - let mut failed_records: Vec = Vec::new(); - let mut retry_checks: Vec = Vec::new(); - - // Decide what must happen to cranks - match result { - Ok(ref result) => { - // Batch completed, update individual crank based on tx status - for (task, res) in result { - if let Err(e) = res { - self.prepare_crank_failure_outcome( - task, - e, - &mut failed_records, - &mut retry_checks, - )?; - } else { - self.prepare_crank_success_outcome( - task, - now_millis, - &mut success_updates, - &mut success_removals, - )?; - } - } - } - Err(ref e) => { - // Whole batch failed, fail all cranks - for task in &batch { - self.prepare_crank_failure_outcome( - task, - e, - &mut failed_records, - &mut retry_checks, - )?; - } + /// Waits until the validator has a usable blockhash, or the scheduler is + /// cancelled, or [`BLOCK_READY_TIMEOUT`] elapses. + async fn wait_for_block_ready(&self) { + let start = Instant::now(); + while start.elapsed() < BLOCK_READY_TIMEOUT { + if self.token.is_cancelled() + || self.block.load().blockhash != Default::default() + { + return; } + tokio::time::sleep(Duration::from_millis(100)).await; } + warn!("Timed out waiting for a usable blockhash before migration"); + } - // Apply db updates for the whole batch - let completion = self - .db - .apply_crank_batch_completion( - &success_updates, - &success_removals, - &failed_records, - &retry_checks, - ) - .await?; - - // Update queue, retries and versions based on decisions - match result { - Ok(result) => { - for (task, res) in &result { - if let Err(e) = res { - if completion.failed_moves.get(&task.id).is_some_and( - |m| m.expected_updated_at == task.updated_at, - ) || completion - .retry_checks - .get(&task.id) - .is_some_and(|c| { - c.current_updated_at == task.updated_at - }) - { - self.apply_crank_failure_outcome(task, e)?; - } - } else if let Some(update) = - completion.success_updates.get(&task.id) - { - if update.expected_updated_at == task.updated_at { - self.apply_crank_success_outcome( - task, - now_millis, - Some(update.new_updated_at), - )?; - } - } else if completion - .success_removals - .get(&task.id) - .is_some_and(|r| { - r.expected_updated_at == task.updated_at - }) - { - self.apply_crank_success_outcome( - task, now_millis, None, - )?; - } - } + /// Processes a [TaskRequest] from the transaction executor. + async fn process_request(&self, request: TaskRequest) { + let task_id = request.id(); + let result = match request { + TaskRequest::Schedule(schedule_request) => { + self.process_schedule_request(schedule_request).await } - Err(ref e) => { - for task in &batch { - if completion.failed_moves.get(&task.id).is_some_and(|m| { - m.expected_updated_at == task.updated_at - }) || completion.retry_checks.get(&task.id).is_some_and( - |c| c.current_updated_at == task.updated_at, - ) { - self.apply_crank_failure_outcome(task, e)?; - } - } + TaskRequest::Cancel(cancel_request) => { + self.process_cancel_request(&cancel_request).await } + }; + if let Err(e) = result { + error!("Failed to process task request {}: {}", task_id, e); } - - Ok(()) } - /// Either: - /// - reschedules crank with remaining iterations - /// - remove exhausted cranks - fn prepare_crank_success_outcome( + /// Schedules a task: creates and funds its hydra crank. + async fn process_schedule_request( &self, - task: &DbTask, - now_millis: i64, - success_updates: &mut Vec, - success_removals: &mut Vec, + task: ScheduleTaskRequest, ) -> TaskSchedulerResult<()> { - let executed_at = next_execution_millis( - task.last_execution_millis, - task.execution_interval_millis, - now_millis, - ); - - if task.executions_left > 1 { - success_updates.push(CrankSuccessUpdate { - task_id: task.id, - last_execution_millis: executed_at, - expected_updated_at: task.updated_at, - }); - } else { - success_removals.push(CrankSuccessRemoval { - task_id: task.id, - expected_updated_at: task.updated_at, - }); + if !is_valid_task_interval(task.execution_interval_millis) { + // Too large or zero: ignore. + return Ok(()); } - + let interval_millis = + task.execution_interval_millis.clamp(1, u32::MAX as i64); + + self.schedule_crank( + &task.authority, + task.id, + interval_millis, + task.iterations, + &task.instructions, + None, + ) + .await?; + debug!("Created hydra crank for task {}", task.id); Ok(()) } - /// Either: - /// - re-queues for retry - /// - queues a permanent failure for SQLite along with clearing retry counters and stale queue keys. - fn prepare_crank_failure_outcome( + /// Cancels a task's hydra crank, if one exists for `(authority, task_id)`. + async fn process_cancel_request( &self, - task: &DbTask, - error: &TaskSchedulerError, - failed_records: &mut Vec, - retry_checks: &mut Vec, + cancel_request: &CancelTaskRequest, ) -> TaskSchedulerResult<()> { - debug!("Failed to execute task {}: {}", task.id, error); + let crank = + crank_pubkey(&cancel_request.authority, cancel_request.task_id); - if !is_retryable_task_execution_error(error) { - // Unretryable crank are moved to failed cranks - failed_records.push(CrankFailedMove { - task_id: task.id, - expected_updated_at: task.updated_at, - error: error.to_string(), - }); - return Ok(()); - } - - let retries = *self.task_execution_retries.get(&task.id).unwrap_or(&0); - if retries >= MAX_TASK_EXECUTION_RETRIES { - // Crank exhausted retries, fail it - failed_records.push(CrankFailedMove { - task_id: task.id, - expected_updated_at: task.updated_at, - error: error.to_string(), - }); - return Ok(()); - } - - // Schedule for retry - retry_checks.push(CrankRetryCheck { - task_id: task.id, - expected_updated_at: task.updated_at, - }); + // Does not check if the crank exists, so it will fail if it does not exist + self.send_cancel(crank).await?; + debug!("Cancelled hydra crank for task {}", cancel_request.task_id); Ok(()) } - /// Updates the delay queue for the next firing after SQLite confirms the same task instance. - fn apply_crank_success_outcome( - &mut self, - task: &DbTask, - now_millis: i64, - updated_at: Option, + /// Creates and funds the hydra crank for a task. If a crank already exists + /// at the deterministic PDA (a reschedule), it is closed first so the new + /// schedule can recreate it. + async fn schedule_crank( + &self, + authority: &Pubkey, + task_id: i64, + interval_millis: i64, + iterations: i64, + instructions: &[Instruction], + last_execution_millis: Option, ) -> TaskSchedulerResult<()> { - if !self.task_version_matches(task) { - debug!( - task_id = task.id, - expected_updated_at = task.updated_at, - "Skipping stale successful crank completion" - ); + if iterations <= 0 { return Ok(()); } - - let executed_at = next_execution_millis( - task.last_execution_millis, - task.execution_interval_millis, - now_millis, - ); - - if let Some(updated_at) = updated_at { - let next_execution = executed_at + task.execution_interval_millis; - let delay = delay_until_millis(next_execution, now_millis); - let new_task = DbTask { - executions_left: task.executions_left - 1, - last_execution_millis: executed_at, - updated_at, - ..task.clone() - }; - let key = self.task_queue.insert(new_task, delay); - self.task_queue_keys.insert(task.id, key); - self.task_versions.insert(task.id, updated_at); - } else { - self.remove_task_runtime_state(task.id); - } - - self.task_execution_retries.remove(&task.id); + self.send_create( + authority, + task_id, + interval_millis, + iterations, + instructions, + last_execution_millis, + ) + .await?; Ok(()) } - /// Either: - /// - re-queues for retry - /// - records a permanent failure in runtime state after SQLite confirms the same task instance. - fn apply_crank_failure_outcome( - &mut self, - task: &DbTask, - error: &TaskSchedulerError, - ) -> TaskSchedulerResult<()> { - if !self.task_version_matches(task) { - debug!( - task_id = task.id, - expected_updated_at = task.updated_at, - "Skipping stale failed crank completion" - ); - return Ok(()); - } + /// Returns whether a hydra-owned crank account currently exists at `crank`. + async fn crank_exists(&self, crank: &Pubkey) -> bool { + matches!( + self.rpc_client.get_account(crank).await, + Ok(account) if account.owner == EPHEMERAL_PROGRAM_ID + ) + } - if !is_retryable_task_execution_error(error) { - self.task_execution_retries.remove(&task.id); - self.remove_task_runtime_state(task.id); - return Ok(()); - } + /// Builds and sends the transaction that creates and funds a hydra crank. + /// It cancels the crank first if it already exists. + async fn send_create( + &self, + authority: &Pubkey, + task_id: i64, + interval_millis: i64, + iterations: i64, + instructions: &[Instruction], + last_execution_millis: Option, + ) -> TaskSchedulerResult { + let crank = crank_pubkey(authority, task_id); + let crank_exists = self.crank_exists(&crank).await; + + let snapshot = self.block.load(); + let start_slot = last_execution_millis + .map(|last_execution_millis| { + legacy_start_slot( + last_execution_millis, + interval_millis, + Utc::now().timestamp_millis(), + snapshot.slot, + self.slot_interval, + ) + }) + .unwrap_or(snapshot.slot); + let blockhash = snapshot.blockhash; + + let interval_slots = + interval_slots(interval_millis, self.slot_interval); + let iterations = iterations as u64; + + let faucet_pubkey = self.faucet.pubkey(); + let create_ix = build_create_ix( + &faucet_pubkey, + authority, + task_id, + crank, + start_slot, + interval_slots, + iterations, + instructions, + ); + let reward_pool = iterations.saturating_mul(CRANKER_REWARD); + let funding = + reward_pool.saturating_add(crank_rent_floor(instructions)); + let fund_ix = solana_system_interface::instruction::transfer( + &faucet_pubkey, + &crank, + funding, + ); - let Some(retries) = ({ - let retries = - self.task_execution_retries.entry(task.id).or_default(); - if *retries >= MAX_TASK_EXECUTION_RETRIES { - None - } else { - *retries += 1; - Some(*retries) - } - }) else { - self.task_execution_retries.remove(&task.id); - self.remove_task_runtime_state(task.id); - return Ok(()); + let validator = validator_authority(); + let ixs = if crank_exists { + let cancel_ix = + ephemeral::cancel(faucet_pubkey, crank, faucet_pubkey); + vec![cancel_ix, create_ix, fund_ix] + } else { + vec![create_ix, fund_ix] }; - - let delay = self.task_execution_retry_delay(retries); - debug!( - task_id = task.id, - retry = retries, - max_retries = MAX_TASK_EXECUTION_RETRIES, - delay_ms = delay.as_millis(), - error = %error, - "Retrying failed task execution" + let tx = Transaction::new_signed_with_payer( + &ixs, + Some(&validator_authority_id()), + &[&validator, &self.faucet], + blockhash, ); - let key = self.task_queue.insert(task.clone(), delay); - self.task_queue_keys.insert(task.id, key); - self.task_versions.insert(task.id, task.updated_at); - - Ok(()) + // Confirm before returning so successive cranks (all sponsored by the + // same faucet) don't race on the faucet's delegated account state. + self.rpc_client + .send_and_confirm_transaction(&tx) + .await + .map_err(Box::new) + .map_err(TaskSchedulerError::from) } - fn task_version_matches(&self, task: &DbTask) -> bool { - self.task_versions.get(&task.id) == Some(&task.updated_at) + /// Sends and confirms a crank cancellation. The crank's remaining lamports + /// are returned to the faucet (the cancel recipient). + async fn send_cancel( + &self, + crank: Pubkey, + ) -> TaskSchedulerResult { + let faucet_pubkey = self.faucet.pubkey(); + let blockhash = self.block.load().blockhash; + let cancel_ix = ephemeral::cancel(faucet_pubkey, crank, faucet_pubkey); + let validator = validator_authority(); + let tx = Transaction::new_signed_with_payer( + &[cancel_ix], + Some(&validator_authority_id()), + &[&validator, &self.faucet], + blockhash, + ); + self.rpc_client + .send_and_confirm_transaction(&tx) + .await + .map_err(Box::new) + .map_err(TaskSchedulerError::from) } +} - /// Removes a task from the queue. - fn remove_task_from_queue(&mut self, task_id: i64) { - if let Some(key) = self.task_queue_keys.remove(&task_id) { - self.task_queue.remove(&key); - } - } +/// Derives the deterministic hydra crank account address for a task. +/// +/// The seed is `hash(authority, task_id)`, so each authority gets its own crank +/// namespace: a different authority scheduling the same `task_id` gets an +/// independent crank, and cancel/reschedule need no database lookup. +pub fn crank_pubkey(authority: &Pubkey, task_id: i64) -> Pubkey { + let seed = solana_sha256_hasher::hashv(&[ + authority.as_ref(), + &task_id.to_le_bytes(), + ]) + .to_bytes(); + ephemeral::find_crank_pda(&seed).0 +} - fn remove_task_runtime_state(&mut self, task_id: i64) { - self.remove_task_from_queue(task_id); - self.task_versions.remove(&task_id); - } +/// Builds the hydra `Create` instruction embedding the task's instructions as +/// the scheduled crank payload. Account signer flags are intentionally dropped: +/// hydra rejects scheduled instructions that declare signers. +/// Rent-exempt minimum for the crank account hydra will allocate for these +/// scheduled instructions. Mirrors hydra's on-chain size accounting +/// (`CRANK_HEADER_SIZE + region_len`, where each scheduled ix serializes to +/// `2 + metas + program_id + 2 + data`). +fn crank_rent_floor(instructions: &[Instruction]) -> u64 { + let region_len: usize = instructions + .iter() + .map(|ix| { + 2 + ix.accounts.len() * SERIALIZED_META_SIZE + + 32 + + 2 + + ix.data.len() + }) + .sum::() + + CRANK_HEADER_SIZE; + let total_size = + (region_len + AccountSharedData::ACCOUNT_STATIC_SIZE as usize) as u64; + total_size * EPHEMERAL_RENT_PER_BYTE +} - /// Calculates the retry delay for the next task execution. - fn task_execution_retry_delay(&self, retry: u32) -> Duration { - let multiplier = 1u32 - .checked_shl(retry.saturating_sub(1)) - .unwrap_or(u32::MAX); - self.slot_interval - .max(TASK_EXECUTION_RETRY_BASE_DELAY) - .checked_mul(multiplier) - .unwrap_or(TASK_EXECUTION_RETRY_MAX_DELAY) - .min(TASK_EXECUTION_RETRY_MAX_DELAY) - } +#[allow(clippy::too_many_arguments)] +fn build_create_ix( + faucet: &Pubkey, + authority: &Pubkey, + task_id: i64, + crank: Pubkey, + start_slot: u64, + interval_slots: u64, + iterations: u64, + instructions: &[Instruction], +) -> Instruction { + let seed = solana_sha256_hasher::hashv(&[ + authority.as_ref(), + &task_id.to_le_bytes(), + ]) + .to_bytes(); + + let metas_per_ix: Vec> = instructions + .iter() + .map(|ix| { + ix.accounts + .iter() + .map(|acc| SchedMeta { + pubkey: acc.pubkey.to_bytes(), + is_writable: acc.is_writable, + }) + .collect() + }) + .collect(); + + let scheduled: Vec = instructions + .iter() + .zip(metas_per_ix.iter()) + .map(|(ix, metas)| ScheduledIx { + program_id: ix.program_id.to_bytes(), + metas: metas.as_slice(), + data: ix.data.as_slice(), + }) + .collect(); + + let args = CreateArgs { + seed, + // The faucet is the sponsor and the cancel authority for the crank. + authority: faucet.to_bytes(), + start_slot, + interval_slots, + remaining: iterations, + priority_tip: 0, + cu_limit: 0, + scheduled: scheduled.as_slice(), + }; + + ephemeral::create(*faucet, crank, &args) } fn is_valid_task_interval(interval: i64) -> bool { interval > 0 && interval < u32::MAX as i64 } -fn next_execution_millis( +/// Computes the hydra start slot for a task migrated from the legacy scheduler. +/// If the legacy task is overdue or has never run, it remains due immediately. +fn legacy_start_slot( last_execution_millis: i64, - execution_interval_millis: i64, - now: i64, -) -> i64 { - if last_execution_millis == 0 { - now - } else { - last_execution_millis + execution_interval_millis + interval_millis: i64, + current_millis: i64, + current_slot: u64, + slot_interval: StdDuration, +) -> u64 { + if last_execution_millis <= 0 { + return current_slot; } -} -fn delay_until_millis(execution_millis: i64, now: i64) -> Duration { - Duration::from_millis(execution_millis.saturating_sub(now).max(0) as u64) + let next_execution_millis = + last_execution_millis.saturating_add(interval_millis.max(0)); + let remaining_millis = next_execution_millis.saturating_sub(current_millis); + if remaining_millis == 0 { + return current_slot; + } + + current_slot.saturating_add(interval_slots(remaining_millis, slot_interval)) } -fn is_retryable_task_execution_error(error: &TaskSchedulerError) -> bool { - // `send_crank_batch` maps Solana send and verification failures to Rpc. - matches!(error, TaskSchedulerError::Rpc(_)) +/// Converts a millisecond execution interval into a slot count (rounding up, +/// with a one-slot minimum) for hydra's slot-based cadence. +fn interval_slots(interval_millis: i64, slot_interval: StdDuration) -> u64 { + let slot_millis = (slot_interval.as_millis() as i64).max(1); + let interval_millis = interval_millis.max(0); + // Ceiling division without the unstable `i64::div_ceil`. + let slots = (interval_millis + slot_millis - 1) / slot_millis; + slots.max(1) as u64 } #[cfg(test)] mod tests { use magicblock_core::coordination_mode::switch_to_primary_mode; - use magicblock_program::{ - args::ScheduleTaskRequest, - validator::generate_validator_authority_if_needed, - }; use serial_test::serial; - use solana_pubkey::Pubkey; use tokio::{sync::mpsc, time::timeout}; use super::*; @@ -791,16 +572,9 @@ mod tests { rpc_client: Arc::new(RpcClient::new( "http://localhost:8899".to_string(), )), + faucet: Keypair::new(), block: LatestBlock::default(), - task_queue: DelayQueue::new(), - task_queue_keys: HashMap::new(), - task_versions: HashMap::new(), - task_execution_retries: HashMap::new(), - tx_counter: Arc::new(AtomicU64::default()), token: CancellationToken::new(), - min_interval: Duration::from_millis(1000), - failed_task_retention: Duration::from_secs(60), - failed_task_cleanup_interval: Duration::from_secs(60), slot_interval: Duration::from_millis(1000), scheduled_tasks, } @@ -808,270 +582,93 @@ mod tests { #[serial] #[test] - fn test_first_execution_anchors_cadence_at_now() { - assert_eq!(next_execution_millis(0, 50, 1_000), 1_000); + fn test_interval_millis_rounds_up_to_slots() { + let slot = StdDuration::from_millis(50); + assert_eq!(interval_slots(1, slot), 1); + assert_eq!(interval_slots(50, slot), 1); + assert_eq!(interval_slots(51, slot), 2); + assert_eq!(interval_slots(100, slot), 2); } #[serial] #[test] - fn test_recurring_execution_preserves_fixed_rate_cadence() { - let executed_at = next_execution_millis(1_000, 50, 1_090); - assert_eq!(executed_at, 1_050); + fn test_legacy_start_slot_preserves_remaining_delay() { + let slot_interval = StdDuration::from_millis(1_000); - let delay = delay_until_millis(executed_at + 50, 1_090); - assert_eq!(delay, Duration::from_millis(10)); + assert_eq!( + legacy_start_slot(10_000, 30_000, 25_000, 100, slot_interval), + 115 + ); + assert_eq!( + legacy_start_slot(10_000, 30_000, 40_000, 100, slot_interval), + 100 + ); + assert_eq!( + legacy_start_slot(0, 30_000, 25_000, 100, slot_interval), + 100 + ); } #[serial] #[test] - fn test_overdue_execution_is_rescheduled_immediately() { - assert_eq!(delay_until_millis(1_100, 1_150), Duration::from_millis(0)); - } - - #[serial] - #[tokio::test] - async fn test_schedule_invalid_tasks() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - generate_validator_authority_if_needed(); - - let (tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - - let service = test_service(db.clone(), rx); - - let handle = service.start().await.unwrap(); - - // Invalid task interval - tx.send(TaskRequest::Schedule(ScheduleTaskRequest { - id: 1, - authority: Pubkey::new_unique(), - execution_interval_millis: u32::MAX as i64, - iterations: 1, - instructions: vec![], - })) - .unwrap(); - // Valid task interval - tx.send(TaskRequest::Schedule(ScheduleTaskRequest { - id: 1, - authority: Pubkey::new_unique(), - execution_interval_millis: u32::MAX as i64 - 1, - iterations: 1, - instructions: vec![], - })) - .unwrap(); - - // After processing the requests, only one task stays in the DB - timeout(Duration::from_secs(1), async move { - loop { - let tasks = db.get_tasks().await.unwrap(); - if tasks.len() > 1 { - return Err::<(), String>(format!( - "Tasks should be 1, got {}", - tasks.len() - )); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .unwrap_err(); - - handle.abort(); + fn test_crank_pubkey_namespaced_by_authority_and_id() { + let a = Pubkey::new_unique(); + let b = Pubkey::new_unique(); + // Deterministic. + assert_eq!(crank_pubkey(&a, 1), crank_pubkey(&a, 1)); + // Different task id -> different crank. + assert_ne!(crank_pubkey(&a, 1), crank_pubkey(&a, 2)); + // Different authority, same id -> different crank (per-authority namespace). + assert_ne!(crank_pubkey(&a, 1), crank_pubkey(&b, 1)); } #[serial] #[tokio::test] - async fn test_remove_invalid_tasks_on_startup() { + async fn test_migration_drops_invalid_tasks_and_empties_db() { magicblock_core::logger::init_for_tests(); switch_to_primary_mode(); let (_tx, rx) = mpsc::unbounded_channel(); let db = SchedulerDatabase::new(":memory:").unwrap(); - // Invalid task interval + // Invalid interval. db.insert_task(&DbTask { id: 1, authority: Pubkey::new_unique(), execution_interval_millis: u32::MAX as i64, executions_left: 1, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, + last_execution_millis: 0, instructions: vec![], }) .await .unwrap(); - // Valid task interval + // Exhausted (no executions left). db.insert_task(&DbTask { id: 2, authority: Pubkey::new_unique(), - execution_interval_millis: u32::MAX as i64 - 1, - executions_left: 1, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, - instructions: vec![], - }) - .await - .unwrap(); - let service = test_service(db.clone(), rx); - - let handle = service.start().await.unwrap(); - - // After starting, only one task should be in the database - timeout(Duration::from_secs(1), async move { - loop { - let tasks = db.get_tasks().await?; - if tasks.len() == 1 { - return Ok::<_, TaskSchedulerError>(()); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .unwrap() - .unwrap(); - handle.abort(); - } - - #[serial] - #[tokio::test] - async fn test_completed_tasks_are_removed_on_startup() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - - let (_tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - db.insert_task(&DbTask { - id: 1, - authority: Pubkey::new_unique(), execution_interval_millis: 50, executions_left: 0, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, - instructions: vec![], - }) - .await - .unwrap(); - db.insert_task(&DbTask { - id: 2, - authority: Pubkey::new_unique(), - execution_interval_millis: 50, - executions_left: 1, - last_execution_millis: chrono::Utc::now().timestamp_millis(), - updated_at: 0, + last_execution_millis: 0, instructions: vec![], }) .await .unwrap(); - let mut service = test_service(db.clone(), rx); - service.min_interval = Duration::from_millis(10); - + let service = test_service(db.clone(), rx); let handle = service.start().await.unwrap(); - timeout(Duration::from_secs(1), async move { + // Migration drops both invalid tasks without any network access, so the + // database empties promptly. + timeout(Duration::from_secs(2), async move { loop { - let tasks = db.get_task_ids().await?; - if tasks == vec![2] { - return Ok::<_, TaskSchedulerError>(()); + if db.get_task_ids().await.unwrap().is_empty() { + return; } - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(50)).await; } }) .await - .unwrap() - .unwrap(); - handle.abort(); - } + .expect("database should empty after migration"); - #[serial] - #[tokio::test] - async fn test_stale_crank_completion_does_not_mutate_replaced_task() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - - let (_tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - let authority = Pubkey::new_unique(); - let mut old_task = DbTask { - id: 1, - authority, - execution_interval_millis: 50, - executions_left: 2, - last_execution_millis: 0, - updated_at: 0, - instructions: vec![], - }; - old_task.updated_at = db.insert_task(&old_task).await.unwrap(); - - let mut service = test_service(db.clone(), rx); - - let mut replacement = DbTask { - executions_left: 5, - ..old_task.clone() - }; - replacement.updated_at = db.insert_task(&replacement).await.unwrap(); - let key = service - .task_queue - .insert(replacement.clone(), Duration::from_secs(10)); - service.task_queue_keys.insert(replacement.id, key); - service - .task_versions - .insert(replacement.id, replacement.updated_at); - - service - .on_crank_batch_completed( - vec![old_task.clone()], - Ok(vec![(old_task, Ok(Signature::new_unique()))]), - ) - .await - .unwrap(); - - let persisted = db.get_task(replacement.id).await.unwrap().unwrap(); - assert_eq!(persisted.executions_left, replacement.executions_left); - assert_eq!(persisted.updated_at, replacement.updated_at); - - let key = service.task_queue_keys.remove(&replacement.id).unwrap(); - let queued = service.task_queue.remove(&key).into_inner(); - assert_eq!(queued.updated_at, replacement.updated_at); - assert_eq!(queued.executions_left, replacement.executions_left); - } - - #[serial] - #[tokio::test] - async fn test_failed_records_are_cleaned_up_periodically() { - magicblock_core::logger::init_for_tests(); - switch_to_primary_mode(); - - let (_tx, rx) = mpsc::unbounded_channel(); - let db = SchedulerDatabase::new(":memory:").unwrap(); - - db.insert_failed_scheduling(1, "schedule failed".to_string()) - .await - .unwrap(); - db.insert_failed_task(2, "task failed".to_string()) - .await - .unwrap(); - tokio::time::sleep(Duration::from_millis(2)).await; - - let mut service = test_service(db.clone(), rx); - service.failed_task_retention = Duration::from_millis(1); - service.failed_task_cleanup_interval = Duration::from_millis(5); - - let handle = service.start().await.unwrap(); - - timeout(Duration::from_secs(1), async move { - loop { - if db.get_failed_schedulings().await?.is_empty() - && db.get_failed_tasks().await?.is_empty() - { - return Ok::<_, TaskSchedulerError>(()); - } - tokio::time::sleep(Duration::from_millis(5)).await; - } - }) - .await - .unwrap() - .unwrap(); handle.abort(); } } diff --git a/programs/magicblock/src/magicblock_processor.rs b/programs/magicblock/src/magicblock_processor.rs index d39ed2486..0ce806c32 100644 --- a/programs/magicblock/src/magicblock_processor.rs +++ b/programs/magicblock/src/magicblock_processor.rs @@ -21,9 +21,7 @@ use crate::{ }, mutate_accounts::process_mutate_accounts, process_scheduled_commit_sent, - schedule_task::{ - process_cancel_task, process_execute_crank, process_schedule_task, - }, + schedule_task::{process_cancel_task, process_schedule_task}, schedule_transactions::{ process_accept_scheduled_commits, process_add_action_callback, process_execute_callback, process_schedule_cloned_account_undelegation, @@ -226,40 +224,6 @@ declare_process_instruction!( remote_slot, authority, ), - ExecuteCrank { - authority, - instructions, - } => process_execute_crank( - signers, - invoke_context, - &authority, - instructions, - ), - } - } -); - -declare_process_instruction!( - CrankEntrypoint, - DEFAULT_COMPUTE_UNITS, - |invoke_context| { - let instruction = deserialize_instruction(invoke_context)?; - let transaction_context = &invoke_context.transaction_context; - let instruction_context = - transaction_context.get_current_instruction_context()?; - let signers = instruction_context.get_signers()?; - - match instruction { - MagicBlockInstruction::ExecuteCrank { - authority, - instructions, - } => process_execute_crank( - signers, - invoke_context, - &authority, - instructions, - ), - _ => Err(InstructionError::InvalidInstructionData), } } ); @@ -317,37 +281,11 @@ declare_process_instruction!( #[cfg(test)] mod test { - use magicblock_magic_program_api::args::ScheduleTaskArgs; use solana_instruction::AccountMeta; use solana_program_runtime::invoke_context::mock_process_instruction; use super::*; - #[test] - fn crank_entrypoint_rejects_non_execute_crank_instructions() { - let data = bincode::serialize(&MagicBlockInstruction::ScheduleTask( - ScheduleTaskArgs { - task_id: 1, - execution_interval_millis: 10, - iterations: 1, - instructions: vec![], - }, - )) - .unwrap(); - - mock_process_instruction( - &crate::CRANK_PROGRAM_ID, - None, - &data, - Vec::new(), - vec![AccountMeta::new_readonly(crate::CRANK_PROGRAM_ID, false)], - Err(InstructionError::InvalidInstructionData), - CrankEntrypoint::vm, - |_invoke_context| {}, - |_invoke_context| {}, - ); - } - #[test] fn callback_entrypoint_rejects_invalid_instruction_data() { let data = vec![0xFF, 0xFF, 0xFF, 0xFF]; diff --git a/programs/magicblock/src/schedule_task/mod.rs b/programs/magicblock/src/schedule_task/mod.rs index f21836594..75f85bcc5 100644 --- a/programs/magicblock/src/schedule_task/mod.rs +++ b/programs/magicblock/src/schedule_task/mod.rs @@ -1,44 +1,31 @@ mod process_cancel_task; -mod process_execute_task; mod process_schedule_task; -use magicblock_magic_program_api::{ - instruction::MagicBlockInstruction, pda::crank_signer_pda, -}; +use magicblock_magic_program_api::instruction::MagicBlockInstruction; pub(crate) use process_cancel_task::*; -pub(crate) use process_execute_task::*; pub(crate) use process_schedule_task::*; use solana_instruction::{error::InstructionError, Instruction}; use solana_log_collector::ic_msg; use solana_program_runtime::invoke_context::InvokeContext; -use solana_pubkey::Pubkey; use crate::validator::effective_validator_authority_id; -// Assert that the task instructions do not have signers aside from the crank signer +// Assert that the task instructions do not have signers // Assert they don't use the validator either // Assert they are not a privileged instruction pub(crate) fn validate_cranks_instructions( invoke_context: &mut InvokeContext, - authority: &Pubkey, instructions: &[Instruction], ) -> Result<(), InstructionError> { - let crank_signer = crank_signer_pda(authority); for instruction in instructions { for account in &instruction.accounts { - if account.is_signer && account.pubkey.ne(&crank_signer) { + if account.is_signer { ic_msg!( invoke_context, - "Crank ERR: only the crank signer PDA can be a signer in cranks (invalid signer: '{}')", + "Crank ERR: scheduled crank instructions cannot include signer accounts (invalid signer: '{}')", account.pubkey, ); return Err(InstructionError::MissingRequiredSignature); - } else if account.is_writable && account.pubkey.eq(&crank_signer) { - ic_msg!( - invoke_context, - "Crank ERR: the crank signer PDA cannot be a writable account in cranks", - ); - return Err(InstructionError::Immutable); } else if account.pubkey.eq(&effective_validator_authority_id()) { ic_msg!( invoke_context, diff --git a/programs/magicblock/src/schedule_task/process_execute_task.rs b/programs/magicblock/src/schedule_task/process_execute_task.rs deleted file mode 100644 index 542bcbc79..000000000 --- a/programs/magicblock/src/schedule_task/process_execute_task.rs +++ /dev/null @@ -1,381 +0,0 @@ -use std::collections::HashSet; - -use magicblock_magic_program_api::{pda::crank_signer_pda, CRANK_PROGRAM_ID}; -use solana_instruction::{error::InstructionError, Instruction}; -use solana_log_collector::ic_msg; -use solana_program_runtime::invoke_context::InvokeContext; -use solana_pubkey::Pubkey; - -use crate::{ - schedule_task::validate_cranks_instructions, - utils::accounts::get_instruction_pubkey_with_idx, - validator::effective_validator_authority_id, -}; - -pub(crate) fn process_execute_crank( - signers: HashSet, - invoke_context: &mut InvokeContext, - authority: &Pubkey, - instructions: Vec, -) -> Result<(), InstructionError> { - const VALIDATOR_IDX: u16 = 0; - const CRANK_SIGNER_IDX: u16 = 1; - let crank_signer = crank_signer_pda(authority); - - const ACCOUNTS_START: usize = CRANK_SIGNER_IDX as usize + 1; - - { - let transaction_context = &*invoke_context.transaction_context; - let ix_ctx = transaction_context.get_current_instruction_context()?; - let ix_accs_len = ix_ctx.get_number_of_instruction_accounts() as usize; - - // Assert crank executor program. - let program_key = ix_ctx.get_program_key()?; - if program_key != &crate::id() && program_key != &CRANK_PROGRAM_ID { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: crank executor program account not found" - ); - return Err(InstructionError::UnsupportedProgramId); - } - - // Assert enough accounts - if ix_accs_len < ACCOUNTS_START { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: not enough accounts to execute crank ({}), need crank signer and instructions", - ix_accs_len - ); - return Err(InstructionError::MissingAccount); - } - - // Assert Validator is signer - // Only the validator can execute a crank - let validator_pubkey = get_instruction_pubkey_with_idx( - transaction_context, - VALIDATOR_IDX, - )?; - let validator_authority = effective_validator_authority_id(); - if validator_pubkey != &validator_authority { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: validator pubkey {} is not the expected validator", - validator_pubkey - ); - return Err(InstructionError::IncorrectAuthority); - } - if !signers.contains(validator_pubkey) { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: validator pubkey {} is not in signers", - validator_pubkey - ); - return Err(InstructionError::MissingRequiredSignature); - } - - // Assert Crank signer is provided - let crank_signer_pubkey = get_instruction_pubkey_with_idx( - transaction_context, - CRANK_SIGNER_IDX, - )?; - if crank_signer_pubkey != &crank_signer { - ic_msg!( - invoke_context, - "ExecuteCrank ERR: crank signer pubkey {} is not the expected Crank signer", - crank_signer_pubkey - ); - return Err(InstructionError::InvalidSeeds); - } - } - - // Already validated when scheduling the task. - // This check prevents the validator from manually sending transactions disguised as cranks. - validate_cranks_instructions(invoke_context, authority, &instructions)?; - - let len = instructions.len(); - for ix in instructions { - invoke_context.native_invoke(ix, &[crank_signer])?; - } - - ic_msg!(invoke_context, "Executed crank with {} instructions", len); - - Ok(()) -} - -#[cfg(test)] -mod test { - use magicblock_magic_program_api::args::ScheduleTaskArgs; - use solana_account::AccountSharedData; - use solana_instruction::AccountMeta; - use solana_keypair::Keypair; - use solana_sdk_ids::system_program; - - use super::*; - use crate::{ - test_utils::process_instruction, - utils::instruction_utils::InstructionUtils, - validator::{init_validator_authority, validator_authority_id}, - }; - - pub fn complex_ix(payer: Pubkey) -> Instruction { - let mut ix = InstructionUtils::noop_instruction(0); - ix.accounts.push(AccountMeta::new(payer, false)); - ix - } - - #[test] - fn test_execute_task_simple() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Ok(()), - ); - } - - #[test] - fn test_execute_task_complex() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let payer = Pubkey::new_unique(); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![complex_ix(payer)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - payer, - AccountSharedData::new(1000000, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Ok(()), - ); - } - - #[test] - fn fail_execute_task_without_crank_signer() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - vec![], - Err(InstructionError::MissingAccount), - ); - } - - #[test] - fn fail_execute_task_wrong_validator() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let wrong_validator = Pubkey::new_unique(); - let transaction_accounts = vec![ - ( - wrong_validator, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::IncorrectAuthority), - ); - } - - #[test] - fn fail_execute_task_validator_not_in_signers() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let mut ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - ix.accounts[0].is_signer = false; - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::MissingRequiredSignature), - ); - } - - #[test] - fn fail_execute_task_wrong_crank_signer() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![InstructionUtils::noop_instruction(0)], - ); - let wrong_crank_signer = Pubkey::new_unique(); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - wrong_crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::InvalidSeeds), - ); - } - - #[test] - fn fail_execute_task_missing_accounts() { - init_validator_authority(Keypair::new()); - let payer = Pubkey::new_unique(); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![complex_ix(payer)], - ); - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(InstructionError::MissingAccount), - ); - } - - #[test] - fn fail_execute_task_with_invalid_instructions() { - init_validator_authority(Keypair::new()); - let authority = Pubkey::new_unique(); - let crank_signer = crank_signer_pda(&authority); - let payer = Pubkey::new_unique(); - let mut inner_ix = InstructionUtils::schedule_task_instruction( - &payer, - ScheduleTaskArgs { - task_id: 0, - execution_interval_millis: 1, - iterations: 1, - instructions: vec![InstructionUtils::noop_instruction(0)], - }, - ); - - for (signer, writable, pubkey, expected) in [ - ( - true, - false, - Pubkey::new_unique(), - InstructionError::MissingRequiredSignature, - ), - (false, true, crank_signer, InstructionError::Immutable), - ( - false, - false, - validator_authority_id(), - InstructionError::IncorrectAuthority, - ), - ] { - inner_ix.accounts[0].is_signer = signer; - inner_ix.accounts[0].is_writable = writable; - inner_ix.accounts[0].pubkey = pubkey; - let ix = InstructionUtils::execute_task_instruction( - authority, - vec![inner_ix.clone()], - ); - - let transaction_accounts = vec![ - ( - validator_authority_id(), - AccountSharedData::new(0, 0, &system_program::id()), - ), - ( - crank_signer, - AccountSharedData::new(0, 0, &system_program::id()), - ), - ]; - - process_instruction( - &ix.data, - transaction_accounts, - ix.accounts, - Err(expected), - ); - } - } -} diff --git a/programs/magicblock/src/schedule_task/process_schedule_task.rs b/programs/magicblock/src/schedule_task/process_schedule_task.rs index dfe20c63b..e478c59e7 100644 --- a/programs/magicblock/src/schedule_task/process_schedule_task.rs +++ b/programs/magicblock/src/schedule_task/process_schedule_task.rs @@ -93,11 +93,7 @@ pub(crate) fn process_schedule_task( return Err(InstructionError::InvalidInstructionData); } - validate_cranks_instructions( - invoke_context, - &payer_pubkey, - &args.instructions, - )?; + validate_cranks_instructions(invoke_context, &args.instructions)?; let schedule_request = ScheduleTaskRequest { id: args.task_id, diff --git a/programs/magicblock/src/utils/instruction_utils.rs b/programs/magicblock/src/utils/instruction_utils.rs index 8a9702b86..53445fdbb 100644 --- a/programs/magicblock/src/utils/instruction_utils.rs +++ b/programs/magicblock/src/utils/instruction_utils.rs @@ -9,9 +9,7 @@ use magicblock_magic_program_api::{ AccountModification, AccountModificationForInstruction, MagicBlockInstruction, PostDelegationActionExecutorInstruction, }, - pda::crank_signer_pda, - CRANK_PROGRAM_ID, MAGIC_CONTEXT_PUBKEY, - POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, + MAGIC_CONTEXT_PUBKEY, POST_DELEGATION_ACTION_EXECUTOR_PROGRAM_ID, }; use solana_hash::Hash; use solana_instruction::{AccountMeta, Instruction}; @@ -253,47 +251,6 @@ impl InstructionUtils { ) } - // ----------------- - // Execute Crank - // ----------------- - pub fn execute_task_instruction( - authority: Pubkey, - instructions: Vec, - ) -> Instruction { - let mut account_metas = vec![ - AccountMeta::new_readonly(validator_authority_id(), true), - AccountMeta::new_readonly(crank_signer_pda(&authority), false), - ]; - for instruction in &instructions { - account_metas - .push(AccountMeta::new_readonly(instruction.program_id, false)); - account_metas.extend(instruction.accounts.iter().map(|account| { - AccountMeta { - pubkey: account.pubkey, - is_signer: false, - is_writable: account.is_writable, - } - })); - } - Instruction::new_with_bincode( - CRANK_PROGRAM_ID, - &MagicBlockInstruction::ExecuteCrank { - authority, - instructions, - }, - account_metas, - ) - } - - pub fn execute_task( - authority: Pubkey, - instructions: Vec, - recent_blockhash: Hash, - ) -> Transaction { - let ix = Self::execute_task_instruction(authority, instructions); - Self::into_transaction(&validator_authority(), ix, recent_blockhash) - } - // ----------------- // Executable Check // ----------------- diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 2d6c2aa7c..c2a650c15 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -120,7 +120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a48ac39333d364f102b03ef00b59cd5b1f878ccb3c5f0f67df20f44824d51f" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "digest 0.10.7", "ed25519-dalek 1.0.1", "libsecp256k1", @@ -153,7 +153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84debd4abe0cbab5a6aac2ee50e3969ef0e0961f7dff7e8f96bda0be7998bca2" dependencies = [ "agave-bls12-381", - "bincode", + "bincode 1.3.3", "libsecp256k1", "num-traits", "solana-account 3.4.0", @@ -786,6 +786,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.69.5" @@ -1939,6 +1958,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ephemeral-rollups-pinocchio" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a09b61c47e5b4eeb7add34e9d43d836784a66d78fa704606bebaa453bef73e21" +dependencies = [ + "bincode 2.0.1", + "pinocchio 0.10.2", + "pinocchio-pubkey", + "pinocchio-system", + "solana-address 2.5.0", +] + [[package]] name = "ephemeral-rollups-sdk" version = "0.14.4" @@ -1946,7 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "838564736408ada011e9986c819b26cf0195bb2dd238e10be258731979213d75" dependencies = [ "base64ct", - "bincode", + "bincode 1.3.3", "bytemuck", "ephemeral-rollups-sdk-attribute-action", "ephemeral-rollups-sdk-attribute-commit", @@ -2509,7 +2541,7 @@ dependencies = [ name = "guinea" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "magicblock-magic-program-api 0.13.2", "serde", "solana-program 3.0.0", @@ -2746,6 +2778,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "hydra-api" +version = "0.1.1" +source = "git+https://github.com/magicblock-labs/hydra.git?rev=449532ee0ee792e9a3809b16fd9d289c2024341a#449532ee0ee792e9a3809b16fd9d289c2024341a" +dependencies = [ + "ephemeral-rollups-pinocchio", + "pinocchio 0.10.2", + "solana-address 2.5.0", + "solana-instruction 3.3.0", + "solana-pubkey 4.1.0", +] + [[package]] name = "hyper" version = "1.10.1" @@ -3500,7 +3544,7 @@ name = "magicblock-account-cloner" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "magicblock-chainlink", "magicblock-core", "magicblock-ledger", @@ -3579,7 +3623,7 @@ dependencies = [ "agave-geyser-plugin-interface", "arc-swap", "base64 0.21.7", - "bincode", + "bincode 1.3.3", "bs58", "fastwebsockets", "futures", @@ -3669,6 +3713,7 @@ dependencies = [ "solana-sha256-hasher 3.1.0", "solana-signature", "solana-signer", + "solana-system-interface 3.1.0", "solana-system-program", "solana-sysvar 3.1.1", "solana-transaction", @@ -3687,7 +3732,7 @@ dependencies = [ "agave-feature-set", "arc-swap", "async-trait", - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "futures-util", "helius-laserstream", @@ -3756,7 +3801,7 @@ name = "magicblock-committor-service" version = "0.13.2" dependencies = [ "async-trait", - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "futures-util", "lru", @@ -3817,7 +3862,7 @@ dependencies = [ name = "magicblock-core" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bytes", "flume", "magicblock-magic-program-api 0.13.2", @@ -3847,7 +3892,7 @@ name = "magicblock-delegation-program-api" version = "0.3.0" source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b#25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "bytemuck", "const-crypto", @@ -3878,7 +3923,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "288904a9950bd20f27f0ef934f320ab1410bd35a6d5c9cf138eca276442b6b2e" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 0.10.4", "borsh 1.7.0", "bytemuck", @@ -3910,7 +3955,7 @@ name = "magicblock-ledger" version = "0.13.2" dependencies = [ "arc-swap", - "bincode", + "bincode 1.3.3", "byteorder", "fs_extra", "libc", @@ -3949,7 +3994,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dc8fba0307c90b91b70c9ed06d4242d6c4159f331b2f05bf8f875c2a94e0e98" dependencies = [ - "bincode", + "bincode 1.3.3", "const-crypto", "serde", "solana-program 3.0.0", @@ -3960,7 +4005,7 @@ dependencies = [ name = "magicblock-magic-program-api" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "const-crypto", "serde", "solana-program 3.0.0", @@ -3989,7 +4034,7 @@ dependencies = [ "agave-feature-set", "agave-precompiles", "agave-syscalls", - "bincode", + "bincode 1.3.3", "blake3", "magicblock-accounts-db", "magicblock-core", @@ -4032,7 +4077,7 @@ dependencies = [ name = "magicblock-program" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "lazy_static", "magicblock-core", "magicblock-delegation-program-api 0.3.0", @@ -4069,7 +4114,7 @@ name = "magicblock-replicator" version = "0.13.2" dependencies = [ "async-nats", - "bincode", + "bincode 1.3.3", "bytes", "futures", "magicblock-accounts-db", @@ -4119,7 +4164,7 @@ dependencies = [ name = "magicblock-services" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "futures-util", "magicblock-core", "magicblock-magic-program-api 0.13.2", @@ -4167,20 +4212,26 @@ dependencies = [ name = "magicblock-task-scheduler" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "futures-util", + "hydra-api", "magicblock-config", "magicblock-core", "magicblock-ledger", "magicblock-program", "rusqlite", + "solana-account 3.4.0", "solana-instruction 3.3.0", + "solana-keypair", "solana-message 3.1.0", "solana-pubkey 4.1.0", "solana-rpc-client", "solana-rpc-client-api", + "solana-sha256-hasher 3.1.0", "solana-signature", + "solana-signer", + "solana-system-interface 3.1.0", "solana-transaction", "solana-transaction-error 3.1.0", "thiserror 2.0.18", @@ -5018,7 +5069,7 @@ dependencies = [ name = "program-flexi-counter" version = "0.0.0" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "ephemeral-rollups-sdk", "magicblock-magic-program-api 0.13.2", @@ -6442,9 +6493,9 @@ dependencies = [ [[package]] name = "solana-account" version = "3.4.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "serde_bytes", @@ -6464,7 +6515,7 @@ checksum = "ff1acff600a621d4445e0610fa71a53bef5390a5e349cfbd30651917593fb57d" dependencies = [ "Inflector", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "bv", "serde", @@ -6519,7 +6570,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8f5152a288ef1912300fc6efa6c2d1f9bb55d9398eb6c72326360b8063987da" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-program-error 2.2.2", "solana-program-memory 2.3.1", @@ -6532,7 +6583,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9cf16495d9eb53e3d04e72366a33bb1c20c24e78c171d8b8f5978357b63ae95" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-address 2.5.0", "solana-program-error 3.0.1", @@ -6589,7 +6640,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673f67efe870b64a65cb39e6194be5b26527691ce5922909939961a6e6b395" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6606,7 +6657,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e8df0b083c10ce32490410f3795016b1b5d9b4d094658c0a5e496753645b7cd" dependencies = [ - "bincode", + "bincode 1.3.3", "bytemuck", "serde", "serde_derive", @@ -6664,7 +6715,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19a3787b8cf9c9fe3dd360800e8b70982b9e5a8af9e11c354b6665dd4a003adc" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "solana-instruction 2.3.3", ] @@ -6675,7 +6726,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "278a1a5bad62cd9da89ac8d4b7ec444e83caa8ae96aa656dfc27684b28d49a5d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde_core", "solana-instruction-error", ] @@ -6754,7 +6805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219bfba64973ac9e64aa181f03fd56ac319e2d50d8a23d16c54bbd7fa9807a47" dependencies = [ "agave-syscalls", - "bincode", + "bincode 1.3.3", "qualifier_attr", "solana-account 3.4.0", "solana-bincode 3.1.0", @@ -6896,7 +6947,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e401ae56aed512821cc7a0adaa412ff97fecd2dff4602be7b1330d2daec0c4" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -7152,7 +7203,7 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43f5c5382b449e8e4e3016fb05e418c53d57782d8b5c30aa372fc265654b956d" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 2.2.1", @@ -7171,7 +7222,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75ca9b5cbb6f500f7fd73db5bd95640f71a83f04d6121a0e59a43b202dca2731" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-account 3.4.0", @@ -7233,7 +7284,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "749eccc960e85c9b33608450093d256006253e1cb436b8380e71777840a3f675" dependencies = [ - "bincode", + "bincode 1.3.3", "chrono", "memmap2 0.5.10", "solana-account 3.4.0", @@ -7319,7 +7370,7 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab5682934bd1f65f8d2c16f21cb532526fcc1a09f796e2cacdb091eee5774ad" dependencies = [ - "bincode", + "bincode 1.3.3", "getrandom 0.2.17", "js-sys", "num-traits", @@ -7337,7 +7388,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97881335fc698deb46c6571945969aae6d93a14e2fff792a368b4fac872f116" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "serde", "serde_derive", @@ -7590,7 +7641,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1796aabce376ff74bf89b78d268fa5e683d7d7a96a0a4e4813ec34de49d5314b" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7613,7 +7664,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0448b1fd891c5f46491e5dc7d9986385ba3c852c340db2911dd29faa01d2b08d" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "lazy_static", "serde", @@ -7803,7 +7854,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98eca145bd3545e2fbb07166e895370576e47a00a7d824e325390d33bf467210" dependencies = [ - "bincode", + "bincode 1.3.3", "blake3", "bs58", "bytemuck", @@ -8022,10 +8073,10 @@ dependencies = [ [[package]] name = "solana-program-runtime" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "cfg-if", "itertools 0.12.1", "log", @@ -8188,7 +8239,7 @@ checksum = "10dd50b329ce569340c1deab3667d21e26a41e65cc6460e8a5bb8b57aff8420d" dependencies = [ "async-trait", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "futures", "indicatif", @@ -8303,7 +8354,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f03df7969f5e723ad31b6c9eadccc209037ac4caa34d8dc259316b05c11e82b" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "serde", "solana-account 3.4.0", @@ -8686,7 +8737,7 @@ dependencies = [ name = "solana-storage-proto" version = "0.13.2" dependencies = [ - "bincode", + "bincode 1.3.3", "bs58", "prost", "protobuf-src", @@ -8706,7 +8757,7 @@ dependencies = [ [[package]] name = "solana-svm" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ "ahash 0.8.12", "log", @@ -8865,7 +8916,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "450479004fee3396c88cc4aa2f9b2b8db9c77be42ee7c1c53e6fac9eaec5fd51" dependencies = [ - "bincode", + "bincode 1.3.3", "log", "serde", "solana-account 3.4.0", @@ -8907,7 +8958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8c3595f95069f3d90f275bb9bd235a1973c4d059028b0a7f81baca2703815db" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -8944,7 +8995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6690d3dd88f15c21edff68eb391ef8800df7a1f5cec84ee3e8d1abf05affdf74" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "lazy_static", @@ -9003,7 +9054,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96697cff5075a028265324255efed226099f6d761ca67342b230d09f72cc48d2" dependencies = [ - "bincode", + "bincode 1.3.3", "serde", "serde_derive", "solana-address 2.5.0", @@ -9022,9 +9073,9 @@ dependencies = [ [[package]] name = "solana-transaction-context" version = "4.0.0" -source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=b275f82#b275f825fdf2c33139b99a9ef9a6db109ef5306a" +source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=24cea3e311ae97c976ae8f3390185d7e9cacb27a#24cea3e311ae97c976ae8f3390185d7e9cacb27a" dependencies = [ - "bincode", + "bincode 1.3.3", "qualifier_attr", "serde", "solana-account 3.4.0", @@ -9067,7 +9118,7 @@ dependencies = [ "Inflector", "agave-reserved-account-keys", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "bs58", "log", @@ -9108,7 +9159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6886b7bb8fbba5937b4a38fa67ed442d7971629244b8fbd95c7963b2126bc9" dependencies = [ "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bs58", "serde", "serde_json", @@ -9145,7 +9196,7 @@ version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b80d57478d6599d30acc31cc5ae7f93ec2361a06aefe8ea79bc81739a08af4c3" dependencies = [ - "bincode", + "bincode 1.3.3", "num-derive", "num-traits", "serde", @@ -9169,7 +9220,7 @@ version = "5.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d444ce30b6b4f9c281ba06061ea96638d063b53c2171b1e41ba02ebff79ed28f" dependencies = [ - "bincode", + "bincode 1.3.3", "cfg_eval", "num-derive", "num-traits", @@ -9196,7 +9247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4537fd6efe65f53ccd28d54d2ad43275b024834a4a8ca4dfa4babfa01e6d11ab" dependencies = [ "agave-feature-set", - "bincode", + "bincode 1.3.3", "log", "num-derive", "num-traits", @@ -9259,7 +9310,7 @@ checksum = "9602bcb1f7af15caef92b91132ec2347e1c51a72ecdbefdaefa3eac4b8711475" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -9296,7 +9347,7 @@ checksum = "09670ff59f60e6ddc2209c2e4353992a9b9f01d4e244f3e9d67bd5146e33d388" dependencies = [ "aes-gcm-siv", "base64 0.22.1", - "bincode", + "bincode 1.3.3", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -9800,7 +9851,7 @@ dependencies = [ name = "test-chainlink" version = "0.0.0" dependencies = [ - "bincode", + "bincode 1.3.3", "borsh 1.7.0", "futures", "integration-test-tools", @@ -10005,6 +10056,7 @@ dependencies = [ "anyhow", "chrono", "cleanass", + "hydra-api", "integration-test-tools", "magicblock-config", "magicblock-program", @@ -10666,6 +10718,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "ureq" version = "2.12.1" @@ -10783,6 +10841,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "void" version = "1.0.2" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index b27717add..60f061186 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -41,6 +41,7 @@ ephemeral-rollups-sdk = { version = "0.14", default-features = false, features = "modular-sdk", ] } futures = "0.3.31" +hydra-api = { git = "https://github.com/magicblock-labs/hydra.git", rev = "449532ee0ee792e9a3809b16fd9d289c2024341a" } integration-test-tools = { path = "test-tools" } isocountry = "0.3.2" lazy_static = "1.4.0" @@ -80,7 +81,8 @@ schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" shlex = "1.3.0" -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } solana-address-lookup-table-interface = "3.0" solana-commitment-config = "3.1.1" solana-compute-budget-interface = "3.0" @@ -121,9 +123,13 @@ url = "2.5.0" solana-storage-proto = { path = "../storage-proto" } # same reason as above rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } -solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } -solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b275f82" } +# TODO: Update to latest version is merged +solana-account = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } +# TODO: Update to latest version is merged +solana-transaction-context = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "24cea3e311ae97c976ae8f3390185d7e9cacb27a" } # Fix libsodium for ARM builds - upstream crates.io version breaks Linux ARM binaries libsodium-rs = { git = "https://github.com/jedisct1/libsodium-rs.git", rev = "0397a6c5785233f9f2ac91f3eedc3cceb74e0060" } diff --git a/test-integration/configs/schedule-task.devnet.toml b/test-integration/configs/schedule-task.devnet.toml index babad3bb1..0ec185729 100644 --- a/test-integration/configs/schedule-task.devnet.toml +++ b/test-integration/configs/schedule-task.devnet.toml @@ -30,5 +30,9 @@ path = "../../target/deploy/magicblock_committor_program.so" id = "9hgprgZiRWmy8KkfvUuaVkDGrqo9GzeXMohwq6BazgUY" path = "../target/deploy/program_schedulecommit.so" +[[programs]] +id = "eHyd5BU8QffvHi4GnXwxrK4WpS7pM2x9UGKHBWii7mf" +path = "../programs/hydra/hydra.so" + [metrics] address = "0.0.0.0:9000" diff --git a/test-integration/programs/hydra/hydra.so b/test-integration/programs/hydra/hydra.so new file mode 100755 index 000000000..7ca74bf57 Binary files /dev/null and b/test-integration/programs/hydra/hydra.so differ diff --git a/test-integration/test-ledger-restore/src/lib.rs b/test-integration/test-ledger-restore/src/lib.rs index e12c674fb..0bd527696 100644 --- a/test-integration/test-ledger-restore/src/lib.rs +++ b/test-integration/test-ledger-restore/src/lib.rs @@ -19,8 +19,7 @@ use integration_test_tools::{ use magicblock_config::{ config::{ accounts::AccountsDbConfig, ledger::LedgerConfig, - scheduler::TaskSchedulerConfig, validator::ValidatorConfig, - LifecycleMode, LoadableProgram, + validator::ValidatorConfig, LifecycleMode, LoadableProgram, }, consts::DEFAULT_LEDGER_BLOCK_TIME_MS, types::{crypto::SerdePubkey, network::Remote, StorageDirectory}, @@ -146,10 +145,6 @@ pub fn setup_validator_with_local_remote_and_resume_strategy( }, accountsdb: accountsdb_config.clone(), programs, - task_scheduler: TaskSchedulerConfig { - reset: reset_ledger, - ..Default::default() - }, lifecycle: LifecycleMode::Ephemeral, remotes: vec![ Remote::from_str(IntegrationTestContext::url_chain()).unwrap(), diff --git a/test-integration/test-task-scheduler/Cargo.toml b/test-integration/test-task-scheduler/Cargo.toml index 4ac807154..892730a92 100644 --- a/test-integration/test-task-scheduler/Cargo.toml +++ b/test-integration/test-task-scheduler/Cargo.toml @@ -7,13 +7,16 @@ edition = "2021" anyhow = { workspace = true } chrono = { workspace = true } cleanass = { workspace = true } +hydra-api = { workspace = true } integration-test-tools = { workspace = true } tracing = { workspace = true } magicblock-task-scheduler = { path = "../../magicblock-task-scheduler" } magicblock-program = { path = "../../programs/magicblock" } magicblock-config = { path = "../../magicblock-config" } program-flexi-counter = { path = "../programs/flexi-counter" } -program-schedulecommit = { path = "../programs/schedulecommit", features = ["no-entrypoint"] } +program-schedulecommit = { path = "../programs/schedulecommit", features = [ + "no-entrypoint", +] } solana-sdk = { workspace = true } solana-rpc-client = { workspace = true } tempfile = { workspace = true } diff --git a/test-integration/test-task-scheduler/src/lib.rs b/test-integration/test-task-scheduler/src/lib.rs index e3422bb40..641bdce95 100644 --- a/test-integration/test-task-scheduler/src/lib.rs +++ b/test-integration/test-task-scheduler/src/lib.rs @@ -1,10 +1,12 @@ use std::{ + path::PathBuf, process::Child, str::FromStr, time::{Duration, Instant}, }; use cleanass::assert; +use hydra_api::ephemeral::ID as HYDRA_EPHEMERAL_PROGRAM_ID; use integration_test_tools::{ expect, loaded_accounts::LoadedAccounts, @@ -21,10 +23,11 @@ use magicblock_config::{ scheduler::TaskSchedulerConfig, validator::ValidatorConfig, LifecycleMode, }, - types::{network::Remote, StorageDirectory}, + types::{crypto::SerdeKeypair, network::Remote, StorageDirectory}, ValidatorParams, }; use magicblock_program::Pubkey; +use magicblock_task_scheduler::{db::DbTask, SchedulerDatabase}; use program_flexi_counter::{ instruction::{ create_delegate_ix_with_commit_frequency_ms, create_init_ix, @@ -33,26 +36,22 @@ use program_flexi_counter::{ }; use program_schedulecommit::MainAccount; use solana_sdk::{ - signature::Keypair, signer::Signer, transaction::Transaction, + native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, + transaction::Transaction, }; use tempfile::TempDir; +use tokio::runtime::Runtime; pub const TASK_SCHEDULER_TICK_MILLIS: u64 = 50; -pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { - let (default_tmpdir, temp_dir) = resolve_tmp_dir(TMP_DIR_CONFIG); - - let config = ValidatorParams { +fn validator_config(temp_dir: PathBuf, faucet: &Keypair) -> ValidatorParams { + ValidatorParams { lifecycle: LifecycleMode::Ephemeral, remotes: vec![ Remote::from_str(IntegrationTestContext::url_chain()).unwrap(), Remote::from_str(IntegrationTestContext::ws_url_chain()).unwrap(), ], accountsdb: AccountsDbConfig::default(), - task_scheduler: TaskSchedulerConfig { - reset: true, - ..Default::default() - }, validator: ValidatorConfig { ..Default::default() }, @@ -61,9 +60,34 @@ pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { block_time: Duration::from_millis(TASK_SCHEDULER_TICK_MILLIS), ..Default::default() }, - storage: StorageDirectory(temp_dir.clone()), + task_scheduler: TaskSchedulerConfig { + faucet_keypair: Some( + SerdeKeypair::from_str(&faucet.to_base58_string()).unwrap(), + ), + }, + storage: StorageDirectory(temp_dir), ..Default::default() - }; + } +} + +fn airdrop_faucet(faucet: &Keypair) { + let chain_ctx = IntegrationTestContext::try_new_chain_only() + .expect("failed to connect to base chain to fund faucet"); + chain_ctx + .airdrop_chain(&faucet.pubkey(), 100 * LAMPORTS_PER_SOL) + .expect("failed to airdrop to task scheduler faucet"); +} + +fn start_validator( + config: ValidatorParams, + default_tmpdir: TempDir, + temp_dir: PathBuf, +) -> (TempDir, Child, IntegrationTestContext, Option) { + let faucet_keypair = config + .task_scheduler + .faucet_keypair + .clone() + .map(|k| k.insecure_clone()); let (default_tmpdir_config, Some(mut validator), port) = start_magicblock_validator_with_config_struct_and_temp_dir( config, @@ -79,7 +103,40 @@ pub fn setup_validator() -> (TempDir, Child, IntegrationTestContext) { IntegrationTestContext::try_new_with_ephem_port(port), validator ); - (default_tmpdir_config, validator, ctx) + (default_tmpdir_config, validator, ctx, faucet_keypair) +} + +pub fn setup_validator( +) -> (TempDir, Child, IntegrationTestContext, Option) { + setup_validator_with_migration_tasks(&[]) +} + +pub fn setup_validator_with_migration_tasks( + tasks: &[DbTask], +) -> (TempDir, Child, IntegrationTestContext, Option) { + let (default_tmpdir, temp_dir) = resolve_tmp_dir(TMP_DIR_CONFIG); + + // Seed the migration database before the validator opens it. The handle is + // dropped (flushing the WAL) before startup. + { + let db_path = SchedulerDatabase::path(&temp_dir); + let db = SchedulerDatabase::new(&db_path) + .expect("failed to open seed database"); + let runtime = Runtime::new().expect("failed to create runtime"); + for task in tasks { + runtime + .block_on(db.insert_task(task)) + .expect("failed to seed task"); + } + } + + // The faucet that pays for cranks must be funded before the validator + // delegates it on startup. + let faucet = Keypair::new(); + airdrop_faucet(&faucet); + + let config = validator_config(temp_dir.clone(), &faucet); + start_validator(config, default_tmpdir, temp_dir) } pub fn create_delegated_counter( @@ -133,6 +190,89 @@ pub fn create_delegated_counter( expect!(ctx.wait_for_delta_slot_ephem(10), validator); } +pub fn wait_for_hydra_crank( + ctx: &IntegrationTestContext, + crank_pda: &Pubkey, + max_timeout: Duration, + validator: &mut Child, +) { + let now = Instant::now(); + while now.elapsed() < max_timeout { + let maybe_account = ctx + .try_ephem_client() + .ok() + .and_then(|client| client.get_account(crank_pda).ok()); + if let Some(account) = maybe_account { + assert!( + account.owner.to_bytes() + == HYDRA_EPHEMERAL_PROGRAM_ID.to_bytes(), + cleanup(validator), + "crank account {} not owned by hydra program (owner: {})", + crank_pda, + account.owner + ); + return; + } + expect!(ctx.wait_for_next_slot_ephem(), validator); + } + assert!( + false, + cleanup(validator), + "hydra crank account {} was not created before timeout", crank_pda + ); +} + +pub fn wait_for_hydra_crank_closed( + ctx: &IntegrationTestContext, + crank_pda: &Pubkey, + max_timeout: Duration, + validator: &mut Child, +) { + let now = Instant::now(); + while now.elapsed() < max_timeout { + let closed = match ctx + .try_ephem_client() + .ok() + .and_then(|client| client.get_account(crank_pda).ok()) + { + // Account fully removed. + None => true, + // Closed ephemeral accounts are drained to zero lamports. + Some(account) => account.lamports == 0, + }; + if closed { + return; + } + expect!(ctx.wait_for_next_slot_ephem(), validator); + } + assert!( + false, + cleanup(validator), + "hydra crank account {} was not closed before timeout", crank_pda + ); +} + +pub fn wait_for_empty_db( + runtime: &Runtime, + db: &SchedulerDatabase, + max_timeout: Duration, + validator: &mut Child, +) { + let now = Instant::now(); + while now.elapsed() < max_timeout { + let ids = expect!(runtime.block_on(db.get_task_ids()), validator); + if ids.is_empty() { + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + assert!( + false, + cleanup(validator), + "migration database was not emptied before timeout" + ); +} + pub fn wait_for_incremented_counter( ctx: &IntegrationTestContext, counter_pda: &Pubkey, diff --git a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs index 951f98239..39dbcd8d3 100644 --- a/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs +++ b/test-integration/test-task-scheduler/tests/test_cancel_ongoing_task.rs @@ -1,6 +1,7 @@ -use cleanass::{assert, assert_eq}; +use std::time::Duration; + use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, @@ -9,16 +10,17 @@ use solana_sdk::{ native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, transaction::Transaction, }; -use test_task_scheduler::{create_delegated_counter, setup_validator}; -use tokio::runtime::Runtime; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, +}; #[test] fn test_cancel_ongoing_task() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); expect!( ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), @@ -33,7 +35,7 @@ fn test_cancel_ongoing_task() { // Schedule a task let task_id = 3; let execution_interval_millis = 100; - let iterations = 1000000; + let iterations = 1000; let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -67,10 +69,16 @@ fn test_cancel_ongoing_task() { validator ); - // Wait for the task to be scheduled - expect!(ctx.wait_for_delta_slot_ephem(2), validator); + // The crank is created and funded for all scheduled iterations. + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + wait_for_hydra_crank( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); - // Cancel the task + // Cancel the task while it is still ongoing. let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -97,65 +105,12 @@ fn test_cancel_ongoing_task() { validator ); - // Wait for the task to be cancelled - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Check that the task was cancelled - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - - let task = expect!(runtime.block_on(db.get_task(task_id)), validator); - assert!(task.is_none(), cleanup(&mut validator)); - - // Check that the counter was incremented but not as much as the number of executions - let counter_account = expect!( - ctx.try_ephem_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let counter = - expect!(FlexiCounter::try_decode(&counter_account.data), validator); - assert!( - counter.count < iterations as u64, - cleanup(&mut validator), - "counter.count: {}", - counter.count, - ); - assert!( - counter.count > 0, - cleanup(&mut validator), - "counter.count: {}", - counter.count, + // Cancelling closes the crank. + wait_for_hydra_crank_closed( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, ); cleanup(&mut validator); diff --git a/test-integration/test-task-scheduler/tests/test_independent_authority.rs b/test-integration/test-task-scheduler/tests/test_independent_authority.rs new file mode 100644 index 000000000..0ac09cd74 --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_independent_authority.rs @@ -0,0 +1,132 @@ +use std::time::Duration; + +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::crank_pubkey; +use program_flexi_counter::instruction::{ + create_cancel_task_ix, create_schedule_task_ix, +}; +use solana_sdk::{ + native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, + transaction::Transaction, +}; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, +}; + +#[test] +fn test_independent_cranks_per_authority() { + let (_temp_dir, mut validator, ctx, _) = setup_validator(); + + let payer = Keypair::new(); + let other = Keypair::new(); + + expect!( + ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + expect!( + ctx.airdrop_chain(&other.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + + create_delegated_counter(&ctx, &payer, &mut validator, 0); + create_delegated_counter(&ctx, &other, &mut validator, 0); + + let ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + + // Both authorities schedule the same task_id with different iteration counts. + let task_id = 1; + let payer_iterations = 3; + let other_iterations = 6; + for (signer, iterations) in + [(&payer, payer_iterations), (&other, other_iterations)] + { + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_schedule_task_ix( + signer.pubkey(), + task_id, + 100, + iterations, + false, + false, + )], + Some(&signer.pubkey()), + &[signer], + ephem_blockhash, + ), + &[signer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + } + + // Each authority owns its own independent crank at a distinct PDA, funded + // for its own iteration count. + let payer_crank = crank_pubkey(&payer.pubkey(), task_id); + let other_crank = crank_pubkey(&other.pubkey(), task_id); + assert_ne!(payer_crank, other_crank); + wait_for_hydra_crank( + &ctx, + &payer_crank, + Duration::from_secs(10), + &mut validator, + ); + wait_for_hydra_crank( + &ctx, + &other_crank, + Duration::from_secs(10), + &mut validator, + ); + + // Cancelling one authority's task closes only its crank; the other remains. + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_cancel_task_ix(payer.pubkey(), task_id,)], + Some(&payer.pubkey()), + &[&payer], + ephem_blockhash, + ), + &[&payer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + wait_for_hydra_crank_closed( + &ctx, + &payer_crank, + Duration::from_secs(10), + &mut validator, + ); + // The other authority's crank is unaffected. + wait_for_hydra_crank( + &ctx, + &other_crank, + Duration::from_secs(10), + &mut validator, + ); + + cleanup(&mut validator); +} diff --git a/test-integration/test-task-scheduler/tests/test_migration.rs b/test-integration/test-task-scheduler/tests/test_migration.rs new file mode 100644 index 000000000..0dab1b88a --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_migration.rs @@ -0,0 +1,63 @@ +use std::time::Duration; + +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::{crank_pubkey, db::DbTask, SchedulerDatabase}; +use solana_sdk::{ + instruction::{AccountMeta, Instruction}, + pubkey::Pubkey, + signature::Keypair, + signer::Signer, +}; +use test_task_scheduler::{ + setup_validator_with_migration_tasks, wait_for_empty_db, + wait_for_hydra_crank, +}; +use tokio::runtime::Runtime; + +/// Tasks persisted by the legacy (validator-funded) scheduler are migrated onto +/// hydra at startup: a funded hydra crank is created for each, and the +/// migration database is emptied. The database is used only for this one-time +/// migration. +#[test] +fn test_migration_reschedules_tasks_and_empties_db() { + let authority = Keypair::new().pubkey(); + let task_id = 42; + let iterations = 2; + // A simple, signer-free instruction is enough for the crank to be created; + // migration does not execute it. + let instructions = vec![Instruction { + program_id: program_flexi_counter::ID, + accounts: vec![AccountMeta::new(Pubkey::new_unique(), false)], + data: vec![0], + }]; + let task = DbTask { + id: task_id, + authority, + execution_interval_millis: 100, + executions_left: iterations, + last_execution_millis: 0, + instructions, + }; + + let (temp_dir, mut validator, ctx, _) = + setup_validator_with_migration_tasks(&[task]); + + // Migration creates a funded hydra crank for the persisted task... + let crank_pda = crank_pubkey(&authority, task_id); + wait_for_hydra_crank( + &ctx, + &crank_pda, + Duration::from_secs(15), + &mut validator, + ); + + // ...and empties the migration database. + let db = expect!( + SchedulerDatabase::new(SchedulerDatabase::path(temp_dir.path())), + validator + ); + let runtime = expect!(Runtime::new(), validator); + wait_for_empty_db(&runtime, &db, Duration::from_secs(15), &mut validator); + + cleanup(&mut validator); +} diff --git a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs index f85928981..57510f310 100644 --- a/test-integration/test-task-scheduler/tests/test_reschedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_reschedule_task.rs @@ -1,8 +1,7 @@ use std::time::Duration; -use cleanass::assert_eq; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, @@ -12,17 +11,16 @@ use solana_sdk::{ transaction::Transaction, }; use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_incremented_counter, + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, }; -use tokio::runtime::Runtime; #[test] fn test_reschedule_task() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); expect!( ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), @@ -34,9 +32,9 @@ fn test_reschedule_task() { let ephem_blockhash = expect!(ctx.try_get_latest_blockhash_ephem(), validator); - // Schedule a task let task_id = 1; let execution_interval_millis = 100; + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); let iterations = 2; let sig = expect!( ctx.send_transaction_ephem_with_preflight( @@ -66,12 +64,16 @@ fn test_reschedule_task() { .ok_or_else(|| anyhow::anyhow!("Transaction failed")), validator ); - - // Wait for the task to be scheduled - expect!(ctx.wait_for_delta_slot_ephem(5), validator); + wait_for_hydra_crank( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); // Reschedule the task let new_execution_interval_millis = 200; + let new_iterations = 5; let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -79,7 +81,7 @@ fn test_reschedule_task() { payer.pubkey(), task_id, new_execution_interval_millis, - iterations, + new_iterations, false, false, )], @@ -101,51 +103,16 @@ fn test_reschedule_task() { validator ); - // Wait for the rescheduled task to finish all remaining executions. - wait_for_incremented_counter( + // Funding now covers the larger iteration count, which the original crank + // could not have held, proving the crank was recreated. + wait_for_hydra_crank( &ctx, - &counter_pda, - 2 * iterations as u64, + &crank_pda, Duration::from_secs(10), &mut validator, ); - // Check that the completed task was removed from the database - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - let task = expect!(runtime.block_on(db.get_task(task_id)), validator); - assert_eq!(task, None, cleanup(&mut validator)); - - // Cancel the task + // Cancel and confirm the active crank is closed. let sig = expect!( ctx.send_transaction_ephem_with_preflight( &mut Transaction::new_signed_with_payer( @@ -167,12 +134,12 @@ fn test_reschedule_task() { .ok_or_else(|| anyhow::anyhow!("Transaction failed")), validator ); - - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Check that the task was cancelled - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!(tasks.len(), 0, cleanup(&mut validator)); + wait_for_hydra_crank_closed( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); cleanup(&mut validator); } diff --git a/test-integration/test-task-scheduler/tests/test_schedule_error.rs b/test-integration/test-task-scheduler/tests/test_schedule_error.rs deleted file mode 100644 index 0a624e678..000000000 --- a/test-integration/test-task-scheduler/tests/test_schedule_error.rs +++ /dev/null @@ -1,173 +0,0 @@ -use std::{ - thread::sleep, - time::{Duration, Instant}, -}; - -use cleanass::{assert, assert_eq}; -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; -use program_flexi_counter::{ - instruction::{create_cancel_task_ix, create_schedule_task_ix}, - state::FlexiCounter, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{create_delegated_counter, setup_validator}; -use tokio::runtime::Runtime; - -// Test that a task with an error is unscheduled -#[test] -fn test_schedule_error() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); - - let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 2; - let execution_interval_millis = 100; - let iterations = 3; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - true, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that the task eventually exhausts retries - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - let started = Instant::now(); - let failed_tasks = loop { - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - if failed_tasks.len() == 1 - || started.elapsed() >= Duration::from_secs(45) - { - break failed_tasks; - } - sleep(Duration::from_millis(200)); - }; - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - assert_eq!( - failed_tasks.len(), - 1, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks, - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - - assert!( - expect!(runtime.block_on(db.get_task(task_id)), validator).is_none(), - cleanup(&mut validator) - ); - - // Check that the counter was not incremented - let counter_account = expect!( - ctx.try_ephem_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let counter = - expect!(FlexiCounter::try_decode(&counter_account.data), validator); - assert!( - counter.count == 0, - cleanup(&mut validator), - "counter.count: {}", - counter.count, - ); - - // Cancel the task - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_cancel_task_ix(payer.pubkey(), task_id,)], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - expect!(ctx.wait_for_delta_slot_ephem(2), validator); - - // Check that the task was cancelled - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - - cleanup(&mut validator); -} diff --git a/test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs b/test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs deleted file mode 100644 index 554f49e3a..000000000 --- a/test-integration/test-task-scheduler/tests/test_schedule_magic_cpi_crank.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::time::Duration; - -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_program::{ - args::ScheduleTaskArgs, instruction_utils::InstructionUtils, - pda::crank_signer_pda, MAGIC_CONTEXT_PUBKEY, -}; -use program_schedulecommit::{ - api::{ - delegate_account_cpi_instruction, init_account_instruction, - schedule_commit_cpi_instruction, UserSeeds, - }, - ScheduleCommitType, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, signature::Keypair, - signer::Signer, transaction::Transaction, -}; -use test_task_scheduler::{setup_validator, wait_for_committed_count}; - -#[test] -fn test_crank_can_execute_program_that_cpis_into_magic() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let player = Keypair::new(); - let (committee, _) = Pubkey::find_program_address( - &[ - UserSeeds::MagicScheduleCommit.bytes(), - player.pubkey().as_ref(), - ], - &program_schedulecommit::ID, - ); - - expect!( - ctx.airdrop_chain(&player.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - let chain_blockhash = expect!( - ctx.try_chain_client().and_then(|client| client - .get_latest_blockhash() - .map_err(|e| anyhow::anyhow!( - "Failed to get latest chain blockhash: {}", - e - ))), - validator - ); - - expect!( - ctx.send_transaction_chain( - &mut Transaction::new_signed_with_payer( - &[init_account_instruction( - player.pubkey(), - player.pubkey(), - committee, - )], - Some(&player.pubkey()), - &[&player], - chain_blockhash, - ), - &[&player] - ), - validator - ); - - expect!( - ctx.send_transaction_chain( - &mut Transaction::new_signed_with_payer( - &[delegate_account_cpi_instruction( - player.pubkey(), - None, - player.pubkey(), - UserSeeds::MagicScheduleCommit, - )], - Some(&player.pubkey()), - &[&player], - chain_blockhash, - ), - &[&player] - ), - validator - ); - - expect!(ctx.wait_for_delta_slot_ephem(10), validator); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - let task_id = 17; - let crank_signer = crank_signer_pda(&player.pubkey()); - let mut crank_ix = schedule_commit_cpi_instruction( - crank_signer, - magicblock_program::ID, - MAGIC_CONTEXT_PUBKEY, - None, - &[player.pubkey()], - &[committee], - ScheduleCommitType::Commit, - ); - crank_ix.accounts[0].is_writable = false; - - let schedule_sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[InstructionUtils::schedule_task_instruction( - &player.pubkey(), - ScheduleTaskArgs { - task_id, - execution_interval_millis: 10, - iterations: 1, - instructions: vec![crank_ix], - }, - )], - Some(&player.pubkey()), - &[&player], - ephem_blockhash, - ), - &[&player] - ), - validator - ); - let schedule_status = - expect!(ctx.get_transaction_ephem(&schedule_sig), validator); - expect!( - schedule_status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Scheduling transaction failed")), - validator - ); - - wait_for_committed_count( - &ctx, - &committee, - 1, - Duration::from_secs(15), - &mut validator, - ); - - cleanup(&mut validator); -} diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task.rs b/test-integration/test-task-scheduler/tests/test_schedule_task.rs index 2d974906e..8b5cdd9af 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task.rs @@ -1,8 +1,7 @@ use std::time::Duration; -use cleanass::assert_eq; use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; +use magicblock_task_scheduler::crank_pubkey; use program_flexi_counter::{ instruction::{create_cancel_task_ix, create_schedule_task_ix}, state::FlexiCounter, @@ -12,17 +11,16 @@ use solana_sdk::{ transaction::Transaction, }; use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_incremented_counter, + create_delegated_counter, setup_validator, wait_for_hydra_crank, + wait_for_hydra_crank_closed, }; -use tokio::runtime::Runtime; #[test] fn test_schedule_task() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + let (_counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); expect!( ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), @@ -67,51 +65,15 @@ fn test_schedule_task() { validator ); - // Wait for the task to be scheduled and executed - // Check that the counter was incremented - wait_for_incremented_counter( + // The crank is created by hydra and funded for every iteration. + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + wait_for_hydra_crank( &ctx, - &counter_pda, - iterations as u64, + &crank_pda, Duration::from_secs(10), &mut validator, ); - // Check that the completed task was removed from the database - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - 0, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!( - tasks.len(), - 0, - cleanup(&mut validator), - "tasks: {:?}", - tasks - ); - let task = expect!(runtime.block_on(db.get_task(task_id)), validator); - assert_eq!(task, None, cleanup(&mut validator)); - // Cancel the task let sig = expect!( ctx.send_transaction_ephem_with_preflight( @@ -135,11 +97,13 @@ fn test_schedule_task() { validator ); - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Check that the task was cancelled - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!(tasks.len(), 0, cleanup(&mut validator)); + // Cancelling closes the hydra crank. + wait_for_hydra_crank_closed( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); cleanup(&mut validator); } diff --git a/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs b/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs index 55bb2f3f5..2f84a1763 100644 --- a/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs +++ b/test-integration/test-task-scheduler/tests/test_schedule_task_signed.rs @@ -12,7 +12,7 @@ use test_task_scheduler::{create_delegated_counter, setup_validator}; /// Test that a task can be scheduled and executed when it has multiple signers #[test] fn test_schedule_task_signed() { - let (_temp_dir, mut validator, ctx) = setup_validator(); + let (_temp_dir, mut validator, ctx, _) = setup_validator(); let payer = Keypair::new(); expect!( diff --git a/test-integration/test-task-scheduler/tests/test_scheduled_commits.rs b/test-integration/test-task-scheduler/tests/test_scheduled_commits.rs deleted file mode 100644 index 0722724ae..000000000 --- a/test-integration/test-task-scheduler/tests/test_scheduled_commits.rs +++ /dev/null @@ -1,126 +0,0 @@ -use cleanass::assert; -use integration_test_tools::{expect, validator::cleanup}; -use program_flexi_counter::{ - instruction::create_schedule_task_ix, state::FlexiCounter, -}; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{create_delegated_counter, setup_validator}; - -#[test] -#[ignore] -fn test_scheduled_commits() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - let commit_frequency_ms = 400; - create_delegated_counter(&ctx, &payer, &mut validator, commit_frequency_ms); - - eprintln!("Delegated counter: {:?}", counter_pda); - - // Schedule a task - let task_id = 5; - // Interval matching mainnet block time - let execution_interval_millis = 400; - let iterations = 3; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - false, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that the counter value is properly incremented on mainnet - const MAX_TRIES: u32 = 30; - let mut tries = 0; - let mut chain_values = vec![1, 2]; - let mut ephem_values = vec![1, 2]; - loop { - let chain_counter_account = expect!( - ctx.try_chain_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let chain_counter = expect!( - FlexiCounter::try_decode(&chain_counter_account.data), - validator - ); - - let ephem_counter_account = expect!( - ctx.try_ephem_client().and_then(|client| client - .get_account(&counter_pda) - .map_err(|e| anyhow::anyhow!("Failed to get account: {}", e))), - validator - ); - let ephem_counter = expect!( - FlexiCounter::try_decode(&ephem_counter_account.data), - validator - ); - - let chain_value_index = - chain_values.iter().position(|x| x == &chain_counter.count); - if let Some(index) = chain_value_index { - chain_values.remove(index); - } - - let ephem_value_index = - ephem_values.iter().position(|x| x == &ephem_counter.count); - if let Some(index) = ephem_value_index { - ephem_values.remove(index); - } - - if chain_values.is_empty() && ephem_values.is_empty() { - break; - } - - tries += 1; - if tries > MAX_TRIES { - assert!( - false, - cleanup(&mut validator), - "Missed some values: ephem_values: {:?}, chain_values: {:?}", - ephem_values, - chain_values - ); - } - - std::thread::sleep(std::time::Duration::from_millis(50)); - } - - cleanup(&mut validator); -} diff --git a/test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs b/test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs deleted file mode 100644 index 4390422d8..000000000 --- a/test-integration/test-task-scheduler/tests/test_unauthorized_reschedule.rs +++ /dev/null @@ -1,200 +0,0 @@ -use std::time::{Duration, Instant}; - -use cleanass::{assert, assert_eq}; -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_task_scheduler::SchedulerDatabase; -use program_flexi_counter::instruction::create_schedule_task_ix; -use solana_sdk::{ - native_token::LAMPORTS_PER_SOL, signature::Keypair, signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{create_delegated_counter, setup_validator}; -use tokio::runtime::Runtime; - -#[test] -fn test_unauthorized_reschedule() { - let (temp_dir, mut validator, ctx) = setup_validator(); - let db_path = SchedulerDatabase::path(temp_dir.path()); - - let payer = Keypair::new(); - let different_payer = Keypair::new(); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - expect!( - ctx.airdrop_chain(&different_payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - create_delegated_counter(&ctx, &different_payer, &mut validator, 0); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 1; - let execution_interval_millis = 100; - let iterations = 1000; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - payer.pubkey(), - task_id, - execution_interval_millis, - iterations, - false, - false, - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Wait for the task to be scheduled - expect!(ctx.wait_for_delta_slot_ephem(5), validator); - - // Reschedule the same task with a different payer - let new_execution_interval_millis = 200; - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[create_schedule_task_ix( - different_payer.pubkey(), - task_id, - new_execution_interval_millis, - iterations, - false, - false, - )], - Some(&different_payer.pubkey()), - &[&different_payer], - ephem_blockhash, - ), - &[&different_payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that one task is scheduled but another one is failed to schedule - let db = expect!(SchedulerDatabase::new(db_path), validator); - let runtime = expect!(Runtime::new(), validator); - - let failed_scheduling = wait_for_failed_schedulings( - &db, - &runtime, - 1, - Duration::from_secs(10), - &ctx, - &mut validator, - ); - assert_eq!( - failed_scheduling.len(), - 1, - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - assert_eq!( - failed_scheduling[0].task_id, task_id, - cleanup(&mut validator), - "failed_scheduling: {:?}", failed_scheduling, - ); - assert!( - failed_scheduling[0].error.contains("UnauthorizedReplacing"), - cleanup(&mut validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - - let failed_tasks = - expect!(runtime.block_on(db.get_failed_tasks()), validator); - assert_eq!( - failed_tasks.len(), - 0, - cleanup(&mut validator), - "failed_tasks: {:?}", - failed_tasks - ); - - let tasks = expect!(runtime.block_on(db.get_task_ids()), validator); - assert_eq!(tasks.len(), 1, cleanup(&mut validator)); - - let task = expect!( - runtime - .block_on(db.get_task(task_id)) - .ok() - .flatten() - .ok_or(anyhow::anyhow!("Task not found")), - validator - ); - assert_eq!(task.authority, payer.pubkey(), cleanup(&mut validator)); - assert_eq!( - task.execution_interval_millis, execution_interval_millis, - cleanup(&mut validator) - ); - assert!( - task.executions_left > 0, - cleanup(&mut validator), - "task.executions_left: {}", - task.executions_left - ); - - cleanup(&mut validator); -} - -fn wait_for_failed_schedulings( - db: &SchedulerDatabase, - runtime: &Runtime, - expected_len: usize, - timeout: Duration, - ctx: &integration_test_tools::IntegrationTestContext, - validator: &mut std::process::Child, -) -> Vec { - let start = Instant::now(); - while start.elapsed() < timeout { - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - if failed_scheduling.len() == expected_len { - return failed_scheduling; - } - - expect!(ctx.wait_for_next_slot_ephem(), validator); - } - - let failed_scheduling = - expect!(runtime.block_on(db.get_failed_schedulings()), validator); - assert_eq!( - failed_scheduling.len(), - expected_len, - cleanup(validator), - "failed_scheduling: {:?}", - failed_scheduling, - ); - failed_scheduling -} diff --git a/test-integration/test-task-scheduler/tests/test_undrained_faucet.rs b/test-integration/test-task-scheduler/tests/test_undrained_faucet.rs new file mode 100644 index 000000000..d50972641 --- /dev/null +++ b/test-integration/test-task-scheduler/tests/test_undrained_faucet.rs @@ -0,0 +1,163 @@ +use std::time::Duration; + +use hydra_api::instruction::ephemeral; +use integration_test_tools::{expect, validator::cleanup}; +use magicblock_task_scheduler::crank_pubkey; +use program_flexi_counter::{ + instruction::{create_schedule_task_ix, FlexiCounterInstruction}, + state::FlexiCounter, +}; +use solana_sdk::{ + message::{AccountMeta, Instruction}, + native_token::LAMPORTS_PER_SOL, + signature::Keypair, + signer::Signer, + transaction::Transaction, +}; +use test_task_scheduler::{ + create_delegated_counter, setup_validator, wait_for_hydra_crank, +}; + +/// The validator identity must not be drained to pay for hydra cranks: a +/// dedicated, delegated faucet account pays instead. Scheduling a task creates +/// and funds a crank, and the validator identity's balance is left untouched. +#[test] +fn test_undrained_faucet() { + let (_temp_dir, mut validator, ctx, faucet_keypair) = setup_validator(); + + let payer = Keypair::new(); + let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); + + expect!( + ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), + validator + ); + + create_delegated_counter(&ctx, &payer, &mut validator, 0); + + // The default validator identity keypair (see consts::DEFAULT_VALIDATOR_KEYPAIR). + let faucet_keypair = expect!( + faucet_keypair + .ok_or_else(|| anyhow::anyhow!("Faucet keypair not found")), + validator + ); + let faucet_pk = faucet_keypair.pubkey(); + let faucet_balance_before = + expect!(ctx.fetch_ephem_account_balance(&faucet_pk), validator); + + let mut ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + + // Schedule a task + let task_id = 1; + let execution_interval_millis = 100; + let iterations = 3; + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[create_schedule_task_ix( + payer.pubkey(), + task_id, + execution_interval_millis, + iterations, + false, + false, + )], + Some(&payer.pubkey()), + &[&payer], + ephem_blockhash, + ), + &[&payer] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + // The crank is created by hydra and funded for every iteration (paid for by + // the faucet, not the validator identity). + let crank_pda = crank_pubkey(&payer.pubkey(), task_id); + wait_for_hydra_crank( + &ctx, + &crank_pda, + Duration::from_secs(10), + &mut validator, + ); + + // The faucet executes the crank + for _ in 0..iterations { + ephem_blockhash = + expect!(ctx.try_get_latest_blockhash_ephem(), validator); + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[ + ephemeral::trigger(crank_pda, faucet_pk,), + Instruction::new_with_borsh( + program_flexi_counter::ID, + &FlexiCounterInstruction::AddUnsigned { count: 1 }, + vec![AccountMeta::new(counter_pda, false)], + ) + ], + Some(&faucet_pk), + &[&faucet_keypair], + ephem_blockhash, + ), + &[&faucet_keypair] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + expect!(ctx.wait_for_next_slot_ephem(), validator); + } + + // Close the crank + let sig = expect!( + ctx.send_transaction_ephem_with_preflight( + &mut Transaction::new_signed_with_payer( + &[ephemeral::close(faucet_pk, crank_pda, faucet_pk)], + Some(&faucet_pk), + &[&faucet_keypair], + ephem_blockhash, + ), + &[&faucet_keypair] + ), + validator + ); + let status = expect!(ctx.get_transaction_ephem(&sig), validator); + expect!( + status + .transaction + .meta + .and_then(|m| m.status.ok()) + .ok_or_else(|| anyhow::anyhow!("Transaction failed")), + validator + ); + + // The validator identity must not have been drained to pay for the crank. + let faucet_balance_after = + expect!(ctx.fetch_ephem_account_balance(&faucet_pk), validator); + assert!( + faucet_balance_after >= faucet_balance_before, + "faucet was drained paying for cranks: {} < {}", + faucet_balance_after, + faucet_balance_before + ); + + cleanup(&mut validator); +} diff --git a/test-integration/test-task-scheduler/tests/test_use_crank_signer.rs b/test-integration/test-task-scheduler/tests/test_use_crank_signer.rs deleted file mode 100644 index e35c8e477..000000000 --- a/test-integration/test-task-scheduler/tests/test_use_crank_signer.rs +++ /dev/null @@ -1,91 +0,0 @@ -use std::time::Duration; - -use integration_test_tools::{expect, validator::cleanup}; -use magicblock_program::{ - args::ScheduleTaskArgs, instruction_utils::InstructionUtils, - pda::crank_signer_pda, -}; -use program_flexi_counter::{ - instruction::FlexiCounterInstruction, state::FlexiCounter, -}; -use solana_sdk::{ - instruction::{AccountMeta, Instruction}, - native_token::LAMPORTS_PER_SOL, - signature::Keypair, - signer::Signer, - transaction::Transaction, -}; -use test_task_scheduler::{ - create_delegated_counter, setup_validator, wait_for_incremented_counter, -}; - -#[test] -fn test_use_crank_signer() { - let (_temp_dir, mut validator, ctx) = setup_validator(); - - let payer = Keypair::new(); - let (counter_pda, _) = FlexiCounter::pda(&payer.pubkey()); - - expect!( - ctx.airdrop_chain(&payer.pubkey(), 10 * LAMPORTS_PER_SOL), - validator - ); - - create_delegated_counter(&ctx, &payer, &mut validator, 0); - - let ephem_blockhash = - expect!(ctx.try_get_latest_blockhash_ephem(), validator); - - // Schedule a task - let task_id = 9; - let execution_interval_millis = 10; - let iterations = 5; - let crank_signer = crank_signer_pda(&payer.pubkey()); - let sig = expect!( - ctx.send_transaction_ephem_with_preflight( - &mut Transaction::new_signed_with_payer( - &[InstructionUtils::schedule_task_instruction( - &payer.pubkey(), - ScheduleTaskArgs { - task_id, - execution_interval_millis, - iterations, - instructions: vec![Instruction::new_with_borsh( - program_flexi_counter::ID, - &FlexiCounterInstruction::AddUnsigned { count: 1 }, - vec![ - AccountMeta::new(counter_pda, false), - AccountMeta::new_readonly(crank_signer, true) - ], - )], - } - )], - Some(&payer.pubkey()), - &[&payer], - ephem_blockhash, - ), - &[&payer] - ), - validator - ); - let status = expect!(ctx.get_transaction_ephem(&sig), validator); - expect!( - status - .transaction - .meta - .and_then(|m| m.status.ok()) - .ok_or_else(|| anyhow::anyhow!("Transaction failed")), - validator - ); - - // Check that the counter was incremented - wait_for_incremented_counter( - &ctx, - &counter_pda, - iterations as u64, - Duration::from_secs(10), - &mut validator, - ); - - cleanup(&mut validator); -}