diff --git a/.agents/context/crate-map.md b/.agents/context/crate-map.md index 1904ca0cd..39a0af0a7 100644 --- a/.agents/context/crate-map.md +++ b/.agents/context/crate-map.md @@ -36,16 +36,16 @@ The validator is performance-sensitive. When changing any crate on RPC, account | Crate | Purpose | Depends on | Used by | Notes | |---|---|---|---|---| -| `magicblock-chainlink` | Base-chain account/delegation coordination. | accounts-db, AML, config, core, magic-program API, metrics | account-cloner, accounts, aperture, API, magic program | Checks/clones remote accounts, tracks delegation state, coordinates base-layer reads. See `.agents/context/crates/magicblock-chainlink.md` before changing this crate. | -| `magicblock-account-cloner` | Fetches and injects base-layer accounts/programs into local validator state. | accounts-db, chainlink, committor-service, config, core, ledger, magic-program API, magic program, rpc-client | accounts, aperture, API | Distinguishes fee payer, delegated, and undelegated accounts; handles large/program clone paths. See `.agents/context/crates/magicblock-account-cloner.md` before changing this crate. | -| `magicblock-accounts` | Account manager and scheduled commit processing glue. | account-cloner, accounts-db, chainlink, committor-service, core, metrics, magic program | `magicblock-api` | Current active role is scheduled commit processing and pending intent recovery; see `.agents/context/crates/magicblock-accounts.md` before changing this crate. | +| `magicblock-chainlink` | Base-chain account/delegation coordination. | accounts-db, AML, config, core, magic-program API, metrics | account-cloner, aperture, API, magic program, services | Checks/clones remote accounts, tracks delegation state, coordinates base-layer reads and observed undelegation requests. See `.agents/context/crates/magicblock-chainlink.md` before changing this crate. | +| `magicblock-account-cloner` | Fetches and injects base-layer accounts/programs into local validator state. | accounts-db, chainlink, committor-service, config, core, ledger, magic-program API, magic program, rpc-client | aperture, API, services | Distinguishes fee payer, delegated, and undelegated accounts; handles large/program clone paths. See `.agents/context/crates/magicblock-account-cloner.md` before changing this crate. | +| `magicblock-accounts` | Legacy account config/error/trait compatibility crate. | committor-service | no active runtime consumer | Do not add new services here; current account lifecycle and scheduled-intent behavior live in chainlink/cloner, services, and committor-service. See `.agents/context/crates/magicblock-accounts.md` before changing this crate. | | `magicblock-aml` | External/cached risk-scoring integration. | `magicblock-config` (dev: `magicblock-core`) | `magicblock-chainlink` | Optional Range risk checks for post-delegation action signers; see `.agents/context/crates/magicblock-aml.md` before changing this crate. | ## Commit and base-layer settlement crates | Crate | Purpose | Depends on | Used by | Notes | |---|---|---|---|---| -| `magicblock-committor-service` | Executes scheduled base-layer intents: commit, undelegate, finalize, action. | committor program, core, metrics, magic program, rpc-client, table-mania | account-cloner, accounts, API | Durable commit pipeline; handles scheduling, transaction prep, buffers, ALTs, confirmations. See `.agents/context/crates/magicblock-committor-service.md` before changing this crate. | +| `magicblock-committor-service` | Executes scheduled base-layer intents: commit, undelegate, finalize, action. | committor program, core, metrics, magic program, rpc-client, table-mania | account-cloner, API | Durable commit pipeline; accepts scheduled intents, handles recovery, transaction prep, buffers, ALTs, confirmations. See `.agents/context/crates/magicblock-committor-service.md` before changing this crate. | | `magicblock-committor-program` | On-chain committor program. | none | `magicblock-committor-service` | Base-layer program side for changeset buffers/commit application; see `.agents/context/crates/magicblock-committor-program.md` before changing this crate. | | `magicblock-table-mania` | Address lookup table management. | metrics, rpc-client | `magicblock-committor-service` | Creates/extends/deactivates/closes ALTs needed by commit transactions. See `.agents/context/crates/magicblock-table-mania.md` before changing this crate. | @@ -63,7 +63,7 @@ The validator is performance-sensitive. When changing any crate on RPC, account |---|---|---|---|---| | `magicblock-task-scheduler` | Program-scheduled task/crank service. | config, core, ledger, magic program | `magicblock-api` | SQLite-backed delay queue, retries/backoff, scheduled transaction submission. See `.agents/context/crates/magicblock-task-scheduler.md` before changing this crate. | | `magicblock-replicator` | Primary/replica event replication over NATS JetStream. | accounts-db, config, core, ledger | `magicblock-api` | Preserves HA/replica replay behavior; primary and replica modes differ intentionally. | -| `magicblock-services` | Shared service utilities/adapters. | core, magic-program API | `magicblock-api` | Common service abstractions; keep generic. See `.agents/context/crates/magicblock-services.md` before changing this crate. | +| `magicblock-services` | Shared validator services/adapters. | account-cloner, chainlink, core, magic-program API, metrics, magic program | `magicblock-api` | Callback adapter and owner-program undelegation request observer. See `.agents/context/crates/magicblock-services.md` before changing this crate. | | `magicblock-metrics` | Metrics helpers and instrumentation. | none | RPC, ledger, processor, chainlink, committor, table-mania, API | Prefer adding observability here rather than ad-hoc metrics code. See `.agents/context/crates/magicblock-metrics.md` before changing this crate. | ## Tools and test support @@ -78,8 +78,8 @@ The validator is performance-sensitive. When changing any crate on RPC, account ## How to use this map - For transaction correctness, start with `magicblock-processor`, then inspect `magicblock-accounts-db`, `magicblock-ledger`, and `magicblock-program` interactions. -- For delegation or account cloning bugs, start with `magicblock-chainlink`, `magicblock-account-cloner`, and `magicblock-accounts`. -- For commit or undelegation bugs, start with `magicblock-program`, `magicblock-accounts`, and `magicblock-committor-service`. +- For delegation or account cloning bugs, start with `magicblock-chainlink` and `magicblock-account-cloner`; include `magicblock-services` when observed undelegation requests are involved. +- For commit or undelegation bugs, start with `magicblock-program`, `magicblock-committor-service`, and `magicblock-services` for request-triggered scheduling. - For RPC behavior, start with `magicblock-aperture`; check `magicblock-chainlink` if reads trigger cloning. - For validator lifecycle/startup/shutdown, start with `magicblock-api` and `magicblock-validator`. - When adding, removing, renaming, or repurposing a crate, update this file and `AGENTS.md` in the same change. diff --git a/.agents/context/crates/magicblock-accounts.md b/.agents/context/crates/magicblock-accounts.md index ac6379fbb..7edc614ff 100644 --- a/.agents/context/crates/magicblock-accounts.md +++ b/.agents/context/crates/magicblock-accounts.md @@ -2,354 +2,169 @@ ## Purpose -`magicblock-accounts` is the validator-side glue between Magic Program scheduled intents, Chainlink account lifecycle tracking, and the committor service. In the current implementation it does **not** own the account-cloning manager described by its historical README; account fetching/cloning is owned by `magicblock-chainlink` and `magicblock-account-cloner`. This crate's active runtime responsibility is scheduled commit processing. +`magicblock-accounts` is currently a legacy compatibility crate. It remains in +the workspace because older account/scheduled-commit types are still exported, +but no production validator service is wired through this crate. -At a high level it: +Current source exports: -- exposes the `ScheduledCommitsProcessor` trait used by `magicblock-api`'s slot ticker; -- implements `ScheduledCommitsProcessorImpl`, which drains accepted `ScheduledIntentBundle`s from the Magic Program global transaction scheduler; -- forwards scheduled and recovered intent bundles to `magicblock-committor-service`; -- tracks per-intent metadata needed to signal `ScheduledCommitSent` back into local validator execution after base-layer intent execution finishes; -- notifies Chainlink when accounts are about to be undelegated so base-layer undelegation completion can be watched; -- starts a one-shot recovery pass for persisted pending intent bundles after ledger replay and account-bank reset. +- `config::*`, including the local historical `LifecycleMode`; +- `errors`, including legacy account and scheduled-commit error types; +- `traits::*`, including the legacy `ScheduledCommitsProcessor` trait. -This crate sits on the commit/undelegation settlement path and interacts with the transaction scheduler. It is not part of ordinary SVM transaction execution, but its ticker-triggered work can affect slot progression, local commit lifecycle state, Chainlink subscription state, and committor throughput. Avoid adding blocking work, unbounded locks, duplicate intent scheduling, or heavy per-intent processing without an explicit performance tradeoff. +This crate does **not** own active account fetching/cloning, scheduled intent +execution, owner-program undelegation request handling, or committor delivery. -## Update requirement +Current owners: -Update this document in the same change whenever behavior in `magicblock-accounts` changes, or when another crate changes the flows/contracts consumed here. Update it for changes to: +- account fetching/cloning: `magicblock-chainlink` and + `magicblock-account-cloner`; +- scheduled intent acceptance, result handling, and pending-intent recovery: + `magicblock-committor-service/src/service.rs` and + `magicblock-committor-service/src/service/intent_client.rs`; +- owner-program `UndelegationRequest` observation and local + `ScheduleCommitAndUndelegate` submission: + `magicblock-services/src/undelegation_request_service.rs`; +- validator startup/shutdown wiring: `magicblock-api/src/magic_validator.rs`. -- `ScheduledCommitsProcessor` trait methods or `ScheduledCommitsProcessorImpl::new` wiring; -- scheduled intent draining, metadata tracking, result processing, or `SentCommit` construction; -- pending intent recovery timing, delegation checks, or recovered scheduling behavior; -- Chainlink undelegation notification behavior; -- Magic Program scheduled intent types, `TransactionScheduler` global state, `ScheduledCommitSent`, or `SentCommit` fields; -- committor result subscription, broadcast error handling, patched-error/callback-report handling, or execution-output variants; -- startup/shutdown ordering in `magicblock-api` that affects this processor; -- validation commands or integration suites relevant to schedule intents, commits, or undelegation; -- performance characteristics of scheduled commit processing, result handling, or recovery. +Do not add new runtime services here just because the crate name says +`accounts`. Prefer the current owner crate for the behavior being changed. -For the general documentation-update rule, see `.agents/memory/agent-memory-and-docs.md`. +## Update Requirement -## Where it sits in the repository +Update this document in the same change when: -Primary files and nearby contracts: +- this crate gains or loses exports; +- the legacy `ScheduledCommitsProcessor` trait or account error surface changes; +- production wiring starts or stops depending on this crate; +- historical account-manager remnants are removed or intentionally revived. + +For the general documentation-update rule, see +`.agents/memory/agent-memory-and-docs.md`. + +## Where It Sits | Path | Role | |---|---| -| `magicblock-accounts/Cargo.toml` | Crate dependencies. Pulls in Chainlink/cloner aliases, committor service, core scheduler types, metrics, and Magic Program scheduled intent types. | -| `magicblock-accounts/README.md` | Historical notes about an `AccountsManager`/`ensure_accounts` design. Treat source and this guide as canonical for current behavior. | -| `magicblock-accounts/src/lib.rs` | Public exports: `config::*`, `traits::*`, `errors`, and `scheduled_commits_processor`. | -| `magicblock-accounts/src/config.rs` | Defines a local `LifecycleMode` enum and `requires_ephemeral_validation`; currently no tracked consumer uses this type. Do not confuse it with `magicblock-config::config::LifecycleMode`, which is the validator config type in use. | -| `magicblock-accounts/src/traits.rs` | Defines the `ScheduledCommitsProcessor` async trait consumed by the API slot ticker. | -| `magicblock-accounts/src/scheduled_commits_processor.rs` | Main implementation: scheduled intent draining, committor scheduling, pending recovery, Chainlink undelegation notifications, result subscription loop, and `SentCommit` signaling. | -| `magicblock-accounts/src/errors.rs` | Error enums and result aliases for scheduled commit processing plus older account/committor error variants. | -| `magicblock-api/src/magic_validator.rs` | Production wiring. Constructs `ScheduledCommitsProcessorImpl`, starts recovery after replay/reset, passes it to the slot ticker, and stops it before stopping the committor service. | -| `magicblock-api/src/tickers.rs` | Slot ticker checks `MagicContext::has_scheduled_commits`, executes `AcceptScheduleCommits`, then calls `ScheduledCommitsProcessor::process`. | -| `programs/magicblock/src/magic_context.rs` | `MagicContext` storage and `has_scheduled_commits` fast check used by the slot ticker. | -| `programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs` | Magic Program instruction that moves scheduled intents from `MagicContext` into global `TransactionScheduler` state. | -| `programs/magicblock/src/magic_scheduled_base_intent.rs` | Defines `ScheduledIntentBundle`, `MagicIntentBundle`, and helpers such as `get_all_committed_pubkeys` / `has_undelegate_intent`. | -| `magicblock-committor-service/src/service.rs` | `BaseIntentCommittor` trait and `CommittorService` oneshot APIs used by this crate. | -| `magicblock-committor-service/src/committor_processor.rs` | Persists and schedules normal intents; loads pending intent bundles for recovery; schedules recovered bundles without re-persisting. | -| `test-integration/test-schedule-intent/` and `test-integration/schedulecommit/` | Integration coverage for schedule intent, commit, and commit-and-undelegate behavior. | - -Main consumers: - -- `magicblock-api` is the only production consumer of `ScheduledCommitsProcessorImpl`. -- `magicblock-api::tickers` depends on the `ScheduledCommitsProcessor` trait rather than the concrete implementation. -- The committor service is the downstream executor for base-layer intents. -- Chainlink is notified when accounts are entering undelegation so it can watch/update base-layer state. -- The Magic Program is both upstream (schedules/accepts intents) and downstream (receives local `ScheduledCommitSent` signals). - -Important boundaries: - -- This crate does not fetch or clone transaction accounts for ordinary execution; use `magicblock-chainlink` / `magicblock-account-cloner` for account availability changes. -- This crate does not build base-layer commit transactions; that belongs to `magicblock-committor-service`. -- This crate does not enforce final SVM writable-account access rules; that belongs to the processor/SVM path. - -## Public API shape / Main public types and APIs +| `magicblock-accounts/Cargo.toml` | Minimal dependencies for the legacy config, traits, and errors. | +| `magicblock-accounts/src/lib.rs` | Public surface: `config::*`, `errors`, and `traits::*`. | +| `magicblock-accounts/src/config.rs` | Historical `LifecycleMode` and `requires_ephemeral_validation` helper. Current validator lifecycle config comes from `magicblock-config`, not this local enum. | +| `magicblock-accounts/src/traits.rs` | Legacy `ScheduledCommitsProcessor` trait. There is no current production implementation in this crate. | +| `magicblock-accounts/src/errors.rs` | Legacy account and scheduled-commit error types. Check for real consumers before adding variants. | +| `magicblock-accounts/README.md` | Historical account-manager notes. Treat it as stale unless source confirms the described API still exists. | + +The crate is still a workspace member and workspace dependency alias, but there +is no active production Rust consumer at the time of writing. + +## Public API Shape `magicblock-accounts/src/lib.rs` exports: ```rust mod config; pub mod errors; -pub mod scheduled_commits_processor; mod traits; pub use config::*; pub use traits::*; ``` -### `ScheduledCommitsProcessor` trait - -Defined in `src/traits.rs`: - -- `async fn process(&self) -> ScheduledCommitsProcessorResult<()>` drains accepted scheduled intents and hands them to the committor; -- `fn scheduled_commits_len(&self) -> usize` returns the count of accepted intents in Magic Program global scheduler state; -- `fn clear_scheduled_commits(&self)` clears that global scheduler state; -- `fn stop(&self)` cancels processor background work. - -The trait is `Send + Sync + 'static` and is used by `magicblock-api/src/tickers.rs` to keep slot ticker code generic. - -### `ScheduledCommitsProcessorImpl` - -Defined in `src/scheduled_commits_processor.rs` and constructed with: - -```rust -pub fn new( - committor: Arc, - chainlink: Arc, - internal_transaction_scheduler: TransactionSchedulerHandle, - latest_block: impl LatestBlockProvider, -) -> Self -``` - -Stored state: - -- `committor: Arc` schedules intent bundles and provides result broadcasts / pending persisted intents; -- `chainlink: Arc` receives `undelegation_requested(pubkey)` calls before commit-and-undelegate execution; -- `cancellation_token: CancellationToken` stops the result-processing loop; -- `intents_meta_map: Arc>>` maps intent IDs to local metadata needed when committor results return; -- `transaction_scheduler: magicblock_program::TransactionScheduler` accesses the Magic Program's global scheduled action store. - -Important public method outside the trait: - -- `spawn_pending_intents_recovery(self: &Arc)` starts a one-shot task that loads persisted pending intent bundles from committor storage, filters them through Chainlink delegation checks, and schedules recoverable bundles. It must run only after ledger replay and account-bank reset. - -### Type aliases - -- `InnerChainlinkImpl = ProdInnerChainlink` -- `ChainlinkImpl = ProdChainlink` - -These encode the current production Chainlink/cloner stack. If Chainlink generic wiring changes, update these aliases and this guide together. - -### Errors - -`ScheduledCommitsProcessorError` wraps: - -- `tokio::sync::oneshot::error::RecvError` when committor oneshot requests fail; -- boxed `CommittorServiceError` from scheduling, recovery, or result-subscription operations. - -`AccountsError` still contains broader account/committor/cloner variants from older APIs. Check whether a variant is actually consumed before relying on it for new behavior. - -## Runtime flows - -### Normal scheduled commit flow - -```text -program invokes Magic Program schedule instruction - -> MagicContext stores ScheduledIntentBundle(s) - -> slot ticker sees MagicContext::has_scheduled_commits - -> validator-signed AcceptScheduleCommits transaction runs locally - -> Magic Program moves bundles into global TransactionScheduler state - -> ScheduledCommitsProcessorImpl::process drains them - -> committor service persists/schedules base-layer intent execution - -> committor broadcasts result - -> processor registers SentCommit and schedules local ScheduledCommitSent transaction -``` - -Ordered details: - -1. User/program code schedules commit, commit-and-undelegate, commit-finalize, and/or action intent bundles through the Magic Program. -2. `magicblock-api::init_slot_ticker` periodically reads `MAGIC_CONTEXT_PUBKEY` from `AccountsDb` and calls `MagicContext::has_scheduled_commits`. -3. If there are pending scheduled commits, `handle_scheduled_commits` builds `InstructionUtils::accept_scheduled_commits(latest_block.blockhash)` and submits it through the internal `TransactionSchedulerHandle`. -4. `process_accept_scheduled_commits` validates the validator authority signer, drains `MagicContext::scheduled_base_intents`, and calls `TransactionScheduler::default().accept_scheduled_base_intent(...)`. -5. `ScheduledCommitsProcessorImpl::process` calls `take_scheduled_intent_bundles()` on its `magicblock_program::TransactionScheduler` handle. Empty drains are no-ops. -6. For non-empty drains it increments `magicblock_metrics::metrics::inc_committor_intents_count_by` and calls `process_intent_bundles`. -7. `prepare_intent_bundles_for_scheduling` stores `ScheduledBaseIntentMeta` for every intent ID and gathers pubkeys from undelegation intents. -8. `process_undelegation_requests` concurrently calls `chainlink.undelegation_requested(pubkey)` for gathered pubkeys; failures are logged but do not abort the commit. -9. The committor receives the bundles via `CommittorService::schedule_intent_bundles` and returns through a oneshot when scheduling is accepted. -10. The background `result_processor` receives `BroadcastedIntentExecutionResult`s from the committor broadcast channel. -11. `process_intent_result` removes the intent metadata, builds/registers a `SentCommit`, creates or reuses the `ScheduledCommitSent` transaction, encodes it with `with_encoded`, and submits it to the internal scheduler. - -Caveats: - -- The slot ticker has a TODO about possible delay between accepting and processing scheduled commits. Do not add extra sleeps or slow work to this path. -- `process_undelegation_requests` logs subscription failures and continues. That can leave undelegating accounts in a problematic local state; changing this policy is a lifecycle decision, not a local cleanup. -- `intents_meta_map` is keyed by intent ID. Duplicate IDs or unexpected duplicate results can cause missing metadata and are logged as errors. - -### Pending intent recovery flow - -```text -validator start/restart - -> ledger replay - -> account-bank reset/cleanup - -> spawn_pending_intents_recovery - -> committor loads persisted pending bundles - -> Chainlink verifies accounts delegated on base and ER - -> recoverable bundles schedule through committor without re-persisting -``` - -Ordered details: - -1. `magicblock-api::MagicValidator::start` processes ledger replay and clears Magic Program global scheduled actions before starting the normal slot ticker. -2. After account-bank reset, and only in the branch where replay/reset is performed, the API calls `processor.spawn_pending_intents_recovery()`. -3. `recover_pending_intents` asks the committor for `get_pending_intent_bundles().await??`. -4. `recoverable_intent_bundles` checks every bundle's committed pubkeys with `chainlink.accounts_delegated_on_base_and_er(&pubkeys, AccountFetchOrigin::GetAccount)`. -5. Bundles with any non-delegated account, or bundles whose checks error, are skipped and logged. -6. Recoverable bundles go through `process_intent_bundles` with `CommittorService::schedule_recovered_intent_bundles`, which schedules without re-persisting rows. -7. If scheduling recovered bundles fails, metadata for those intent IDs is removed to avoid stale entries. - -Caveats: - -- Recovery must happen after replay/reset because Chainlink delegation checks read local bank state. -- Recovery currently only runs when `MagicValidator::start` enters the replay/reset path. Changing startup branches can affect pending intent durability. -- Filtering requires all committed accounts in a bundle to be delegated on both base and ER; this protects against re-sending stale or already-undelegated work. +### Legacy `ScheduledCommitsProcessor` -### Result-to-local-signal flow +`src/traits.rs` defines: -When the committor completes an intent: +- `process`; +- `scheduled_commits_len`; +- `clear_scheduled_commits`; +- `stop`. -1. `BroadcastedIntentExecutionResult` includes the intent ID, success/error output, patched errors, and callback scheduling report. -2. `ScheduledBaseIntentMeta` supplies the original slot, blockhash, payer, committed pubkeys, optional prebuilt `sent_transaction`, and undelegation flag. -3. `build_sent_commit` converts execution output into chain signatures: - - `ExecutionOutput::SingleStage(signature)` becomes one signature; - - `ExecutionOutput::TwoStage { commit_signature, finalize_signature }` becomes two signatures; - - errors try to expose any available commit/finalize signatures through `err.signatures()`. -4. Patched errors and callback scheduling results are stringified into `SentCommit` fields. -5. `register_scheduled_commit_sent(sent_commit)` stores the result for the Magic Program's local `ScheduledCommitSent` processor. -6. The internal transaction scheduler executes the validator-signed `ScheduledCommitSent` transaction so local execution can observe the sent-commit result. +The active scheduled-intent service no longer consumes this trait. It lives in +`magicblock-committor-service` and accepts scheduled intents directly through an +internal ER intent client. -If the original intent did not carry a signed `sent_transaction`, the processor builds a new one with the current latest blockhash. This fallback is important for recovered intents reconstructed from persistence. +### Legacy Errors -### Shutdown flow +`src/errors.rs` still exposes `AccountsError`, +`ScheduledCommitsProcessorError`, and result aliases. Several variants refer to +historical account-manager or scheduled-commit flows. Before reusing one for new +behavior, confirm there is an active caller that will receive and handle it. -`MagicValidator::stop` cancels the validator token, then calls `scheduled_commits_processor.stop()` before `committor_service.stop()`. Preserve this ordering: the result processor must be told to stop while the committor is still available enough to shut down cleanly, and the committor is intentionally stopped last among these services. +## Important Caveats -## Important internals and caveats +### Do Not Revive Runtime Ownership Accidentally -### Magic Program global scheduler state +This crate was previously used for account/scheduled-commit glue, but that +responsibility has moved. New services should be placed where the active runtime +boundary is: -`magicblock_program::TransactionScheduler::default()` is used as a handle to global scheduled action state. The `AcceptScheduleCommits` instruction writes into that state, and `ScheduledCommitsProcessorImpl::process` drains it. During ledger replay, `magicblock-api` clears this state to avoid re-committing accepted intents replayed from the ledger. +- `magicblock-services` for small validator background services and adapters; +- `magicblock-committor-service` for base-layer intent execution and recovery; +- `magicblock-chainlink` / `magicblock-account-cloner` for account lifecycle and + cloning; +- `magicblock-api` for startup/shutdown orchestration. -Do not replace this with an ordinary per-instance queue unless the Magic Program and replay semantics are updated together. +### Historical README/API Drift -### Metadata map and locking +The README describes older names such as `AccountsManager`, +`ExternalAccountsManager`, `BankAccountProvider`, `RemoteAccountCloner`, +`Transwise`, and `ensure_accounts`. These are not present in the current crate +source. Do not implement new features against those names without first checking +current Chainlink/cloner/API ownership and updating or removing stale docs. -`intents_meta_map` is protected by a standard `Mutex`. Current critical sections are intentionally small: insert metadata and remove metadata by intent ID. Avoid holding this lock across `.await`, committor calls, Chainlink calls, scheduler execution, or expensive logging/formatting. +### Local `LifecycleMode` Drift -The code uses `expect(POISONED_MUTEX_MSG)` in the normal processing paths. One recovery cleanup path handles poisoning by logging and taking the inner map. Treat mutex poisoning as a serious invariant violation. +`magicblock-accounts/src/config.rs` defines a `LifecycleMode` separate from +`magicblock-config::config::LifecycleMode`. Current repository usages of +validator lifecycle mode use `magicblock-config`, not this local type. Avoid +adding new configuration wiring through the local enum unless that duplication +is intentional and documented. -### Undelegation notifications are best-effort +## Change Guidance -Before scheduling an intent bundle, the processor calls `chainlink.undelegation_requested` for accounts from commit-and-undelegate intents. Errors are aggregated and logged but do not fail scheduling. This favors settlement progress over local watcher correctness; if you change it, document how accounts that are already locally immutable/undelegating recover from missed base-layer subscriptions. - -### Historical README/API drift - -The README describes `AccountsManager`, `ExternalAccountsManager`, `BankAccountProvider`, `RemoteAccountCloner`, `Transwise`, and `ensure_accounts`. These are not present in the current crate source. Do not implement new features against those names without first checking current Chainlink/cloner/API ownership and updating/removing stale docs. - -### Local `LifecycleMode` drift - -`magicblock-accounts/src/config.rs` defines a `LifecycleMode` separate from `magicblock-config::config::LifecycleMode`. Current repository usages of validator lifecycle mode use `magicblock-config`, not this local type. Avoid adding new configuration wiring through the local enum unless that duplication is intentional and documented. - -## Important invariants - -1. `process()` must drain only accepted scheduled intent bundles from Magic Program global scheduler state; it must not re-read `MagicContext` directly. -2. `AcceptScheduleCommits` must be executed before `process()` so MagicContext intents are moved and cleared atomically by the Magic Program. -3. Intent metadata must be inserted before committor scheduling so result processing can build a correct `SentCommit`. -4. Intent metadata must be removed on result handling and on failed recovered scheduling; stale entries can make later duplicate IDs or results misleading. -5. Recovery must run only after ledger replay and local account-bank reset, because delegation checks depend on current local account state. -6. Recovered bundles must be filtered so all committed accounts are delegated on base and ER before scheduling. -7. Recovered bundles must use the committor recovered scheduling path, not the normal persistence path, to avoid duplicating persisted rows. -8. Commit-and-undelegate intents must notify Chainlink before scheduling whenever possible so base-layer undelegation completion can be tracked. -9. The result processor must not block indefinitely on slow local scheduler work without observing cancellation between results. -10. Broadcast lag from committor results is unexpected and requires investigation; silently dropping lagged results would leave local sent-commit state incomplete. -11. `SentCommit` fields must stay aligned with Magic Program `ScheduledCommitSent` expectations, including chain signatures, patched errors, callback reports, included pubkeys, payer, slot, blockhash, and undelegation flag. -12. Documentation and code must not describe this crate as the active account ensure/cloning owner unless that behavior is restored in source. - -## Common change areas and what to inspect - -### Changing scheduled commit acceptance or slot ticker behavior - -Inspect first: - -- `magicblock-api/src/tickers.rs` (`init_slot_ticker`, `handle_scheduled_commits`); -- `programs/magicblock/src/magic_context.rs` (`has_scheduled_commits`, `take_scheduled_commits`); -- `programs/magicblock/src/schedule_transactions/process_accept_scheduled_commits.rs`; -- `magicblock-accounts/src/scheduled_commits_processor.rs::process`. - -Risks: - -- accepting without processing can leave intents stranded in global scheduler state; -- processing without accepting misses MagicContext-staged intents; -- replay can re-populate global scheduled actions unless startup clear semantics are preserved. - -### Changing committor scheduling or result handling - -Inspect first: - -- `ScheduledCommitsProcessorImpl::process_intent_bundles`; -- `prepare_intent_bundles_for_scheduling`; -- `result_processor`, `process_intent_result`, and `build_sent_commit`; -- `magicblock-committor-service/src/service.rs` `BaseIntentCommittor` methods; -- `magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs` result fields. - -Risks: - -- missing result subscriptions can prevent local `ScheduledCommitSent` signaling; -- new `ExecutionOutput` variants must be converted into `SentCommit.chain_signatures` deliberately; -- callback or patched-error fields are user/operator-visible through Magic Program result state. - -### Changing undelegation lifecycle behavior - -Inspect first: - -- `prepare_intent_bundles_for_scheduling` and `process_undelegation_requests`; -- `ScheduledIntentBundle::get_undelegate_intent_pubkeys` / `has_undelegate_intent`; -- `magicblock-chainlink::undelegation_requested`; -- Magic Program code that marks accounts undelegating/immutable when scheduling commit-and-undelegate. - -Risks: - -- missing Chainlink notification can leave local state out of sync with base-layer undelegation; -- failing the whole commit on notification errors may affect settlement availability; -- ordinary ER execution must not continue mutating accounts after commit-and-undelegate has been scheduled. - -### Changing pending intent recovery +### Removing Legacy Surface Inspect first: -- `spawn_pending_intents_recovery`, `recover_pending_intents`, `recoverable_intent_bundles`, and `process_recovered_intent_bundles`; -- `magicblock-api/src/magic_validator.rs` startup ordering around replay/reset; -- `magicblock-committor-service/src/committor_processor.rs::pending_intent_bundles`; -- Chainlink `accounts_delegated_on_base_and_er` behavior. +- workspace and integration `Cargo.toml` files; +- `rg "magicblock_accounts|magicblock-accounts"`; +- `magicblock-api/src/errors.rs`; +- public downstream crates or examples if this crate is published. Risks: -- running recovery before local bank repair can schedule invalid or stale commits; -- skipping delegation checks can re-send intents for accounts no longer delegated to this ER; -- normal scheduling can duplicate persistence rows for recovered work. - -### Cleaning up stale account-manager remnants +- deleting exported symbols can still be a public API break even when no + current workspace consumer uses them; +- tests or external users may still compile against the legacy traits/errors. -Inspect first: +### Adding New Account Lifecycle Behavior -- `magicblock-accounts/README.md`; -- `src/config.rs` and `src/errors.rs` for unused historical APIs; -- `.agents/context/crate-map.md` and this guide; -- current owners in `magicblock-chainlink`, `magicblock-account-cloner`, and `magicblock-api`. +Start with: -Risks: +- `magicblock-chainlink`; +- `magicblock-account-cloner`; +- `magicblock-services`; +- `magicblock-api/src/magic_validator.rs`. -- deleting exported symbols can be a public API break even if no current workspace consumer uses them; -- account availability behavior belongs outside this crate in the current architecture. +Do not put new account synchronization, DLP request scanning, or commit +scheduling loops in this crate without an explicit ownership decision. -## Tests and validation +## Tests And Validation -- Markdown-only guide changes: run `git diff --check` for this file; no Rust checks are needed. -- Rust changes in this crate: use `.agents/rules/testing-and-validation.md` or `mbv-check`; include focused package checks for `magicblock-accounts`. -- Related package checks by change area: `magicblock-api`, `magicblock-program`, and `magicblock-committor-service` for scheduled commits, pending recovery, or undelegation behavior. -- Relevant integration suites: `test-schedule-intents`, `test-committor`, and `test-task-scheduler`; use `.agents/rules/testing-and-validation.md` for exact setup/test commands. -- Performance validation intent: changes to slot-ticker commit processing, intent metadata handling, Chainlink undelegation notification, or result handling should report whether they add work to periodic slot processing, committor scheduling, or transaction-scheduler submission. +- Markdown-only guide changes: run `git diff --check`. +- Rust changes in this crate: run `cargo check -p magicblock-accounts`. +- If removing exports, also run the smallest affected consumer package checks + found by `rg`. +- If moving behavior out of or into this crate, also run checks for the new + owner crate and `magicblock-api`. -## Adjacent implementation references +## Adjacent References -- `.agents/context/crates/magicblock-account-cloner.md` — current account-cloner guide; useful because this crate aliases the production cloner stack through Chainlink. -- `.agents/context/crates/magicblock-chainlink.md` — Chainlink lifecycle/delegation guide, especially `undelegation_requested` and delegation checks. -- `magicblock-accounts/README.md` — historical summary; verify against source before using. -- `magicblock-api/src/tickers.rs` — slot ticker that invokes this crate. -- `magicblock-api/src/magic_validator.rs` — startup/shutdown and recovery wiring. -- `programs/magicblock/src/schedule_transactions/` — Magic Program scheduling and acceptance processors. -- `magicblock-committor-service/` — base-layer intent execution, persistence, and recovery source. -- `test-integration/test-schedule-intent/` and `test-integration/schedulecommit/` — integration coverage for schedule intent, commit, and commit-and-undelegate behavior. +- `.agents/context/crates/magicblock-services.md` - background service adapters, + including owner-program undelegation request observation. +- `.agents/context/crates/magicblock-committor-service.md` - scheduled intent + acceptance, execution, result notification, and recovery. +- `.agents/context/crates/magicblock-chainlink.md` - delegation/account lifecycle + coordination. +- `.agents/context/crates/magicblock-account-cloner.md` - base account/program + cloning. +- `.agents/context/crates/magicblock-api.md` - validator startup/shutdown wiring. diff --git a/.agents/context/crates/magicblock-api.md b/.agents/context/crates/magicblock-api.md index 29ba84ae7..63c650573 100644 --- a/.agents/context/crates/magicblock-api.md +++ b/.agents/context/crates/magicblock-api.md @@ -9,7 +9,7 @@ High-level responsibilities: - initialize persistent ledger and AccountsDb state; - sync and verify the validator keypair stored beside the ledger; - build genesis/sys accounts required by the local runtime; -- create Chainlink/account-cloning, committor, scheduled-commit, replication, metrics, task-scheduler, and Aperture RPC services; +- create Chainlink/account-cloning, committor/intent-execution, undelegation-request, replication, metrics, task-scheduler, and Aperture RPC services; - spawn transaction execution, RPC runtime, slot/system tickers, ledger truncation, and optional replication service threads/tasks; - transition the scheduler out of `StartingUp` after replay/reset/recovery; - register/unregister the validator in the Magic Domain Program and manage validator fee-vault setup in standalone ephemeral mode; @@ -24,7 +24,7 @@ Update this document in the same change whenever `magicblock-api` behavior or co - `MagicValidator::try_from_config`, `start`, `stop`, `prepare_ledger_for_shutdown`, or validator registration/unregistration behavior; - startup/shutdown ordering, service ownership, cancellation semantics, or thread/runtime spawning; - ledger, AccountsDb, keypair, genesis, MagicContext, ephemeral vault, or native mint initialization; -- Chainlink, committor, scheduled commit recovery, replication, task scheduler, RPC/Aperture, metrics, or ledger truncator wiring; +- Chainlink, committor/intent-execution, undelegation request service, replication, task scheduler, RPC/Aperture, metrics, or ledger truncator wiring; - scheduler mode transitions for standalone, primary, or replica roles; - MagicSys/commit nonce lookup behavior; - domain registry, fee vault, claim-fees, or other base-layer operator flows; @@ -43,7 +43,7 @@ For the general documentation-update rule, see `.agents/memory/agent-memory-and- | `magicblock-api/src/fund_account.rs` | Local genesis/runtime account seeding for validator identity, `MagicContext`, and ephemeral vault. | | `magicblock-api/src/genesis_utils.rs` | Local genesis account construction, native mint setup, and feature activation helpers. | | `magicblock-api/src/magic_sys_adapter.rs` | Adapter from Magic Program `MagicSys` trait calls to `CommittorService` commit-nonce lookups. | -| `magicblock-api/src/tickers.rs` | Slot ticker for accepting scheduled commits and system metrics ticker for storage/account gauges. | +| `magicblock-api/src/tickers.rs` | System metrics ticker for storage/account gauges. | | `magicblock-api/src/errors.rs` | `ApiError` and `ApiResult`, including boxed conversions from downstream service errors. | | `magicblock-api/README.md` | Short public usage example for constructing and starting `MagicValidator`. | | `magicblock-validator/src/main.rs` | Primary consumer: parses config, creates `MagicValidator`, takes the ledger lock, starts/stops the validator, and drives TUI/headless lifecycle. | @@ -56,7 +56,7 @@ Main consumers: - `test-integration/test-magicblock-api` directly uses `DomainRegistryManager` and exercises runtime behavior through integration contexts. - Other crates are mostly downstream services wired by this crate rather than consumers of it. -Important downstream dependencies include `magicblock-accounts-db`, `magicblock-ledger`, `magicblock-processor`, `magicblock-aperture`, `magicblock-chainlink`, `magicblock-account-cloner`, `magicblock-accounts`, `magicblock-committor-service`, `magicblock-replicator`, `magicblock-task-scheduler`, `magicblock-metrics`, `magicblock-services`, `magicblock-program`, and `magicblock-validator-admin`. +Important downstream dependencies include `magicblock-accounts-db`, `magicblock-ledger`, `magicblock-processor`, `magicblock-aperture`, `magicblock-chainlink`, `magicblock-account-cloner`, `magicblock-committor-service`, `magicblock-replicator`, `magicblock-task-scheduler`, `magicblock-metrics`, `magicblock-services`, `magicblock-program`, and `magicblock-validator-admin`. ## Public API shape / Main public types and APIs @@ -82,7 +82,6 @@ The exported public API is intentionally small for production use: Private but important internal APIs: - `MagicSysAdapter` implements `magicblock_core::traits::MagicSys` so Magic Program execution can synchronously ask the committor service for current commit nonces. -- `init_slot_ticker` periodically accepts scheduled commits through the transaction scheduler and then asks `ScheduledCommitsProcessor` to process them. - `init_system_metrics_ticker` periodically updates storage/account gauges. ## Runtime flows @@ -97,7 +96,7 @@ ValidatorParams -> chainlink/account cloner -> replication service (optional) -> metrics + system metrics ticker - -> scheduled commit processor + -> undelegation request service -> program loading + validator authority init -> transaction scheduler/execution thread -> Aperture RPC runtime thread @@ -116,7 +115,7 @@ Current order matters: 8. Create replication service if a broker was configured. 9. Insert missing genesis accounts, initialize validator identity, MagicContext, and ephemeral vault accounts. 10. Start metrics service and system metrics ticker. -11. Create the scheduled-commit processor if a committor service exists. +11. Create the undelegation request service for non-replica validators. 12. Load configured upgradeable programs and initialize Magic Program validator authority/override. 13. Build the SVM environment, transaction scheduler state, and executor count; spawn the transaction execution thread. 14. Build Aperture shared state and start the RPC server on its own Tokio runtime thread. @@ -135,28 +134,40 @@ Caveats: 3. After replay, Magic Program scheduled actions are cleared so replayed accept-commit transactions do not re-commit. 4. Optional AccountsDb defragmentation runs before normal work starts; this is explicitly marked safe only before cleanup, scheduler mode changes, replication, slot ticks, or task recovery. 5. Standalone/primary nodes reset the bank and, for primaries, send a replication `Message::Reset`. -6. Pending commit intent recovery runs only after replay and reset. +6. The intent execution service starts only after replay/reset and handles pending commit intent recovery before its normal accept loop. 7. Standalone nodes switch the scheduler to `SchedulerMode::Primary`; primary/replica modes spawn the replication service instead. 8. Standalone ephemeral nodes start background base-layer setup: funding check, magic fee vault init/delegation, optional startup fee claim, and optional domain registration. -9. Claim-fees periodic task, slot ticker, ledger truncator, and primary-only task scheduler are started. +9. The undelegation request service, claim-fees periodic task, intent execution service, ledger truncator, and primary-only task scheduler are started. -### Scheduled commit tick flow +### Scheduled intent service flow ```text -slot ticker sleep +intent execution service interval -> read MagicContext from AccountsDb -> if scheduled commits exist, execute AcceptScheduleCommits tx through scheduler - -> ScheduledCommitsProcessor::process() - -> committor service schedules base-layer work + -> take accepted ScheduledIntentBundle(s) + -> committor processor schedules base-layer work + -> result processor submits ScheduledCommitSent locally ``` -The ticker uses the same transaction scheduler path as normal execution for the validator-signed accept transaction. It currently has a known TODO about possible delay between accept and processing; do not hide or worsen that behavior without an explicit fix. +The accept transaction uses the same transaction scheduler path as normal execution. The scheduling/result loop lives in `magicblock-committor-service::service`, while this crate owns construction, start, and stop ordering. + +### Undelegation request service flow + +```text +Chainlink observes DLP UndelegationRequest + -> magicblock-services::UndelegationRequestService validates request/delegation + -> submit local ScheduleCommitAndUndelegate transaction + -> scheduled intent service accepts and executes the resulting intent +``` + +This service is started only for non-replica validators. ### Shutdown flow: `MagicValidator::stop` 1. Send unregister transaction in the background when standalone ephemeral on-chain coordination is enabled. 2. Set the local `exit` flag and cancel the shared cancellation token. -3. Stop scheduled commit processor, then committor service. The code comment says the committor service should be stopped last, but the current implementation stops it early after scheduled-commit processing; treat this ordering as compatibility-sensitive and inspect in-flight intent behavior before changing it. +3. Stop the undelegation request service, then the intent execution service. 4. Stop claim-fees task. 5. Join RPC thread, slot ticker, ledger truncator, replication thread, and transaction execution thread. 6. Flush AccountsDb. @@ -203,7 +214,7 @@ The ledger stores a validator keypair file beside the blockstore parent. With `v 1. Ledger replay, optional defragmentation, bank reset, pending intent recovery, and scheduler mode transition must remain ordered so execution does not race recovery or cleanup. 2. Replicas must not use enabled Chainlink or local cleanup in ways that diverge from primary-replicated state. -3. Scheduled commit recovery must run only after replay and reset, because it reads local account state for delegation checks. +3. Pending intent recovery must run only after replay and reset, because it reads local account state for delegation checks. 4. `MagicContext` must exist before the slot ticker can run, and it must remain Magic Program-owned and delegated locally. 5. Ephemeral vault initialization must preserve the ephemeral flag and Magic Program owner. 6. Validator identity must remain privileged in local AccountsDb. @@ -235,7 +246,7 @@ Inspect first: - `MagicValidator::stop`; - `prepare_ledger_for_shutdown` and `magicblock-validator/src/main.rs` shutdown sequence; -- downstream service `stop`/`join` semantics for scheduled commits, committor, ledger truncator, replication, transaction scheduler, task scheduler, and RPC. +- downstream service `stop`/`join` semantics for undelegation requests, intent execution, ledger truncator, replication, transaction scheduler, task scheduler, and RPC. Risks: @@ -270,7 +281,7 @@ Inspect first: Risks: -- missing local sys accounts causing startup or scheduled-commit ticker panics; +- missing local sys accounts causing startup or scheduled-intent accept loop panics; - changing account flags that SVM access validation relies on; - primary/replica genesis state divergence. @@ -298,7 +309,7 @@ Risks: ## Adjacent implementation references - `.agents/context/crates/magicblock-aperture.md` — RPC/pubsub service details wired by this crate. -- `.agents/context/crates/magicblock-account-cloner.md`, `.agents/context/crates/magicblock-accounts.md`, and `.agents/context/crates/magicblock-chainlink.md` — account sync and scheduled-commit dependencies wired by this crate. +- `.agents/context/crates/magicblock-account-cloner.md`, `.agents/context/crates/magicblock-chainlink.md`, `.agents/context/crates/magicblock-committor-service.md`, and `.agents/context/crates/magicblock-services.md` — account sync, scheduled-intent, and request-observer dependencies wired by this crate. - `magicblock-api/README.md` — short usage example for `MagicValidator`. - `magicblock-validator/src/main.rs` — production binary lifecycle around `MagicValidator`. - `test-integration/test-magicblock-api/` — integration tests for API-owned flows. diff --git a/.agents/context/crates/magicblock-committor-service.md b/.agents/context/crates/magicblock-committor-service.md index 9e5ce5c20..5023b3670 100644 --- a/.agents/context/crates/magicblock-committor-service.md +++ b/.agents/context/crates/magicblock-committor-service.md @@ -6,9 +6,9 @@ High-level responsibilities: -- expose `CommittorService` / `BaseIntentCommittor` as the async service boundary used by `magicblock-api`, `magicblock-accounts`, and account cloning; +- expose `CommittorService` / `BaseIntentCommittor` as the async service boundary used by `magicblock-api` and account cloning; - schedule intent bundles without executing mutually conflicting committed accounts in parallel; -- fetch Delegation Program metadata, commit nonces, rent reimbursements, and base accounts needed for task construction; +- fetch Delegation Program metadata, including commit nonces and rent payer data, plus base accounts needed for task construction; - choose commit delivery strategies: state args, diff args, state buffers, diff buffers, and optional ALTs; - prepare and clean up committor-program buffer accounts and TableMania lookup-table reservations; - execute single-stage or two-stage base-layer transaction flows and schedule action callbacks; @@ -55,7 +55,7 @@ For the general documentation-update rule, see .agents/memory/agent-memory-and-d | `src/persist/` | SQLite persistence for commit rows, bundle signatures, status/strategy enums, and conversion utilities. | | `src/stubs/` | Feature-gated dev/test stub committor behind `dev-context-only-utils`. | | `magicblock-api/src/magic_validator.rs` | Starts the service at validator initialization with `committor_service.sqlite`, validator keypair, RPC URL, websocket URL, compute-unit price, and action callback scheduler. | -| `magicblock-accounts/src/scheduled_commits_processor.rs` | Main runtime producer/consumer: takes scheduled intent bundles from the transaction scheduler, schedules them with the committor, consumes result broadcasts, and performs pending-intent recovery after ledger replay. | +| `src/service.rs` and `src/service/intent_client.rs` | Runtime producer/consumer: accepts scheduled intent bundles from `MagicContext`, schedules them with the committor, consumes result broadcasts, submits `ScheduledCommitSent`, and performs pending-intent recovery after ledger replay. | | `magicblock-account-cloner/src/account_cloner.rs` | Uses `BaseIntentCommittor` for lookup-table reservation around account cloning and diagnostic mapping of committor errors. | | `magicblock-api/src/magic_sys_adapter.rs` | Fetches current commit nonces through the committor service for Magic syscalls. | | `test-integration/test-committor-service/` | Integration coverage for delivery preparators, transaction preparators, intent executor flows, and local commit execution. | @@ -136,14 +136,15 @@ magicblock-api::MagicValidator::init_committor_service -> actor run loop spawned on Tokio ``` -The service is initialized before the account manager starts pending-intent recovery. Pending recovery must run after ledger replay so local accounts reflect delegated state before recovered intents are checked. +The intent execution service is started after ledger replay/reset so pending recovery sees local accounts that reflect current delegated state before recovered intents are checked. ### Fresh scheduled intent flow ```text Magic Program schedules intent in ER - -> transaction scheduler exposes ScheduledIntentBundle(s) - -> magicblock-accounts::ScheduledCommitsProcessor::process + -> MagicContext stores ScheduledIntentBundle(s) + -> InternalIntentRpcClient executes AcceptScheduleCommits locally + -> TransactionScheduler exposes accepted ScheduledIntentBundle(s) -> CommittorService::schedule_intent_bundles -> CommittorProcessor::schedule_intent_bundle -> IntentPersisterImpl::start_base_intents @@ -152,19 +153,19 @@ Magic Program schedules intent in ER -> IntentScheduler blocks conflicts by committed pubkeys -> IntentExecutorImpl executes selected intent -> broadcast result - -> ScheduledCommitsProcessor consumes result and updates local/metadata state + -> IntentExecutionService consumes result and submits ScheduledCommitSent locally ``` `CommittorProcessor::schedule_intent_bundle` logs persistence failures but still tries to execute. This is intentionally loud because losing persistence weakens restart recovery; do not hide or downgrade that error path. ### Recovery flow for pending intents -1. `magicblock-accounts` calls `get_pending_intent_bundles()` after replay. +1. `IntentExecutionService` calls `pending_intent_bundles()` after replay/reset when it starts. 2. `CommittorProcessor::pending_intent_bundles` loads SQLite rows with `CommitStatus::Pending` and `created_at` inside the 14-day recovery window. 3. It fetches the current base-layer slot and reconstructs `ScheduledIntentBundle`s grouped by `message_id`. 4. Rows for a message must agree on ER slot and ER blockhash; otherwise that message is skipped. 5. Data-account rows without stored data are skipped because they cannot reconstruct a `CommittedAccount` safely. -6. `magicblock-accounts` filters recovered bundles against current delegated state, then calls `schedule_recovered_intent_bundles` so rows are not inserted again. +6. `IntentExecutionService` filters recovered bundles against current delegated state, then calls `schedule_recovered_intent_bundles` so rows are not inserted again. Preserve the no-repersist path for recovered intents. Re-inserting rows can violate primary keys or duplicate status history. @@ -189,7 +190,7 @@ The scheduler blocks on the union of `ScheduledIntentBundle::get_all_committed_p IntentExecutorImpl::execute -> mark persisted rows Pending -> TaskBuilderImpl::commit_tasks + finalize_tasks - -> fetch next commit nonces and diffable base accounts using max(remote_slot) + -> fetch next commit nonces, delegation metadata, and diffable base accounts using max(remote_slot) -> persist commit_id for each committed account -> create commit, commit-finalize, undelegate, finalize, and action tasks -> TaskStrategist::build_execution_strategy @@ -203,6 +204,10 @@ IntentExecutorImpl::execute -> reset nonce cache for all committed pubkeys on errors, or only undelegated pubkeys on successful undelegation ``` +Current DLP finalize instructions do not undelegate and require only the validator plus delegated account, so `FinalizeTask` and `CommitFinalizeTask` no longer fetch or carry owner/rent metadata. Finalize-stage `UndelegateTask`s complete ownership return through standalone DLP `Undelegate`. For owner-program requests, task building detects `DelegationMetadata.undelegation_requester = OwnerProgram` and includes the derived request PDA; DLP validates that request account and uses `DelegationMetadata.rent_payer` to close both delegation and request accounts. For validator-requested undelegation, or metadata still showing `None` because commit and finalize task lists are built before the commit-stage transaction records the validator requester on base, no request account is included. + +Transaction fit is not only packet size. `TaskStrategist` currently checks whether a single-stage transaction or each two-stage transaction fits the wire size, optionally after switching commits to buffers and adding ALTs, but it does not split a transaction stage by compute units. Each commit, finalize, commit-finalize, and undelegate task currently advertises `120_000` CU, while Agave caps a transaction at `1_400_000` CU. Keep task bundles below that transaction-level cap unless the strategist and executor/output/persistence model are extended to split, record, and confirm multiple transactions for the affected stage. + For committed accounts with `data.len() > COMMIT_STATE_SIZE_THRESHOLD` (`256`), the task builder fetches the base account and may use diff-in-args delivery. If the base-account fetch fails, it falls back to full state args and logs a warning. This can increase transaction size and trigger buffer/ALT strategy later. ### Delivery preparation and cleanup flow @@ -229,7 +234,7 @@ Do not remove sorted lock acquisition or the retiring map without replacing the ### `min_context_slot` and freshness -Task-info RPC reads use the maximum `remote_slot` across committed accounts as `min_context_slot` when fetching delegation metadata and diffable base accounts. This helps avoid building commits against base-layer state older than the ER account snapshot. The fetcher retries `Minimum context slot not reached` up to five times with short sleeps. Preserve this freshness check unless the broader account-sync/settlement contract changes. +Task-info RPC reads use the maximum `remote_slot` across committed accounts as `min_context_slot` when fetching delegation metadata and diffable base accounts. This helps avoid building commits or standalone undelegate tasks against base-layer state older than the ER account snapshot, including stale rent-payer or owner-program requester metadata. The fetcher retries `Minimum context slot not reached` up to five times with short sleeps. Preserve this freshness check unless the broader account-sync/settlement contract changes. ### Persistence is both status API and recovery state @@ -268,7 +273,7 @@ The public service API uses nonblocking `try_send`. If the service channel is fu ### Changing service API, startup, or shutdown -Start with `src/service.rs`, `src/committor_processor.rs`, and `magicblock-api/src/magic_validator.rs`. Then inspect `magicblock-accounts/src/scheduled_commits_processor.rs`, `magicblock-account-cloner/src/account_cloner.rs`, and `magicblock-api/src/magic_sys_adapter.rs`. Check oneshot behavior, channel capacity/backpressure, cancellation, and whether consumers need errors instead of logged-only failures. +Start with `src/service.rs`, `src/service/intent_client.rs`, `src/committor_processor.rs`, and `magicblock-api/src/magic_validator.rs`. Then inspect `magicblock-account-cloner/src/account_cloner.rs` and `magicblock-api/src/magic_sys_adapter.rs`. Check oneshot behavior, channel capacity/backpressure, cancellation, and whether consumers need errors instead of logged-only failures. ### Changing scheduling or concurrency @@ -288,7 +293,7 @@ Start with `src/transaction_preparator/mod.rs` and `delivery_preparator.rs`, the ### Changing persistence or recovery -Start with `src/persist/db.rs`, `src/persist/commit_persister.rs`, `src/persist/types/`, and `src/committor_processor.rs` recovery helpers. Then inspect `magicblock-accounts/src/scheduled_commits_processor.rs`. Preserve schema compatibility, enum string values, `u64`/`i64` conversions, row grouping by `message_id`, 14-day recovery window, and no-repersist recovery scheduling. +Start with `src/persist/db.rs`, `src/persist/commit_persister.rs`, `src/persist/types/`, `src/committor_processor.rs` recovery helpers, and `src/service.rs` recovery filtering. Preserve schema compatibility, enum string values, `u64`/`i64` conversions, row grouping by `message_id`, 14-day recovery window, and no-repersist recovery scheduling. ### Changing metrics or observability @@ -308,6 +313,6 @@ Start with metric calls in `intent_execution_engine.rs`, `delivery_preparator.rs - `.agents/context/crates/magicblock-committor-program.md` — buffer/chunks on-chain helper contracts. - `.agents/context/crates/magicblock-rpc-client.md` — base-layer send/confirm and RPC helper behavior. - `.agents/context/crates/magicblock-table-mania.md` — ALT lifecycle and finalized-read semantics. -- `.agents/context/crates/magicblock-accounts.md` — scheduled commit processing and pending-intent recovery call sites. +- `.agents/context/crates/magicblock-services.md` — request-driven local scheduling that can create commit-and-undelegate intents. - `magicblock-committor-service/README.md` — high-level implementation notes. - `test-integration/test-committor-service/` — integration coverage of delivery and intent execution. diff --git a/.agents/context/crates/magicblock-core.md b/.agents/context/crates/magicblock-core.md index 06f23deae..3c34739df 100644 --- a/.agents/context/crates/magicblock-core.md +++ b/.agents/context/crates/magicblock-core.md @@ -55,7 +55,7 @@ Main consumers include: - `magicblock-processor`, which consumes `ValidatorChannelEndpoints`, executes `SchedulerCommand`s, drains `ExecutionTlsStash`, and emits account/status/replication messages; - `magicblock-replicator`, which consumes and publishes `link::replication::Message` and uses `wait_for_idle()` during checksum/reset operations; - `magicblock-ledger`, which replays persisted transactions through `TransactionSchedulerHandle::replay`; -- `magicblock-accounts`, `magicblock-account-cloner`, and `magicblock-task-scheduler`, which submit validator-internal transactions or consume scheduled tasks; +- `magicblock-committor-service`, `magicblock-services`, `magicblock-account-cloner`, and `magicblock-task-scheduler`, which submit validator-internal transactions, consume scheduled intents, or consume scheduled tasks; - `programs/magicblock`, which uses `MagicSys`, `CommittedAccount`, `BaseActionCallback`, coordination mode, and `ExecutionTlsStash`; - `magicblock-committor-service`, which consumes committed-account/action types and callback scheduling traits; - `magicblock-chainlink`, which uses token/eATA helpers and consolidated logging helpers. @@ -346,7 +346,8 @@ Start with: - `magicblock-core/src/traits.rs` - `programs/magicblock/src/magic_sys.rs` - `programs/magicblock/src/magic_scheduled_base_intent.rs` -- `magicblock-accounts/src/scheduled_commits_processor.rs` +- `magicblock-committor-service/src/service/intent_client.rs` +- `magicblock-services/src/undelegation_request_service.rs` - `magicblock-committor-service/src/` Check serialization, persistence, commit nonce behavior, callback error mapping, and base-layer settlement compatibility. @@ -389,6 +390,7 @@ Avoid adding high-cardinality or noisy logs to hot loops. Keep test logging idem - `.agents/context/crates/magicblock-api.md` — service wiring consumer of core endpoints. - `.agents/context/crates/magicblock-aperture.md` — RPC/account/status event consumer. - `.agents/context/crates/magicblock-account-cloner.md` — internal transaction submission consumer. -- `.agents/context/crates/magicblock-accounts.md` — scheduled commit consumer of core payloads. +- `.agents/context/crates/magicblock-committor-service.md` — scheduled intent consumer of core payloads. +- `.agents/context/crates/magicblock-services.md` — service adapters that submit validator-internal transactions. - `.agents/context/crates/magicblock-chainlink.md` — token/eATA helper and logging consumer. - `.agents/context/crates/magicblock-config.md` — config types that drive startup mode and service wiring. diff --git a/.agents/context/crates/magicblock-services.md b/.agents/context/crates/magicblock-services.md index 97c33d568..f0496e524 100644 --- a/.agents/context/crates/magicblock-services.md +++ b/.agents/context/crates/magicblock-services.md @@ -2,29 +2,32 @@ ## Purpose -`magicblock-services` contains small reusable service adapters that run alongside the validator and communicate through existing validator/RPC contracts. Its current responsibility is the action-callback adapter used by the committor pipeline to notify user callback programs about Magic Action results. +`magicblock-services` contains small reusable services and adapters that run alongside the validator and communicate through existing validator/RPC contracts. It currently owns the action-callback adapter used by the committor pipeline and the owner-program undelegation request service that turns observed Delegation Program requests into local `ScheduleCommitAndUndelegate` transactions. High-level responsibilities: - implement `magicblock_core::traits::ActionsCallbackScheduler` for validator-runtime use; - turn `BaseActionCallback` payloads from committed Magic Actions into local callback transactions; - wrap callback results in the `magicblock-magic-program-api` `MagicResponse::V1` wire shape; -- submit callback transactions asynchronously through Solana's nonblocking `RpcClient`. +- submit callback transactions asynchronously through Solana's nonblocking `RpcClient`; +- subscribe to Chainlink-observed Delegation Program `UndelegationRequest` accounts; +- validate request PDAs and delegated-account state before scheduling local validator-signed `ScheduleCommitAndUndelegate` transactions. -This crate is settlement-adjacent and can affect Magic Action user experience. It is not the committor itself, does not own intent execution or persistence, and must stay generic enough to be used as a service adapter rather than a protocol owner. +This crate is settlement-adjacent and can affect Magic Action user experience and undelegation liveness. It is not the committor itself, does not own base-layer intent execution or persistence, and should stay limited to service/adaptor boundaries rather than becoming a protocol owner. -End-to-end commit/undelegation semantics live in .agents/specs/validator-specification.md; this crate owns fire-and-forget local Magic Action callback transaction construction and scheduling. +End-to-end commit/undelegation semantics live in .agents/specs/validator-specification.md; this crate owns callback transaction construction/scheduling and request-observer submission of local schedule transactions. ## Update requirement Update this guide in the same change whenever behavior or contracts in `magicblock-services` change. In particular, update it for changes to: -- exported modules or public constructors in `magicblock-services/src/lib.rs` or `actions_callback_service.rs`; +- exported modules or public constructors in `magicblock-services/src/lib.rs`, `actions_callback_service.rs`, or `undelegation_request_service.rs`; - the `ActionsCallbackService` transaction layout, signer handling, blockhash source, or callback response encoding; +- the `UndelegationRequestService` request validation, retry policy, Chainlink checks, transaction layout, signer handling, blockhash source, or subscription/polling behavior; - how callback signatures, errors, receipts, discriminators, payloads, or account metas are propagated; - asynchronous send behavior, logging, retry/confirmation semantics, or Tokio runtime assumptions; -- call-site wiring in `magicblock-api` or committor expectations around `ActionsCallbackScheduler`; -- validation commands or integration suites relevant to action callbacks. +- call-site wiring in `magicblock-api`, committor expectations around `ActionsCallbackScheduler`, or Chainlink observer contracts consumed by the undelegation request service; +- validation commands or integration suites relevant to action callbacks or owner-program undelegation requests. Because callbacks are a cross-crate contract between Magic Program scheduling, committor execution, and user callback programs, also update this file when another crate changes `BaseActionCallback`, `MagicResponse`, `CallbackInstruction`, or `ActionsCallbackScheduler` semantics. @@ -34,19 +37,23 @@ For the general documentation-update rule, see .agents/memory/agent-memory-and-d | Path | Role | |---|---| -| `magicblock-services/Cargo.toml` | Package metadata and dependencies. Depends on `magicblock-core`, `magicblock-magic-program-api`, Solana transaction/RPC crates, Tokio, and tracing. | -| `magicblock-services/src/lib.rs` | Public module surface. Currently exports only `actions_callback_service`. | +| `magicblock-services/Cargo.toml` | Package metadata and dependencies. Depends on shared core/Magic Program types, Chainlink/cloner wiring, Solana transaction/RPC crates, Tokio, and tracing. | +| `magicblock-services/src/lib.rs` | Public module surface. Exports `actions_callback_service` and `undelegation_request_service`. | | `magicblock-services/src/actions_callback_service.rs` | Implements `ActionsCallbackService` and `ActionsCallbackScheduler`. Builds and sends callback transactions. | -| `magicblock-api/src/magic_validator.rs` | Runtime wiring. `init_committor_service` creates `ActionsCallbackService` with the validator keypair, local latest-block provider, and an RPC client pointed at the validator aperture HTTP endpoint. | +| `magicblock-services/src/undelegation_request_service.rs` | Implements `UndelegationRequestService`. Consumes observed DLP request accounts and schedules local `ScheduleCommitAndUndelegate` transactions. | +| `magicblock-api/src/magic_validator.rs` | Runtime wiring. `init_committor_processor` creates `ActionsCallbackService`; validator startup creates and starts `UndelegationRequestService` for non-replica validators. | | `magicblock-core/src/traits.rs` | Defines `ActionsCallbackScheduler`, `ActionResult`, `ActionError`, and `CallbackScheduleError`. | | `magicblock-core/src/intent.rs` | Defines `BaseActionCallback`, the callback payload this crate consumes. | +| `magicblock-chainlink` | Supplies `subscribe_undelegation_requests`, delegation-state checks, and `undelegation_requested` lifecycle notification. | | `magicblock-committor-service/src/intent_executor/utils.rs` | Extracts callbacks from action tasks and invokes the scheduler after success, action failure, or timeout. | | `programs/magicblock/src/schedule_transactions/process_add_action_callback.rs` | Magic Program path that attaches callbacks to scheduled base actions. | +| `programs/magicblock/src/schedule_transactions/process_schedule_commit.rs` | Magic Program path reached by the local `ScheduleCommitAndUndelegate` transaction. | Main consumers: -- `magicblock-api`, which constructs the concrete callback service for the validator; +- `magicblock-api`, which constructs the concrete callback service and undelegation request service for the validator; - `magicblock-committor-service`, which is generic over `ActionsCallbackScheduler` and calls it from intent execution; +- Chainlink, which publishes observed undelegation request updates consumed by this crate; - callback-capable Magic Action flows scheduled through `programs/magicblock`. There are no crate-local tests or README files for `magicblock-services` at the time of writing. Use the committor integration tests when callback behavior changes. @@ -58,8 +65,9 @@ There are no crate-local tests or README files for `magicblock-services` at the `src/lib.rs` exposes: - `pub mod actions_callback_service`. +- `pub mod undelegation_request_service`. -Keep this surface small. New shared services should be added only when they are truly generic validator service adapters and not better owned by API orchestration, committor, RPC-client, or Magic Program crates. +Keep this surface small. New shared services should be added only when they are truly validator background services/adapters and not better owned by API orchestration, committor, RPC-client, Chainlink, or Magic Program crates. ### `ActionsCallbackService` @@ -87,6 +95,35 @@ The service implements `Clone` when `L: Clone`. Cloning uses `authority.insecure The return value reports scheduling/build success, not confirmed on-chain callback execution. The spawned task logs send failures but does not retry or update the returned result after the fact. +### `UndelegationRequestService` + +`UndelegationRequestService` stores: + +- `Arc>` for observed request subscription, delegation checks, and undelegation tracking notification; +- `TransactionSchedulerHandle` for submitting local validator transactions; +- validator `Keypair` authority used as payer and signer; +- a `LatestBlockProvider` wrapper used to sign each local transaction with a fresh ER blockhash; +- a `CancellationToken` used by `stop`. + +Public API: + +```rust +UndelegationRequestService::new(chainlink, scheduler, authority, latest_block) +service.start() +service.stop() +``` + +`start` subscribes to Chainlink observed undelegation requests and spawns one background task. The service does not own request discovery itself; Chainlink owns base-layer subscription/scanning and emits `ObservedUndelegationRequest` values. + +For each observed request, the service: + +1. verifies the request PDA matches the delegated account; +2. checks the delegated account is delegated on base and ER; +3. best-effort notifies Chainlink with `undelegation_requested`; +4. submits a local validator-signed `ScheduleCommitAndUndelegate` transaction with payer, `MAGIC_CONTEXT_PUBKEY`, and delegated account. + +Transient delegation-check and local-scheduling failures are retried three times with short exponential backoff. Invalid request PDAs and non-delegated accounts are skipped. + ## Runtime flows ### Validator startup wiring @@ -98,6 +135,20 @@ The return value reports scheduling/build success, not confirmed on-chain callba Preserve this separation. `magicblock-services` should not reach back into validator orchestration or committor internals. +### Undelegation Request Service Startup + +```text +magicblock-api::MagicValidator::try_from_config + -> create UndelegationRequestService for non-replica validators +magicblock-api::MagicValidator::start + -> service.start() + -> Chainlink observed request subscription + -> local ScheduleCommitAndUndelegate transaction + -> committor scheduled-intent service handles the resulting Magic Program intent +``` + +Replica validators do not start this service because replica Chainlink is disabled and ownership/lifecycle decisions should come from primary state replication. + ### Callback scheduling and transaction construction ```text @@ -159,6 +210,14 @@ The callback instruction data combines a program-specific discriminator with a b `CallbackScheduleError` only covers local serialization/signing failures. RPC send failures are logged asynchronously with the callback transaction signature and do not flow back into `IntentExecutionReport` after `schedule` returns. +### Undelegation request observer boundary + +The undelegation request service is a trigger: it only schedules the local Magic Program intent. It does not build or send base-layer commit/undelegate transactions, persist intent rows, or decide committor task strategy. After local scheduling, the normal committor intent service accepts and executes the resulting scheduled intent. + +The service validates the request PDA from delegated account before scheduling. It must not trust request-local owner, rent payer, or commit nonce fields; current request data carries the delegated account and expiry slot only. + +The service logs expired requests but still schedules normal undelegation when the delegated account is still valid. This preserves the best chance of committing ER state and clearing lifecycle state instead of leaving timeout/rollback handling to a stale request. + ## Important invariants 1. `schedule` must return exactly one `Result` for each input callback, preserving input order. @@ -168,7 +227,13 @@ The callback instruction data combines a program-specific discriminator with a b 5. Inner account meta writability must be propagated from `BaseActionCallback`; this crate should not reinterpret callback account authorization. 6. The `Noop(counter)` uniqueness instruction must keep otherwise duplicate callback transactions from producing identical signatures. 7. Do not add blocking RPC confirmation or retry loops to the committor hot path without an explicit architecture decision and performance review. -8. Keep the crate dependency-light. Generic service adapters should not pull in validator orchestration, persistence, or large protocol owners. +8. Keep service dependencies scoped to the adapter. Do not pull validator orchestration or persistence into this crate. +9. `UndelegationRequestService` must verify the request PDA before scheduling. +10. `UndelegationRequestService` must schedule only when the delegated account is delegated on both base and ER. +11. `UndelegationRequestService` must use the validator authority as local transaction payer and signer. +12. Request observation must remain non-replica-only unless replica lifecycle semantics are redesigned. +13. The request service must not bypass the Magic Program scheduled-intent path or call the committor directly for owner-program requests. +14. Missing Chainlink `undelegation_requested` notification should be logged but must not prevent local scheduling under the current best-effort policy. ## Common change areas and what to inspect @@ -199,11 +264,24 @@ Do not make `schedule` block on network confirmation unless the committor timeou Start with: - `magicblock-api/src/magic_validator.rs::init_committor_service`; +- `magicblock-api/src/magic_validator.rs` construction/start/stop of `UndelegationRequestService`; - `magicblock-config` endpoint settings used by `config.aperture.listen.http()` and `config.rpc_url()`; - `magicblock-committor-service::CommittorService::try_start` generic bounds. Be explicit about whether callbacks should be sent to local aperture or base-layer RPC. +### Changing owner-program undelegation request handling + +Start with: + +- `magicblock-services/src/undelegation_request_service.rs`; +- `magicblock-chainlink` observed undelegation request subscription and delegation checks; +- `programs/magicblock/src/schedule_transactions/process_schedule_commit.rs`; +- `magicblock-committor-service/src/service.rs` and task building for accepted commit-and-undelegate intents; +- DLP request PDA and metadata semantics in `magicblock-delegation-program-api`. + +Check PDA validation, replica gating, retry behavior, local transaction accounts, blockhash freshness, and whether failures should be skipped, retried, or surfaced. + ### Adding another shared service adapter Start with `magicblock-services/src/lib.rs` and ask whether the new adapter belongs here. Prefer this crate only for small generic services that implement shared traits. Put orchestration in `magicblock-api`, settlement logic in `magicblock-committor-service`, RPC policy in `magicblock-rpc-client`, and protocol wire types in `magicblock-magic-program-api`. @@ -212,8 +290,8 @@ Start with `magicblock-services/src/lib.rs` and ask whether the new adapter belo - Markdown-only guide changes: run `git diff --check` for this file; no Rust checks are needed. - Rust changes in this crate: use `.agents/rules/testing-and-validation.md` or `mbv-check`; include focused package checks for `magicblock-services`. -- Consumer validation intent: because this crate has no crate-local tests, include relevant `magicblock-committor-service` checks for callback behavior changes. -- Relevant integration suites: committor suites that exercise action callbacks and timeouts; use `.agents/rules/testing-and-validation.md` for exact setup/test commands. +- Consumer validation intent: because this crate has limited crate-local coverage, include relevant `magicblock-committor-service` checks for callback behavior changes and request-driven undelegation integration checks for undelegation request changes. +- Relevant integration suites: committor suites that exercise action callbacks/timeouts, plus schedule-commit or undelegation request suites when `UndelegationRequestService` changes; use `.agents/rules/testing-and-validation.md` for exact setup/test commands. - Performance-risk validation intent: report any added synchronous RPC work, retries, confirmation, persistence, extra serialization, or unbounded spawning/logging and its effect on committor throughput and callback latency. @@ -221,7 +299,10 @@ Start with `magicblock-services/src/lib.rs` and ask whether the new adapter belo - `.agents/context/crates/magicblock-core.md` — `ActionsCallbackScheduler`, `BaseActionCallback`, and related shared trait/type contracts. - `.agents/context/crates/magicblock-api.md` — validator startup and service wiring. +- `.agents/context/crates/magicblock-chainlink.md` — observed undelegation request subscription and delegation lifecycle checks. +- `.agents/context/crates/magicblock-committor-service.md` — scheduled intent acceptance/execution after request service submits the local schedule transaction. - `.agents/context/crates/magicblock-magic-program-api.md` — callback instruction and response wire types. - `.agents/context/crates/magicblock-rpc-client.md` — relevant if callback delivery begins using shared send/confirm helpers. - `magicblock-committor-service/src/intent_executor/utils.rs` — committor callback scheduling call site. - `programs/magicblock/src/schedule_transactions/process_add_action_callback.rs` — Magic Program callback attachment path. +- `programs/magicblock/src/schedule_transactions/process_schedule_commit.rs` — Magic Program schedule path used by owner-program undelegation requests. diff --git a/.agents/context/crates/magicblock-validator-admin.md b/.agents/context/crates/magicblock-validator-admin.md index 08e3ec73e..9dad839b8 100644 --- a/.agents/context/crates/magicblock-validator-admin.md +++ b/.agents/context/crates/magicblock-validator-admin.md @@ -129,8 +129,8 @@ The periodic task uses the validator's configured RPC URL. Do not move it onto t ```text MagicValidator::stop - -> stop scheduled-commit processor - -> stop committor service + -> stop undelegation request service + -> stop intent execution service -> claim_fees_task.stop().await -> join RPC thread and remaining validator services ``` diff --git a/.agents/rules/testing-and-validation.md b/.agents/rules/testing-and-validation.md index a0b57cc9a..24f47b6ec 100644 --- a/.agents/rules/testing-and-validation.md +++ b/.agents/rules/testing-and-validation.md @@ -98,6 +98,19 @@ RUN_TESTS=committor_intent_executor make test The integration runner builds required SBF programs via `make programs` as dependencies of `make test`. +When updating `magicblock-delegation-program-api` to a new Delegation Program +commit, also update the DLP program binary used by integration validators: +`test-integration/schedulecommit/elfs/dlp.so`. A mismatched API/ELF can compile +but fail at runtime when instruction account layouts change, for example DLP +`TooManyAccountKeys` on `CommitFinalize`. + +Avoid using DLP transaction log strings as the primary success oracle in +committor integration tests. Local validator transaction history/log retrieval +can be inconsistent across DLP or validator revisions, and DLP may change opcode +logging without changing validator behavior. Prefer persisted committor status, +strategy, signatures, and final account owner/data checks; keep log fetches for +diagnostics on failure. + ## Isolating one integration test For fast debugging, start only the validators needed by a suite in one terminal, then run the desired Rust test directly in another terminal. diff --git a/.agents/specs/validator-specification.md b/.agents/specs/validator-specification.md index 71c4ff794..41dd98eda 100644 --- a/.agents/specs/validator-specification.md +++ b/.agents/specs/validator-specification.md @@ -257,6 +257,29 @@ When `ScheduleIntentBundle` includes undelegation, the Magic Program marks each This is a critical lifecycle invariant. Do not allow normal ER transactions to keep mutating an account after commit-and-undelegate has been scheduled. +Owner-program requested undelegation is recorded by the Delegation Program in +`DelegationMetadata.undelegation_requester = OwnerProgram` and an +`UndelegationRequest` PDA. This marker is not a validator-side completion +signal: the validator must still schedule commit-and-undelegate so the latest +ER state is committed/finalized before ownership returns to the owner program. +The request account data carries the delegated account and expiry slot only; +owner, rent payer, and commit nonce information must come from the delegation +record/metadata. Validators must still verify that the request PDA matches the +delegated account before scheduling. + +The current Delegation Program does not undelegate from finalize instructions. +Commit/finalize-style instructions only commit/finalize state and record or +preserve `DelegationMetadata.undelegation_requester`. The validator-side +committor completes undelegation by sending a standalone DLP `Undelegate` +instruction in the finalize stage. When metadata says the requester is +`OwnerProgram`, the committor must also include the undelegation request PDA in +`Undelegate`; the delegation metadata rent payer is used for both delegation +and request account cleanup. When metadata says `Validator` or is still `None` +at task-build time, no request account is passed. `None` can be valid for +validator-requested undelegation because commit and finalize task lists are +built before the commit-stage transaction records the validator requester on +base. + ### Callback discriminator The public docs specify an undelegation callback discriminator: @@ -278,7 +301,7 @@ The committor service realizes scheduled base-layer intents. Its inputs are sche The documented commit pipeline is: 1. Magic Program schedules intents in MagicContext. -2. `magicblock-accounts` / scheduled commit processing picks up intents each slot. +2. `magicblock-committor-service` intent execution service accepts scheduled intents from `MagicContext` on its interval. 3. `magicblock-committor-service` schedules and executes intents. 4. Task building creates atomic base-layer tasks such as commit, undelegate, finalize, and action. 5. Task strategy packs tasks into valid transactions. diff --git a/Cargo.lock b/Cargo.lock index e4c21d437..630f30984 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,7 +280,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -291,7 +291,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -900,17 +900,40 @@ dependencies = [ "subtle", ] +[[package]] +name = "borsh" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115e54d64eb62cdebad391c19efc9dce4981c690c85a33a12199d99bb9546fee" +dependencies = [ + "borsh-derive 0.10.4", + "hashbrown 0.13.2", +] + [[package]] name = "borsh" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" dependencies = [ - "borsh-derive", + "borsh-derive 1.6.1", "bytes", "cfg_aliases", ] +[[package]] +name = "borsh-derive" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831213f80d9423998dd696e2c5345aba6be7a0bd8cd19e31c5243e13df1cef89" +dependencies = [ + "borsh-derive-internal", + "borsh-schema-derive-internal", + "proc-macro-crate 0.1.5", + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "borsh-derive" version = "1.6.1" @@ -918,12 +941,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" dependencies = [ "once_cell", - "proc-macro-crate", + "proc-macro-crate 3.5.0", "proc-macro2", "quote", "syn 2.0.118", ] +[[package]] +name = "borsh-derive-internal" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d6ba50644c98714aa2a70d13d7df3cd75cd2b523a2b452bf010443800976b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276691d96f063427be83e6692b86148e488ebba9f48f77788724ca027ba3b6d4" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "brotli" version = "8.0.4" @@ -2021,7 +2066,7 @@ dependencies = [ "atomic", "pear", "serde", - "toml", + "toml 0.8.23", "uncased", "version_check", ] @@ -3422,7 +3467,7 @@ name = "magic-domain-program" version = "0.3.0" source = "git+https://github.com/magicblock-labs/magic-domain-program.git?rev=6cb903d#6cb903da837ebafeecb59a3861a457b7a2092e55" dependencies = [ - "borsh", + "borsh 1.6.1", "bytemuck_derive", "solana-program 3.0.0", "solana-system-interface 3.1.0", @@ -3565,11 +3610,10 @@ version = "0.13.3" dependencies = [ "agave-feature-set", "anyhow", - "borsh", + "borsh 1.6.1", "fd-lock", "magic-domain-program", "magicblock-account-cloner", - "magicblock-accounts", "magicblock-accounts-db", "magicblock-aperture", "magicblock-chainlink", @@ -3631,7 +3675,7 @@ dependencies = [ "assert_matches", "async-trait", "bincode", - "borsh", + "borsh 1.6.1", "futures-util", "helius-laserstream", "lru 0.16.4", @@ -3684,7 +3728,7 @@ dependencies = [ name = "magicblock-committor-program" version = "0.13.3" dependencies = [ - "borsh", + "borsh 1.6.1", "paste", "solana-account 3.4.0", "solana-account-info 3.1.1", @@ -3701,7 +3745,7 @@ version = "0.13.3" dependencies = [ "async-trait", "bincode", - "borsh", + "borsh 1.6.1", "futures-util", "lru 0.16.4", "magicblock-account-cloner", @@ -3758,7 +3802,7 @@ dependencies = [ "solana-pubkey 4.1.0", "solana-signer", "tempfile", - "toml", + "toml 0.8.23", "url", ] @@ -3794,11 +3838,12 @@ dependencies = [ [[package]] name = "magicblock-delegation-program-api" -version = "0.3.0" -source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b#25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" +version = "3.1.0" +source = "git+https://github.com/magicblock-labs/delegation-program.git?tag=v3.1.0#27f7fd8178630b88e2be45d92b174b8f2ef4661e" dependencies = [ "bincode", - "borsh", + "borsh 0.10.4", + "borsh 1.6.1", "bytemuck", "const-crypto", "libsodium-rs", @@ -3813,6 +3858,7 @@ dependencies = [ "solana-instruction 3.3.0", "solana-loader-v3-interface 6.1.1", "solana-program 3.0.0", + "solana-pubkey 2.4.0", "solana-sdk", "solana-sdk-ids 3.1.0", "solana-sha256-hasher 3.1.0", @@ -4045,8 +4091,14 @@ version = "0.13.3" dependencies = [ "bincode", "futures-util", + "magicblock-account-cloner", + "magicblock-chainlink", "magicblock-core", + "magicblock-delegation-program-api", "magicblock-magic-program-api", + "magicblock-metrics", + "magicblock-program", + "solana-hash 4.2.0", "solana-instruction 3.3.0", "solana-keypair", "solana-message 3.1.0", @@ -4055,8 +4107,10 @@ dependencies = [ "solana-signature 3.3.0", "solana-signer", "solana-transaction", + "solana-transaction-error 3.1.0", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", ] @@ -4402,7 +4456,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4553,7 +4607,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 3.5.0", "proc-macro2", "quote", "syn 2.0.118", @@ -4907,6 +4961,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml 0.5.11", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -4970,7 +5033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -4991,7 +5054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6247,7 +6310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6388,7 +6451,7 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f08236dacd0e6dc8234becef58e27c8567856644ef509d7e97957d55a91dc72" dependencies = [ - "borsh", + "borsh 1.6.1", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -6568,7 +6631,7 @@ version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c04abbae16f57178a163125805637b8a076175bb5c0002fb04f4792bea901cf7" dependencies = [ - "borsh", + "borsh 1.6.1", ] [[package]] @@ -6700,7 +6763,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8292c436b269ad23cecc8b24f7da3ab07ca111661e25e00ce0e1d22771951ab9" dependencies = [ - "borsh", + "borsh 1.6.1", "solana-instruction 3.3.0", "solana-sdk-ids 3.1.0", ] @@ -7116,7 +7179,7 @@ version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8064ea1d591ec791be95245058ca40f4f5345d390c200069d0f79bbf55bfae55" dependencies = [ - "borsh", + "borsh 1.6.1", "bytemuck", "bytemuck_derive", "five8 1.0.0", @@ -7162,7 +7225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97881335fc698deb46c6571945969aae6d93a14e2fff792a368b4fac872f116" dependencies = [ "bincode", - "borsh", + "borsh 1.6.1", "serde", "serde_derive", "solana-define-syscall 5.1.0", @@ -7790,7 +7853,7 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f04fa578707b3612b095f0c8e19b66a1233f7c42ca8082fcb3b745afcc0add6" dependencies = [ - "borsh", + "borsh 1.6.1", "serde", "serde_derive", ] @@ -7894,6 +7957,8 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b62adb9c3261a052ca1f999398c388f1daf558a1b492f60a6d9e64857db4ff1" dependencies = [ + "borsh 0.10.4", + "borsh 1.6.1", "bytemuck", "bytemuck_derive", "curve25519-dalek 4.1.3", @@ -8904,7 +8969,7 @@ dependencies = [ "agave-reserved-account-keys", "base64 0.22.1", "bincode", - "borsh", + "borsh 1.6.1", "bs58", "log", "serde", @@ -9065,7 +9130,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea15126ebdc7e270c50d43884369af9f51d2308156d46a18e351522a164844d" dependencies = [ - "borsh", + "borsh 1.6.1", "bytemuck", "bytemuck_derive", ] @@ -9222,7 +9287,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6433917b60441d68d99a17e121d9db0ea15a9a69c0e5afa34649cf5ba12612f" dependencies = [ - "borsh", + "borsh 1.6.1", "solana-instruction 3.3.0", "solana-pubkey 3.0.0", ] @@ -9289,7 +9354,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f9c6e142cdf1e7e77f480053ec9f0ce989890768ddf91f619b50f39d1b456f5" dependencies = [ - "borsh", + "borsh 1.6.1", "bytemuck", "bytemuck_derive", "num-derive", @@ -9407,7 +9472,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c467c7c3bd056f8fe60119e7ec34ddd6f23052c2fa8f1f51999098063b72676" dependencies = [ - "borsh", + "borsh 1.6.1", "num-derive", "num-traits", "solana-borsh", @@ -9852,6 +9917,15 @@ dependencies = [ "webpki-roots 0.26.11", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.8.23" diff --git a/Cargo.toml b/Cargo.toml index 451715d7d..1e59bfe95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ magicblock-committor-program = { path = "./magicblock-committor-program", featur magicblock-committor-service = { path = "./magicblock-committor-service" } magicblock-config = { path = "./magicblock-config" } magicblock-core = { path = "./magicblock-core" } -magicblock-delegation-program-api = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" } +magicblock-delegation-program-api = { git = "https://github.com/magicblock-labs/delegation-program.git", tag = "v3.1.0" } magicblock-ledger = { path = "./magicblock-ledger" } magicblock-magic-program-api = { path = "./magicblock-magic-program-api" } magicblock-metrics = { path = "./magicblock-metrics" } diff --git a/docs/architecture.md b/docs/architecture.md index 0e77bbd3d..a0f51479d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -179,9 +179,9 @@ loader-specific paths (V1→V3 conversion, V4 buffer + deploy). in the 5 MB `MagicContext` account. A second validator-signed instruction (`AcceptScheduleCommits`) moves staged intents to a global map — a deliberate two-stage flow across slot boundaries. -2. [`magicblock-accounts`](../magicblock-accounts) - (`ScheduledCommitsProcessorImpl`) picks intents up each slot (driven by the slot - ticker) and feeds [`magicblock-committor-service`](../magicblock-committor-service). +2. [`magicblock-committor-service`](../magicblock-committor-service) runs the + intent execution service, which periodically accepts staged intents from + `MagicContext`, schedules them, and reports `ScheduledCommitSent` locally. 3. The committor service builds base-chain transactions: [`tasks/task_strategist.rs`](../magicblock-committor-service/src/tasks/task_strategist.rs) packs commit tasks under CPI-depth limits (falling back to a two-stage diff --git a/magicblock-api/Cargo.toml b/magicblock-api/Cargo.toml index 1ddbc66de..76828a769 100644 --- a/magicblock-api/Cargo.toml +++ b/magicblock-api/Cargo.toml @@ -16,7 +16,6 @@ tracing = { workspace = true } magicblock-delegation-program-api = { workspace = true } magic-domain-program = { workspace = true } magicblock-account-cloner = { workspace = true } -magicblock-accounts = { workspace = true } magicblock-accounts-db = { workspace = true } magicblock-aperture = { workspace = true } magicblock-chainlink = { workspace = true } diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index 4f80a6b72..f04f30d8c 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -13,9 +13,6 @@ pub enum ApiError { #[error("Aperture service error: {0}")] Aperture(#[from] magicblock_aperture::error::ApertureError), - #[error("Accounts error: {0}")] - AccountsError(Box), - #[error("Ledger error: {0}")] LedgerError(Box), @@ -113,12 +110,6 @@ pub enum ApiError { Replication(#[from] magicblock_replicator::Error), } -impl From for ApiError { - fn from(e: magicblock_accounts::errors::AccountsError) -> Self { - Self::AccountsError(Box::new(e)) - } -} - impl From for ApiError { fn from(e: magicblock_ledger::errors::LedgerError) -> Self { Self::LedgerError(Box::new(e)) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index b4d569ab8..e23f806f8 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -57,7 +57,10 @@ use magicblock_program::{ TransactionScheduler as ActionTransactionScheduler, }; use magicblock_replicator::{nats::Broker, BrokerSource, ReplicationService}; -use magicblock_services::actions_callback_service::ActionsCallbackService; +use magicblock_services::{ + actions_callback_service::ActionsCallbackService, + undelegation_request_service::UndelegationRequestService, +}; use magicblock_task_scheduler::{SchedulerDatabase, TaskSchedulerService}; use magicblock_validator_admin::claim_fees::{claim_fees, ClaimFeesTask}; use mdp::state::{ @@ -113,6 +116,7 @@ pub struct MagicValidator { ledger_truncator: LedgerTruncator, intent_execution_service: IntentExecutionServiceImpl, replication_service: Option, + undelegation_request_service: Option>, rpc_handle: thread::JoinHandle<()>, identity: Pubkey, transaction_scheduler: TransactionSchedulerHandle, @@ -323,6 +327,19 @@ impl MagicValidator { ); log_timing("startup", "system_metrics_ticker_start", step_start); + let undelegation_request_service = (!matches!( + config.validator.replication_mode, + ReplicationMode::Replica { .. } + )) + .then(|| { + Arc::new(UndelegationRequestService::new( + chainlink.clone(), + dispatch.transaction_scheduler.clone(), + identity_keypair.insecure_clone(), + ledger.latest_block().clone(), + )) + }); + let step_start = Instant::now(); load_upgradeable_programs( &accountsdb, @@ -444,6 +461,7 @@ impl MagicValidator { _metrics: (metrics_service, system_metrics_ticker), intent_execution_service, replication_service, + undelegation_request_service, token, ledger, ledger_truncator, @@ -1037,6 +1055,15 @@ impl MagicValidator { } } + if !matches!( + self.config.validator.replication_mode, + ReplicationMode::Replica { .. } + ) { + if let Some(service) = self.undelegation_request_service.as_ref() { + service.start(); + } + } + // Now we are ready to start all services and are ready to accept transactions if let Some(frequency) = self .config @@ -1122,9 +1149,20 @@ impl MagicValidator { self.start_unregister_validator_on_chain().await; self.exit.store(true, Ordering::Relaxed); - // Ordering is important here - // Commitor service shall be stopped last + // Stop request ingress before stopping intent execution so shutdown + // does not admit new local undelegation scheduling work. self.token.cancel(); + if let Some(ref undelegation_request_service) = + self.undelegation_request_service + { + let step_start = Instant::now(); + undelegation_request_service.stop(); + log_timing( + "shutdown", + "undelegation_request_service_stop", + step_start, + ); + } let step_start = Instant::now(); if let Err(err) = self.intent_execution_service.stop().await { diff --git a/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs b/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs index 2546c98b2..f2e330763 100644 --- a/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs +++ b/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs @@ -9,7 +9,8 @@ use std::{ }; use dlp_api::{ - pda::delegation_record_pda_from_delegated_account, state::DelegationRecord, + pda::delegation_record_pda_from_delegated_account, + state::{DelegationRecord, UndelegationRequest}, }; use lru::LruCache; use magicblock_accounts_db::traits::AccountsBank; @@ -35,7 +36,7 @@ use solana_sdk_ids::system_program; use solana_signature::Signature; use solana_signer::Signer; use tokio::{ - sync::{mpsc, oneshot, Semaphore}, + sync::{broadcast, mpsc, oneshot, Semaphore}, task, task::JoinSet, }; @@ -76,6 +77,7 @@ use crate::{ blacklisted_accounts::{ blacklisted_accounts, programs_not_to_subscribe, }, + ObservedUndelegationRequest, }, cloner::{ errors::{ClonerError, ClonerResult}, @@ -145,6 +147,8 @@ where /// Risk checker for post-delegation action addresses. risk_service: Option>, + + undelegation_request_sender: broadcast::Sender, } struct PendingUndelegationGuard { @@ -209,6 +213,9 @@ where .pending_operation_timeout_ms .clone(), risk_service: self.risk_service.clone(), + undelegation_request_sender: self + .undelegation_request_sender + .clone(), } } } @@ -230,6 +237,33 @@ where subscription_updates_rx: mpsc::Receiver, allowed_programs: Option>, risk_service: Option>, + ) -> Arc { + let (undelegation_request_sender, _) = broadcast::channel(1024); + Self::new_with_undelegation_request_sender( + remote_account_provider, + accounts_bank, + cloner, + validator_keypair, + subscription_updates_rx, + allowed_programs, + risk_service, + undelegation_request_sender, + ) + } + + /// Create FetchCloner with subscription updates and request notifications connected. + #[allow(clippy::too_many_arguments)] + pub fn new_with_undelegation_request_sender( + remote_account_provider: &Arc>, + accounts_bank: &Arc, + cloner: &Arc, + validator_keypair: Keypair, + subscription_updates_rx: mpsc::Receiver, + allowed_programs: Option>, + risk_service: Option>, + undelegation_request_sender: broadcast::Sender< + ObservedUndelegationRequest, + >, ) -> Arc { let validator_pubkey = validator_keypair.pubkey(); let blacklisted_accounts = blacklisted_accounts(&validator_pubkey); @@ -258,6 +292,7 @@ where FETCH_CLONE_OPERATION_TIMEOUT.as_millis() as u64, )), risk_service, + undelegation_request_sender, }); let accounts_bank_for_eviction = accounts_bank.clone(); @@ -2147,6 +2182,37 @@ where // use, etc. (None, None, DelegationActions::default()) } + } else if let Ok(request) = + UndelegationRequest::try_from_bytes_with_discriminator( + account.data(), + ) + { + let observed = ObservedUndelegationRequest { + request_pda: pubkey, + delegated_account: request.delegated_account, + expires_at_slot: request.expires_at_slot, + observed_slot: account.remote_slot(), + }; + trace!( + request_pda = %observed.request_pda, + delegated_account = %observed.delegated_account, + expires_at_slot = observed.expires_at_slot, + "Observed DLP undelegation request" + ); + if let Err(broadcast::error::SendError(observed)) = + self.undelegation_request_sender.send(observed) + { + warn!( + request_pda = %observed.request_pda, + delegated_account = %observed.delegated_account, + "Dropped observed DLP undelegation request because no subscribers are active" + ); + } + ( + Some(account.into_account_shared_data()), + None, + DelegationActions::default(), + ) } else if is_internal_dlp_account_data(account.data()) { ( Some(account.into_account_shared_data()), diff --git a/magicblock-chainlink/src/chainlink/mod.rs b/magicblock-chainlink/src/chainlink/mod.rs index 2f4a844c3..991800c84 100644 --- a/magicblock-chainlink/src/chainlink/mod.rs +++ b/magicblock-chainlink/src/chainlink/mod.rs @@ -15,7 +15,10 @@ use solana_commitment_config::CommitmentConfig; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_transaction::sanitized::SanitizedTransaction; -use tokio::{sync::mpsc, task}; +use tokio::{ + sync::{broadcast, mpsc}, + task, +}; use tracing::*; use crate::{ @@ -56,6 +59,14 @@ pub type ProdChainlink = ReplicationModeAwareChainlink< C, >; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ObservedUndelegationRequest { + pub request_pda: Pubkey, + pub delegated_account: Pubkey, + pub expires_at_slot: u64, + pub observed_slot: u64, +} + // ----------------- // Chainlink // ----------------- @@ -67,6 +78,7 @@ pub struct InnerChainlink< > { accounts_bank: Arc, fetch_cloner: Option>>, + undelegation_request_sender: broadcast::Sender, /// The subscription to events for each account that is removed from /// the accounts tracked by the provider. /// In that case we also remove it from the bank since it is no longer @@ -188,6 +200,17 @@ impl Self::Disabled => false, } } + + pub fn subscribe_undelegation_requests( + &self, + ) -> Option> { + match self { + Self::Enabled(chainlink) => { + Some(chainlink.subscribe_undelegation_requests()) + } + Self::Disabled => None, + } + } } impl @@ -196,6 +219,21 @@ impl pub fn try_new( accounts_bank: &Arc, fetch_cloner: Option>>, + ) -> ChainlinkResult { + let (undelegation_request_sender, _) = broadcast::channel(1024); + Self::try_new_with_undelegation_request_sender( + accounts_bank, + fetch_cloner, + undelegation_request_sender, + ) + } + + pub fn try_new_with_undelegation_request_sender( + accounts_bank: &Arc, + fetch_cloner: Option>>, + undelegation_request_sender: broadcast::Sender< + ObservedUndelegationRequest, + >, ) -> ChainlinkResult { let removed_accounts_sub = if let Some(fetch_cloner) = &fetch_cloner { let removed_accounts_rx = @@ -213,6 +251,7 @@ impl Ok(Self { accounts_bank: accounts_bank.clone(), fetch_cloner, + undelegation_request_sender, removed_accounts_sub, }) } @@ -254,6 +293,7 @@ impl Some(chain_slot), ) .await?; + let (undelegation_request_sender, _) = broadcast::channel(1024); let fetch_cloner = if let Some(provider) = account_provider { let provider = Arc::new(provider); let risk_service = RiskService::try_from_config( @@ -261,21 +301,27 @@ impl ledger_path, )? .map(Arc::new); - let fetch_cloner = FetchCloner::new( - &provider, - accounts_bank, - cloner, - validator_keypair, - rx, - chainlink_config.allowed_programs.clone(), - risk_service, - ); + let fetch_cloner = + FetchCloner::new_with_undelegation_request_sender( + &provider, + accounts_bank, + cloner, + validator_keypair, + rx, + chainlink_config.allowed_programs.clone(), + risk_service, + undelegation_request_sender.clone(), + ); Some(fetch_cloner) } else { None }; - InnerChainlink::try_new(accounts_bank, fetch_cloner) + InnerChainlink::try_new_with_undelegation_request_sender( + accounts_bank, + fetch_cloner, + undelegation_request_sender, + ) } fn subscribe_account_removals( @@ -594,6 +640,12 @@ impl .map(|provider| provider.is_watching(pubkey)) .unwrap_or(false) } + + pub fn subscribe_undelegation_requests( + &self, + ) -> broadcast::Receiver { + self.undelegation_request_sender.subscribe() + } } // ----------------- diff --git a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs b/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs index b4f26fe1d..ba138ed16 100644 --- a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs +++ b/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs @@ -46,12 +46,12 @@ pub trait TaskInfoFetcher: Send + Sync + 'static { min_context_slot: u64, ) -> TaskInfoFetcherResult>; - /// Fetches rent reimbursement address for pubkeys - async fn fetch_rent_reimbursements( + /// Fetches delegation metadata keyed by delegated account pubkey. + async fn fetch_delegation_metadata( &self, pubkeys: &[Pubkey], min_context_slot: u64, - ) -> TaskInfoFetcherResult>; + ) -> TaskInfoFetcherResult>; async fn get_base_accounts( &self, @@ -241,7 +241,7 @@ impl TaskInfoFetcher for RpcTaskInfoFetcher { ) .await? .into_iter() - .map(|m| m.last_update_nonce + 1); + .map(|m| m.last_commit_id + 1); Ok(pubkeys.iter().copied().zip(nonces).collect()) } @@ -261,25 +261,28 @@ impl TaskInfoFetcher for RpcTaskInfoFetcher { ) .await? .into_iter() - .map(|m| m.last_update_nonce); + .map(|m| m.last_commit_id); Ok(pubkeys.iter().copied().zip(nonces).collect()) } - async fn fetch_rent_reimbursements( + async fn fetch_delegation_metadata( &self, pubkeys: &[Pubkey], min_context_slot: u64, - ) -> TaskInfoFetcherResult> { - Ok(Self::fetch_metadata_with_retries( - &self.rpc_client, - pubkeys, - min_context_slot, - NUM_FETCH_RETRIES, - ) - .await? - .into_iter() - .map(|m| m.rent_payer) - .collect()) + ) -> TaskInfoFetcherResult> { + Ok(pubkeys + .iter() + .copied() + .zip( + Self::fetch_metadata_with_retries( + &self.rpc_client, + pubkeys, + min_context_slot, + NUM_FETCH_RETRIES, + ) + .await?, + ) + .collect()) } async fn get_base_accounts( @@ -613,13 +616,13 @@ impl TaskInfoFetcher for CacheTaskInfoFetcher { Ok(result) } - async fn fetch_rent_reimbursements( + async fn fetch_delegation_metadata( &self, pubkeys: &[Pubkey], min_context_slot: u64, - ) -> TaskInfoFetcherResult> { + ) -> TaskInfoFetcherResult> { self.inner - .fetch_rent_reimbursements(pubkeys, min_context_slot) + .fetch_delegation_metadata(pubkeys, min_context_slot) .await } @@ -1007,12 +1010,27 @@ mod tests { .collect()) } - async fn fetch_rent_reimbursements( + async fn fetch_delegation_metadata( &self, - _: &[Pubkey], + pubkeys: &[Pubkey], _: u64, - ) -> TaskInfoFetcherResult> { - unimplemented!() + ) -> TaskInfoFetcherResult> + { + Ok(pubkeys + .iter() + .map(|pubkey| { + ( + *pubkey, + DelegationMetadata { + last_commit_id: 0, + undelegation_requester: + dlp_api::state::UndelegationRequester::None, + seeds: vec![], + rent_payer: *pubkey, + }, + ) + }) + .collect()) } async fn get_base_accounts( diff --git a/magicblock-committor-service/src/tasks/commit_finalize_task.rs b/magicblock-committor-service/src/tasks/commit_finalize_task.rs index 6a174f582..33f48c0f6 100644 --- a/magicblock-committor-service/src/tasks/commit_finalize_task.rs +++ b/magicblock-committor-service/src/tasks/commit_finalize_task.rs @@ -1,7 +1,9 @@ use dlp_api::{ args::CommitFinalizeArgs, diff::compute_diff, - instruction_builder::{commit_diff_size_budget, commit_size_budget}, + instruction_builder::{ + commit_finalize_from_buffer_size_budget, commit_finalize_size_budget, + }, AccountSizeClass, }; use magicblock_committor_program::Chunks; @@ -241,16 +243,16 @@ impl BaseTask for CommitFinalizeTask { fn accounts_size_budget(&self) -> u32 { match &self.delivery { CommitDelivery::StateInArgs => { - commit_size_budget(AccountSizeClass::Dynamic( + commit_finalize_size_budget(AccountSizeClass::Dynamic( self.committed_account.account.data.len() as u32, )) } CommitDelivery::StateInBuffer { .. } | CommitDelivery::DiffInBuffer { .. } => { - commit_size_budget(AccountSizeClass::Huge) + commit_finalize_from_buffer_size_budget(AccountSizeClass::Huge) } CommitDelivery::DiffInArgs { .. } => { - commit_diff_size_budget(AccountSizeClass::Dynamic( + commit_finalize_size_budget(AccountSizeClass::Dynamic( self.committed_account.account.data.len() as u32, )) } diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 7640ff74e..845b11680 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -95,10 +95,16 @@ impl BaseTask for BaseTaskImpl { AccountSizeClass::Huge, ) } - Self::Undelegate(_) => { - dlp_api::instruction_builder::undelegate_size_budget( - AccountSizeClass::Huge, - ) + Self::Undelegate(value) => { + if value.include_undelegation_request { + dlp_api::instruction_builder::undelegate_with_request_size_budget( + AccountSizeClass::Huge, + ) + } else { + dlp_api::instruction_builder::undelegate_size_budget( + AccountSizeClass::Huge, + ) + } } } } @@ -177,16 +183,26 @@ pub struct UndelegateTask { pub delegated_account: Pubkey, pub owner_program: Pubkey, pub rent_reimbursement: Pubkey, + pub include_undelegation_request: bool, } impl UndelegateTask { pub fn instruction(&self, validator: &Pubkey) -> Instruction { - dlp_api::instruction_builder::undelegate( - *validator, - self.delegated_account, - self.owner_program, - self.rent_reimbursement, - ) + if self.include_undelegation_request { + dlp_api::instruction_builder::undelegate_with_request( + *validator, + self.delegated_account, + self.owner_program, + self.rent_reimbursement, + ) + } else { + dlp_api::instruction_builder::undelegate( + *validator, + self.delegated_account, + self.owner_program, + self.rent_reimbursement, + ) + } } } @@ -548,6 +564,10 @@ impl CleanupTask { #[cfg(test)] mod serialization_safety_test { + use dlp_api::{ + discriminator::DlpDiscriminator, + pda::undelegation_request_pda_from_delegated_account, + }; use magicblock_core::intent::CommittedAccount; use magicblock_program::{ args::ShortAccountMeta, magic_scheduled_base_intent::ProgramArgs, @@ -612,6 +632,7 @@ mod serialization_safety_test { delegated_account: Pubkey::new_unique(), owner_program: Pubkey::new_unique(), rent_reimbursement: Pubkey::new_unique(), + include_undelegation_request: false, } .into(); assert_serializable(&undelegate_task.instruction(&validator)); @@ -661,6 +682,29 @@ mod serialization_safety_test { assert_serializable(&base_action_v2.instruction(&validator)); } + #[test] + fn test_undelegate_task_uses_request_account_when_included() { + let delegated_account = Pubkey::new_unique(); + + let ix = UndelegateTask { + delegated_account, + owner_program: Pubkey::new_unique(), + rent_reimbursement: Pubkey::new_unique(), + include_undelegation_request: true, + } + .instruction(&Pubkey::new_unique()); + + assert_eq!(ix.program_id, dlp_api::id()); + assert_eq!(ix.data, DlpDiscriminator::Undelegate.to_vec()); + assert_eq!(ix.accounts.len(), 13); + assert_eq!( + ix.accounts[12].pubkey, + undelegation_request_pda_from_delegated_account(&delegated_account) + ); + assert!(ix.accounts[12].is_writable); + assert!(!ix.accounts[12].is_signer); + } + fn make_buffer_commit_task( commit_id: u64, allow_undelegation: bool, diff --git a/magicblock-committor-service/src/tasks/task_builder.rs b/magicblock-committor-service/src/tasks/task_builder.rs index 515e925f5..5efbeb24b 100644 --- a/magicblock-committor-service/src/tasks/task_builder.rs +++ b/magicblock-committor-service/src/tasks/task_builder.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; -use futures_util::future::try_join_all; +use dlp_api::state::{DelegationMetadata, UndelegationRequester}; use magicblock_core::intent::CommittedAccount; use magicblock_program::magic_scheduled_base_intent::{ BaseAction, CommitAndUndelegate, CommitType, ScheduledIntentBundle, @@ -137,6 +137,21 @@ impl TaskBuilderImpl { .await } + fn get_finalize_stage_metadata_pubkeys( + intent_bundle: &ScheduledIntentBundle, + ) -> Vec { + [ + intent_bundle.get_undelegate_intent_pubkeys(), + intent_bundle + .intent_bundle + .get_commit_finalize_and_undelegate_intent_pubkeys(), + ] + .into_iter() + .flatten() + .flatten() + .collect() + } + async fn fetch_commit_stage_info( intent_bundle: &ScheduledIntentBundle, task_info_fetcher: &Arc, @@ -299,11 +314,13 @@ impl TasksBuilder for TaskBuilderImpl { fn undelegate_task( account: &CommittedAccount, rent_reimbursement: &Pubkey, + include_undelegation_request: bool, ) -> BaseTaskImpl { UndelegateTask { delegated_account: account.pubkey, owner_program: account.account.owner, rent_reimbursement: *rent_reimbursement, + include_undelegation_request, } .into() } @@ -330,33 +347,31 @@ impl TasksBuilder for TaskBuilderImpl { } } - async fn create_undelegate_tasks( + fn create_undelegate_tasks( commit_and_undelegate: &CommitAndUndelegate, - info_fetcher: &Arc, + delegation_metadata: &HashMap, ) -> TaskBuilderResult> { - // Get rent reimbursments for undelegated accounts let accounts = commit_and_undelegate.get_committed_accounts(); - let mut min_context_slot = 0; - let pubkeys = accounts - .iter() - .map(|account| { - min_context_slot = - std::cmp::max(min_context_slot, account.remote_slot); - account.pubkey - }) - .collect::>(); - let rent_reimbursements = info_fetcher - .fetch_rent_reimbursements(&pubkeys, min_context_slot) - .await - .map_err(TaskBuilderError::FinalizedTasksBuildError)?; let mut tasks = accounts .iter() - .zip(rent_reimbursements) - .map(|(account, rent_reimbursement)| { - undelegate_task(account, &rent_reimbursement) + .map(|account| { + let rent_reimbursement = rent_reimbursement( + delegation_metadata, + account.pubkey, + )?; + let include_undelegation_request = + should_include_undelegation_request( + delegation_metadata, + account.pubkey, + )?; + Ok(undelegate_task( + account, + &rent_reimbursement, + include_undelegation_request, + )) }) - .collect::>(); + .collect::>>()?; if let UndelegateType::WithBaseActions(actions) = &commit_and_undelegate.undelegate_action @@ -367,7 +382,21 @@ impl TasksBuilder for TaskBuilderImpl { } let mut tasks = Vec::new(); - let mut futures = Vec::with_capacity(2); + let finalize_metadata_pubkeys = + Self::get_finalize_stage_metadata_pubkeys(intent_bundle); + let min_context_slot = intent_bundle + .get_all_committed_accounts() + .iter() + .map(|account| account.remote_slot) + .max() + .unwrap_or(0); + let delegation_metadata = info_fetcher + .fetch_delegation_metadata( + &finalize_metadata_pubkeys, + min_context_slot, + ) + .await + .map_err(TaskBuilderError::FinalizedTasksBuildError)?; if let Some(ref value) = intent_bundle.intent_bundle.commit { tasks.extend(create_finalize_tasks(value)); @@ -377,17 +406,15 @@ impl TasksBuilder for TaskBuilderImpl { intent_bundle.intent_bundle.commit_and_undelegate { tasks.extend(create_finalize_tasks(&value.commit_action)); - futures.push(create_undelegate_tasks(value, info_fetcher)); + tasks.extend(create_undelegate_tasks(value, &delegation_metadata)?); } if let Some(ref value) = intent_bundle.intent_bundle.commit_finalize_and_undelegate { - futures.push(create_undelegate_tasks(value, info_fetcher)); + tasks.extend(create_undelegate_tasks(value, &delegation_metadata)?); } - tasks.extend(try_join_all(futures).await?.into_iter().flatten()); - Ok(tasks) } } @@ -523,12 +550,34 @@ fn take_commit_nonce( }) } +fn rent_reimbursement( + delegation_metadata: &HashMap, + pubkey: Pubkey, +) -> TaskBuilderResult { + delegation_metadata + .get(&pubkey) + .map(|metadata| metadata.rent_payer) + .ok_or(TaskBuilderError::MissingDelegationMetadata(pubkey)) +} + +fn should_include_undelegation_request( + delegation_metadata: &HashMap, + pubkey: Pubkey, +) -> TaskBuilderResult { + let metadata = delegation_metadata + .get(&pubkey) + .ok_or(TaskBuilderError::MissingDelegationMetadata(pubkey))?; + Ok(metadata.undelegation_requester == UndelegationRequester::OwnerProgram) +} + #[derive(thiserror::Error, Debug)] pub enum TaskBuilderError { #[error("CommitIdFetchError: {0}")] CommitTasksBuildError(#[source] TaskInfoFetcherError), #[error("FinalizedTasksBuildError: {0}")] FinalizedTasksBuildError(#[source] TaskInfoFetcherError), + #[error("Missing delegation metadata for {0}")] + MissingDelegationMetadata(Pubkey), } impl TaskBuilderError { @@ -536,6 +585,7 @@ impl TaskBuilderError { match self { Self::CommitTasksBuildError(err) => err.signature(), Self::FinalizedTasksBuildError(err) => err.signature(), + Self::MissingDelegationMetadata(_) => None, } } } diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index b55cbd7dc..b142addb4 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -472,6 +472,7 @@ pub type TaskStrategistResult = Result; mod tests { use std::{collections::HashMap, sync::Arc}; + use dlp_api::state::{DelegationMetadata, UndelegationRequester}; use magicblock_core::intent::CommittedAccount; use magicblock_program::magic_scheduled_base_intent::{ BaseAction, ProgramArgs, @@ -497,7 +498,10 @@ mod tests { test_utils, }; - struct MockInfoFetcher; + #[derive(Default)] + struct MockInfoFetcher { + delegation_metadata: HashMap, + } #[async_trait::async_trait] impl TaskInfoFetcher for MockInfoFetcher { @@ -517,12 +521,31 @@ mod tests { Ok(pubkeys.iter().map(|pubkey| (*pubkey, 0)).collect()) } - async fn fetch_rent_reimbursements( + async fn fetch_delegation_metadata( &self, pubkeys: &[Pubkey], _: u64, - ) -> TaskInfoFetcherResult> { - Ok(pubkeys.iter().map(|_| Pubkey::new_unique()).collect()) + ) -> TaskInfoFetcherResult> + { + Ok(pubkeys + .iter() + .map(|pubkey| { + let (undelegation_requester, rent_payer) = self + .delegation_metadata + .get(pubkey) + .copied() + .unwrap_or((UndelegationRequester::None, *pubkey)); + ( + *pubkey, + DelegationMetadata { + last_commit_id: 0, + undelegation_requester, + seeds: vec![], + rent_payer, + }, + ) + }) + .collect()) } async fn get_base_accounts( @@ -607,6 +630,7 @@ mod tests { delegated_account: Pubkey::new_unique(), owner_program: Pubkey::default(), rent_reimbursement: Pubkey::new_unique(), + include_undelegation_request: false, } } @@ -862,12 +886,36 @@ mod tests { assert!(!strategy.lookup_tables_keys.is_empty()); } + #[tokio::test] + async fn test_finalize_tasks_include_request_for_owner_program_undelegate() + { + let delegated_account = Pubkey::new_unique(); + let intent = create_test_intent(0, &[delegated_account], true); + let info_fetcher = Arc::new(MockInfoFetcher { + delegation_metadata: HashMap::from([( + delegated_account, + (UndelegationRequester::OwnerProgram, delegated_account), + )]), + }); + + let tasks = TaskBuilderImpl::finalize_tasks(&info_fetcher, &intent) + .await + .unwrap(); + + let BaseTaskImpl::Undelegate(task) = &tasks[1] else { + panic!("expected undelegate task"); + }; + assert_eq!(task.delegated_account, delegated_account); + assert_eq!(task.rent_reimbursement, delegated_account); + assert!(task.include_undelegation_request); + } + #[tokio::test] async fn test_build_single_stage_mode() { let pubkey = [Pubkey::new_unique()]; let intent = create_test_intent(0, &pubkey, false); - let info_fetcher = Arc::new(MockInfoFetcher); + let info_fetcher = Arc::new(MockInfoFetcher::default()); let commit_task = TaskBuilderImpl::commit_tasks( &info_fetcher, &intent, @@ -895,11 +943,12 @@ mod tests { } #[tokio::test] - async fn test_build_two_stage_mode_no_alts() { - let pubkeys: [_; 3] = std::array::from_fn(|_| Pubkey::new_unique()); + async fn test_build_two_stage_mode_when_task_count_exceeds_single_stage_limit( + ) { + let pubkeys: [_; 8] = std::array::from_fn(|_| Pubkey::new_unique()); let intent = create_test_intent(0, &pubkeys, true); - let info_fetcher = Arc::new(MockInfoFetcher); + let info_fetcher = Arc::new(MockInfoFetcher::default()); let commit_task = TaskBuilderImpl::commit_tasks( &info_fetcher, &intent, @@ -920,15 +969,9 @@ mod tests { ) .expect("Execution mode created"); - let StrategyExecutionMode::TwoStage { - commit_stage, - finalize_stage, - } = execution_mode - else { + let StrategyExecutionMode::TwoStage { .. } = execution_mode else { panic!("Unexpected execution mode"); }; - assert!(!commit_stage.uses_alts()); - assert!(!finalize_stage.uses_alts()); } #[tokio::test] @@ -936,7 +979,7 @@ mod tests { let pubkeys: [_; 8] = std::array::from_fn(|_| Pubkey::new_unique()); let intent = create_test_intent(0, &pubkeys, false); - let info_fetcher = Arc::new(MockInfoFetcher); + let info_fetcher = Arc::new(MockInfoFetcher::default()); let commit_task = TaskBuilderImpl::commit_tasks( &info_fetcher, &intent, diff --git a/magicblock-services/Cargo.toml b/magicblock-services/Cargo.toml index 2c4134370..da9fe6a0b 100644 --- a/magicblock-services/Cargo.toml +++ b/magicblock-services/Cargo.toml @@ -11,9 +11,15 @@ description = "Services that run on top of the MagicBlock validator and communic [dependencies] bincode = { workspace = true } futures-util = { workspace = true } +magicblock-account-cloner = { workspace = true } +magicblock-chainlink = { workspace = true } magicblock-core = { workspace = true } +magicblock-delegation-program-api = { workspace = true } magicblock-magic-program-api = { workspace = true } +magicblock-metrics = { workspace = true } +magicblock-program = { workspace = true } thiserror = { workspace = true } +solana-hash = { workspace = true } solana-instruction = { workspace = true } solana-keypair = { workspace = true } solana-message = { workspace = true } @@ -22,5 +28,7 @@ solana-rpc-client = { workspace = true } solana-signature = { workspace = true } solana-signer = { workspace = true } solana-transaction = { workspace = true } +solana-transaction-error = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tracing = { workspace = true } diff --git a/magicblock-services/src/lib.rs b/magicblock-services/src/lib.rs index a3b1ee7f5..ae39af350 100644 --- a/magicblock-services/src/lib.rs +++ b/magicblock-services/src/lib.rs @@ -1 +1,2 @@ pub mod actions_callback_service; +pub mod undelegation_request_service; diff --git a/magicblock-services/src/undelegation_request_service.rs b/magicblock-services/src/undelegation_request_service.rs new file mode 100644 index 000000000..2f1196174 --- /dev/null +++ b/magicblock-services/src/undelegation_request_service.rs @@ -0,0 +1,327 @@ +use std::{fmt, sync::Arc, time::Duration}; + +use dlp_api::pda::undelegation_request_pda_from_delegated_account; +use magicblock_account_cloner::ChainlinkCloner; +use magicblock_chainlink::{ObservedUndelegationRequest, ProdChainlink}; +use magicblock_core::{ + link::transactions::TransactionSchedulerHandle, traits::LatestBlockProvider, +}; +use magicblock_metrics::metrics::AccountFetchOrigin; +use magicblock_program::{ + instruction::MagicBlockInstruction, MAGIC_CONTEXT_PUBKEY, +}; +use solana_hash::Hash; +use solana_instruction::{AccountMeta, Instruction}; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_signer::Signer; +use solana_transaction::Transaction; +use solana_transaction_error::TransactionError; +use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +const UNDELEGATION_REQUEST_MAX_ATTEMPTS: usize = 3; +const UNDELEGATION_REQUEST_RETRY_BASE_DELAY: Duration = + Duration::from_millis(100); + +pub type ChainlinkImpl = ProdChainlink; + +#[derive(Debug)] +enum ObservedUndelegationRequestError { + Transient(&'static str), + Schedule(TransactionError), +} + +impl ObservedUndelegationRequestError { + fn retryable(&self) -> bool { + matches!(self, Self::Transient(_) | Self::Schedule(_)) + } +} + +impl fmt::Display for ObservedUndelegationRequestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Transient(reason) => write!(f, "{reason}"), + Self::Schedule(err) => { + write!(f, "failed to schedule request: {err}") + } + } + } +} + +pub struct UndelegationRequestService { + chainlink: Arc, + cancellation_token: CancellationToken, + internal_transaction_scheduler: TransactionSchedulerHandle, + validator_authority: Arc, + latest_block_reader: Arc, +} + +trait LatestBlockReader: Send + Sync { + fn blockhash(&self) -> Hash; +} + +impl LatestBlockReader for T +where + T: LatestBlockProvider, +{ + fn blockhash(&self) -> Hash { + LatestBlockProvider::blockhash(self) + } +} + +impl UndelegationRequestService { + pub fn new( + chainlink: Arc, + internal_transaction_scheduler: TransactionSchedulerHandle, + validator_authority: Keypair, + latest_block: impl LatestBlockProvider, + ) -> Self { + Self { + chainlink, + cancellation_token: CancellationToken::new(), + internal_transaction_scheduler, + validator_authority: Arc::new(validator_authority), + latest_block_reader: Arc::new(latest_block.clone()), + } + } + + async fn undelegation_request_processor( + mut requests: broadcast::Receiver, + cancellation_token: CancellationToken, + chainlink: Arc, + internal_transaction_scheduler: TransactionSchedulerHandle, + validator_authority: Arc, + latest_block: Arc, + ) { + loop { + let request = tokio::select! { + biased; + _ = cancellation_token.cancelled() => { + info!("Shutting down undelegation request processor"); + return; + } + request = requests.recv() => { + match request { + Ok(request) => request, + Err(broadcast::error::RecvError::Closed) => { + info!("Undelegation request subscription closed"); + return; + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + error!( + skipped_count = skipped, + "Lagged behind undelegation request updates" + ); + continue; + } + } + } + }; + + Self::process_observed_undelegation_request_with_retries( + request, + &chainlink, + &internal_transaction_scheduler, + validator_authority.as_ref(), + latest_block.as_ref(), + &cancellation_token, + ) + .await; + } + } + + async fn process_observed_undelegation_request_with_retries( + request: ObservedUndelegationRequest, + chainlink: &ChainlinkImpl, + internal_transaction_scheduler: &TransactionSchedulerHandle, + validator_authority: &Keypair, + latest_block: &dyn LatestBlockReader, + cancellation_token: &CancellationToken, + ) { + let mut attempt = 1; + loop { + let result = Self::process_observed_undelegation_request( + request.clone(), + chainlink, + internal_transaction_scheduler, + validator_authority, + latest_block, + ) + .await; + match result { + Ok(()) => return, + Err(err) + if err.retryable() + && attempt < UNDELEGATION_REQUEST_MAX_ATTEMPTS => + { + let delay = UNDELEGATION_REQUEST_RETRY_BASE_DELAY + * 2_u32.pow((attempt - 1) as u32); + warn!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + attempt, + max_attempts = UNDELEGATION_REQUEST_MAX_ATTEMPTS, + retry_delay_ms = delay.as_millis(), + error = %err, + "Retrying observed undelegation request after transient failure" + ); + tokio::select! { + biased; + _ = cancellation_token.cancelled() => { + info!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + "Stopping undelegation request retry because processor is shutting down" + ); + return; + } + _ = tokio::time::sleep(delay) => {} + } + attempt += 1; + } + Err(err) => { + error!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + attempt, + max_attempts = UNDELEGATION_REQUEST_MAX_ATTEMPTS, + error = %err, + "Failed to process observed undelegation request" + ); + return; + } + } + } + } + + async fn process_observed_undelegation_request( + request: ObservedUndelegationRequest, + chainlink: &ChainlinkImpl, + internal_transaction_scheduler: &TransactionSchedulerHandle, + validator_authority: &Keypair, + latest_block: &dyn LatestBlockReader, + ) -> Result<(), ObservedUndelegationRequestError> { + if request.observed_slot >= request.expires_at_slot { + warn!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + observed_slot = request.observed_slot, + expires_at_slot = request.expires_at_slot, + "Observed expired undelegation request; scheduling normal undelegation anyway to avoid timeout carry-over rollback when possible" + ); + } + + let expected_request_pda = + undelegation_request_pda_from_delegated_account( + &request.delegated_account, + ); + if expected_request_pda != request.request_pda { + error!( + request_pda = %request.request_pda, + expected_request_pda = %expected_request_pda, + delegated_account = %request.delegated_account, + "Skipping undelegation request with invalid PDA" + ); + return Ok(()); + } + + let delegated = match chainlink + .accounts_delegated_on_base_and_er( + &[request.delegated_account], + AccountFetchOrigin::GetAccount, + ) + .await + { + Ok(value) => value.into_iter().next().unwrap_or(false), + Err(err) => { + error!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + error = ?err, + "Failed to verify requested undelegation account" + ); + return Err(ObservedUndelegationRequestError::Transient( + "failed to verify requested undelegation account", + )); + } + }; + if !delegated { + debug!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + "Skipping request because account is not delegated on base and ER" + ); + return Ok(()); + } + + if let Err(err) = chainlink + .undelegation_requested(request.delegated_account) + .await + { + error!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + error = ?err, + "Failed to start undelegation tracking" + ); + } + + let ix = Self::schedule_commit_and_undelegate_instruction( + &validator_authority.pubkey(), + request.delegated_account, + ); + let tx = Transaction::new_signed_with_payer( + &[ix], + Some(&validator_authority.pubkey()), + &[validator_authority], + latest_block.blockhash(), + ); + + if let Err(err) = internal_transaction_scheduler.execute(tx).await { + return Err(ObservedUndelegationRequestError::Schedule(err)); + } + + info!( + request_pda = %request.request_pda, + delegated_account = %request.delegated_account, + "Scheduled requested undelegation via ScheduleCommitAndUndelegate" + ); + Ok(()) + } + + fn schedule_commit_and_undelegate_instruction( + payer: &Pubkey, + delegated_account: Pubkey, + ) -> Instruction { + Instruction::new_with_bincode( + magicblock_program::id(), + &MagicBlockInstruction::ScheduleCommitAndUndelegate, + vec![ + AccountMeta::new(*payer, true), + AccountMeta::new(MAGIC_CONTEXT_PUBKEY, false), + AccountMeta::new(delegated_account, false), + ], + ) + } + + pub fn start(self: &Arc) { + let Some(requests) = self.chainlink.subscribe_undelegation_requests() + else { + return; + }; + tokio::spawn(Self::undelegation_request_processor( + requests, + self.cancellation_token.clone(), + self.chainlink.clone(), + self.internal_transaction_scheduler.clone(), + self.validator_authority.clone(), + self.latest_block_reader.clone(), + )); + } + + pub fn stop(&self) { + self.cancellation_token.cancel(); + } +} diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 04305a12b..16d623cfd 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -289,7 +289,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -300,7 +300,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2041,7 +2041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2102,7 +2102,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3050,7 +3050,7 @@ dependencies = [ "borsh 1.7.0", "color-backtrace", "magicblock-config", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "rand 0.8.6", "rayon", "serde", @@ -3522,19 +3522,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "magicblock-accounts" -version = "0.13.3" -dependencies = [ - "async-trait", - "magicblock-committor-service", - "solana-pubkey 4.1.0", - "solana-transaction-error 3.1.0", - "thiserror 2.0.18", - "tokio", - "url", -] - [[package]] name = "magicblock-accounts-db" version = "0.13.3" @@ -3627,14 +3614,13 @@ dependencies = [ "fd-lock", "magic-domain-program", "magicblock-account-cloner", - "magicblock-accounts", "magicblock-accounts-db", "magicblock-aperture", "magicblock-chainlink", "magicblock-committor-service", "magicblock-config", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-ledger", "magicblock-magic-program-api 0.13.3", "magicblock-metrics", @@ -3696,7 +3682,7 @@ dependencies = [ "magicblock-aml", "magicblock-config", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", "magicblock-metrics", "parking_lot", @@ -3765,7 +3751,7 @@ dependencies = [ "magicblock-chainlink", "magicblock-committor-program", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-metrics", "magicblock-program", "magicblock-rpc-client", @@ -3844,10 +3830,12 @@ dependencies = [ [[package]] name = "magicblock-delegation-program-api" -version = "0.3.0" -source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=25386a7c1d406d06b8d07a4d5b0fd37d5e74213b#25386a7c1d406d06b8d07a4d5b0fd37d5e74213b" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "288904a9950bd20f27f0ef934f320ab1410bd35a6d5c9cf138eca276442b6b2e" dependencies = [ "bincode", + "borsh 0.10.4", "borsh 1.7.0", "bytemuck", "const-crypto", @@ -3863,6 +3851,7 @@ dependencies = [ "solana-instruction 3.3.0", "solana-loader-v3-interface 6.1.1", "solana-program 3.0.0", + "solana-pubkey 2.4.0", "solana-sdk", "solana-sdk-ids 3.1.0", "solana-sha256-hasher 3.1.0", @@ -3874,9 +3863,8 @@ dependencies = [ [[package]] name = "magicblock-delegation-program-api" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288904a9950bd20f27f0ef934f320ab1410bd35a6d5c9cf138eca276442b6b2e" +version = "3.1.0" +source = "git+https://github.com/magicblock-labs/delegation-program.git?tag=v3.1.0#27f7fd8178630b88e2be45d92b174b8f2ef4661e" dependencies = [ "bincode", "borsh 0.10.4", @@ -4035,7 +4023,7 @@ dependencies = [ "bincode", "lazy_static", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", "num-derive", "num-traits", @@ -4121,8 +4109,14 @@ version = "0.13.3" dependencies = [ "bincode", "futures-util", + "magicblock-account-cloner", + "magicblock-chainlink", "magicblock-core", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", + "magicblock-metrics", + "magicblock-program", + "solana-hash 4.2.0", "solana-instruction 3.3.0", "solana-keypair", "solana-message 3.1.0", @@ -4131,8 +4125,10 @@ dependencies = [ "solana-signature", "solana-signer", "solana-transaction", + "solana-transaction-error 3.1.0", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", ] @@ -4193,7 +4189,7 @@ dependencies = [ name = "magicblock-validator-admin" version = "0.13.3" dependencies = [ - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-program", "magicblock-rpc-client", "solana-commitment-config", @@ -4451,7 +4447,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5042,7 +5038,7 @@ version = "0.0.0" dependencies = [ "borsh 1.7.0", "ephemeral-rollups-sdk", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", "rkyv 0.7.46", "solana-program 3.0.0", @@ -5339,7 +5335,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -5801,7 +5797,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5930,7 +5926,7 @@ dependencies = [ "anyhow", "borsh 1.7.0", "integration-test-tools", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "program-schedulecommit", "solana-commitment-config", "solana-compute-budget-interface", @@ -5952,7 +5948,7 @@ dependencies = [ "magicblock-committor-program", "magicblock-committor-service", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-program", "magicblock-rpc-client", "magicblock-table-mania", @@ -5980,7 +5976,7 @@ dependencies = [ "ephemeral-rollups-sdk", "integration-test-tools", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", "magicblock-program", "program-schedulecommit", @@ -6001,7 +5997,7 @@ name = "schedulecommit-test-security" version = "0.0.0" dependencies = [ "integration-test-tools", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", "program-schedulecommit", "program-schedulecommit-security", @@ -6423,7 +6419,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9759,7 +9755,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -9780,7 +9776,7 @@ dependencies = [ "integration-test-tools", "magicblock-config", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "program-flexi-counter", "program-mini", "solana-loader-v4-interface 3.1.0", @@ -9807,7 +9803,7 @@ dependencies = [ "magicblock-aml", "magicblock-chainlink", "magicblock-config", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "program-flexi-counter", "program-mini", "solana-account 3.4.0", @@ -9833,7 +9829,7 @@ version = "0.0.0" dependencies = [ "integration-test-tools", "magicblock-core", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "program-flexi-counter", "program-mini", "solana-loader-v4-interface 3.1.0", @@ -9903,7 +9899,7 @@ dependencies = [ "integration-test-tools", "magicblock-accounts-db", "magicblock-config", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "program-flexi-counter", "solana-commitment-config", "solana-rpc-client", @@ -9926,7 +9922,7 @@ dependencies = [ "magic-domain-program", "magicblock-api", "magicblock-config", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-program", "magicblock-validator-admin", "solana-commitment-config", @@ -9972,7 +9968,7 @@ dependencies = [ "ephemeral-rollups-sdk", "integration-test-tools", "log", - "magicblock-delegation-program-api 0.3.0", + "magicblock-delegation-program-api 3.1.0", "magicblock-magic-program-api 0.13.3", "program-flexi-counter", "solana-rpc-client-api", @@ -10944,7 +10940,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -11099,15 +11095,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index 05da5a293..c98198e31 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -63,7 +63,7 @@ magicblock-committor-program = { path = "../magicblock-committor-program", featu magicblock-committor-service = { path = "../magicblock-committor-service" } magicblock-config = { path = "../magicblock-config" } magicblock-core = { path = "../magicblock-core" } -magicblock-delegation-program-api = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "25386a7c1d406d06b8d07a4d5b0fd37d5e74213b", default-features = false } +magicblock-delegation-program-api = { git = "https://github.com/magicblock-labs/delegation-program.git", tag = "v3.1.0", default-features = false } magicblock-magic-program-api = { path = "../magicblock-magic-program-api" } magicblock-program = { path = "../programs/magicblock" } magicblock-rpc-client = { path = "../magicblock-rpc-client" } diff --git a/test-integration/programs/schedulecommit/src/api.rs b/test-integration/programs/schedulecommit/src/api.rs index 0dfe0931b..ed50202f6 100644 --- a/test-integration/programs/schedulecommit/src/api.rs +++ b/test-integration/programs/schedulecommit/src/api.rs @@ -1,4 +1,11 @@ -use dlp_api::args::{DelegateArgs, DelegateEphemeralBalanceArgs}; +use dlp_api::{ + args::{DelegateArgs, DelegateEphemeralBalanceArgs}, + pda::{ + delegation_metadata_pda_from_delegated_account, + delegation_record_pda_from_delegated_account, + undelegation_request_pda_from_delegated_account, + }, +}; use ephemeral_rollups_sdk::delegate_args::{ DelegateAccountMetas, DelegateAccounts, }; @@ -164,6 +171,37 @@ pub fn delegate_account_cpi_instruction( ) } +pub fn request_undelegation_cpi_instruction( + payer: Pubkey, + player: Pubkey, + committee: Pubkey, +) -> Instruction { + let program_id = crate::id(); + let request_pda = + undelegation_request_pda_from_delegated_account(&committee); + let delegation_record = + delegation_record_pda_from_delegated_account(&committee); + let delegation_metadata = + delegation_metadata_pda_from_delegated_account(&committee); + + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new_readonly(committee, false), + AccountMeta::new_readonly(program_id, false), + AccountMeta::new(request_pda, false), + AccountMeta::new_readonly(delegation_record, false), + AccountMeta::new(delegation_metadata, false), + AccountMeta::new_readonly(system_program::id(), false), + AccountMeta::new_readonly(dlp_api::id(), false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::RequestUndelegationCpi(player), + account_metas, + ) +} + /// Creates an instruction that calls the _legit_ program which owns /// the PDAs to be commited via CPI into the MagicBlock program. /// It provides the following account metas to the invoked program: diff --git a/test-integration/programs/schedulecommit/src/lib.rs b/test-integration/programs/schedulecommit/src/lib.rs index 414219861..6caa9b031 100644 --- a/test-integration/programs/schedulecommit/src/lib.rs +++ b/test-integration/programs/schedulecommit/src/lib.rs @@ -198,6 +198,9 @@ pub enum ScheduleCommitInstruction { ScheduleCommitWithVaultAndOrderBookCpi(ScheduleCommitWithOrderBookArgs), ScheduleCommitForOrderBook(ScheduleCommitType), + + /// CPI into DLP to request undelegation for a schedulecommit-owned PDA. + RequestUndelegationCpi(Pubkey), } /// @@ -348,6 +351,9 @@ pub fn process_instruction<'a>( ScheduleCommitForOrderBook(commit_type) => { process_schedulecommit_for_orderbook(accounts, commit_type) } + RequestUndelegationCpi(player) => { + process_request_undelegation_cpi(program_id, accounts, player) + } ScheduleCommitWithVaultAndOrderBookCpi(args) => { process_schedulecommit_with_vault_and_order_book_cpi( accounts, &args, @@ -646,6 +652,66 @@ pub fn process_delegate_cpi( Ok(()) } +fn process_request_undelegation_cpi( + program_id: &Pubkey, + accounts: &[AccountInfo], + player: Pubkey, +) -> Result<(), ProgramError> { + msg!("Processing request_undelegation_cpi instruction"); + + let [payer, delegated_account, owner_program, undelegation_request, delegation_record, delegation_metadata, system_program, dlp_program] = + accounts + else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + assert_keys_equal(owner_program.key, program_id, || { + format!( + "Owner program '{}' does not match schedulecommit program '{}'", + owner_program.key, program_id + ) + })?; + assert_keys_equal(dlp_program.key, &dlp_api::id(), || { + format!( + "DLP program '{}' does not match expected program '{}'", + dlp_program.key, + dlp_api::id() + ) + })?; + + let (expected_pda, bump) = pda_and_bump(&player); + assert_keys_equal(delegated_account.key, &expected_pda, || { + format!( + "Delegated account '{}' does not match PDA '{}' for player '{}'", + delegated_account.key, expected_pda, player + ) + })?; + + let ix = dlp_api::instruction_builder::request_undelegation( + *payer.key, + *delegated_account.key, + *owner_program.key, + ); + let bump_arr = [bump]; + let signer_seeds = pda_seeds_with_bump(&player, &bump_arr); + + invoke_signed( + &ix, + &[ + payer.clone(), + delegated_account.clone(), + owner_program.clone(), + undelegation_request.clone(), + delegation_record.clone(), + delegation_metadata.clone(), + system_program.clone(), + dlp_program.clone(), + ], + &[&signer_seeds], + ) +} + pub struct ProcessSchedulecommitCpiArgs { pub modify_accounts: bool, pub commit_payer: bool, diff --git a/test-integration/schedulecommit/elfs/dlp.so b/test-integration/schedulecommit/elfs/dlp.so index 248510aaa..b8fa734fe 100755 Binary files a/test-integration/schedulecommit/elfs/dlp.so and b/test-integration/schedulecommit/elfs/dlp.so differ diff --git a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs index dbf3e8515..aa7419525 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs @@ -1,3 +1,9 @@ +use std::{ + thread::sleep, + time::{Duration, Instant}, +}; + +use dlp_api::pda::undelegation_request_pda_from_delegated_account; use integration_test_tools::{ conversions::stringify_simulation_result, run_test, scheduled_commits::extract_scheduled_commit_sent_signature_from_logs, @@ -5,14 +11,15 @@ use integration_test_tools::{ }; use program_schedulecommit::{ api::{ - increase_count_instruction, + increase_count_instruction, request_undelegation_cpi_instruction, schedule_commit_and_undelegate_cpi_instruction, schedule_commit_and_undelegate_cpi_twice, schedule_commit_and_undelegate_cpi_with_mod_after_instruction, schedule_commit_instruction_for_order_book, set_count_instruction, update_order_book_instruction, UserSeeds, }, - BookUpdate, OrderLevel, ScheduleCommitType, FAIL_UNDELEGATION_COUNT, + BookUpdate, MainAccount, OrderLevel, ScheduleCommitType, + FAIL_UNDELEGATION_COUNT, }; use rand::{RngCore, SeedableRng}; use schedulecommit_client::{ @@ -49,6 +56,9 @@ use crate::utils::{ mod utils; +const REQUESTED_UNDELEGATION_MAX_WAIT: Duration = Duration::from_secs(30); +const REQUESTED_UNDELEGATION_RETRY_DELAY: Duration = Duration::from_millis(250); + fn commit_and_undelegate_one_account( modify_after: bool, ) -> ( @@ -285,6 +295,52 @@ fn commit_and_undelegate_two_accounts_twice() -> ( (ctx, *sig, tx_res) } +fn wait_for_requested_undelegation_on_chain( + ctx: &ScheduleCommitTestContext, + pda: Pubkey, + request_pda: Pubkey, + expected_count: u64, +) { + let start = Instant::now(); + let mut last_state = String::new(); + + while start.elapsed() < REQUESTED_UNDELEGATION_MAX_WAIT { + let owner_result = ctx.fetch_chain_account_owner(pda); + let owner_restored = matches!( + &owner_result, + Ok(owner) if *owner == program_schedulecommit::id() + ); + let owner_state = owner_result + .map(|owner| owner.to_string()) + .unwrap_or_else(|err| format!("error: {err:?}")); + + let request_exists = ctx.fetch_chain_account(request_pda).is_ok(); + let count_result = ctx + .fetch_chain_account_data(pda) + .map_err(|err| err.to_string()) + .and_then(|data| { + MainAccount::try_decode(&data) + .map(|account| account.count) + .map_err(|err| err.to_string()) + }); + let count_matches = + matches!(&count_result, Ok(count) if *count == expected_count); + + if owner_restored && !request_exists && count_matches { + return; + } + + last_state = format!( + "owner={owner_state}, request_exists={request_exists}, count={count_result:?}" + ); + sleep(REQUESTED_UNDELEGATION_RETRY_DELAY); + } + + panic!( + "timed out waiting for requested undelegation of {pda}; last state: {last_state}" + ); +} + #[test] fn test_committing_and_undelegating_one_account() { run_test!({ @@ -300,6 +356,75 @@ fn test_committing_and_undelegating_one_account() { }); } +#[test] +fn test_request_undelegation_commits_and_undelegates_one_account() { + run_test!({ + let ctx = get_context_with_delegated_committees( + 1, + UserSeeds::MagicScheduleCommit, + ); + let ScheduleCommitTestContextFields { + payer_ephem, + payer_chain, + committees, + commitment, + ephem_client, + .. + } = ctx.fields(); + let (player, committee_pda) = &committees[0]; + let request_pda = + undelegation_request_pda_from_delegated_account(committee_pda); + + assert!( + ctx.fetch_chain_account(request_pda).is_err(), + "request PDA should not exist before request" + ); + + assert_can_increase_committee_count( + *committee_pda, + payer_ephem, + ephem_client, + commitment, + ); + + let ix = request_undelegation_cpi_instruction( + payer_chain.pubkey(), + player.pubkey(), + *committee_pda, + ); + let request_tx = Transaction::new_signed_with_payer( + &[ix], + Some(&payer_chain.pubkey()), + &[payer_chain], + ctx.try_chain_blockhash().unwrap(), + ); + let request_sig = *request_tx.get_signature(); + let request_res = ctx + .try_chain_client() + .unwrap() + .send_and_confirm_transaction_with_spinner_and_config( + &request_tx, + *commitment, + RpcSendTransactionConfig { + skip_preflight: true, + ..Default::default() + }, + ); + ctx.dump_chain_logs(request_sig); + assert!( + request_res.is_ok(), + "request undelegation transaction failed: {request_res:?}" + ); + + wait_for_requested_undelegation_on_chain( + &ctx, + *committee_pda, + request_pda, + 1, + ); + }); +} + #[test] fn test_commit_huge_order_book_account() { run_test_for_commit_huge_order_book_account(ScheduleCommitType::Commit); diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 66f896211..bbf90d47e 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -7,6 +7,7 @@ use std::{ }; use async_trait::async_trait; +use dlp_api::state::{DelegationMetadata, UndelegationRequester}; use magicblock_committor_service::{ intent_executor::{ task_info_fetcher::{ @@ -184,12 +185,25 @@ impl TaskInfoFetcher for MockTaskInfoFetcher { Ok(pubkeys.iter().map(|pubkey| (*pubkey, 0)).collect()) } - async fn fetch_rent_reimbursements( + async fn fetch_delegation_metadata( &self, pubkeys: &[Pubkey], _: u64, - ) -> TaskInfoFetcherResult> { - Ok(pubkeys.to_vec()) + ) -> TaskInfoFetcherResult> { + Ok(pubkeys + .iter() + .map(|pubkey| { + ( + *pubkey, + DelegationMetadata { + last_commit_id: 0, + undelegation_requester: UndelegationRequester::None, + seeds: vec![], + rent_payer: *pubkey, + }, + ) + }) + .collect()) } async fn get_base_accounts( diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 72ae07121..f7e5df7a8 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -24,7 +24,7 @@ use solana_sdk::{ use test_kit::init_logger; use tokio::task::JoinSet; use tracing::*; -use utils::transactions::{print_tx_logs, tx_logs_contain}; +use utils::transactions::print_tx_logs; use self::utils::transactions::init_and_delegate_order_book_on_chain; use crate::utils::{ @@ -564,9 +564,9 @@ async fn test_commit_20_accounts_1kb_bundle_size_6() { } #[tokio::test] -async fn test_commit_20_accounts_1kb_bundle_size_20() { +async fn test_commit_20_accounts_1kb_bundle_size_5() { commit_20_accounts_1kb( - 20, + 5, expect_strategies(&[(CommitStrategy::DiffBufferWithLookupTable, 20)]), CommitIntentKind::Commit, ) @@ -1014,39 +1014,7 @@ async fn ix_commit_local( let commit_finalized_and_undelegated_accounts = base_intent.get_commit_finalize_and_undelegate_intent_accounts(); - let has_commit_flow = - committed_accounts.is_some() || undelegated_accounts.is_some(); - let has_commit_finalize_flow = committed_finalize_accounts.is_some() - || commit_finalized_and_undelegated_accounts.is_some(); - if has_commit_flow { - assert!( - tx_logs_contain(&rpc_client, &commit_signature, "CommitState") - .await - ); - assert!( - tx_logs_contain(&rpc_client, &finalize_signature, "Finalize") - .await - ); - } - if has_commit_finalize_flow { - assert!( - tx_logs_contain( - &rpc_client, - &commit_signature, - "CommitFinalize" - ) - .await - ); - } - let has_undelegate = base_intent.has_undelegate_intent(); - if has_undelegate { - // Undelegate is part of atomic Finalization Stage - assert!( - tx_logs_contain(&rpc_client, &finalize_signature, "Undelegate") - .await - ); - } let mut committed_accounts: HashMap = [ (false, committed_accounts), diff --git a/test-integration/test-committor-service/tests/test_transaction_preparator.rs b/test-integration/test-committor-service/tests/test_transaction_preparator.rs index 207f3fc48..e5cfb01b0 100644 --- a/test-integration/test-committor-service/tests/test_transaction_preparator.rs +++ b/test-integration/test-committor-service/tests/test_transaction_preparator.rs @@ -287,6 +287,7 @@ async fn test_prepare_finalize_tx_with_undelegate_with_atls() { delegated_account: committed_account.pubkey, owner_program: Pubkey::new_unique(), rent_reimbursement: Pubkey::new_unique(), + include_undelegation_request: false, } .into(), ];