refactor: generalized convergence for host boot interface configuration

crates/api-core/src/cfg/README.md

@@ -347,7 +347,7 @@ Extends `StateControllerConfig` with:
 | `dpu_up_threshold` | `Duration` | `5m`    | Max time without DPU health report before assuming it's down. |
 | `scout_reporting_timeout` | `Duration` | `5m`    | Duration without scout report before host is unhealthy. |
 | `uefi_boot_wait` | `Duration` | `5m`    | Wait time for UEFI boot completion after host reboot. |
-| `max_bios_config_retries` | `u32` | `3` | Max HandleBiosJobFailure recovery cycles during BIOS configuration. |
+| `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. |
 | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. |
 | `controller` | `StateControllerConfig` | *(default)* | Common state controller timing (see [StateControllerConfig](#statecontrollerconfig)). |
 

crates/api-core/src/tests/common/api_fixtures/mod.rs

@@ -476,6 +476,10 @@ impl TestEnv {
                         MachineValidatingState::RebootHost { .. } => state.clone(),
                         MachineValidatingState::PrepareBootRepair { .. }
                         | MachineValidatingState::UnlockForBootRepair { .. }
+                        | MachineValidatingState::CheckBootConfigForRepair { .. }
+                        | MachineValidatingState::ConfigureBootBios { .. }
+                        | MachineValidatingState::WaitingForBootBiosJob { .. }
+                        | MachineValidatingState::PollingBootBiosSetup { .. }
                         | MachineValidatingState::RepairBootConfig { .. }
                         | MachineValidatingState::LockAfterBootRepair { .. } => state.clone(),
                     }

crates/api-core/src/tests/dpu_reprovisioning.rs

@@ -28,8 +28,8 @@ use libredfish::{EnabledDisabled, SystemPowerControl};
 use model::instance::status::tenant::TenantState;
 use model::machine::{
     DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState,
-    Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, ReprovisionState,
-    SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState,
+    Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState,
+    ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState,
 };
 use model::test_support::HardwareInfoTemplate;
 use rpc::forge::MachineArchitecture;
@@ -86,8 +86,6 @@ fn reprovision_host_boot_repair_states(
             unlock_host_state: UnlockHostState::DisableLockdown,
         },
         ReprovisionState::CheckHostBootConfig,
-        ReprovisionState::ConfigureHostBoot { retry_count: 0 },
-        ReprovisionState::PollingHostBiosSetup { retry_count: 0 },
         reprovision_set_host_boot_order_state(SetBootOrderState::SetBootOrder),
         reprovision_set_host_boot_order_state(SetBootOrderState::WaitForSetBootOrderJobScheduled),
         reprovision_set_host_boot_order_state(SetBootOrderState::RebootHost),
@@ -137,6 +135,7 @@ async fn assert_dpu_reprovision_host_boot_repair(
     expected_states: Vec<ManagedHostState>,
 ) -> Machine {
     env.redfish_sim.set_lockdown(EnabledDisabled::Enabled);
+    env.redfish_sim.set_is_bios_setup(true);
     env.redfish_sim.set_is_boot_order_setup(false);
 
     let redfish_timepoint = env.redfish_sim.timepoint();
@@ -179,27 +178,56 @@ async fn assert_dpu_reprovision_host_boot_repair(
         }
     }
 
-    // machine_setup enables the bootable DPU interface before boot-order promotion.
     let actions = env
         .redfish_sim
         .actions_since(&redfish_timepoint)
         .all_hosts();
-    let machine_setup_pos = actions
-        .iter()
-        .position(|action| matches!(action, RedfishSimAction::MachineSetup { .. }))
-        .expect("expected DPU reprovision boot repair to call machine_setup");
-    let set_boot_order_pos = actions
+    let (set_boot_order_pos, set_boot_order_mac) = actions
         .iter()
-        .position(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. }))
+        .enumerate()
+        .find_map(|(position, action)| match action {
+            RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } => {
+                Some((position, boot_interface_mac))
+            }
+            _ => None,
+        })
         .expect("expected DPU reprovision boot repair to set DPU-first boot order");
-    let check_boot_order_pos = actions
+    let boot_order_checks = actions
         .iter()
-        .rposition(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. }))
-        .expect("expected DPU reprovision boot repair to verify boot order");
+        .enumerate()
+        .filter_map(|(position, action)| match action {
+            RedfishSimAction::IsBootOrderSetup { boot_interface_mac } => {
+                Some((position, boot_interface_mac))
+            }
+            _ => None,
+        })
+        .collect::<Vec<_>>();
 
     assert!(
-        machine_setup_pos < set_boot_order_pos && set_boot_order_pos < check_boot_order_pos,
-        "expected machine_setup before set_boot_order_dpu_first before is_boot_order_setup; got: {actions:?}"
+        actions
+            .iter()
+            .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })),
+        "expected order-only DPU reprovision boot repair to preserve the configured HTTP boot device; got: {actions:?}"
+    );
+    assert!(
+        boot_order_checks
+            .iter()
+            .filter(|(position, _)| *position < set_boot_order_pos)
+            .count()
+            >= 2,
+        "expected both CheckHostBootConfig and SetBootOrder to inspect the order before writing it; got: {actions:?}"
+    );
+    assert!(
+        boot_order_checks
+            .iter()
+            .any(|(position, _)| *position > set_boot_order_pos),
+        "expected boot-order verification after the write; got: {actions:?}"
+    );
+    assert!(
+        boot_order_checks
+            .iter()
+            .all(|(_, boot_interface_mac)| *boot_interface_mac == set_boot_order_mac),
+        "expected every order check and write to target the same interface; got: {actions:?}"
     );
 
     let rebooting_machine = machine.next_iteration_machine(env).await;
@@ -564,6 +592,136 @@ async fn test_dpu_reprovision_viking_skips_boot_order_when_bios_setup(pool: sqlx
     );
 }
 
+#[crate::sqlx_test]
+async fn test_dpu_reprovision_viking_finishes_parked_boot_order_recovery(pool: sqlx::PgPool) {
+    let env = create_test_env(pool).await;
+    let mh = create_managed_host_with_hardware_info_template(
+        &env,
+        HardwareInfoTemplate::Custom(DGX_H100_INFO_JSON),
+    )
+    .await;
+    let dpu_machine = prepare_dpu_reprovision_host_boot_check(&env, &mh).await;
+
+    // A controller upgrade can find a Viking in a persisted recovery substate
+    // created before boot-order remediation was disabled for this platform.
+    // Finish restoring host power before taking the safe terminal shortcut.
+    let parked_recovery = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state(
+        SetBootOrderState::HandleJobFailure {
+            failure: "persisted failed boot-order job".to_string(),
+            power_state: PowerState::Off,
+        },
+    ));
+    let mut txn = env.pool.begin().await.unwrap();
+    db::machine::update_state(&mut txn, &mh.id, &parked_recovery)
+        .await
+        .unwrap();
+    txn.commit().await.unwrap();
+
+    let redfish_timepoint = env.redfish_sim.timepoint();
+    let dpu = dpu_machine.next_iteration_machine(&env).await;
+    assert_eq!(dpu.current_state(), &parked_recovery);
+    assert_eq!(
+        env.redfish_sim
+            .actions_since(&redfish_timepoint)
+            .all_hosts(),
+        vec![RedfishSimAction::Power(SystemPowerControl::ForceOff)],
+        "Viking policy must not abandon an in-flight power recovery"
+    );
+
+    let recovering_power_on = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state(
+        SetBootOrderState::HandleJobFailure {
+            failure: "persisted failed boot-order job".to_string(),
+            power_state: PowerState::On,
+        },
+    ));
+    let redfish_timepoint = env.redfish_sim.timepoint();
+    let dpu = dpu_machine.next_iteration_machine(&env).await;
+    assert_eq!(dpu.current_state(), &recovering_power_on);
+    assert_eq!(
+        env.redfish_sim
+            .actions_since(&redfish_timepoint)
+            .all_hosts(),
+        vec![RedfishSimAction::BmcReset],
+        "parked recovery should reset the BMC after the host powers off"
+    );
+
+    // ForceOff records last_reboot_requested; backdate it so power_down_wait
+    // elapses in-process before the controller restores host power.
+    {
+        let mut txn = env.db_txn().await;
+        let host = mh.host().db_machine(&mut txn).await;
+        update_time_params(&env.pool, &host, 1, None).await;
+    }
+
+    let redfish_timepoint = env.redfish_sim.timepoint();
+    let dpu = dpu_machine.next_iteration_machine(&env).await;
+    assert_eq!(dpu.current_state(), &recovering_power_on);
+    assert_eq!(
+        env.redfish_sim
+            .actions_since(&redfish_timepoint)
+            .all_hosts(),
+        vec![RedfishSimAction::Power(SystemPowerControl::On)],
+        "parked recovery should restore host power after the BMC reset"
+    );
+
+    let safe_terminal = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state(
+        SetBootOrderState::CheckBootOrder,
+    ));
+    let redfish_timepoint = env.redfish_sim.timepoint();
+    let dpu = dpu_machine.next_iteration_machine(&env).await;
+    assert_eq!(dpu.current_state(), &safe_terminal);
+    assert!(
+        env.redfish_sim
+            .actions_since(&redfish_timepoint)
+            .all_hosts()
+            .is_empty(),
+        "completed recovery should naturally reach CheckBootOrder"
+    );
+
+    // CheckBootOrder has no unfinished operation behind it, but its preceding
+    // reboot may have reverted the HTTP boot device. Repair that BIOS drift
+    // without touching the unsupported Viking boot-order APIs, and spend one
+    // attempt from the shared convergence budget.
+    env.redfish_sim.set_is_bios_setup(false);
+
+    let redfish_timepoint = env.redfish_sim.timepoint();
+    let dpu = dpu_machine.next_iteration_machine(&env).await;
+    assert_eq!(
+        dpu.current_state(),
+        &mh.new_dpu_reprovision_state(ReprovisionState::ConfigureHostBoot { retry_count: 1 })
+    );
+    assert!(
+        env.redfish_sim
+            .actions_since(&redfish_timepoint)
+            .all_hosts()
+            .is_empty(),
+        "Viking BIOS recovery must not read or write boot order"
+    );
+
+    // Once BIOS is intact, the same safe terminal skips boot-order
+    // verification and completes the repair.
+    env.redfish_sim.set_is_bios_setup(true);
+    let mut txn = env.pool.begin().await.unwrap();
+    db::machine::update_state(&mut txn, &mh.id, &safe_terminal)
+        .await
+        .unwrap();
+    txn.commit().await.unwrap();
+
+    let redfish_timepoint = env.redfish_sim.timepoint();
+    let dpu = dpu_machine.next_iteration_machine(&env).await;
+    assert_eq!(
+        dpu.current_state(),
+        &mh.new_dpu_reprovision_state(ReprovisionState::LockHostAfterBootRepair)
+    );
+    assert!(
+        env.redfish_sim
+            .actions_since(&redfish_timepoint)
+            .all_hosts()
+            .is_empty(),
+        "safe Viking completion should not touch Redfish boot order"
+    );
+}
+
 #[crate::sqlx_test]
 async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgPool) {
     let env = create_test_env(pool).await;