diff --git a/Cargo.lock b/Cargo.lock index 972be69c6d..05035dd040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2378,6 +2378,7 @@ dependencies = [ "carbide-secrets", "carbide-state-controller-common", "carbide-test-harness", + "carbide-test-support", "carbide-utils", "carbide-uuid", "chrono", diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index dd6800ea7e..41bbbe142d 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -347,7 +347,7 @@ Extends `StateControllerConfig` with: | `dpu_up_threshold` | `Duration` | `5m` | Max time without DPU health report before assuming it's down. | | `scout_reporting_timeout` | `Duration` | `5m` | Duration without scout report before host is unhealthy. | | `uefi_boot_wait` | `Duration` | `5m` | Wait time for UEFI boot completion after host reboot. | -| `max_bios_config_retries` | `u32` | `3` | Max HandleBiosJobFailure recovery cycles during BIOS configuration. | +| `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. | | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. | | `controller` | `StateControllerConfig` | *(default)* | Common state controller timing (see [StateControllerConfig](#statecontrollerconfig)). | diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 5a04a00f5f..18ff8e3b69 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -476,6 +476,10 @@ impl TestEnv { MachineValidatingState::RebootHost { .. } => state.clone(), MachineValidatingState::PrepareBootRepair { .. } | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } | MachineValidatingState::RepairBootConfig { .. } | MachineValidatingState::LockAfterBootRepair { .. } => state.clone(), } diff --git a/crates/api-core/src/tests/dpu_reprovisioning.rs b/crates/api-core/src/tests/dpu_reprovisioning.rs index 6871251981..d59e7b12ce 100644 --- a/crates/api-core/src/tests/dpu_reprovisioning.rs +++ b/crates/api-core/src/tests/dpu_reprovisioning.rs @@ -28,8 +28,8 @@ use libredfish::{EnabledDisabled, SystemPowerControl}; use model::instance::status::tenant::TenantState; use model::machine::{ DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState, - Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, ReprovisionState, - SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, + Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, + ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, }; use model::test_support::HardwareInfoTemplate; use rpc::forge::MachineArchitecture; @@ -86,8 +86,6 @@ fn reprovision_host_boot_repair_states( unlock_host_state: UnlockHostState::DisableLockdown, }, ReprovisionState::CheckHostBootConfig, - ReprovisionState::ConfigureHostBoot { retry_count: 0 }, - ReprovisionState::PollingHostBiosSetup { retry_count: 0 }, reprovision_set_host_boot_order_state(SetBootOrderState::SetBootOrder), reprovision_set_host_boot_order_state(SetBootOrderState::WaitForSetBootOrderJobScheduled), reprovision_set_host_boot_order_state(SetBootOrderState::RebootHost), @@ -137,6 +135,7 @@ async fn assert_dpu_reprovision_host_boot_repair( expected_states: Vec, ) -> Machine { env.redfish_sim.set_lockdown(EnabledDisabled::Enabled); + env.redfish_sim.set_is_bios_setup(true); env.redfish_sim.set_is_boot_order_setup(false); let redfish_timepoint = env.redfish_sim.timepoint(); @@ -179,27 +178,56 @@ async fn assert_dpu_reprovision_host_boot_repair( } } - // machine_setup enables the bootable DPU interface before boot-order promotion. let actions = env .redfish_sim .actions_since(&redfish_timepoint) .all_hosts(); - let machine_setup_pos = actions - .iter() - .position(|action| matches!(action, RedfishSimAction::MachineSetup { .. })) - .expect("expected DPU reprovision boot repair to call machine_setup"); - let set_boot_order_pos = actions + let (set_boot_order_pos, set_boot_order_mac) = actions .iter() - .position(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })) + .enumerate() + .find_map(|(position, action)| match action { + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) .expect("expected DPU reprovision boot repair to set DPU-first boot order"); - let check_boot_order_pos = actions + let boot_order_checks = actions .iter() - .rposition(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })) - .expect("expected DPU reprovision boot repair to verify boot order"); + .enumerate() + .filter_map(|(position, action)| match action { + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) + .collect::>(); assert!( - machine_setup_pos < set_boot_order_pos && set_boot_order_pos < check_boot_order_pos, - "expected machine_setup before set_boot_order_dpu_first before is_boot_order_setup; got: {actions:?}" + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "expected order-only DPU reprovision boot repair to preserve the configured HTTP boot device; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .filter(|(position, _)| *position < set_boot_order_pos) + .count() + >= 2, + "expected both CheckHostBootConfig and SetBootOrder to inspect the order before writing it; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .any(|(position, _)| *position > set_boot_order_pos), + "expected boot-order verification after the write; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .all(|(_, boot_interface_mac)| *boot_interface_mac == set_boot_order_mac), + "expected every order check and write to target the same interface; got: {actions:?}" ); let rebooting_machine = machine.next_iteration_machine(env).await; @@ -564,6 +592,136 @@ async fn test_dpu_reprovision_viking_skips_boot_order_when_bios_setup(pool: sqlx ); } +#[crate::sqlx_test] +async fn test_dpu_reprovision_viking_finishes_parked_boot_order_recovery(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let mh = create_managed_host_with_hardware_info_template( + &env, + HardwareInfoTemplate::Custom(DGX_H100_INFO_JSON), + ) + .await; + let dpu_machine = prepare_dpu_reprovision_host_boot_check(&env, &mh).await; + + // A controller upgrade can find a Viking in a persisted recovery substate + // created before boot-order remediation was disabled for this platform. + // Finish restoring host power before taking the safe terminal shortcut. + let parked_recovery = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::Off, + }, + )); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &parked_recovery) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &parked_recovery); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::ForceOff)], + "Viking policy must not abandon an in-flight power recovery" + ); + + let recovering_power_on = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::On, + }, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::BmcReset], + "parked recovery should reset the BMC after the host powers off" + ); + + // ForceOff records last_reboot_requested; backdate it so power_down_wait + // elapses in-process before the controller restores host power. + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + update_time_params(&env.pool, &host, 1, None).await; + } + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::On)], + "parked recovery should restore host power after the BMC reset" + ); + + let safe_terminal = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::CheckBootOrder, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &safe_terminal); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "completed recovery should naturally reach CheckBootOrder" + ); + + // CheckBootOrder has no unfinished operation behind it, but its preceding + // reboot may have reverted the HTTP boot device. Repair that BIOS drift + // without touching the unsupported Viking boot-order APIs, and spend one + // attempt from the shared convergence budget. + env.redfish_sim.set_is_bios_setup(false); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::ConfigureHostBoot { retry_count: 1 }) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "Viking BIOS recovery must not read or write boot order" + ); + + // Once BIOS is intact, the same safe terminal skips boot-order + // verification and completes the repair. + env.redfish_sim.set_is_bios_setup(true); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &safe_terminal) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::LockHostAfterBootRepair) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "safe Viking completion should not touch Redfish boot order" + ); +} + #[crate::sqlx_test] async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgPool) { let env = create_test_env(pool).await; diff --git a/crates/api-core/src/tests/machine_states.rs b/crates/api-core/src/tests/machine_states.rs index d946a6f041..7a6044cdd2 100644 --- a/crates/api-core/src/tests/machine_states.rs +++ b/crates/api-core/src/tests/machine_states.rs @@ -63,6 +63,7 @@ use model::machine::{ MachineValidatingState, ManagedHostState, MeasuringState, PowerState, SetBootOrderInfo, SetBootOrderState, SetSecureBootState, SpdmMeasuringState, StateMachineArea, ValidationState, }; +use model::machine_validation::MachineValidationState; use model::network_segment::NetworkSegmentType; use model::site_explorer::{EndpointExplorationReport, ExploredDpu, ExploredManagedHost}; use model::test_support::{DpuConfig, ManagedHostConfig}; @@ -2605,6 +2606,77 @@ async fn test_hpc_polling_bios_setup_stuck_enters_handle_bios_job_failure(pool: ); } +/// Assigned platform configuration should repair boot-order-only drift without +/// rerunning broad BIOS setup. +#[crate::sqlx_test] +async fn test_hpc_order_only_drift_skips_bios_configuration(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = common::api_fixtures::create_managed_host(&env).await; + let segment_id = env.create_vpc_and_tenant_segment().await; + create_instance(&env, &mh, false, segment_id).await; + let host_id = mh.host().id; + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::CheckHostConfig, + }, + }, + 0, + ) + .await; + // CheckHostConfig requires a DPU observation associated with this host + // state before Redfish results are trusted. + mh.network_configured(&env).await; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + let checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + .. + }, + }, + }, + } + ), + "order-only drift should route directly to SetBootOrder, got: {:?}", + host.current_state() + ); + } + + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .any(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })), + "assigned platform configuration should restore the drifted order, got: {actions:?}" + ); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "order-only drift should not rerun machine_setup, got: {actions:?}" + ); +} + /// Stuck HostInit/PollingBiosSetup recovery re-runs machine_setup and reaches HostInit/SetBootOrder. #[crate::sqlx_test] async fn test_polling_bios_setup_full_recovery_reruns_machine_setup_and_succeeds( @@ -2707,25 +2779,27 @@ async fn test_polling_bios_setup_full_recovery_reruns_machine_setup_and_succeeds /// so a zero-DPU host's in-band NIC takes a real `machine_interfaces` row that /// the boot-order phase can resolve. Mirrors `test_dhcp_allows_zero_dpu_host`. async fn create_zero_dpu_test_env(pool: sqlx::PgPool) -> TestEnv { - let env = create_test_env_with_overrides( - pool, - TestEnvOverrides { - site_prefixes: Some(vec![ - IpNetwork::new( - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - IpNetwork::new( - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - ]), - ..Default::default() - }, - ) - .await; + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::default()).await +} + +async fn create_zero_dpu_test_env_with_overrides( + pool: sqlx::PgPool, + mut overrides: TestEnvOverrides, +) -> TestEnv { + overrides.site_prefixes = Some(vec![ + IpNetwork::new( + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + IpNetwork::new( + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + ]); + + let env = create_test_env_with_overrides(pool, overrides).await; create_host_inband_network_segment(&env.api, None).await; env } @@ -2937,14 +3011,265 @@ async fn test_set_boot_order_sets_order_without_reasserting_when_device_configur ); } -/// Recovery path: when the HTTP-boot device is reverted (the boot NIC dropped off -/// the BMC's Redfish inventory on a reboot, so `is_bios_setup` reads false), -/// SetBootOrder re-asserts `machine_setup` and reboots to apply it *before* the -/// boot-order set -- the two BIOS writes never share one pass (they can't share -/// one Dell config job). Once the device verifies again, the order is set and the -/// host advances. +/// The reboot that applies a boot-order job can independently revert a managed +/// BIOS setting. Final verification checks BIOS before order, routes back to +/// the shared job-aware stages, and carries the existing convergence budget. +#[crate::sqlx_test] +async fn test_check_boot_order_repairs_bios_drift_after_order_reboot(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 2, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 3 }, + } + ), + "post-order BIOS drift should carry the convergence budget into shared BIOS repair, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "BIOS drift should short-circuit final boot-order verification, got: {actions:?}" + ); +} + +/// Repeated BIOS drift across boot-order reboots consumes the carried budget +/// instead of cycling indefinitely through machine_setup and another reboot. +#[crate::sqlx_test] +async fn test_check_boot_order_bios_drift_exhausts_convergence_budget(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + .. + }, + .. + } + ), + "expected repeated cross-phase drift to exhaust into Failed, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "an exhausted convergence budget must not issue another BIOS write, got: {actions:?}" + ); +} + +/// Boot-order verification uses the same configured convergence budget as +/// BIOS recovery, then remains able to observe an operator's repair while +/// parked at that limit. +#[crate::sqlx_test] +async fn test_check_boot_order_respects_configured_convergence_budget(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 1; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + .. + }), + }, + } + ), + "an exhausted configured budget must not advance to another boot-order attempt, got: {:?}", + host.current_state() + ); + assert!( + matches!( + host.controller_state_outcome.as_ref(), + Some(PersistentStateHandlerOutcome::Error { err, .. }) + if err.contains("Manual intervention required") + && err.contains("max_retries: 1") + ), + "expected boot-order exhaustion to report the configured limit, got: {:?}", + host.controller_state_outcome + ); + } + + env.redfish_sim.set_is_boot_order_setup(true); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, + } + ), + "a manually repaired boot order should complete from the parked verification state, got: {:?}", + host.current_state() + ); +} + +/// A configured budget above the historical fixed ceiling permits the shared +/// boot-config flow to spend the additional retries. +#[crate::sqlx_test] +async fn test_check_boot_order_allows_configured_budget_above_legacy_limit(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 5; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: 4, + .. + }), + }, + } + ), + "a configured budget above three should permit another boot-order attempt, got: {:?}", + host.current_state() + ); +} + +/// If the HTTP-boot device reverts after inspection but before SetBootOrder, +/// the order actuator routes back through the shared BIOS driver. In +/// particular, a Dell-style job returned by machine_setup is persisted and +/// completed before the order write. #[crate::sqlx_test] -async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx::PgPool) { +async fn test_set_boot_order_routes_reverted_bios_through_shared_job_flow(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -2956,12 +3281,22 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx // while the device is gone. env.redfish_sim.set_is_bios_setup(false); env.redfish_sim.set_is_boot_order_setup(false); + // SetBootOrder is reached only after HostInit has opened lockdown for BIOS + // writes; model that precondition when planting this synthetic state. + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_LATE_BOOT_DRIFT".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); set_host_stuck_in_set_boot_order(&env, host_id).await; let redfish_timepoint = env.redfish_sim.timepoint(); - // First pass: re-assert the device and reboot to apply it. It must NOT set the - // boot order in the same pass -- two BIOS writes can't share one config job. + // First pass only detects the late drift and moves back to the shared BIOS + // stage. It must not issue either BIOS write from the order actuator. env.run_machine_state_controller_iteration().await; let first_pass = env .redfish_sim @@ -2970,8 +3305,8 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert!( first_pass .iter() - .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), - "a reverted HTTP-boot device should be re-asserted at SetBootOrder, got: {first_pass:?}" + .all(|a| !matches!(a, RedfishSimAction::MachineSetup { .. })), + "SetBootOrder should delegate late BIOS drift to the shared driver, got: {first_pass:?}" ); assert!( !first_pass @@ -2980,9 +3315,6 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx "the boot order must not be set in the same pass as the re-assert (shared BIOS job), got: {first_pass:?}" ); - // The re-assert is committed once. While the device is still reverted (the - // apply reboot hasn't landed yet), the host polls in - // WaitForHttpBootDeviceApplied without re-running machine_setup. { let mut txn = env.db_txn().await; let host = mh.host().db_machine(&mut txn).await; @@ -2990,31 +3322,17 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - .. - }), - }, + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 1 }, } ), - "after the re-assert the host should wait for the device to apply, got: {:?}", + "late BIOS drift should return to shared configuration, got: {:?}", host.current_state() ); } - let after_reassert = env.redfish_sim.timepoint(); - env.run_machine_state_controller_iteration().await; - let second_pass = env.redfish_sim.actions_since(&after_reassert).all_hosts(); - assert!( - !second_pass - .iter() - .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), - "machine_setup must not re-run while waiting for the re-asserted device to apply, got: {second_pass:?}" - ); - // Within the wait window the host stays parked in - // WaitForHttpBootDeviceApplied -- it neither advances nor bounces back to - // SetBootOrder for another re-assert. + // The shared driver persists the returned job instead of discarding it and + // rebooting immediately. + env.run_machine_state_controller_iteration().await; { let mut txn = env.db_txn().await; let host = mh.host().db_machine(&mut txn).await; @@ -3022,20 +3340,21 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + machine_state: MachineState::WaitingForBiosJob { + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + retry_count: 1, .. - }), + }, }, - } + } if job_id == "JID_LATE_BOOT_DRIFT" ), - "the host should keep waiting for the device to apply within the wait window, got: {:?}", + "the late-drift BIOS job should be persisted, got: {:?}", host.current_state() ); } - // Model the reboot applying the re-asserted config: the device now verifies. + // Model the BIOS job applying the restored device. env.redfish_sim.set_is_bios_setup(true); // With the device restored, the host sets the boot order and advances. @@ -3053,14 +3372,12 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } -/// When the re-asserted HTTP-boot device does not verify within the wait -/// window (the boot NIC can drop off the BMC's Redfish inventory again on the -/// apply reboot itself), the host returns to `SetBootOrder` for a fresh -/// re-assert -- one `machine_setup` per window, not per pass. Once the state -/// machine's retry budget is exhausted it stops re-asserting and surfaces the -/// host for manual intervention. +/// Hosts persisted in the legacy HTTP-device wait state migrate to the shared +/// BIOS driver when their wait expires. The existing retry count becomes the +/// shared convergence budget, and an exhausted budget still stops for manual +/// intervention. #[crate::sqlx_test] -async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx::PgPool) { +async fn test_legacy_http_boot_wait_migrates_to_shared_bios_repair(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -3069,6 +3386,8 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: // The device stays reverted throughout: every verify reads false. env.redfish_sim.set_is_bios_setup(false); env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); let stuck_waiting = |retry_count: u32| ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { @@ -3081,8 +3400,8 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: }; // Plant the host mid-wait with the window already expired (backdated past - // the 10-minute apply window). Pass 1 returns it to SetBootOrder; pass 2 - // re-asserts once and parks it back in the wait substate. + // the 10-minute apply window). Pass 1 migrates its persisted state; pass 2 + // runs machine_setup through the common BIOS driver. set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(0), 11).await; let checkpoint = env.redfish_sim.timepoint(); env.run_machine_state_controller_iteration().await; @@ -3103,16 +3422,10 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - retry_count: 1, - .. - }), - }, + machine_state: MachineState::PollingBiosSetup { retry_count: 1 }, } ), - "the fresh re-assert should park the host back in the wait substate with the retry spent, got: {:?}", + "the legacy wait should carry its retry count into shared BIOS polling, got: {:?}", host.current_state() ); } @@ -3171,9 +3484,16 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool env.redfish_sim.set_is_boot_order_setup(false); env.redfish_sim .set_lockdown(libredfish::EnabledDisabled::Enabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_VALIDATION_BOOT_REPAIR".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); // Park the host mid-validation, waiting on a reboot that cannot land // (the state is newer than the host's last reported boot). + let validation_id = MachineValidationId::new(); set_host_controller_state_stuck_in( &env, host_id, @@ -3181,7 +3501,7 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool validation_state: ValidationState::MachineValidation { machine_validation: MachineValidatingState::MachineValidating { context: "Discovery".to_string(), - id: MachineValidationId::new(), + id: validation_id, completed: 1, total: 1, is_enabled: true, @@ -3221,6 +3541,29 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool .contains(&libredfish::EnabledDisabled::Disabled), "boot repair should unlock the BMC before writing BIOS settings" ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::WaitingForBootBiosJob { + validation_id: current_validation_id, + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + .. + }, + }, + }, + } if *current_validation_id == validation_id + && job_id == "JID_VALIDATION_BOOT_REPAIR" + ), + "validation should preserve the BIOS job before rebooting, got: {:?}", + host.current_state() + ); + } // The repair reboot applies the re-asserted device. env.redfish_sim.set_is_bios_setup(true); @@ -3236,9 +3579,11 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool host.current_state(), ManagedHostState::Validation { validation_state: ValidationState::MachineValidation { - machine_validation: MachineValidatingState::RebootHost { .. }, + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, }, - } + } if *current_validation_id == validation_id ) { resumed = true; break; @@ -3261,6 +3606,183 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } +/// Boot-order-only drift (the HTTP boot device is still configured, but its +/// priority changed) is detected while validation waits for a reboot. The +/// repair restores only the order, re-locks the BMC, and resumes validation. +#[crate::sqlx_test] +async fn test_machine_validation_repairs_drifted_boot_order(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // The HTTP boot device is intact; only the order drifted. Set the BIOS + // status explicitly so this regression does not rely on simulator defaults. + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + + let validation_id = MachineValidationId::new(); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::MachineValidating { + context: "Discovery".to_string(), + id: validation_id, + completed: 1, + total: 1, + is_enabled: true, + }, + }, + }, + 0, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + + let mut unlocked = false; + let mut resumed = false; + for _ in 0..20 { + env.run_machine_state_controller_iteration().await; + unlocked |= env + .redfish_sim + .lockdown_states() + .contains(&libredfish::EnabledDisabled::Disabled); + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + if matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, + }, + } if *current_validation_id == validation_id + ) { + resumed = true; + break; + } + } + + assert!( + resumed, + "order-only drift should be repaired before validation resumes" + ); + assert!( + unlocked, + "validation should unlock the BMC before repairing boot-order drift" + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "validation should check the boot order for the boot NIC, got: {actions:?}" + ); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "the drifted boot order should be restored for the boot NIC, got: {actions:?}" + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "order-only repair should preserve the configured HTTP boot device, got: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|l| *l == libredfish::EnabledDisabled::Enabled), + "the BMC should be re-locked once the boot order is repaired" + ); +} + +/// Exhausting the shared BIOS recovery budget during validation is terminal +/// for both the host repair and the active validation run. +#[crate::sqlx_test] +async fn test_validation_bios_setup_exhaustion_completes_active_validation(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = create_managed_host(&env).await; + let host_id = mh.host().id; + let validation_id = + on_demand_machine_validation(&env, host_id, Vec::new(), Vec::new(), false, Vec::new()) + .await + .validation_id + .expect("on-demand validation should return an id"); + + env.redfish_sim.set_is_bios_setup(false); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count: 3, + }, + }, + }, + 16, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + .. + }, + .. + } + ), + "expected validation boot repair exhaustion to fail the host, got: {:?}", + host.current_state() + ); + assert_eq!( + host.on_demand_machine_validation_request, + Some(false), + "terminal boot repair should clear the validation request" + ); + + let validation = db::machine_validation::find_by_id(&env.pool, &validation_id) + .await + .expect("validation run should still be queryable"); + assert!( + matches!( + validation.status.as_ref().map(|status| &status.state), + Some(&MachineValidationState::Failed) + ), + "validation run should be marked failed, got: {:?}", + validation.status + ); + assert!( + validation.end_time.is_some(), + "terminal validation run should record an end time" + ); +} + /// When HostInit/PollingBiosSetup retry budget is exhausted, enter Failed and recover via is_bios_setup. #[crate::sqlx_test] async fn test_polling_bios_setup_exhausted_enters_failed_and_recovers_when_bios_setup_true( diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 3eb8e8dbc5..d70973297e 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1308,6 +1308,23 @@ pub enum MachineValidatingState { validation_id: MachineValidationId, unlock_host_state: UnlockHostState, }, + CheckBootConfigForRepair { + validation_id: MachineValidationId, + }, + ConfigureBootBios { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, + WaitingForBootBiosJob { + validation_id: MachineValidationId, + bios_config_info: BiosConfigInfo, + }, + PollingBootBiosSetup { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, RepairBootConfig { validation_id: MachineValidationId, set_boot_order_info: SetBootOrderInfo, @@ -1907,7 +1924,8 @@ pub struct BiosConfigInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub bios_job_id: Option, pub bios_config_state: BiosConfigState, - /// Full configure_host_bios retry count across HandleBiosJobFailure recovery cycles. + /// Shared host boot-configuration convergence retry count, including BIOS + /// job-failure recovery cycles. #[serde(default)] pub retry_count: u32, } @@ -1933,7 +1951,8 @@ pub struct SetBootOrderInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub set_boot_order_jid: Option, pub set_boot_order_state: SetBootOrderState, - /// Retry counter for SetBootOrder state machine. Defaults to 0 for backwards compatibility. + /// Shared host boot-configuration convergence retry count. Defaults to 0 + /// for backwards compatibility. #[serde(default)] pub retry_count: u32, } @@ -1943,9 +1962,9 @@ pub struct SetBootOrderInfo { #[serde(tag = "state", rename_all = "lowercase")] pub enum SetBootOrderState { SetBootOrder, - /// A reverted HTTP-boot device was re-asserted (`machine_setup`) and the - /// host restarted to apply it; polls the device across that reboot, then - /// returns to `SetBootOrder` to set the boot order. + /// Legacy persisted state from the former inline HTTP-device repair flow. + /// It polls the device across the already-started reboot, then either + /// resumes `SetBootOrder` or migrates to the shared BIOS repair stages. WaitForHttpBootDeviceApplied, WaitForSetBootOrderJobScheduled, RebootHost, @@ -2695,6 +2714,10 @@ pub fn state_sla( } MachineValidatingState::PrepareBootRepair { .. } | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } | MachineValidatingState::RepairBootConfig { .. } | MachineValidatingState::LockAfterBootRepair { .. } => { StateSla::with_sla(slas::VALIDATION, time_in_state) diff --git a/crates/machine-controller/Cargo.toml b/crates/machine-controller/Cargo.toml index 57ba2d66ad..2aba63d481 100644 --- a/crates/machine-controller/Cargo.toml +++ b/crates/machine-controller/Cargo.toml @@ -77,6 +77,7 @@ version-compare = { workspace = true } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-redfish = { path = "../redfish", features = ["test-support"] } carbide-test-harness = { path = "../test-harness" } +carbide-test-support = { path = "../test-support" } figment = { workspace = true, features = ["env", "test", "toml"] } regex = { workspace = true } diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index 76e964eaaf..4af4ecb2bf 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -70,7 +70,9 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub uefi_boot_wait: Duration, - /// Max configure_host_bios retry cycles through HandleBiosJobFailure recovery. + /// Retry budget for automated host boot-configuration convergence, shared + /// across BIOS recovery and boot-order verification. The field name is + /// retained for configuration compatibility. #[serde(default = "MachineStateControllerConfig::max_bios_config_retries_default")] pub max_bios_config_retries: u32, /// How long PollingBiosSetup may sit on Ok(false) before escalating into diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 3f01c4a6ce..efb8395ca5 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -119,21 +119,23 @@ mod bios_config; mod dpf; mod firmware_artifact; mod helpers; +mod host_boot_config; mod machine_validation; mod power; mod sku; #[cfg(test)] mod test_machine_setup; -use bios_config::{ - BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, - advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, - handle_bios_setup_failed_recovery, -}; +use bios_config::handle_bios_setup_failed_recovery; use helpers::{ DpuDiscoveringStateHelper, DpuInitStateHelper, ManagedHostStateHelper, NextState, ReprovisionStateHelper, all_equal, }; +use host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, + initial_set_boot_order_info, run_host_boot_config_stage, should_skip_boot_order_remediation, +}; use state_controller::db_write_batch::DbWriteBatch; use crate::config::{BomValidationConfig, PowerManagerOptions}; @@ -1472,11 +1474,7 @@ impl MachineStateHandler { FailureCause::BiosSetupFailed { .. } if machine_id.machine_type().is_host() => { let recovered = ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + set_boot_order_info: Some(initial_set_boot_order_info()), }, }; handle_bios_setup_failed_recovery(ctx, mh_snapshot, recovered).await @@ -3137,243 +3135,70 @@ async fn handle_dpu_reprovision( // WaitingForNetworkConfig already accepted the DPU observation. Do // not require a newer observation just because the host state // version advanced while entering host boot repair. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::AlreadyValidated, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::CheckHostBootConfigAfterHostReboot => { // This path rebooted the host after unlocking, so require a DPU // observation newer than that reboot before trusting boot checks. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::SinceLastHostRebootRequest, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::ConfigureHostBoot { retry_count } => { - // Run machine_setup only after the reprovisioned DPU is healthy; it - // may patch BIOS settings and trigger host-impacting recovery. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match configure_host_bios( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - *retry_count, + reachability_params, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: *retry_count, - }, - )?, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } ReprovisionState::WaitingForHostBiosJob { bios_config_info } => { - // Poll vendor BIOS jobs before verifying the setup and boot order. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match advance_bios_config_job( + handle_dpu_reprovision_host_boot_config_stage( ctx, - redfish_client.as_ref(), state, - bios_config_info.clone(), + reachability_params, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { - bios_config_info: updated, - }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: bios_config_info.retry_count, - }, - )?, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => Ok( - StateHandlerOutcome::transition(dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - )), - ), - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::ConfigureHostBoot { retry_count }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::PollingHostBiosSetup { retry_count } => { - // Verify machine_setup effects before promoting the DPU boot option. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let predictions = load_boot_predictions(ctx, &state.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_stage( + ctx, state, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - let next_state = if should_skip_boot_order_remediation(state) { - ReprovisionState::LockHostAfterBootRepair - } else { - ReprovisionState::SetHostBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - PollingBiosSetupOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( - dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - ), - )), - PollingBiosSetupOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::SetHostBootOrder { set_boot_order_info, } => { - // Promote the selected DPU boot option after machine_setup has enabled it. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match set_host_boot_order( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - set_boot_order_info.clone(), + reachability_params, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::SetHostBootOrder { - set_boot_order_info: boot_order_info, - }, - )?, - )) - } - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::LockHostAfterBootRepair, - )?, - )), - SetBootOrderOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::LockHostAfterBootRepair => { // Preserve expected-machine lockdown policy after temporarily @@ -3499,6 +3324,117 @@ async fn handle_dpu_reprovision( } } +/// Checks host boot configuration for DPU reprovisioning and maps the shared +/// decision back into the reprovision state while preserving all active DPU +/// targets. The caller selects the observation-freshness requirement because +/// only the post-unlock path has rebooted the host. +async fn handle_dpu_reprovision_host_boot_config_check( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + let next_state = match check_host_boot_config( + redfish_client.as_ref(), + state, + reachability_params, + dpu_freshness, + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + ReprovisionState::ConfigureHostBoot { retry_count: 0 } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + ReprovisionState::SetHostBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + ReprovisionState::LockHostAfterBootRepair + } + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, next_state)?, + )) +} + +/// Handles a host boot-configuration stage owned by DPU reprovisioning. +/// +/// Called by `handle_dpu_reprovision` for its host BIOS configuration, +/// vendor-job, polling, and boot-order substates. This adapter preserves the +/// active DPU reprovision targets, continues to `LockHostAfterBootRepair` on +/// completion, and attributes terminal failure to the lifecycle that owns the +/// reprovision. +async fn handle_dpu_reprovision_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + reachability_params, + redfish_client.as_ref(), + state, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let reprovision_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + ReprovisionState::ConfigureHostBoot { retry_count } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + ReprovisionState::WaitingForHostBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + ReprovisionState::PollingHostBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => ReprovisionState::SetHostBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, reprovision_state)?, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state( + state, + ReprovisionState::LockHostAfterBootRepair, + )?, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + dpu_reprovision_host_boot_failed_state( + &state.managed_state, + state.host_snapshot.id, + failure, + ), + )), + } +} + /// Build the correct failed state for host boot repair during DPU reprovision. fn dpu_reprovision_host_boot_failed_state( current_state: &ManagedHostState, @@ -3555,152 +3491,6 @@ async fn load_boot_predictions( Ok(predictions) } -/// Check whether host BIOS and DPU-first boot order remediation is required. -async fn check_host_boot_config( - redfish_client: &dyn Redfish, - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> Result { - // Wait for DPUs only when this caller needs a fresh observation. DPU - // reprovision already validated DPU health before entering host boot repair. - if should_wait_for_dpus_before_host_boot_config( - mh_snapshot, - reachability_params, - dpu_freshness, - ctx, - ) - .await - { - return Ok(HostBootConfigDecision::Wait( - "Waiting for DPUs to come up.".to_string(), - )); - } - - // Resolve the interface whose boot option should be first in host UEFI. A - // zero-DPU host whose boot NIC has not taken its first HostInband lease yet - // falls back to its predicted boot NIC, and only waits when even that is - // unavailable. - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - let boot_interface = match require_boot_interface( - mh_snapshot, - &predictions, - "configuring boot", - HostBootConfigDecision::Wait, - )? { - RequiredBootInterface::Ready(target) => target, - RequiredBootInterface::Wait(decision) => return Ok(decision), - }; - - let vendor = mh_snapshot.host_snapshot.bmc_vendor(); - - log_host_config(redfish_client, mh_snapshot).await; - - let is_bios_setup = boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await - .map_err(|e| redfish_error("is_bios_setup", e))?; - - if should_skip_boot_order_remediation(mh_snapshot) { - if is_bios_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Skipping boot order remediation on Viking (known FW/BMC issue)" - ); - return Ok(HostBootConfigDecision::LockHost); - } - - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup is not configured properly on Viking; running BIOS repair before skipping boot order remediation" - ); - return Ok(HostBootConfigDecision::ConfigureBoot); - } - - let is_boot_order_setup = boot_interface - .run(|bi| redfish_client.is_boot_order_setup(bi)) - .await - .map_err(|e| redfish_error("is_boot_order_setup", e))?; - - if is_bios_setup && is_boot_order_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup and boot order are configured properly" - ); - Ok(HostBootConfigDecision::LockHost) - } else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - is_bios_setup, - is_boot_order_setup, - "Host BIOS setup or boot order is not configured properly" - ); - Ok(HostBootConfigDecision::ConfigureBoot) - } -} - -/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair still applies. -fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { - mh_snapshot - .host_snapshot - .hardware_info - .as_ref() - .is_some_and(|hw| hw.is_dgx_h100()) -} - -async fn should_wait_for_dpus_before_host_boot_config( - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> bool { - if !mh_snapshot.has_managed_dpus() { - return false; - } - - match dpu_freshness { - HostBootConfigDpuFreshness::AlreadyValidated => false, - HostBootConfigDpuFreshness::CurrentHostState => { - !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await - } - HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { - let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested - else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - "No host reboot request timestamp found before post-reboot host boot config check" - ); - return false; - }; - - for dpu_snapshot in &mh_snapshot.dpu_snapshots { - if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { - match trigger_reboot_if_needed( - dpu_snapshot, - mh_snapshot, - None, - reachability_params, - ctx, - ) - .await - { - Ok(_) => {} - Err(e) => tracing::warn!("could not reboot dpu {}: {e}", dpu_snapshot.id), - } - return true; - } - } - - false - } - } -} - // Returns true if update_manager flagged this managed host as needing its firmware examined fn host_reprovisioning_requested(state: &ManagedHostStateSnapshot) -> bool { state.host_snapshot.host_reprovision_requested.is_some() @@ -4930,6 +4720,9 @@ pub struct RebootStatus { /// Outcome of set_host_boot_order function. enum SetBootOrderOutcome { Continue(SetBootOrderInfo), + /// BIOS drifted after the caller's inspection; return to the shared BIOS + /// configuration stages so any Redfish job ID is persisted and polled. + ConfigureBios, Done, WaitingForReboot(String), /// No boot interface to act on yet -- e.g. a zero-DPU host whose boot NIC @@ -4938,20 +4731,6 @@ enum SetBootOrderOutcome { Wait(String), } -/// Decision from checking whether host boot repair is still required. -enum HostBootConfigDecision { - ConfigureBoot, - LockHost, - Wait(String), -} - -/// DPU observation freshness required before checking host boot config. -enum HostBootConfigDpuFreshness { - AlreadyValidated, - CurrentHostState, - SinceLastHostRebootRequest, -} - /// Outcome of [`require_boot_interface`]: the resolved boot NIC, or the /// caller's wait outcome for a zero-DPU host still discovering its boot NIC. #[derive(Debug)] @@ -5409,63 +5188,63 @@ fn check_host_health_for_alerts(state: &ManagedHostStateSnapshot) -> Result<(), } } -async fn handle_host_boot_order_setup( +/// Handles a shared boot-configuration stage during host initialization. +/// +/// Called by `HostMachineStateHandler::handle_object_state` for its platform +/// configuration, vendor-job, polling, and boot-order states. This adapter maps +/// continued stages back into `MachineState`, proceeds to measurements on +/// completion, and reports terminal failure as a host-init BIOS setup failure. +async fn handle_host_init_boot_config_stage( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, - host_handler_params: HostHandlerParams, - mh_snapshot: &mut ManagedHostStateSnapshot, - set_boot_order_info: Option, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, ) -> Result, StateHandlerError> { - tracing::info!( - "Starting Boot Order Configuration for {}: {set_boot_order_info:#?}", - mh_snapshot.host_snapshot.id - ); - - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) - .await?; - - let next_state = match set_boot_order_info { - Some(info) => { - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, - redfish_client.as_ref(), - mh_snapshot, - info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(boot_order_info), - }, - }, - SetBootOrderOutcome::Done => ManagedHostState::HostInit { - machine_state: MachineState::Measuring { - measuring_state: MeasuringState::WaitingForMeasurements, - }, - }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineState::WaitingForPlatformConfiguration { retry_count } } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineState::WaitingForBiosJob { bios_config_info } } - } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineState::SetBootOrder { + set_boot_order_info: Some(set_boot_order_info), + }, + }; + Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { machine_state }, + )) } - None => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, }, - }, - }; - - Ok(StateHandlerOutcome::transition(next_state)) + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + }, + machine_id: mh_snapshot.host_snapshot.id, + retry_count: 0, + })) + } + } } /// TODO: we need to handle the case where the job is deleted for some reason @@ -5849,147 +5628,76 @@ impl StateHandler for HostMachineStateHandler { } } - match configure_host_bios( + handle_host_init_boot_config_stage( ctx, + mh_snapshot, &self.host_handler_params.reachability_params, redfish_client.as_ref(), - mh_snapshot, - *retry_count, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: *retry_count, - }, - }, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::WaitingForBiosJob { bios_config_info } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match advance_bios_config_job( + handle_host_init_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), - ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { - bios_config_info: updated, - }, - }), - ), - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - }, - }, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::WaitingForPlatformConfiguration { - retry_count, - }, - }, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } - } - MachineState::PollingBiosSetup { retry_count } => { - let next_state = ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), }, - }; - + ) + .await + } + MachineState::PollingBiosSetup { retry_count } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - let predictions = - load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_host_init_boot_config_stage( + ctx, mh_snapshot, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - Ok(StateHandlerOutcome::transition(next_state)) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - PollingBiosSetupOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - PollingBiosSetupOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::SetBootOrder { set_boot_order_info, - } => Ok(handle_host_boot_order_setup( - ctx, - self.host_handler_params.clone(), - mh_snapshot, - set_boot_order_info.clone(), - ) - .await?), + } => { + let Some(set_boot_order_info) = set_boot_order_info.clone() else { + return Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(initial_set_boot_order_info()), + }, + }, + )); + }; + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_host_init_boot_config_stage( + ctx, + mh_snapshot, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }, + ) + .await + } MachineState::Measuring { measuring_state } => { if !self.host_handler_params.attestation_enabled { return Ok(StateHandlerOutcome::transition( @@ -7274,11 +6982,7 @@ impl StateHandler for InstanceStateHandler { instance_state: InstanceState::HostPlatformConfiguration { platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, + set_boot_order_info: initial_set_boot_order_info(), }, }, }; @@ -10954,6 +10658,75 @@ async fn log_host_config(redfish_client: &dyn Redfish, mh_snapshot: &ManagedHost ); } +/// Handles a shared boot-configuration stage during assigned-host platform +/// configuration. +/// +/// Called by `handle_instance_host_platform_config` for its BIOS configuration, +/// vendor-job, polling, and boot-order states. This adapter maps continued +/// stages back into `HostPlatformConfigurationState`, proceeds to `LockHost` on +/// completion, and scopes terminal failure to the assigned instance. +async fn handle_instance_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let platform_config_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + HostPlatformConfigurationState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state, + }, + }, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + }, + }, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::AssignedInstance), + }, + machine_id: mh_snapshot.host_snapshot.id, + }, + }, + )), + } +} + async fn handle_instance_host_platform_config( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, mh_snapshot: &mut ManagedHostStateSnapshot, @@ -11182,18 +10955,29 @@ async fn handle_instance_host_platform_config( ) .await? { - HostBootConfigDecision::Wait(reason) => { + HostBootConfigCheckOutcome::Wait(reason) => { return Ok(StateHandlerOutcome::wait(reason)); } - HostBootConfigDecision::ConfigureBoot => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: 0, - }, - }, - HostBootConfigDecision::LockHost => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, - }, + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count: 0, + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + } + } } } HostPlatformConfigurationState::ConfigureBios { @@ -11214,174 +10998,48 @@ async fn handle_instance_host_platform_config( )); } - let next_platform = match configure_host_bios( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - retry_count, + HostBootConfigStage::ConfigureBios { retry_count }, ) - .await? - { - BiosConfigOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { retry_count } - } - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } - } - BiosConfigOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } => { - let next_platform = match advance_bios_config_job( + return handle_instance_host_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info: updated, - } - } - BiosConfigJobAdvanceOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - } - } - BiosConfigJobAdvanceOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { - retry_count: next_count, - } => HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: next_count, - }, - BiosConfigJobAdvanceOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::PollingBiosSetup { retry_count } => { - let next_instance_state = InstanceState::HostPlatformConfiguration { - platform_config_state: if should_skip_boot_order_remediation(mh_snapshot) { - HostPlatformConfigurationState::LockHost - } else { - HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }, - }; - - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + return handle_instance_host_boot_config_stage( + ctx, mh_snapshot, - retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { retry_count }, ) - .await? - { - PollingBiosSetupOutcome::Verified => next_instance_state, - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info, - }, - }, - }, - )); - } - PollingBiosSetupOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - PollingBiosSetupOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + .await; } HostPlatformConfigurationState::SetBootOrder { set_boot_order_info, } => { - match set_host_boot_order( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - set_boot_order_info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: boot_order_info, - }, - } - } - SetBootOrderOutcome::Done => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, + HostBootConfigStage::SetBootOrder { + set_boot_order_info, }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + ) + .await; } HostPlatformConfigurationState::LockHost => { if mh_snapshot.host_snapshot.host_profile.disable_lockdown { @@ -11442,51 +11100,36 @@ async fn set_host_boot_order( RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; - // Don't re-apply a boot config that's already in place. `SetBootOrder` - // re-asserts the HTTP-boot device (`machine_setup`) and then sets the - // boot order, but on Dell two BIOS config writes can't share one - // pending job: re-asserting commits a config job, and - // `set_boot_order_dpu_first` then can't set the order (iDRAC `SYS011`, - // "a config job is already committed"), so re-applying an - // already-correct config collides and loops. Check first -- if the - // config is already set, skip straight to verification. Only when the - // HTTP-boot device was actually reverted (the boot NIC dropped off the - // BMC's Redfish inventory on a reboot) do we re-assert it, and then - // reboot to apply it before the boot-order set, so the two writes - // never land in one pass. + // Don't re-apply a boot config that's already in place. On Dell two + // BIOS config writes cannot share one pending job: machine_setup can + // commit a job that collides with set_boot_order_dpu_first (iDRAC + // `SYS011`). If the HTTP device drifted since the owning lifecycle's + // inspection, return to the shared BIOS/job stages before attempting + // the order write. let is_bios_setup = boot_interface .run(|bi| redfish_client.is_bios_setup(Some(bi))) .await .map_err(|e| redfish_error("is_bios_setup", e))?; if !is_bios_setup { - // The HTTP-boot device was reverted (the boot NIC dropped off the - // BMC's Redfish inventory on a reboot), so re-assert it with - // `machine_setup` and restart the host to apply it *before* - // setting the boot order -- the two BIOS writes never share one - // pass (they can't share one Dell config job). The re-assert - // commits once: `WaitForHttpBootDeviceApplied` polls the device - // across the reboot instead of this arm re-writing the staged - // settings and re-creating the config job every pass while the - // reboot lands. - call_machine_setup_and_handle_no_dpu_error( - redfish_client, - Some(&boot_interface), - mh_snapshot.host_snapshot.associated_dpu_machine_ids().len(), - &ctx.services.site_config, - ) - .await - .map_err(|e| redfish_error("machine_setup", e))?; - - handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart) - .await?; - log_host_config(redfish_client, mh_snapshot).await; - - return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - retry_count: set_boot_order_info.retry_count, - })); + // The HTTP-boot device drifted after the owning lifecycle's + // inspection. Route back through the shared BIOS driver rather + // than issuing machine_setup here: vendor jobs must be persisted + // and polled before boot-order work can safely continue. + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // Keep the vendor safety policy at the actuator boundary. This is + // intentionally after the BIOS check: validation reuses this state + // to restore a reverted HTTP boot device on Viking, where BIOS + // repair is supported but boot-order remediation is not. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order remediation on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); } // HTTP-boot device is already set, so `machine_setup` was not re-run @@ -11577,14 +11220,9 @@ async fn set_host_boot_order( })) } SetBootOrderState::WaitForHttpBootDeviceApplied => { - // The SetBootOrder substate re-asserted the reverted HTTP-boot device - // and restarted the host; the staged BIOS settings apply across that - // reboot. Poll until the device verifies, then return to SetBootOrder - // to set the boot order. If it still hasn't applied once the reboot - // has had time to land (the boot NIC can drop off the BMC's Redfish - // inventory again on the apply reboot itself), return to SetBootOrder - // for a fresh re-assert -- re-commits stay bounded to one per wait - // window instead of one per pass. + // Backward compatibility for hosts persisted in the former inline + // machine_setup flow. Once its wait window expires, migrate into the + // shared BIOS/job stages so any new vendor job is retained. const HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES: i64 = 10; const MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES: u32 = 3; @@ -11623,11 +11261,10 @@ async fn set_host_boot_order( .num_minutes(); if minutes_waiting >= HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES { - // Each expired window spends one retry from the SetBootOrder - // state machine's shared budget. Once it's exhausted the device - // is not coming back on its own -- stop re-asserting and surface - // the host for an operator, the same terminal escalation the - // reboot pacer applies to a host that never comes up. + // Preserve the terminal behavior of a legacy state whose old + // retry budget was already exhausted. Any nonterminal state + // carries its existing count into the shared convergence + // budget. if set_boot_order_info.retry_count >= MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES { return Err(StateHandlerError::ManualInterventionRequired(format!( "HTTP boot device on host {} still not applied after {} re-asserts; manual intervention required", @@ -11637,13 +11274,9 @@ async fn set_host_boot_order( tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, minutes_waiting, - "Re-asserted HTTP boot device still not applied after the wait window; returning to SetBootOrder to re-assert", + "Re-asserted HTTP boot device still not applied after the wait window; migrating to shared BIOS repair", ); - return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: set_boot_order_info.retry_count + 1, - })); + return Ok(SetBootOrderOutcome::ConfigureBios); } Ok(SetBootOrderOutcome::WaitingForReboot(format!( @@ -11673,7 +11306,8 @@ async fn set_host_boot_order( })) } SetBootOrderState::RebootHost => { - // Host needs to be rebooted to pick up the changes after calling machine_setup + // Reboot before final verification so any accepted boot-order + // change or vendor job can apply. handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart).await?; Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11857,10 +11491,14 @@ async fn set_host_boot_order( } } SetBootOrderState::CheckBootOrder => { - const MAX_BOOT_ORDER_RETRIES: u32 = 3; const CHECK_BOOT_ORDER_TIMEOUT_MINUTES: i64 = 30; let retry_count = set_boot_order_info.retry_count; + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; let boot_interface = match require_boot_interface( mh_snapshot, @@ -11872,6 +11510,36 @@ async fn set_host_boot_order( RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; + // The boot-order job/recovery reboot can itself revert the HTTP + // device or another managed BIOS attribute. Verify BIOS first for + // every platform and return to the shared job-aware repair before + // declaring the overall boot configuration complete. + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Host BIOS drifted during boot-order recovery; returning to shared BIOS repair" + ); + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // A controller upgrade can resume a Viking after an old boot-order + // job or power-recovery substate. Once the operation and BIOS + // verification are complete, skip the unsupported order read. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order verification on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); + } + let boot_order_configured = boot_interface .run(|bi| redfish_client.is_boot_order_setup(bi)) .await @@ -11896,16 +11564,23 @@ async fn set_host_boot_order( time_since_state_change.num_minutes() ); - // If we've been stuck for 30+ minutes and haven't exhausted retries, retry SetBootOrder - if time_since_state_change.num_minutes() >= CHECK_BOOT_ORDER_TIMEOUT_MINUTES - && retry_count < MAX_BOOT_ORDER_RETRIES - { - tracing::info!( - "Boot order check timed out for {} after {} minutes, retrying SetBootOrder (retry {} of {})", + if time_since_state_change.num_minutes() < CHECK_BOOT_ORDER_TIMEOUT_MINUTES { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, - time_since_state_change.num_minutes(), - retry_count + 1, - MAX_BOOT_ORDER_RETRIES + retry_count, + max_retries, + time_since_state_change.num_minutes() + ))); + } + + if retry_count < max_retries { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + time_in_state_minutes = time_since_state_change.num_minutes(), + retry_count = retry_count + 1, + max_retries, + "Boot order check timed out; retrying SetBootOrder" ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11915,11 +11590,13 @@ async fn set_host_boot_order( })); } - // Either still within timeout window or exhausted retries - return error - Err(StateHandlerError::GenericError(eyre::eyre!( - "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, time_in_state: {} minutes)", + // Keep the state parked so an operator can repair the BMC and the + // next verification can complete without resetting the lifecycle. + Err(StateHandlerError::ManualInterventionRequired(format!( + "Boot order is not configured properly for host {} after SetBootOrder completed; automated boot-config convergence exhausted (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, retry_count, + max_retries, time_since_state_change.num_minutes() ))) } diff --git a/crates/machine-controller/src/handler/bios_config.rs b/crates/machine-controller/src/handler/bios_config.rs index b94aaa5992..b8644f809e 100644 --- a/crates/machine-controller/src/handler/bios_config.rs +++ b/crates/machine-controller/src/handler/bios_config.rs @@ -363,7 +363,7 @@ fn try_bios_recovery_attempt( ); return Ok(BiosRecoveryAttemptOutcome::Failed { failure: format!( - "{failure} (automated BIOS recovery exhausted after {} attempts)", + "{failure} (automated BIOS recovery exhausted after {} retries)", machine_controller_config.max_bios_config_retries ), }); diff --git a/crates/machine-controller/src/handler/host_boot_config.rs b/crates/machine-controller/src/handler/host_boot_config.rs new file mode 100644 index 0000000000..076f6a1527 --- /dev/null +++ b/crates/machine-controller/src/handler/host_boot_config.rs @@ -0,0 +1,542 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Shared host boot-configuration convergence. +//! +//! The lifecycle state machines retain ownership of their persisted state +//! shapes. [`HostBootConfigStage`] is only a runtime projection of those +//! states, so this module can drive the common BIOS/job/poll/boot-order +//! sequence without introducing another serialized controller wrapper. +//! +//! [`run_host_boot_config_stage`] owns the lifecycle-neutral Redfish work. Its +//! four lifecycle adapters stay beside the state machines that own their +//! persisted state and terminal behavior: +//! - `handle_host_init_boot_config_stage` in `handler.rs`; +//! - `handle_instance_host_boot_config_stage` in `handler.rs`; +//! - `handle_dpu_reprovision_host_boot_config_stage` in `handler.rs`; +//! - `handle_validation_boot_config_stage` in `machine_validation.rs`. + +use carbide_redfish::boot_interface::BootInterfaceTarget; +use carbide_redfish::libredfish::error::state_handler_redfish_error as redfish_error; +use libredfish::Redfish; +use model::machine::{ + BiosConfigInfo, ManagedHostStateSnapshot, SetBootOrderInfo, SetBootOrderState, +}; +use state_controller::state_handler::{StateHandlerContext, StateHandlerError}; + +use super::bios_config::{ + BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, + advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, +}; +use super::{ + ReachabilityParams, RequiredBootInterface, SetBootOrderOutcome, + are_dpus_up_trigger_reboot_if_needed, is_dpu_observed_since, load_boot_predictions, + log_host_config, require_boot_interface, set_host_boot_order, trigger_reboot_if_needed, +}; +use crate::context::MachineStateHandlerContextObjects; + +/// Runtime projection of the persisted BIOS and boot-order states shared by +/// HostInit, assigned platform configuration, DPU reprovision, and validation. +/// +/// This type must remain controller-internal: the owning lifecycle maps each +/// `Continue` outcome back to its existing serialized state variant. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigStage { + ConfigureBios { + retry_count: u32, + }, + WaitingForBiosJob { + bios_config_info: BiosConfigInfo, + }, + PollingBiosSetup { + retry_count: u32, + }, + SetBootOrder { + set_boot_order_info: SetBootOrderInfo, + }, +} + +/// Result of running one common boot-configuration stage. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigOutcome { + Continue(HostBootConfigStage), + Complete, + Wait(String), + Failed { failure: String }, +} + +/// The next remediation required after inspecting the desired and actual host +/// boot configuration. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigDecision { + /// The platform BIOS settings (including the HTTP boot device) drifted. + ConfigureBios, + /// BIOS is already correct; only boot order drifted. + SetBootOrder, + /// BIOS and every NICo-managed boot-order setting are already correct. + Complete, +} + +/// Result of pacing prerequisites and inspecting the host boot config. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigCheckOutcome { + Ready(HostBootConfigDecision), + /// The host cannot be inspected yet, for example while discovering a + /// zero-DPU host's boot NIC or waiting for a fresh DPU observation. + Wait(String), +} + +/// DPU observation freshness required before trusting Redfish boot state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HostBootConfigDpuFreshness { + /// The caller already gated on healthy, synchronized DPUs. + AlreadyValidated, + /// Require observations associated with the current host lifecycle state. + CurrentHostState, + /// Require observations newer than the most recent host reboot request. + SinceLastHostRebootRequest, +} + +/// Actual host boot settings read from Redfish. +/// +/// `boot_order_setup` is `None` when BIOS drift already determines the next +/// action, or when boot order is intentionally not managed on this hardware. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) struct HostBootConfigInspection { + pub is_bios_setup: bool, + pub is_boot_order_setup: Option, +} + +/// Convert an inspection into the smallest remediation that can converge it. +/// BIOS repair takes precedence because it may recreate the boot option whose +/// order would otherwise be inspected or changed. +pub(super) fn decide_host_boot_config( + inspection: HostBootConfigInspection, +) -> HostBootConfigDecision { + if !inspection.is_bios_setup { + HostBootConfigDecision::ConfigureBios + } else if inspection.is_boot_order_setup == Some(false) { + HostBootConfigDecision::SetBootOrder + } else { + HostBootConfigDecision::Complete + } +} + +/// Inspect a host whose boot interface has already been resolved. +/// +/// Keeping interface resolution outside this primitive lets validation retain +/// its existing distinction between a missing interface (hard error), an +/// undiscovered zero-DPU NIC (pace the reboot), and a Redfish read error (log +/// and retry). +pub(super) async fn inspect_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + boot_interface: &BootInterfaceTarget, +) -> Result { + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup || should_skip_boot_order_remediation(mh_snapshot) { + return Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: None, + }); + } + + let is_boot_order_setup = boot_interface + .run(|bi| redfish_client.is_boot_order_setup(bi)) + .await + .map_err(|e| redfish_error("is_boot_order_setup", e))?; + + Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: Some(is_boot_order_setup), + }) +} + +/// Check whether host BIOS or boot order needs remediation. +pub(super) async fn check_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result { + if should_wait_for_dpus_before_host_boot_config( + mh_snapshot, + reachability_params, + dpu_freshness, + ctx, + ) + .await + { + return Ok(HostBootConfigCheckOutcome::Wait( + "Waiting for DPUs to come up.".to_string(), + )); + } + + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "configuring boot", + HostBootConfigCheckOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), + }; + + log_host_config(redfish_client, mh_snapshot).await; + + let inspection = inspect_host_boot_config(redfish_client, mh_snapshot, &boot_interface).await?; + let decision = decide_host_boot_config(inspection); + let vendor = mh_snapshot.host_snapshot.bmc_vendor(); + + match &decision { + HostBootConfigDecision::ConfigureBios => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is not configured properly" + ), + HostBootConfigDecision::SetBootOrder => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is correct but boot order is not configured properly" + ), + HostBootConfigDecision::Complete if should_skip_boot_order_remediation(mh_snapshot) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is configured; skipping boot order remediation on Viking" + ); + } + HostBootConfigDecision::Complete => tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup and boot order are configured properly" + ), + } + + Ok(HostBootConfigCheckOutcome::Ready(decision)) +} + +/// Runs one lifecycle-neutral host boot-configuration stage. +/// +/// Called by the host-init, assigned-instance, DPU-reprovision, and validation +/// adapters. This function performs the Redfish BIOS, vendor-job, polling, or +/// boot-order work for one [`HostBootConfigStage`] and returns a +/// [`HostBootConfigOutcome`]. The caller owns the persisted lifecycle state and +/// maps the outcome back into that state. +pub(super) async fn run_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + stage: HostBootConfigStage, +) -> Result { + match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + match configure_host_bios( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + retry_count, + ) + .await? + { + BiosConfigOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigOutcome::WaitingForReboot(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + let retry_count = bios_config_info.retry_count; + match advance_bios_config_job(ctx, redfish_client, mh_snapshot, bios_config_info) + .await? + { + BiosConfigJobAdvanceOutcome::Continue(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigJobAdvanceOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigJobAdvanceOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + BiosConfigJobAdvanceOutcome::Wait(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + match advance_polling_bios_setup( + redfish_client, + mh_snapshot, + retry_count, + &ctx.services.site_config.machine_state_controller, + &predictions, + ) + .await? + { + PollingBiosSetupOutcome::Verified => { + if should_skip_boot_order_remediation(mh_snapshot) { + Ok(HostBootConfigOutcome::Complete) + } else { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info(retry_count), + }, + )) + } + } + PollingBiosSetupOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + PollingBiosSetupOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => { + let retry_count = set_boot_order_info.retry_count; + match set_host_boot_order( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + set_boot_order_info, + ) + .await? + { + SetBootOrderOutcome::Continue(set_boot_order_info) => Ok( + HostBootConfigOutcome::Continue(HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }), + ), + SetBootOrderOutcome::ConfigureBios => { + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; + let Some(retry_count) = next_boot_config_retry_count(retry_count, max_retries) + else { + return Ok(HostBootConfigOutcome::Failed { + failure: format!( + "BIOS settings repeatedly drifted during boot-order repair; automated boot-config convergence exhausted after {max_retries} retries" + ), + }); + }; + + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + SetBootOrderOutcome::Done => Ok(HostBootConfigOutcome::Complete), + SetBootOrderOutcome::WaitingForReboot(reason) + | SetBootOrderOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + } + } + } +} + +pub(super) fn initial_set_boot_order_info() -> SetBootOrderInfo { + set_boot_order_info(0) +} + +fn set_boot_order_info(retry_count: u32) -> SetBootOrderInfo { + SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count, + } +} + +/// Carry one bounded convergence budget when boot-order work discovers that +/// BIOS must be repaired again. Existing owner states already persist this +/// counter through both phases, so no serialized wrapper is required. +fn next_boot_config_retry_count(retry_count: u32, max_retries: u32) -> Option { + (retry_count < max_retries).then(|| retry_count + 1) +} + +/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair +/// still applies. +pub(super) fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { + mh_snapshot + .host_snapshot + .hardware_info + .as_ref() + .is_some_and(|hw| hw.is_dgx_h100()) +} + +async fn should_wait_for_dpus_before_host_boot_config( + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> bool { + if !mh_snapshot.has_managed_dpus() { + return false; + } + + match dpu_freshness { + HostBootConfigDpuFreshness::AlreadyValidated => false, + HostBootConfigDpuFreshness::CurrentHostState => { + !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await + } + HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { + let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested + else { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "No host reboot request timestamp found before post-reboot host boot config check" + ); + return false; + }; + + for dpu_snapshot in &mh_snapshot.dpu_snapshots { + if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { + match trigger_reboot_if_needed( + dpu_snapshot, + mh_snapshot, + None, + reachability_params, + ctx, + ) + .await + { + Ok(_) => {} + Err(e) => tracing::warn!( + dpu_id = %dpu_snapshot.id, + error = %e, + "Could not reboot DPU while waiting to check host boot config" + ), + } + return true; + } + } + + false + } + } +} + +#[cfg(test)] +mod tests { + use carbide_test_support::value_scenarios; + + use super::*; + + #[test] + fn boot_config_decision_chooses_the_smallest_remediation() { + value_scenarios!(decide_host_boot_config: + "BIOS drift takes precedence" { + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: None, + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::ConfigureBios, + } + + "configured BIOS chooses the narrowest remaining action" { + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::SetBootOrder, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::Complete, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: None, + } => HostBootConfigDecision::Complete, + } + ); + } + + #[test] + fn boot_config_retry_count_is_bounded_across_bios_and_order() { + struct RetryCount { + retry_count: u32, + max_retries: u32, + } + + value_scenarios!( + run = |RetryCount { + retry_count, + max_retries, + }: RetryCount| next_boot_config_retry_count(retry_count, max_retries); + "retry budget remains" { + RetryCount { + retry_count: 0, + max_retries: 1, + } => Some(1), + RetryCount { + retry_count: 4, + max_retries: 5, + } => Some(5), + } + + "retry budget is unavailable" { + RetryCount { + retry_count: 1, + max_retries: 1, + } => None, + RetryCount { + retry_count: 0, + max_retries: 0, + } => None, + RetryCount { + retry_count: 4, + max_retries: 1, + } => None, + } + ); + } +} diff --git a/crates/machine-controller/src/handler/machine_validation.rs b/crates/machine-controller/src/handler/machine_validation.rs index 7950f844cd..d29cfab079 100644 --- a/crates/machine-controller/src/handler/machine_validation.rs +++ b/crates/machine-controller/src/handler/machine_validation.rs @@ -15,10 +15,11 @@ * limitations under the License. */ use carbide_uuid::machine_validation::MachineValidationId; +use chrono::Utc; use libredfish::{EnabledDisabled, RedfishError, SystemPowerControl}; use model::machine::{ - FailureCause, MachineState, MachineValidatingState, ManagedHostState, ManagedHostStateSnapshot, - SetBootOrderInfo, SetBootOrderState, UnlockHostState, ValidationState, + FailureCause, FailureDetails, FailureSource, MachineState, MachineValidatingState, + ManagedHostState, ManagedHostStateSnapshot, StateMachineArea, UnlockHostState, ValidationState, }; use model::machine_validation::{MachineValidationState, MachineValidationStatus}; use state_controller::state_handler::{ @@ -27,10 +28,14 @@ use state_controller::state_handler::{ use super::{HostHandlerParams, is_machine_validation_requested, machine_validation_completed}; use crate::context::{MachineStateHandlerContextObjects, MachineStateHandlerServices}; +use crate::handler::host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, decide_host_boot_config, + initial_set_boot_order_info, inspect_host_boot_config, run_host_boot_config_stage, +}; use crate::handler::{ - RequiredBootInterface, SetBootOrderOutcome, handler_host_power_control, host_power_control, - load_boot_predictions, rebooted, redfish_error, require_boot_interface, set_host_boot_order, - trigger_reboot_if_needed, wait, + RequiredBootInterface, handler_host_power_control, host_power_control, load_boot_predictions, + rebooted, redfish_error, require_boot_interface, trigger_reboot_if_needed, wait, }; /// The validation flavor of the managed-host state, so the repair arms can @@ -41,14 +46,103 @@ fn validating(machine_validation: MachineValidatingState) -> ManagedHostState { } } -/// The starting point for the boot-order flow when validation boot repair -/// re-drives it: the flow itself checks what is already in place and only -/// re-applies what the reboot reverted. -fn boot_repair_start() -> SetBootOrderInfo { - SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, +/// Handles a shared boot-configuration stage for machine validation. +/// +/// Called by `handle_machine_validation_state` for its BIOS configuration, +/// vendor-job, polling, and boot-order repair substates. This adapter preserves +/// the validation ID, maps continued stages back into `MachineValidatingState`, +/// continues to `LockAfterBootRepair` on completion, and closes the validation +/// run before reporting a terminal boot-configuration failure. +async fn handle_validation_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + host_handler_params: &HostHandlerParams, + mh_snapshot: &ManagedHostStateSnapshot, + validation_id: MachineValidationId, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + &host_handler_params.reachability_params, + redfish_client.as_ref(), + mh_snapshot, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_validation = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition(validating( + machine_validation, + ))) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::LockAfterBootRepair { validation_id }, + ))), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + let machine_id = mh_snapshot.host_snapshot.id; + let mut txn = ctx.services.db_pool.begin().await?; + let completed = db::machine_validation::mark_machine_validation_complete( + txn.as_mut(), + &machine_id, + &validation_id, + MachineValidationStatus { + state: MachineValidationState::Failed, + ..MachineValidationStatus::default() + }, + ) + .await?; + + if !completed { + tracing::info!( + %machine_id, + %validation_id, + "Machine validation boot-config failure observed after the run was already terminal" + ); + } + + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + }, + machine_id, + retry_count: 0, + }) + .with_txn(txn)) + } } } @@ -162,11 +256,20 @@ pub(crate) async fn handle_machine_validation_state( .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await + match inspect_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &boot_interface, + ) + .await { - Ok(false) => { + Ok(inspection) + if matches!( + decide_host_boot_config(inspection), + HostBootConfigDecision::ConfigureBios + | HostBootConfigDecision::SetBootOrder + ) => + { tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, "Boot config reads reverted while waiting for the validation reboot; correcting it via boot repair", @@ -175,7 +278,7 @@ pub(crate) async fn handle_machine_validation_state( MachineValidatingState::PrepareBootRepair { validation_id: *id }, ))); } - Ok(true) => {} + Ok(_) => {} Err(e) => { tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, @@ -252,9 +355,8 @@ pub(crate) async fn handle_machine_validation_state( machine_id = %mh_snapshot.host_snapshot.id, "BMC vendor does not support checking lockdown status during validation boot repair", ); - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } Err(e) => { @@ -277,9 +379,8 @@ pub(crate) async fn handle_machine_validation_state( unlock_host_state: UnlockHostState::DisableLockdown, } } - Ok(_) => MachineValidatingState::RepairBootConfig { + Ok(_) => MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), }, }; @@ -322,9 +423,8 @@ pub(crate) async fn handle_machine_validation_state( unlock_host_state: UnlockHostState::RebootHost, } } else { - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } } @@ -359,51 +459,112 @@ pub(crate) async fn handle_machine_validation_state( mh_snapshot.host_snapshot.id ))); } - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } }; Ok(StateHandlerOutcome::transition(validating(next))) } - MachineValidatingState::RepairBootConfig { - validation_id, - set_boot_order_info, - } => { - // Re-drive the boot-order flow: it checks what is already in - // place, re-asserts the reverted HTTP-boot device (rebooting to - // apply it), sets the boot order, and verifies -- the same engine - // the boot-configuration stages use. + MachineValidatingState::CheckBootConfigForRepair { validation_id } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, + let next = match check_host_boot_config( redfish_client.as_ref(), mh_snapshot, - set_boot_order_info.clone(), + &host_handler_params.reachability_params, + HostBootConfigDpuFreshness::CurrentHostState, + ctx, ) .await? { - SetBootOrderOutcome::Continue(info) => Ok(StateHandlerOutcome::transition( - validating(MachineValidatingState::RepairBootConfig { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + MachineValidatingState::ConfigureBootBios { validation_id: *validation_id, - set_boot_order_info: info, - }), - )), - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition(validating( + retry_count: 0, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { MachineValidatingState::LockAfterBootRepair { validation_id: *validation_id, - }, - ))), - SetBootOrderOutcome::WaitingForReboot(reason) - | SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + } + } + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, + ) + .await + } + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, + ) + .await } MachineValidatingState::LockAfterBootRepair { validation_id } => { // Restore the lockdown that boot repair temporarily opened, then diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index e831a4d975..a80411c47b 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -266,6 +266,12 @@ impl StateControllerIO for MachineStateControllerIO { MachineValidatingState::RebootHost { .. } => "reboothost", MachineValidatingState::PrepareBootRepair { .. } => "preparebootrepair", MachineValidatingState::UnlockForBootRepair { .. } => "unlockforbootrepair", + MachineValidatingState::CheckBootConfigForRepair { .. } => { + "checkbootconfigforrepair" + } + MachineValidatingState::ConfigureBootBios { .. } => "configurebootbios", + MachineValidatingState::WaitingForBootBiosJob { .. } => "waitingforbootbiosjob", + MachineValidatingState::PollingBootBiosSetup { .. } => "pollingbootbiossetup", MachineValidatingState::RepairBootConfig { .. } => "repairbootconfig", MachineValidatingState::LockAfterBootRepair { .. } => "lockafterbootrepair", }