From 8e33e6c18f850785e9ffec234af94d431e3baacc Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Mon, 13 Jul 2026 15:05:31 -0700 Subject: [PATCH] refactor: share host boot configuration convergence Host boot configuration now runs through one controller path across host initialization, assigned platform setup, DPU reprovision, and validation. Each lifecycle keeps its own persisted state while sharing BIOS setup, vendor jobs, polling, boot order, and final verification. Primary callouts are: - `host_boot_config` selects the smallest repair and advances the shared flow without introducing a new serialized wrapper. - Order-only drift skips `machine_setup`, while late BIOS drift returns to the job-aware path and carries a bounded convergence budget. - Dell job IDs stay persisted through reboot and recovery; validation keeps its run identity and closes the run on terminal repair failure. - Viking and legacy persisted states keep their safe recovery behavior. Tests updated! This supports https://github.com/NVIDIA/infra-controller/issues/3391 Signed-off-by: Chet Nichols III --- Cargo.lock | 1 + crates/api-core/src/cfg/README.md | 2 +- .../src/tests/common/api_fixtures/mod.rs | 4 + .../api-core/src/tests/dpu_reprovisioning.rs | 190 ++- crates/api-core/src/tests/machine_states.rs | 680 +++++++-- crates/api-model/src/machine/mod.rs | 33 +- crates/machine-controller/Cargo.toml | 1 + .../src/config/controller.rs | 4 +- crates/machine-controller/src/handler.rs | 1233 ++++++----------- .../src/handler/bios_config.rs | 2 +- .../src/handler/host_boot_config.rs | 542 ++++++++ .../src/handler/machine_validation.rs | 259 +++- crates/machine-controller/src/io.rs | 6 + 13 files changed, 2027 insertions(+), 930 deletions(-) create mode 100644 crates/machine-controller/src/handler/host_boot_config.rs diff --git a/Cargo.lock b/Cargo.lock index 972be69c6d..05035dd040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2378,6 +2378,7 @@ dependencies = [ "carbide-secrets", "carbide-state-controller-common", "carbide-test-harness", + "carbide-test-support", "carbide-utils", "carbide-uuid", "chrono", diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index dd6800ea7e..41bbbe142d 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -347,7 +347,7 @@ Extends `StateControllerConfig` with: | `dpu_up_threshold` | `Duration` | `5m` | Max time without DPU health report before assuming it's down. | | `scout_reporting_timeout` | `Duration` | `5m` | Duration without scout report before host is unhealthy. | | `uefi_boot_wait` | `Duration` | `5m` | Wait time for UEFI boot completion after host reboot. | -| `max_bios_config_retries` | `u32` | `3` | Max HandleBiosJobFailure recovery cycles during BIOS configuration. | +| `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. | | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. | | `controller` | `StateControllerConfig` | *(default)* | Common state controller timing (see [StateControllerConfig](#statecontrollerconfig)). | diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 5a04a00f5f..18ff8e3b69 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -476,6 +476,10 @@ impl TestEnv { MachineValidatingState::RebootHost { .. } => state.clone(), MachineValidatingState::PrepareBootRepair { .. } | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } | MachineValidatingState::RepairBootConfig { .. } | MachineValidatingState::LockAfterBootRepair { .. } => state.clone(), } diff --git a/crates/api-core/src/tests/dpu_reprovisioning.rs b/crates/api-core/src/tests/dpu_reprovisioning.rs index 6871251981..d59e7b12ce 100644 --- a/crates/api-core/src/tests/dpu_reprovisioning.rs +++ b/crates/api-core/src/tests/dpu_reprovisioning.rs @@ -28,8 +28,8 @@ use libredfish::{EnabledDisabled, SystemPowerControl}; use model::instance::status::tenant::TenantState; use model::machine::{ DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState, - Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, ReprovisionState, - SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, + Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, + ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, }; use model::test_support::HardwareInfoTemplate; use rpc::forge::MachineArchitecture; @@ -86,8 +86,6 @@ fn reprovision_host_boot_repair_states( unlock_host_state: UnlockHostState::DisableLockdown, }, ReprovisionState::CheckHostBootConfig, - ReprovisionState::ConfigureHostBoot { retry_count: 0 }, - ReprovisionState::PollingHostBiosSetup { retry_count: 0 }, reprovision_set_host_boot_order_state(SetBootOrderState::SetBootOrder), reprovision_set_host_boot_order_state(SetBootOrderState::WaitForSetBootOrderJobScheduled), reprovision_set_host_boot_order_state(SetBootOrderState::RebootHost), @@ -137,6 +135,7 @@ async fn assert_dpu_reprovision_host_boot_repair( expected_states: Vec, ) -> Machine { env.redfish_sim.set_lockdown(EnabledDisabled::Enabled); + env.redfish_sim.set_is_bios_setup(true); env.redfish_sim.set_is_boot_order_setup(false); let redfish_timepoint = env.redfish_sim.timepoint(); @@ -179,27 +178,56 @@ async fn assert_dpu_reprovision_host_boot_repair( } } - // machine_setup enables the bootable DPU interface before boot-order promotion. let actions = env .redfish_sim .actions_since(&redfish_timepoint) .all_hosts(); - let machine_setup_pos = actions - .iter() - .position(|action| matches!(action, RedfishSimAction::MachineSetup { .. })) - .expect("expected DPU reprovision boot repair to call machine_setup"); - let set_boot_order_pos = actions + let (set_boot_order_pos, set_boot_order_mac) = actions .iter() - .position(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })) + .enumerate() + .find_map(|(position, action)| match action { + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) .expect("expected DPU reprovision boot repair to set DPU-first boot order"); - let check_boot_order_pos = actions + let boot_order_checks = actions .iter() - .rposition(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })) - .expect("expected DPU reprovision boot repair to verify boot order"); + .enumerate() + .filter_map(|(position, action)| match action { + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) + .collect::>(); assert!( - machine_setup_pos < set_boot_order_pos && set_boot_order_pos < check_boot_order_pos, - "expected machine_setup before set_boot_order_dpu_first before is_boot_order_setup; got: {actions:?}" + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "expected order-only DPU reprovision boot repair to preserve the configured HTTP boot device; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .filter(|(position, _)| *position < set_boot_order_pos) + .count() + >= 2, + "expected both CheckHostBootConfig and SetBootOrder to inspect the order before writing it; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .any(|(position, _)| *position > set_boot_order_pos), + "expected boot-order verification after the write; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .all(|(_, boot_interface_mac)| *boot_interface_mac == set_boot_order_mac), + "expected every order check and write to target the same interface; got: {actions:?}" ); let rebooting_machine = machine.next_iteration_machine(env).await; @@ -564,6 +592,136 @@ async fn test_dpu_reprovision_viking_skips_boot_order_when_bios_setup(pool: sqlx ); } +#[crate::sqlx_test] +async fn test_dpu_reprovision_viking_finishes_parked_boot_order_recovery(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let mh = create_managed_host_with_hardware_info_template( + &env, + HardwareInfoTemplate::Custom(DGX_H100_INFO_JSON), + ) + .await; + let dpu_machine = prepare_dpu_reprovision_host_boot_check(&env, &mh).await; + + // A controller upgrade can find a Viking in a persisted recovery substate + // created before boot-order remediation was disabled for this platform. + // Finish restoring host power before taking the safe terminal shortcut. + let parked_recovery = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::Off, + }, + )); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &parked_recovery) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &parked_recovery); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::ForceOff)], + "Viking policy must not abandon an in-flight power recovery" + ); + + let recovering_power_on = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::On, + }, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::BmcReset], + "parked recovery should reset the BMC after the host powers off" + ); + + // ForceOff records last_reboot_requested; backdate it so power_down_wait + // elapses in-process before the controller restores host power. + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + update_time_params(&env.pool, &host, 1, None).await; + } + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::On)], + "parked recovery should restore host power after the BMC reset" + ); + + let safe_terminal = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::CheckBootOrder, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &safe_terminal); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "completed recovery should naturally reach CheckBootOrder" + ); + + // CheckBootOrder has no unfinished operation behind it, but its preceding + // reboot may have reverted the HTTP boot device. Repair that BIOS drift + // without touching the unsupported Viking boot-order APIs, and spend one + // attempt from the shared convergence budget. + env.redfish_sim.set_is_bios_setup(false); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::ConfigureHostBoot { retry_count: 1 }) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "Viking BIOS recovery must not read or write boot order" + ); + + // Once BIOS is intact, the same safe terminal skips boot-order + // verification and completes the repair. + env.redfish_sim.set_is_bios_setup(true); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &safe_terminal) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::LockHostAfterBootRepair) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "safe Viking completion should not touch Redfish boot order" + ); +} + #[crate::sqlx_test] async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgPool) { let env = create_test_env(pool).await; diff --git a/crates/api-core/src/tests/machine_states.rs b/crates/api-core/src/tests/machine_states.rs index d946a6f041..7a6044cdd2 100644 --- a/crates/api-core/src/tests/machine_states.rs +++ b/crates/api-core/src/tests/machine_states.rs @@ -63,6 +63,7 @@ use model::machine::{ MachineValidatingState, ManagedHostState, MeasuringState, PowerState, SetBootOrderInfo, SetBootOrderState, SetSecureBootState, SpdmMeasuringState, StateMachineArea, ValidationState, }; +use model::machine_validation::MachineValidationState; use model::network_segment::NetworkSegmentType; use model::site_explorer::{EndpointExplorationReport, ExploredDpu, ExploredManagedHost}; use model::test_support::{DpuConfig, ManagedHostConfig}; @@ -2605,6 +2606,77 @@ async fn test_hpc_polling_bios_setup_stuck_enters_handle_bios_job_failure(pool: ); } +/// Assigned platform configuration should repair boot-order-only drift without +/// rerunning broad BIOS setup. +#[crate::sqlx_test] +async fn test_hpc_order_only_drift_skips_bios_configuration(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = common::api_fixtures::create_managed_host(&env).await; + let segment_id = env.create_vpc_and_tenant_segment().await; + create_instance(&env, &mh, false, segment_id).await; + let host_id = mh.host().id; + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::CheckHostConfig, + }, + }, + 0, + ) + .await; + // CheckHostConfig requires a DPU observation associated with this host + // state before Redfish results are trusted. + mh.network_configured(&env).await; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + let checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + .. + }, + }, + }, + } + ), + "order-only drift should route directly to SetBootOrder, got: {:?}", + host.current_state() + ); + } + + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .any(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })), + "assigned platform configuration should restore the drifted order, got: {actions:?}" + ); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "order-only drift should not rerun machine_setup, got: {actions:?}" + ); +} + /// Stuck HostInit/PollingBiosSetup recovery re-runs machine_setup and reaches HostInit/SetBootOrder. #[crate::sqlx_test] async fn test_polling_bios_setup_full_recovery_reruns_machine_setup_and_succeeds( @@ -2707,25 +2779,27 @@ async fn test_polling_bios_setup_full_recovery_reruns_machine_setup_and_succeeds /// so a zero-DPU host's in-band NIC takes a real `machine_interfaces` row that /// the boot-order phase can resolve. Mirrors `test_dhcp_allows_zero_dpu_host`. async fn create_zero_dpu_test_env(pool: sqlx::PgPool) -> TestEnv { - let env = create_test_env_with_overrides( - pool, - TestEnvOverrides { - site_prefixes: Some(vec![ - IpNetwork::new( - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - IpNetwork::new( - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - ]), - ..Default::default() - }, - ) - .await; + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::default()).await +} + +async fn create_zero_dpu_test_env_with_overrides( + pool: sqlx::PgPool, + mut overrides: TestEnvOverrides, +) -> TestEnv { + overrides.site_prefixes = Some(vec![ + IpNetwork::new( + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + IpNetwork::new( + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + ]); + + let env = create_test_env_with_overrides(pool, overrides).await; create_host_inband_network_segment(&env.api, None).await; env } @@ -2937,14 +3011,265 @@ async fn test_set_boot_order_sets_order_without_reasserting_when_device_configur ); } -/// Recovery path: when the HTTP-boot device is reverted (the boot NIC dropped off -/// the BMC's Redfish inventory on a reboot, so `is_bios_setup` reads false), -/// SetBootOrder re-asserts `machine_setup` and reboots to apply it *before* the -/// boot-order set -- the two BIOS writes never share one pass (they can't share -/// one Dell config job). Once the device verifies again, the order is set and the -/// host advances. +/// The reboot that applies a boot-order job can independently revert a managed +/// BIOS setting. Final verification checks BIOS before order, routes back to +/// the shared job-aware stages, and carries the existing convergence budget. +#[crate::sqlx_test] +async fn test_check_boot_order_repairs_bios_drift_after_order_reboot(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 2, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 3 }, + } + ), + "post-order BIOS drift should carry the convergence budget into shared BIOS repair, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "BIOS drift should short-circuit final boot-order verification, got: {actions:?}" + ); +} + +/// Repeated BIOS drift across boot-order reboots consumes the carried budget +/// instead of cycling indefinitely through machine_setup and another reboot. +#[crate::sqlx_test] +async fn test_check_boot_order_bios_drift_exhausts_convergence_budget(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + .. + }, + .. + } + ), + "expected repeated cross-phase drift to exhaust into Failed, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "an exhausted convergence budget must not issue another BIOS write, got: {actions:?}" + ); +} + +/// Boot-order verification uses the same configured convergence budget as +/// BIOS recovery, then remains able to observe an operator's repair while +/// parked at that limit. +#[crate::sqlx_test] +async fn test_check_boot_order_respects_configured_convergence_budget(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 1; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + .. + }), + }, + } + ), + "an exhausted configured budget must not advance to another boot-order attempt, got: {:?}", + host.current_state() + ); + assert!( + matches!( + host.controller_state_outcome.as_ref(), + Some(PersistentStateHandlerOutcome::Error { err, .. }) + if err.contains("Manual intervention required") + && err.contains("max_retries: 1") + ), + "expected boot-order exhaustion to report the configured limit, got: {:?}", + host.controller_state_outcome + ); + } + + env.redfish_sim.set_is_boot_order_setup(true); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, + } + ), + "a manually repaired boot order should complete from the parked verification state, got: {:?}", + host.current_state() + ); +} + +/// A configured budget above the historical fixed ceiling permits the shared +/// boot-config flow to spend the additional retries. +#[crate::sqlx_test] +async fn test_check_boot_order_allows_configured_budget_above_legacy_limit(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 5; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: 4, + .. + }), + }, + } + ), + "a configured budget above three should permit another boot-order attempt, got: {:?}", + host.current_state() + ); +} + +/// If the HTTP-boot device reverts after inspection but before SetBootOrder, +/// the order actuator routes back through the shared BIOS driver. In +/// particular, a Dell-style job returned by machine_setup is persisted and +/// completed before the order write. #[crate::sqlx_test] -async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx::PgPool) { +async fn test_set_boot_order_routes_reverted_bios_through_shared_job_flow(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -2956,12 +3281,22 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx // while the device is gone. env.redfish_sim.set_is_bios_setup(false); env.redfish_sim.set_is_boot_order_setup(false); + // SetBootOrder is reached only after HostInit has opened lockdown for BIOS + // writes; model that precondition when planting this synthetic state. + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_LATE_BOOT_DRIFT".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); set_host_stuck_in_set_boot_order(&env, host_id).await; let redfish_timepoint = env.redfish_sim.timepoint(); - // First pass: re-assert the device and reboot to apply it. It must NOT set the - // boot order in the same pass -- two BIOS writes can't share one config job. + // First pass only detects the late drift and moves back to the shared BIOS + // stage. It must not issue either BIOS write from the order actuator. env.run_machine_state_controller_iteration().await; let first_pass = env .redfish_sim @@ -2970,8 +3305,8 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert!( first_pass .iter() - .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), - "a reverted HTTP-boot device should be re-asserted at SetBootOrder, got: {first_pass:?}" + .all(|a| !matches!(a, RedfishSimAction::MachineSetup { .. })), + "SetBootOrder should delegate late BIOS drift to the shared driver, got: {first_pass:?}" ); assert!( !first_pass @@ -2980,9 +3315,6 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx "the boot order must not be set in the same pass as the re-assert (shared BIOS job), got: {first_pass:?}" ); - // The re-assert is committed once. While the device is still reverted (the - // apply reboot hasn't landed yet), the host polls in - // WaitForHttpBootDeviceApplied without re-running machine_setup. { let mut txn = env.db_txn().await; let host = mh.host().db_machine(&mut txn).await; @@ -2990,31 +3322,17 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - .. - }), - }, + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 1 }, } ), - "after the re-assert the host should wait for the device to apply, got: {:?}", + "late BIOS drift should return to shared configuration, got: {:?}", host.current_state() ); } - let after_reassert = env.redfish_sim.timepoint(); - env.run_machine_state_controller_iteration().await; - let second_pass = env.redfish_sim.actions_since(&after_reassert).all_hosts(); - assert!( - !second_pass - .iter() - .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), - "machine_setup must not re-run while waiting for the re-asserted device to apply, got: {second_pass:?}" - ); - // Within the wait window the host stays parked in - // WaitForHttpBootDeviceApplied -- it neither advances nor bounces back to - // SetBootOrder for another re-assert. + // The shared driver persists the returned job instead of discarding it and + // rebooting immediately. + env.run_machine_state_controller_iteration().await; { let mut txn = env.db_txn().await; let host = mh.host().db_machine(&mut txn).await; @@ -3022,20 +3340,21 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + machine_state: MachineState::WaitingForBiosJob { + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + retry_count: 1, .. - }), + }, }, - } + } if job_id == "JID_LATE_BOOT_DRIFT" ), - "the host should keep waiting for the device to apply within the wait window, got: {:?}", + "the late-drift BIOS job should be persisted, got: {:?}", host.current_state() ); } - // Model the reboot applying the re-asserted config: the device now verifies. + // Model the BIOS job applying the restored device. env.redfish_sim.set_is_bios_setup(true); // With the device restored, the host sets the boot order and advances. @@ -3053,14 +3372,12 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } -/// When the re-asserted HTTP-boot device does not verify within the wait -/// window (the boot NIC can drop off the BMC's Redfish inventory again on the -/// apply reboot itself), the host returns to `SetBootOrder` for a fresh -/// re-assert -- one `machine_setup` per window, not per pass. Once the state -/// machine's retry budget is exhausted it stops re-asserting and surfaces the -/// host for manual intervention. +/// Hosts persisted in the legacy HTTP-device wait state migrate to the shared +/// BIOS driver when their wait expires. The existing retry count becomes the +/// shared convergence budget, and an exhausted budget still stops for manual +/// intervention. #[crate::sqlx_test] -async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx::PgPool) { +async fn test_legacy_http_boot_wait_migrates_to_shared_bios_repair(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -3069,6 +3386,8 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: // The device stays reverted throughout: every verify reads false. env.redfish_sim.set_is_bios_setup(false); env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); let stuck_waiting = |retry_count: u32| ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { @@ -3081,8 +3400,8 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: }; // Plant the host mid-wait with the window already expired (backdated past - // the 10-minute apply window). Pass 1 returns it to SetBootOrder; pass 2 - // re-asserts once and parks it back in the wait substate. + // the 10-minute apply window). Pass 1 migrates its persisted state; pass 2 + // runs machine_setup through the common BIOS driver. set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(0), 11).await; let checkpoint = env.redfish_sim.timepoint(); env.run_machine_state_controller_iteration().await; @@ -3103,16 +3422,10 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - retry_count: 1, - .. - }), - }, + machine_state: MachineState::PollingBiosSetup { retry_count: 1 }, } ), - "the fresh re-assert should park the host back in the wait substate with the retry spent, got: {:?}", + "the legacy wait should carry its retry count into shared BIOS polling, got: {:?}", host.current_state() ); } @@ -3171,9 +3484,16 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool env.redfish_sim.set_is_boot_order_setup(false); env.redfish_sim .set_lockdown(libredfish::EnabledDisabled::Enabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_VALIDATION_BOOT_REPAIR".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); // Park the host mid-validation, waiting on a reboot that cannot land // (the state is newer than the host's last reported boot). + let validation_id = MachineValidationId::new(); set_host_controller_state_stuck_in( &env, host_id, @@ -3181,7 +3501,7 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool validation_state: ValidationState::MachineValidation { machine_validation: MachineValidatingState::MachineValidating { context: "Discovery".to_string(), - id: MachineValidationId::new(), + id: validation_id, completed: 1, total: 1, is_enabled: true, @@ -3221,6 +3541,29 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool .contains(&libredfish::EnabledDisabled::Disabled), "boot repair should unlock the BMC before writing BIOS settings" ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::WaitingForBootBiosJob { + validation_id: current_validation_id, + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + .. + }, + }, + }, + } if *current_validation_id == validation_id + && job_id == "JID_VALIDATION_BOOT_REPAIR" + ), + "validation should preserve the BIOS job before rebooting, got: {:?}", + host.current_state() + ); + } // The repair reboot applies the re-asserted device. env.redfish_sim.set_is_bios_setup(true); @@ -3236,9 +3579,11 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool host.current_state(), ManagedHostState::Validation { validation_state: ValidationState::MachineValidation { - machine_validation: MachineValidatingState::RebootHost { .. }, + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, }, - } + } if *current_validation_id == validation_id ) { resumed = true; break; @@ -3261,6 +3606,183 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } +/// Boot-order-only drift (the HTTP boot device is still configured, but its +/// priority changed) is detected while validation waits for a reboot. The +/// repair restores only the order, re-locks the BMC, and resumes validation. +#[crate::sqlx_test] +async fn test_machine_validation_repairs_drifted_boot_order(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // The HTTP boot device is intact; only the order drifted. Set the BIOS + // status explicitly so this regression does not rely on simulator defaults. + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + + let validation_id = MachineValidationId::new(); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::MachineValidating { + context: "Discovery".to_string(), + id: validation_id, + completed: 1, + total: 1, + is_enabled: true, + }, + }, + }, + 0, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + + let mut unlocked = false; + let mut resumed = false; + for _ in 0..20 { + env.run_machine_state_controller_iteration().await; + unlocked |= env + .redfish_sim + .lockdown_states() + .contains(&libredfish::EnabledDisabled::Disabled); + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + if matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, + }, + } if *current_validation_id == validation_id + ) { + resumed = true; + break; + } + } + + assert!( + resumed, + "order-only drift should be repaired before validation resumes" + ); + assert!( + unlocked, + "validation should unlock the BMC before repairing boot-order drift" + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "validation should check the boot order for the boot NIC, got: {actions:?}" + ); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "the drifted boot order should be restored for the boot NIC, got: {actions:?}" + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "order-only repair should preserve the configured HTTP boot device, got: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|l| *l == libredfish::EnabledDisabled::Enabled), + "the BMC should be re-locked once the boot order is repaired" + ); +} + +/// Exhausting the shared BIOS recovery budget during validation is terminal +/// for both the host repair and the active validation run. +#[crate::sqlx_test] +async fn test_validation_bios_setup_exhaustion_completes_active_validation(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = create_managed_host(&env).await; + let host_id = mh.host().id; + let validation_id = + on_demand_machine_validation(&env, host_id, Vec::new(), Vec::new(), false, Vec::new()) + .await + .validation_id + .expect("on-demand validation should return an id"); + + env.redfish_sim.set_is_bios_setup(false); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count: 3, + }, + }, + }, + 16, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + .. + }, + .. + } + ), + "expected validation boot repair exhaustion to fail the host, got: {:?}", + host.current_state() + ); + assert_eq!( + host.on_demand_machine_validation_request, + Some(false), + "terminal boot repair should clear the validation request" + ); + + let validation = db::machine_validation::find_by_id(&env.pool, &validation_id) + .await + .expect("validation run should still be queryable"); + assert!( + matches!( + validation.status.as_ref().map(|status| &status.state), + Some(&MachineValidationState::Failed) + ), + "validation run should be marked failed, got: {:?}", + validation.status + ); + assert!( + validation.end_time.is_some(), + "terminal validation run should record an end time" + ); +} + /// When HostInit/PollingBiosSetup retry budget is exhausted, enter Failed and recover via is_bios_setup. #[crate::sqlx_test] async fn test_polling_bios_setup_exhausted_enters_failed_and_recovers_when_bios_setup_true( diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 3eb8e8dbc5..d70973297e 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1308,6 +1308,23 @@ pub enum MachineValidatingState { validation_id: MachineValidationId, unlock_host_state: UnlockHostState, }, + CheckBootConfigForRepair { + validation_id: MachineValidationId, + }, + ConfigureBootBios { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, + WaitingForBootBiosJob { + validation_id: MachineValidationId, + bios_config_info: BiosConfigInfo, + }, + PollingBootBiosSetup { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, RepairBootConfig { validation_id: MachineValidationId, set_boot_order_info: SetBootOrderInfo, @@ -1907,7 +1924,8 @@ pub struct BiosConfigInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub bios_job_id: Option, pub bios_config_state: BiosConfigState, - /// Full configure_host_bios retry count across HandleBiosJobFailure recovery cycles. + /// Shared host boot-configuration convergence retry count, including BIOS + /// job-failure recovery cycles. #[serde(default)] pub retry_count: u32, } @@ -1933,7 +1951,8 @@ pub struct SetBootOrderInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub set_boot_order_jid: Option, pub set_boot_order_state: SetBootOrderState, - /// Retry counter for SetBootOrder state machine. Defaults to 0 for backwards compatibility. + /// Shared host boot-configuration convergence retry count. Defaults to 0 + /// for backwards compatibility. #[serde(default)] pub retry_count: u32, } @@ -1943,9 +1962,9 @@ pub struct SetBootOrderInfo { #[serde(tag = "state", rename_all = "lowercase")] pub enum SetBootOrderState { SetBootOrder, - /// A reverted HTTP-boot device was re-asserted (`machine_setup`) and the - /// host restarted to apply it; polls the device across that reboot, then - /// returns to `SetBootOrder` to set the boot order. + /// Legacy persisted state from the former inline HTTP-device repair flow. + /// It polls the device across the already-started reboot, then either + /// resumes `SetBootOrder` or migrates to the shared BIOS repair stages. WaitForHttpBootDeviceApplied, WaitForSetBootOrderJobScheduled, RebootHost, @@ -2695,6 +2714,10 @@ pub fn state_sla( } MachineValidatingState::PrepareBootRepair { .. } | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } | MachineValidatingState::RepairBootConfig { .. } | MachineValidatingState::LockAfterBootRepair { .. } => { StateSla::with_sla(slas::VALIDATION, time_in_state) diff --git a/crates/machine-controller/Cargo.toml b/crates/machine-controller/Cargo.toml index 57ba2d66ad..2aba63d481 100644 --- a/crates/machine-controller/Cargo.toml +++ b/crates/machine-controller/Cargo.toml @@ -77,6 +77,7 @@ version-compare = { workspace = true } carbide-instrument = { path = "../instrument", features = ["test-support"] } carbide-redfish = { path = "../redfish", features = ["test-support"] } carbide-test-harness = { path = "../test-harness" } +carbide-test-support = { path = "../test-support" } figment = { workspace = true, features = ["env", "test", "toml"] } regex = { workspace = true } diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index 76e964eaaf..4af4ecb2bf 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -70,7 +70,9 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub uefi_boot_wait: Duration, - /// Max configure_host_bios retry cycles through HandleBiosJobFailure recovery. + /// Retry budget for automated host boot-configuration convergence, shared + /// across BIOS recovery and boot-order verification. The field name is + /// retained for configuration compatibility. #[serde(default = "MachineStateControllerConfig::max_bios_config_retries_default")] pub max_bios_config_retries: u32, /// How long PollingBiosSetup may sit on Ok(false) before escalating into diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 3f01c4a6ce..efb8395ca5 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -119,21 +119,23 @@ mod bios_config; mod dpf; mod firmware_artifact; mod helpers; +mod host_boot_config; mod machine_validation; mod power; mod sku; #[cfg(test)] mod test_machine_setup; -use bios_config::{ - BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, - advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, - handle_bios_setup_failed_recovery, -}; +use bios_config::handle_bios_setup_failed_recovery; use helpers::{ DpuDiscoveringStateHelper, DpuInitStateHelper, ManagedHostStateHelper, NextState, ReprovisionStateHelper, all_equal, }; +use host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, + initial_set_boot_order_info, run_host_boot_config_stage, should_skip_boot_order_remediation, +}; use state_controller::db_write_batch::DbWriteBatch; use crate::config::{BomValidationConfig, PowerManagerOptions}; @@ -1472,11 +1474,7 @@ impl MachineStateHandler { FailureCause::BiosSetupFailed { .. } if machine_id.machine_type().is_host() => { let recovered = ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + set_boot_order_info: Some(initial_set_boot_order_info()), }, }; handle_bios_setup_failed_recovery(ctx, mh_snapshot, recovered).await @@ -3137,243 +3135,70 @@ async fn handle_dpu_reprovision( // WaitingForNetworkConfig already accepted the DPU observation. Do // not require a newer observation just because the host state // version advanced while entering host boot repair. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::AlreadyValidated, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::CheckHostBootConfigAfterHostReboot => { // This path rebooted the host after unlocking, so require a DPU // observation newer than that reboot before trusting boot checks. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::SinceLastHostRebootRequest, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::ConfigureHostBoot { retry_count } => { - // Run machine_setup only after the reprovisioned DPU is healthy; it - // may patch BIOS settings and trigger host-impacting recovery. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match configure_host_bios( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - *retry_count, + reachability_params, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: *retry_count, - }, - )?, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } ReprovisionState::WaitingForHostBiosJob { bios_config_info } => { - // Poll vendor BIOS jobs before verifying the setup and boot order. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match advance_bios_config_job( + handle_dpu_reprovision_host_boot_config_stage( ctx, - redfish_client.as_ref(), state, - bios_config_info.clone(), + reachability_params, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { - bios_config_info: updated, - }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: bios_config_info.retry_count, - }, - )?, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => Ok( - StateHandlerOutcome::transition(dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - )), - ), - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::ConfigureHostBoot { retry_count }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::PollingHostBiosSetup { retry_count } => { - // Verify machine_setup effects before promoting the DPU boot option. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let predictions = load_boot_predictions(ctx, &state.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_stage( + ctx, state, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - let next_state = if should_skip_boot_order_remediation(state) { - ReprovisionState::LockHostAfterBootRepair - } else { - ReprovisionState::SetHostBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - PollingBiosSetupOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( - dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - ), - )), - PollingBiosSetupOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::SetHostBootOrder { set_boot_order_info, } => { - // Promote the selected DPU boot option after machine_setup has enabled it. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match set_host_boot_order( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - set_boot_order_info.clone(), + reachability_params, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::SetHostBootOrder { - set_boot_order_info: boot_order_info, - }, - )?, - )) - } - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::LockHostAfterBootRepair, - )?, - )), - SetBootOrderOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::LockHostAfterBootRepair => { // Preserve expected-machine lockdown policy after temporarily @@ -3499,6 +3324,117 @@ async fn handle_dpu_reprovision( } } +/// Checks host boot configuration for DPU reprovisioning and maps the shared +/// decision back into the reprovision state while preserving all active DPU +/// targets. The caller selects the observation-freshness requirement because +/// only the post-unlock path has rebooted the host. +async fn handle_dpu_reprovision_host_boot_config_check( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + let next_state = match check_host_boot_config( + redfish_client.as_ref(), + state, + reachability_params, + dpu_freshness, + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + ReprovisionState::ConfigureHostBoot { retry_count: 0 } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + ReprovisionState::SetHostBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + ReprovisionState::LockHostAfterBootRepair + } + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, next_state)?, + )) +} + +/// Handles a host boot-configuration stage owned by DPU reprovisioning. +/// +/// Called by `handle_dpu_reprovision` for its host BIOS configuration, +/// vendor-job, polling, and boot-order substates. This adapter preserves the +/// active DPU reprovision targets, continues to `LockHostAfterBootRepair` on +/// completion, and attributes terminal failure to the lifecycle that owns the +/// reprovision. +async fn handle_dpu_reprovision_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + reachability_params, + redfish_client.as_ref(), + state, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let reprovision_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + ReprovisionState::ConfigureHostBoot { retry_count } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + ReprovisionState::WaitingForHostBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + ReprovisionState::PollingHostBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => ReprovisionState::SetHostBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, reprovision_state)?, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state( + state, + ReprovisionState::LockHostAfterBootRepair, + )?, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + dpu_reprovision_host_boot_failed_state( + &state.managed_state, + state.host_snapshot.id, + failure, + ), + )), + } +} + /// Build the correct failed state for host boot repair during DPU reprovision. fn dpu_reprovision_host_boot_failed_state( current_state: &ManagedHostState, @@ -3555,152 +3491,6 @@ async fn load_boot_predictions( Ok(predictions) } -/// Check whether host BIOS and DPU-first boot order remediation is required. -async fn check_host_boot_config( - redfish_client: &dyn Redfish, - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> Result { - // Wait for DPUs only when this caller needs a fresh observation. DPU - // reprovision already validated DPU health before entering host boot repair. - if should_wait_for_dpus_before_host_boot_config( - mh_snapshot, - reachability_params, - dpu_freshness, - ctx, - ) - .await - { - return Ok(HostBootConfigDecision::Wait( - "Waiting for DPUs to come up.".to_string(), - )); - } - - // Resolve the interface whose boot option should be first in host UEFI. A - // zero-DPU host whose boot NIC has not taken its first HostInband lease yet - // falls back to its predicted boot NIC, and only waits when even that is - // unavailable. - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - let boot_interface = match require_boot_interface( - mh_snapshot, - &predictions, - "configuring boot", - HostBootConfigDecision::Wait, - )? { - RequiredBootInterface::Ready(target) => target, - RequiredBootInterface::Wait(decision) => return Ok(decision), - }; - - let vendor = mh_snapshot.host_snapshot.bmc_vendor(); - - log_host_config(redfish_client, mh_snapshot).await; - - let is_bios_setup = boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await - .map_err(|e| redfish_error("is_bios_setup", e))?; - - if should_skip_boot_order_remediation(mh_snapshot) { - if is_bios_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Skipping boot order remediation on Viking (known FW/BMC issue)" - ); - return Ok(HostBootConfigDecision::LockHost); - } - - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup is not configured properly on Viking; running BIOS repair before skipping boot order remediation" - ); - return Ok(HostBootConfigDecision::ConfigureBoot); - } - - let is_boot_order_setup = boot_interface - .run(|bi| redfish_client.is_boot_order_setup(bi)) - .await - .map_err(|e| redfish_error("is_boot_order_setup", e))?; - - if is_bios_setup && is_boot_order_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup and boot order are configured properly" - ); - Ok(HostBootConfigDecision::LockHost) - } else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - is_bios_setup, - is_boot_order_setup, - "Host BIOS setup or boot order is not configured properly" - ); - Ok(HostBootConfigDecision::ConfigureBoot) - } -} - -/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair still applies. -fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { - mh_snapshot - .host_snapshot - .hardware_info - .as_ref() - .is_some_and(|hw| hw.is_dgx_h100()) -} - -async fn should_wait_for_dpus_before_host_boot_config( - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> bool { - if !mh_snapshot.has_managed_dpus() { - return false; - } - - match dpu_freshness { - HostBootConfigDpuFreshness::AlreadyValidated => false, - HostBootConfigDpuFreshness::CurrentHostState => { - !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await - } - HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { - let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested - else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - "No host reboot request timestamp found before post-reboot host boot config check" - ); - return false; - }; - - for dpu_snapshot in &mh_snapshot.dpu_snapshots { - if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { - match trigger_reboot_if_needed( - dpu_snapshot, - mh_snapshot, - None, - reachability_params, - ctx, - ) - .await - { - Ok(_) => {} - Err(e) => tracing::warn!("could not reboot dpu {}: {e}", dpu_snapshot.id), - } - return true; - } - } - - false - } - } -} - // Returns true if update_manager flagged this managed host as needing its firmware examined fn host_reprovisioning_requested(state: &ManagedHostStateSnapshot) -> bool { state.host_snapshot.host_reprovision_requested.is_some() @@ -4930,6 +4720,9 @@ pub struct RebootStatus { /// Outcome of set_host_boot_order function. enum SetBootOrderOutcome { Continue(SetBootOrderInfo), + /// BIOS drifted after the caller's inspection; return to the shared BIOS + /// configuration stages so any Redfish job ID is persisted and polled. + ConfigureBios, Done, WaitingForReboot(String), /// No boot interface to act on yet -- e.g. a zero-DPU host whose boot NIC @@ -4938,20 +4731,6 @@ enum SetBootOrderOutcome { Wait(String), } -/// Decision from checking whether host boot repair is still required. -enum HostBootConfigDecision { - ConfigureBoot, - LockHost, - Wait(String), -} - -/// DPU observation freshness required before checking host boot config. -enum HostBootConfigDpuFreshness { - AlreadyValidated, - CurrentHostState, - SinceLastHostRebootRequest, -} - /// Outcome of [`require_boot_interface`]: the resolved boot NIC, or the /// caller's wait outcome for a zero-DPU host still discovering its boot NIC. #[derive(Debug)] @@ -5409,63 +5188,63 @@ fn check_host_health_for_alerts(state: &ManagedHostStateSnapshot) -> Result<(), } } -async fn handle_host_boot_order_setup( +/// Handles a shared boot-configuration stage during host initialization. +/// +/// Called by `HostMachineStateHandler::handle_object_state` for its platform +/// configuration, vendor-job, polling, and boot-order states. This adapter maps +/// continued stages back into `MachineState`, proceeds to measurements on +/// completion, and reports terminal failure as a host-init BIOS setup failure. +async fn handle_host_init_boot_config_stage( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, - host_handler_params: HostHandlerParams, - mh_snapshot: &mut ManagedHostStateSnapshot, - set_boot_order_info: Option, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, ) -> Result, StateHandlerError> { - tracing::info!( - "Starting Boot Order Configuration for {}: {set_boot_order_info:#?}", - mh_snapshot.host_snapshot.id - ); - - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) - .await?; - - let next_state = match set_boot_order_info { - Some(info) => { - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, - redfish_client.as_ref(), - mh_snapshot, - info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(boot_order_info), - }, - }, - SetBootOrderOutcome::Done => ManagedHostState::HostInit { - machine_state: MachineState::Measuring { - measuring_state: MeasuringState::WaitingForMeasurements, - }, - }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineState::WaitingForPlatformConfiguration { retry_count } } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineState::WaitingForBiosJob { bios_config_info } } - } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineState::SetBootOrder { + set_boot_order_info: Some(set_boot_order_info), + }, + }; + Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { machine_state }, + )) } - None => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, }, - }, - }; - - Ok(StateHandlerOutcome::transition(next_state)) + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + }, + machine_id: mh_snapshot.host_snapshot.id, + retry_count: 0, + })) + } + } } /// TODO: we need to handle the case where the job is deleted for some reason @@ -5849,147 +5628,76 @@ impl StateHandler for HostMachineStateHandler { } } - match configure_host_bios( + handle_host_init_boot_config_stage( ctx, + mh_snapshot, &self.host_handler_params.reachability_params, redfish_client.as_ref(), - mh_snapshot, - *retry_count, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: *retry_count, - }, - }, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::WaitingForBiosJob { bios_config_info } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match advance_bios_config_job( + handle_host_init_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), - ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { - bios_config_info: updated, - }, - }), - ), - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - }, - }, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::WaitingForPlatformConfiguration { - retry_count, - }, - }, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } - } - MachineState::PollingBiosSetup { retry_count } => { - let next_state = ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), }, - }; - + ) + .await + } + MachineState::PollingBiosSetup { retry_count } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - let predictions = - load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_host_init_boot_config_stage( + ctx, mh_snapshot, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - Ok(StateHandlerOutcome::transition(next_state)) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - PollingBiosSetupOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - PollingBiosSetupOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::SetBootOrder { set_boot_order_info, - } => Ok(handle_host_boot_order_setup( - ctx, - self.host_handler_params.clone(), - mh_snapshot, - set_boot_order_info.clone(), - ) - .await?), + } => { + let Some(set_boot_order_info) = set_boot_order_info.clone() else { + return Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(initial_set_boot_order_info()), + }, + }, + )); + }; + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_host_init_boot_config_stage( + ctx, + mh_snapshot, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }, + ) + .await + } MachineState::Measuring { measuring_state } => { if !self.host_handler_params.attestation_enabled { return Ok(StateHandlerOutcome::transition( @@ -7274,11 +6982,7 @@ impl StateHandler for InstanceStateHandler { instance_state: InstanceState::HostPlatformConfiguration { platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, + set_boot_order_info: initial_set_boot_order_info(), }, }, }; @@ -10954,6 +10658,75 @@ async fn log_host_config(redfish_client: &dyn Redfish, mh_snapshot: &ManagedHost ); } +/// Handles a shared boot-configuration stage during assigned-host platform +/// configuration. +/// +/// Called by `handle_instance_host_platform_config` for its BIOS configuration, +/// vendor-job, polling, and boot-order states. This adapter maps continued +/// stages back into `HostPlatformConfigurationState`, proceeds to `LockHost` on +/// completion, and scopes terminal failure to the assigned instance. +async fn handle_instance_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let platform_config_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + HostPlatformConfigurationState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state, + }, + }, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + }, + }, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::AssignedInstance), + }, + machine_id: mh_snapshot.host_snapshot.id, + }, + }, + )), + } +} + async fn handle_instance_host_platform_config( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, mh_snapshot: &mut ManagedHostStateSnapshot, @@ -11182,18 +10955,29 @@ async fn handle_instance_host_platform_config( ) .await? { - HostBootConfigDecision::Wait(reason) => { + HostBootConfigCheckOutcome::Wait(reason) => { return Ok(StateHandlerOutcome::wait(reason)); } - HostBootConfigDecision::ConfigureBoot => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: 0, - }, - }, - HostBootConfigDecision::LockHost => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, - }, + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count: 0, + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + } + } } } HostPlatformConfigurationState::ConfigureBios { @@ -11214,174 +10998,48 @@ async fn handle_instance_host_platform_config( )); } - let next_platform = match configure_host_bios( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - retry_count, + HostBootConfigStage::ConfigureBios { retry_count }, ) - .await? - { - BiosConfigOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { retry_count } - } - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } - } - BiosConfigOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } => { - let next_platform = match advance_bios_config_job( + return handle_instance_host_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info: updated, - } - } - BiosConfigJobAdvanceOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - } - } - BiosConfigJobAdvanceOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { - retry_count: next_count, - } => HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: next_count, - }, - BiosConfigJobAdvanceOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::PollingBiosSetup { retry_count } => { - let next_instance_state = InstanceState::HostPlatformConfiguration { - platform_config_state: if should_skip_boot_order_remediation(mh_snapshot) { - HostPlatformConfigurationState::LockHost - } else { - HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }, - }; - - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + return handle_instance_host_boot_config_stage( + ctx, mh_snapshot, - retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { retry_count }, ) - .await? - { - PollingBiosSetupOutcome::Verified => next_instance_state, - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info, - }, - }, - }, - )); - } - PollingBiosSetupOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - PollingBiosSetupOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + .await; } HostPlatformConfigurationState::SetBootOrder { set_boot_order_info, } => { - match set_host_boot_order( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - set_boot_order_info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: boot_order_info, - }, - } - } - SetBootOrderOutcome::Done => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, + HostBootConfigStage::SetBootOrder { + set_boot_order_info, }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + ) + .await; } HostPlatformConfigurationState::LockHost => { if mh_snapshot.host_snapshot.host_profile.disable_lockdown { @@ -11442,51 +11100,36 @@ async fn set_host_boot_order( RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; - // Don't re-apply a boot config that's already in place. `SetBootOrder` - // re-asserts the HTTP-boot device (`machine_setup`) and then sets the - // boot order, but on Dell two BIOS config writes can't share one - // pending job: re-asserting commits a config job, and - // `set_boot_order_dpu_first` then can't set the order (iDRAC `SYS011`, - // "a config job is already committed"), so re-applying an - // already-correct config collides and loops. Check first -- if the - // config is already set, skip straight to verification. Only when the - // HTTP-boot device was actually reverted (the boot NIC dropped off the - // BMC's Redfish inventory on a reboot) do we re-assert it, and then - // reboot to apply it before the boot-order set, so the two writes - // never land in one pass. + // Don't re-apply a boot config that's already in place. On Dell two + // BIOS config writes cannot share one pending job: machine_setup can + // commit a job that collides with set_boot_order_dpu_first (iDRAC + // `SYS011`). If the HTTP device drifted since the owning lifecycle's + // inspection, return to the shared BIOS/job stages before attempting + // the order write. let is_bios_setup = boot_interface .run(|bi| redfish_client.is_bios_setup(Some(bi))) .await .map_err(|e| redfish_error("is_bios_setup", e))?; if !is_bios_setup { - // The HTTP-boot device was reverted (the boot NIC dropped off the - // BMC's Redfish inventory on a reboot), so re-assert it with - // `machine_setup` and restart the host to apply it *before* - // setting the boot order -- the two BIOS writes never share one - // pass (they can't share one Dell config job). The re-assert - // commits once: `WaitForHttpBootDeviceApplied` polls the device - // across the reboot instead of this arm re-writing the staged - // settings and re-creating the config job every pass while the - // reboot lands. - call_machine_setup_and_handle_no_dpu_error( - redfish_client, - Some(&boot_interface), - mh_snapshot.host_snapshot.associated_dpu_machine_ids().len(), - &ctx.services.site_config, - ) - .await - .map_err(|e| redfish_error("machine_setup", e))?; - - handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart) - .await?; - log_host_config(redfish_client, mh_snapshot).await; - - return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - retry_count: set_boot_order_info.retry_count, - })); + // The HTTP-boot device drifted after the owning lifecycle's + // inspection. Route back through the shared BIOS driver rather + // than issuing machine_setup here: vendor jobs must be persisted + // and polled before boot-order work can safely continue. + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // Keep the vendor safety policy at the actuator boundary. This is + // intentionally after the BIOS check: validation reuses this state + // to restore a reverted HTTP boot device on Viking, where BIOS + // repair is supported but boot-order remediation is not. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order remediation on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); } // HTTP-boot device is already set, so `machine_setup` was not re-run @@ -11577,14 +11220,9 @@ async fn set_host_boot_order( })) } SetBootOrderState::WaitForHttpBootDeviceApplied => { - // The SetBootOrder substate re-asserted the reverted HTTP-boot device - // and restarted the host; the staged BIOS settings apply across that - // reboot. Poll until the device verifies, then return to SetBootOrder - // to set the boot order. If it still hasn't applied once the reboot - // has had time to land (the boot NIC can drop off the BMC's Redfish - // inventory again on the apply reboot itself), return to SetBootOrder - // for a fresh re-assert -- re-commits stay bounded to one per wait - // window instead of one per pass. + // Backward compatibility for hosts persisted in the former inline + // machine_setup flow. Once its wait window expires, migrate into the + // shared BIOS/job stages so any new vendor job is retained. const HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES: i64 = 10; const MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES: u32 = 3; @@ -11623,11 +11261,10 @@ async fn set_host_boot_order( .num_minutes(); if minutes_waiting >= HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES { - // Each expired window spends one retry from the SetBootOrder - // state machine's shared budget. Once it's exhausted the device - // is not coming back on its own -- stop re-asserting and surface - // the host for an operator, the same terminal escalation the - // reboot pacer applies to a host that never comes up. + // Preserve the terminal behavior of a legacy state whose old + // retry budget was already exhausted. Any nonterminal state + // carries its existing count into the shared convergence + // budget. if set_boot_order_info.retry_count >= MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES { return Err(StateHandlerError::ManualInterventionRequired(format!( "HTTP boot device on host {} still not applied after {} re-asserts; manual intervention required", @@ -11637,13 +11274,9 @@ async fn set_host_boot_order( tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, minutes_waiting, - "Re-asserted HTTP boot device still not applied after the wait window; returning to SetBootOrder to re-assert", + "Re-asserted HTTP boot device still not applied after the wait window; migrating to shared BIOS repair", ); - return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: set_boot_order_info.retry_count + 1, - })); + return Ok(SetBootOrderOutcome::ConfigureBios); } Ok(SetBootOrderOutcome::WaitingForReboot(format!( @@ -11673,7 +11306,8 @@ async fn set_host_boot_order( })) } SetBootOrderState::RebootHost => { - // Host needs to be rebooted to pick up the changes after calling machine_setup + // Reboot before final verification so any accepted boot-order + // change or vendor job can apply. handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart).await?; Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11857,10 +11491,14 @@ async fn set_host_boot_order( } } SetBootOrderState::CheckBootOrder => { - const MAX_BOOT_ORDER_RETRIES: u32 = 3; const CHECK_BOOT_ORDER_TIMEOUT_MINUTES: i64 = 30; let retry_count = set_boot_order_info.retry_count; + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; let boot_interface = match require_boot_interface( mh_snapshot, @@ -11872,6 +11510,36 @@ async fn set_host_boot_order( RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; + // The boot-order job/recovery reboot can itself revert the HTTP + // device or another managed BIOS attribute. Verify BIOS first for + // every platform and return to the shared job-aware repair before + // declaring the overall boot configuration complete. + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Host BIOS drifted during boot-order recovery; returning to shared BIOS repair" + ); + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // A controller upgrade can resume a Viking after an old boot-order + // job or power-recovery substate. Once the operation and BIOS + // verification are complete, skip the unsupported order read. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order verification on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); + } + let boot_order_configured = boot_interface .run(|bi| redfish_client.is_boot_order_setup(bi)) .await @@ -11896,16 +11564,23 @@ async fn set_host_boot_order( time_since_state_change.num_minutes() ); - // If we've been stuck for 30+ minutes and haven't exhausted retries, retry SetBootOrder - if time_since_state_change.num_minutes() >= CHECK_BOOT_ORDER_TIMEOUT_MINUTES - && retry_count < MAX_BOOT_ORDER_RETRIES - { - tracing::info!( - "Boot order check timed out for {} after {} minutes, retrying SetBootOrder (retry {} of {})", + if time_since_state_change.num_minutes() < CHECK_BOOT_ORDER_TIMEOUT_MINUTES { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, - time_since_state_change.num_minutes(), - retry_count + 1, - MAX_BOOT_ORDER_RETRIES + retry_count, + max_retries, + time_since_state_change.num_minutes() + ))); + } + + if retry_count < max_retries { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + time_in_state_minutes = time_since_state_change.num_minutes(), + retry_count = retry_count + 1, + max_retries, + "Boot order check timed out; retrying SetBootOrder" ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11915,11 +11590,13 @@ async fn set_host_boot_order( })); } - // Either still within timeout window or exhausted retries - return error - Err(StateHandlerError::GenericError(eyre::eyre!( - "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, time_in_state: {} minutes)", + // Keep the state parked so an operator can repair the BMC and the + // next verification can complete without resetting the lifecycle. + Err(StateHandlerError::ManualInterventionRequired(format!( + "Boot order is not configured properly for host {} after SetBootOrder completed; automated boot-config convergence exhausted (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, retry_count, + max_retries, time_since_state_change.num_minutes() ))) } diff --git a/crates/machine-controller/src/handler/bios_config.rs b/crates/machine-controller/src/handler/bios_config.rs index b94aaa5992..b8644f809e 100644 --- a/crates/machine-controller/src/handler/bios_config.rs +++ b/crates/machine-controller/src/handler/bios_config.rs @@ -363,7 +363,7 @@ fn try_bios_recovery_attempt( ); return Ok(BiosRecoveryAttemptOutcome::Failed { failure: format!( - "{failure} (automated BIOS recovery exhausted after {} attempts)", + "{failure} (automated BIOS recovery exhausted after {} retries)", machine_controller_config.max_bios_config_retries ), }); diff --git a/crates/machine-controller/src/handler/host_boot_config.rs b/crates/machine-controller/src/handler/host_boot_config.rs new file mode 100644 index 0000000000..076f6a1527 --- /dev/null +++ b/crates/machine-controller/src/handler/host_boot_config.rs @@ -0,0 +1,542 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Shared host boot-configuration convergence. +//! +//! The lifecycle state machines retain ownership of their persisted state +//! shapes. [`HostBootConfigStage`] is only a runtime projection of those +//! states, so this module can drive the common BIOS/job/poll/boot-order +//! sequence without introducing another serialized controller wrapper. +//! +//! [`run_host_boot_config_stage`] owns the lifecycle-neutral Redfish work. Its +//! four lifecycle adapters stay beside the state machines that own their +//! persisted state and terminal behavior: +//! - `handle_host_init_boot_config_stage` in `handler.rs`; +//! - `handle_instance_host_boot_config_stage` in `handler.rs`; +//! - `handle_dpu_reprovision_host_boot_config_stage` in `handler.rs`; +//! - `handle_validation_boot_config_stage` in `machine_validation.rs`. + +use carbide_redfish::boot_interface::BootInterfaceTarget; +use carbide_redfish::libredfish::error::state_handler_redfish_error as redfish_error; +use libredfish::Redfish; +use model::machine::{ + BiosConfigInfo, ManagedHostStateSnapshot, SetBootOrderInfo, SetBootOrderState, +}; +use state_controller::state_handler::{StateHandlerContext, StateHandlerError}; + +use super::bios_config::{ + BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, + advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, +}; +use super::{ + ReachabilityParams, RequiredBootInterface, SetBootOrderOutcome, + are_dpus_up_trigger_reboot_if_needed, is_dpu_observed_since, load_boot_predictions, + log_host_config, require_boot_interface, set_host_boot_order, trigger_reboot_if_needed, +}; +use crate::context::MachineStateHandlerContextObjects; + +/// Runtime projection of the persisted BIOS and boot-order states shared by +/// HostInit, assigned platform configuration, DPU reprovision, and validation. +/// +/// This type must remain controller-internal: the owning lifecycle maps each +/// `Continue` outcome back to its existing serialized state variant. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigStage { + ConfigureBios { + retry_count: u32, + }, + WaitingForBiosJob { + bios_config_info: BiosConfigInfo, + }, + PollingBiosSetup { + retry_count: u32, + }, + SetBootOrder { + set_boot_order_info: SetBootOrderInfo, + }, +} + +/// Result of running one common boot-configuration stage. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigOutcome { + Continue(HostBootConfigStage), + Complete, + Wait(String), + Failed { failure: String }, +} + +/// The next remediation required after inspecting the desired and actual host +/// boot configuration. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigDecision { + /// The platform BIOS settings (including the HTTP boot device) drifted. + ConfigureBios, + /// BIOS is already correct; only boot order drifted. + SetBootOrder, + /// BIOS and every NICo-managed boot-order setting are already correct. + Complete, +} + +/// Result of pacing prerequisites and inspecting the host boot config. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigCheckOutcome { + Ready(HostBootConfigDecision), + /// The host cannot be inspected yet, for example while discovering a + /// zero-DPU host's boot NIC or waiting for a fresh DPU observation. + Wait(String), +} + +/// DPU observation freshness required before trusting Redfish boot state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HostBootConfigDpuFreshness { + /// The caller already gated on healthy, synchronized DPUs. + AlreadyValidated, + /// Require observations associated with the current host lifecycle state. + CurrentHostState, + /// Require observations newer than the most recent host reboot request. + SinceLastHostRebootRequest, +} + +/// Actual host boot settings read from Redfish. +/// +/// `boot_order_setup` is `None` when BIOS drift already determines the next +/// action, or when boot order is intentionally not managed on this hardware. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) struct HostBootConfigInspection { + pub is_bios_setup: bool, + pub is_boot_order_setup: Option, +} + +/// Convert an inspection into the smallest remediation that can converge it. +/// BIOS repair takes precedence because it may recreate the boot option whose +/// order would otherwise be inspected or changed. +pub(super) fn decide_host_boot_config( + inspection: HostBootConfigInspection, +) -> HostBootConfigDecision { + if !inspection.is_bios_setup { + HostBootConfigDecision::ConfigureBios + } else if inspection.is_boot_order_setup == Some(false) { + HostBootConfigDecision::SetBootOrder + } else { + HostBootConfigDecision::Complete + } +} + +/// Inspect a host whose boot interface has already been resolved. +/// +/// Keeping interface resolution outside this primitive lets validation retain +/// its existing distinction between a missing interface (hard error), an +/// undiscovered zero-DPU NIC (pace the reboot), and a Redfish read error (log +/// and retry). +pub(super) async fn inspect_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + boot_interface: &BootInterfaceTarget, +) -> Result { + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup || should_skip_boot_order_remediation(mh_snapshot) { + return Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: None, + }); + } + + let is_boot_order_setup = boot_interface + .run(|bi| redfish_client.is_boot_order_setup(bi)) + .await + .map_err(|e| redfish_error("is_boot_order_setup", e))?; + + Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: Some(is_boot_order_setup), + }) +} + +/// Check whether host BIOS or boot order needs remediation. +pub(super) async fn check_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result { + if should_wait_for_dpus_before_host_boot_config( + mh_snapshot, + reachability_params, + dpu_freshness, + ctx, + ) + .await + { + return Ok(HostBootConfigCheckOutcome::Wait( + "Waiting for DPUs to come up.".to_string(), + )); + } + + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "configuring boot", + HostBootConfigCheckOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), + }; + + log_host_config(redfish_client, mh_snapshot).await; + + let inspection = inspect_host_boot_config(redfish_client, mh_snapshot, &boot_interface).await?; + let decision = decide_host_boot_config(inspection); + let vendor = mh_snapshot.host_snapshot.bmc_vendor(); + + match &decision { + HostBootConfigDecision::ConfigureBios => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is not configured properly" + ), + HostBootConfigDecision::SetBootOrder => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is correct but boot order is not configured properly" + ), + HostBootConfigDecision::Complete if should_skip_boot_order_remediation(mh_snapshot) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is configured; skipping boot order remediation on Viking" + ); + } + HostBootConfigDecision::Complete => tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup and boot order are configured properly" + ), + } + + Ok(HostBootConfigCheckOutcome::Ready(decision)) +} + +/// Runs one lifecycle-neutral host boot-configuration stage. +/// +/// Called by the host-init, assigned-instance, DPU-reprovision, and validation +/// adapters. This function performs the Redfish BIOS, vendor-job, polling, or +/// boot-order work for one [`HostBootConfigStage`] and returns a +/// [`HostBootConfigOutcome`]. The caller owns the persisted lifecycle state and +/// maps the outcome back into that state. +pub(super) async fn run_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + stage: HostBootConfigStage, +) -> Result { + match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + match configure_host_bios( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + retry_count, + ) + .await? + { + BiosConfigOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigOutcome::WaitingForReboot(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + let retry_count = bios_config_info.retry_count; + match advance_bios_config_job(ctx, redfish_client, mh_snapshot, bios_config_info) + .await? + { + BiosConfigJobAdvanceOutcome::Continue(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigJobAdvanceOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigJobAdvanceOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + BiosConfigJobAdvanceOutcome::Wait(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + match advance_polling_bios_setup( + redfish_client, + mh_snapshot, + retry_count, + &ctx.services.site_config.machine_state_controller, + &predictions, + ) + .await? + { + PollingBiosSetupOutcome::Verified => { + if should_skip_boot_order_remediation(mh_snapshot) { + Ok(HostBootConfigOutcome::Complete) + } else { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info(retry_count), + }, + )) + } + } + PollingBiosSetupOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + PollingBiosSetupOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => { + let retry_count = set_boot_order_info.retry_count; + match set_host_boot_order( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + set_boot_order_info, + ) + .await? + { + SetBootOrderOutcome::Continue(set_boot_order_info) => Ok( + HostBootConfigOutcome::Continue(HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }), + ), + SetBootOrderOutcome::ConfigureBios => { + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; + let Some(retry_count) = next_boot_config_retry_count(retry_count, max_retries) + else { + return Ok(HostBootConfigOutcome::Failed { + failure: format!( + "BIOS settings repeatedly drifted during boot-order repair; automated boot-config convergence exhausted after {max_retries} retries" + ), + }); + }; + + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + SetBootOrderOutcome::Done => Ok(HostBootConfigOutcome::Complete), + SetBootOrderOutcome::WaitingForReboot(reason) + | SetBootOrderOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + } + } + } +} + +pub(super) fn initial_set_boot_order_info() -> SetBootOrderInfo { + set_boot_order_info(0) +} + +fn set_boot_order_info(retry_count: u32) -> SetBootOrderInfo { + SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count, + } +} + +/// Carry one bounded convergence budget when boot-order work discovers that +/// BIOS must be repaired again. Existing owner states already persist this +/// counter through both phases, so no serialized wrapper is required. +fn next_boot_config_retry_count(retry_count: u32, max_retries: u32) -> Option { + (retry_count < max_retries).then(|| retry_count + 1) +} + +/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair +/// still applies. +pub(super) fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { + mh_snapshot + .host_snapshot + .hardware_info + .as_ref() + .is_some_and(|hw| hw.is_dgx_h100()) +} + +async fn should_wait_for_dpus_before_host_boot_config( + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> bool { + if !mh_snapshot.has_managed_dpus() { + return false; + } + + match dpu_freshness { + HostBootConfigDpuFreshness::AlreadyValidated => false, + HostBootConfigDpuFreshness::CurrentHostState => { + !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await + } + HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { + let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested + else { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "No host reboot request timestamp found before post-reboot host boot config check" + ); + return false; + }; + + for dpu_snapshot in &mh_snapshot.dpu_snapshots { + if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { + match trigger_reboot_if_needed( + dpu_snapshot, + mh_snapshot, + None, + reachability_params, + ctx, + ) + .await + { + Ok(_) => {} + Err(e) => tracing::warn!( + dpu_id = %dpu_snapshot.id, + error = %e, + "Could not reboot DPU while waiting to check host boot config" + ), + } + return true; + } + } + + false + } + } +} + +#[cfg(test)] +mod tests { + use carbide_test_support::value_scenarios; + + use super::*; + + #[test] + fn boot_config_decision_chooses_the_smallest_remediation() { + value_scenarios!(decide_host_boot_config: + "BIOS drift takes precedence" { + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: None, + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::ConfigureBios, + } + + "configured BIOS chooses the narrowest remaining action" { + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::SetBootOrder, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::Complete, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: None, + } => HostBootConfigDecision::Complete, + } + ); + } + + #[test] + fn boot_config_retry_count_is_bounded_across_bios_and_order() { + struct RetryCount { + retry_count: u32, + max_retries: u32, + } + + value_scenarios!( + run = |RetryCount { + retry_count, + max_retries, + }: RetryCount| next_boot_config_retry_count(retry_count, max_retries); + "retry budget remains" { + RetryCount { + retry_count: 0, + max_retries: 1, + } => Some(1), + RetryCount { + retry_count: 4, + max_retries: 5, + } => Some(5), + } + + "retry budget is unavailable" { + RetryCount { + retry_count: 1, + max_retries: 1, + } => None, + RetryCount { + retry_count: 0, + max_retries: 0, + } => None, + RetryCount { + retry_count: 4, + max_retries: 1, + } => None, + } + ); + } +} diff --git a/crates/machine-controller/src/handler/machine_validation.rs b/crates/machine-controller/src/handler/machine_validation.rs index 7950f844cd..d29cfab079 100644 --- a/crates/machine-controller/src/handler/machine_validation.rs +++ b/crates/machine-controller/src/handler/machine_validation.rs @@ -15,10 +15,11 @@ * limitations under the License. */ use carbide_uuid::machine_validation::MachineValidationId; +use chrono::Utc; use libredfish::{EnabledDisabled, RedfishError, SystemPowerControl}; use model::machine::{ - FailureCause, MachineState, MachineValidatingState, ManagedHostState, ManagedHostStateSnapshot, - SetBootOrderInfo, SetBootOrderState, UnlockHostState, ValidationState, + FailureCause, FailureDetails, FailureSource, MachineState, MachineValidatingState, + ManagedHostState, ManagedHostStateSnapshot, StateMachineArea, UnlockHostState, ValidationState, }; use model::machine_validation::{MachineValidationState, MachineValidationStatus}; use state_controller::state_handler::{ @@ -27,10 +28,14 @@ use state_controller::state_handler::{ use super::{HostHandlerParams, is_machine_validation_requested, machine_validation_completed}; use crate::context::{MachineStateHandlerContextObjects, MachineStateHandlerServices}; +use crate::handler::host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, decide_host_boot_config, + initial_set_boot_order_info, inspect_host_boot_config, run_host_boot_config_stage, +}; use crate::handler::{ - RequiredBootInterface, SetBootOrderOutcome, handler_host_power_control, host_power_control, - load_boot_predictions, rebooted, redfish_error, require_boot_interface, set_host_boot_order, - trigger_reboot_if_needed, wait, + RequiredBootInterface, handler_host_power_control, host_power_control, load_boot_predictions, + rebooted, redfish_error, require_boot_interface, trigger_reboot_if_needed, wait, }; /// The validation flavor of the managed-host state, so the repair arms can @@ -41,14 +46,103 @@ fn validating(machine_validation: MachineValidatingState) -> ManagedHostState { } } -/// The starting point for the boot-order flow when validation boot repair -/// re-drives it: the flow itself checks what is already in place and only -/// re-applies what the reboot reverted. -fn boot_repair_start() -> SetBootOrderInfo { - SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, +/// Handles a shared boot-configuration stage for machine validation. +/// +/// Called by `handle_machine_validation_state` for its BIOS configuration, +/// vendor-job, polling, and boot-order repair substates. This adapter preserves +/// the validation ID, maps continued stages back into `MachineValidatingState`, +/// continues to `LockAfterBootRepair` on completion, and closes the validation +/// run before reporting a terminal boot-configuration failure. +async fn handle_validation_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + host_handler_params: &HostHandlerParams, + mh_snapshot: &ManagedHostStateSnapshot, + validation_id: MachineValidationId, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + &host_handler_params.reachability_params, + redfish_client.as_ref(), + mh_snapshot, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_validation = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition(validating( + machine_validation, + ))) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::LockAfterBootRepair { validation_id }, + ))), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + let machine_id = mh_snapshot.host_snapshot.id; + let mut txn = ctx.services.db_pool.begin().await?; + let completed = db::machine_validation::mark_machine_validation_complete( + txn.as_mut(), + &machine_id, + &validation_id, + MachineValidationStatus { + state: MachineValidationState::Failed, + ..MachineValidationStatus::default() + }, + ) + .await?; + + if !completed { + tracing::info!( + %machine_id, + %validation_id, + "Machine validation boot-config failure observed after the run was already terminal" + ); + } + + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + }, + machine_id, + retry_count: 0, + }) + .with_txn(txn)) + } } } @@ -162,11 +256,20 @@ pub(crate) async fn handle_machine_validation_state( .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await + match inspect_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &boot_interface, + ) + .await { - Ok(false) => { + Ok(inspection) + if matches!( + decide_host_boot_config(inspection), + HostBootConfigDecision::ConfigureBios + | HostBootConfigDecision::SetBootOrder + ) => + { tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, "Boot config reads reverted while waiting for the validation reboot; correcting it via boot repair", @@ -175,7 +278,7 @@ pub(crate) async fn handle_machine_validation_state( MachineValidatingState::PrepareBootRepair { validation_id: *id }, ))); } - Ok(true) => {} + Ok(_) => {} Err(e) => { tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, @@ -252,9 +355,8 @@ pub(crate) async fn handle_machine_validation_state( machine_id = %mh_snapshot.host_snapshot.id, "BMC vendor does not support checking lockdown status during validation boot repair", ); - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } Err(e) => { @@ -277,9 +379,8 @@ pub(crate) async fn handle_machine_validation_state( unlock_host_state: UnlockHostState::DisableLockdown, } } - Ok(_) => MachineValidatingState::RepairBootConfig { + Ok(_) => MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), }, }; @@ -322,9 +423,8 @@ pub(crate) async fn handle_machine_validation_state( unlock_host_state: UnlockHostState::RebootHost, } } else { - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } } @@ -359,51 +459,112 @@ pub(crate) async fn handle_machine_validation_state( mh_snapshot.host_snapshot.id ))); } - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } }; Ok(StateHandlerOutcome::transition(validating(next))) } - MachineValidatingState::RepairBootConfig { - validation_id, - set_boot_order_info, - } => { - // Re-drive the boot-order flow: it checks what is already in - // place, re-asserts the reverted HTTP-boot device (rebooting to - // apply it), sets the boot order, and verifies -- the same engine - // the boot-configuration stages use. + MachineValidatingState::CheckBootConfigForRepair { validation_id } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, + let next = match check_host_boot_config( redfish_client.as_ref(), mh_snapshot, - set_boot_order_info.clone(), + &host_handler_params.reachability_params, + HostBootConfigDpuFreshness::CurrentHostState, + ctx, ) .await? { - SetBootOrderOutcome::Continue(info) => Ok(StateHandlerOutcome::transition( - validating(MachineValidatingState::RepairBootConfig { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + MachineValidatingState::ConfigureBootBios { validation_id: *validation_id, - set_boot_order_info: info, - }), - )), - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition(validating( + retry_count: 0, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { MachineValidatingState::LockAfterBootRepair { validation_id: *validation_id, - }, - ))), - SetBootOrderOutcome::WaitingForReboot(reason) - | SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + } + } + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, + ) + .await + } + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, + ) + .await } MachineValidatingState::LockAfterBootRepair { validation_id } => { // Restore the lockdown that boot repair temporarily opened, then diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index e831a4d975..a80411c47b 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -266,6 +266,12 @@ impl StateControllerIO for MachineStateControllerIO { MachineValidatingState::RebootHost { .. } => "reboothost", MachineValidatingState::PrepareBootRepair { .. } => "preparebootrepair", MachineValidatingState::UnlockForBootRepair { .. } => "unlockforbootrepair", + MachineValidatingState::CheckBootConfigForRepair { .. } => { + "checkbootconfigforrepair" + } + MachineValidatingState::ConfigureBootBios { .. } => "configurebootbios", + MachineValidatingState::WaitingForBootBiosJob { .. } => "waitingforbootbiosjob", + MachineValidatingState::PollingBootBiosSetup { .. } => "pollingbootbiossetup", MachineValidatingState::RepairBootConfig { .. } => "repairbootconfig", MachineValidatingState::LockAfterBootRepair { .. } => "lockafterbootrepair", }