From 2164d6c938458b9d72ee0884ba050f21b1fae544 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Wed, 8 Jul 2026 04:41:52 +0800 Subject: [PATCH 01/10] feat(vmm): multi-NIC networking with per-VM config and RPC support Single-NIC bridge/user networking could not express mixed topologies (e.g. one bridge NIC for external traffic + one user-mode NIC for isolated management). Extend the model to support multiple NICs per VM. - Manifest field: networking (singular, Optional) -> networks (Vec) with backward-compatible deserialization of the legacy field - Proto: add repeated NetworkingConfig in VmConfiguration/UpdateVmRequest, NetworkInterfaceStatus in VmInfo, and NetworkingCapabilities in GetMetaResponse - QEMU: generate one -netdev/-device pair per entry with indexed net IDs and per-NIC deterministic MACs (SHA256 of vm_id:index) - DHCP: match leases against all NICs, persist per-MAC guest IP map - Port forwarding: select the first bridge NIC with forward_service_enabled - Validation: reject bridge mode when the host interface does not exist, reject custom mode via RPC (manifest-only) - Measurement: networking config excluded from VM measurement to allow topology changes without re-attestation --- dstack/vmm/rpc/proto/vmm_rpc.proto | 37 + dstack/vmm/src/app.rs | 267 ++++- dstack/vmm/src/app/qemu.rs | 1715 +++++++++++++++++++--------- dstack/vmm/src/main_service.rs | 213 +++- 4 files changed, 1641 insertions(+), 591 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 8d79fea66..619dcdc9c 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -37,6 +37,28 @@ message VmInfo { string image_version = 13; // Events repeated GuestEvent events = 14; + // Effective network interfaces resolved against node config. + repeated NetworkInterfaceStatus interfaces = 15; +} + +// Runtime status of a resolved VM network interface. +message NetworkInterfaceStatus { + // Product-facing mode: "user", "bridge", or "custom". + string mode = 1; + // QEMU/network backend shape: "slirp", "tap_bridge", or "custom". + string backend = 2; + string mac = 3; + optional string ip = 4; + // Linux bridge name for tap_bridge backend. + optional string bridge_name = 5; + // QEMU netdev id, e.g. "net0". + optional string netdev_id = 6; + // Host-side interface name when known, e.g. tap/macvtap device. + optional string if_name = 7; + // Lower device for macvtap-like backends. + optional string lowerdev = 8; + // PCI BDF for passthrough-like backends. + optional string pci_bdf = 9; } // Structured log or lifecycle event emitted by the guest or runtime. @@ -101,12 +123,16 @@ message VmConfiguration { bool no_tee = 17; // Per-VM networking mode override (if unset, uses global cvm.networking). optional NetworkingConfig networking = 18; + // Per-VM networking overrides. If set, wins over singular networking. + repeated NetworkingConfig networks = 19; } // Per-VM networking configuration. message NetworkingConfig { // Networking mode: "bridge", "user" string mode = 1; + // Per-VM bridge interface name. Empty = node default bridge. + string bridge_name = 2; } // Requested GPU layout for a CVM. @@ -163,6 +189,10 @@ message UpdateVmRequest { optional string image = 17; // Disable or re-enable TEE for an existing VM. optional bool no_tee = 18; + // Optional update networking. + bool update_networking = 19; + // Networking list. Empty + update_networking=true resets to node default. + repeated NetworkingConfig networks = 20; } // Parameters for listing CVMs with pagination and keyword filtering. @@ -262,11 +292,18 @@ message ResourcesSettings { uint32 max_allocable_memory_in_mb = 3; // in MB. } +message NetworkingCapabilities { + repeated string supported_modes = 1; + string default_mode = 2; + bool forward_service_enabled = 3; +} + // Aggregated metadata exposed through GetMeta. message GetMetaResponse { KmsSettings kms = 1; GatewaySettings gateway = 2; ResourcesSettings resources = 3; + NetworkingCapabilities networking = 4; } // Build information for the running VMM binary. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index dd4b527d0..ff78a46ad 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -20,7 +20,7 @@ use guest_api::client::DefaultClient as GuestClient; use id_pool::IdPool; use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; @@ -32,17 +32,12 @@ use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; -pub use qemu::VmConfig; -pub use workdir::VmWorkDir; +pub use qemu::{VmConfig, VmWorkDir}; -mod host_share; mod id_pool; mod image; -mod mr_config; mod qemu; pub(crate) mod registry; -mod vm_info; -mod workdir; #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { @@ -52,7 +47,7 @@ pub struct PortMapping { pub to: u16, } -#[derive(Deserialize, Serialize, Clone, Builder, Debug)] +#[derive(Serialize, Clone, Builder, Debug)] pub struct Manifest { pub id: String, pub name: String, @@ -75,8 +70,69 @@ pub struct Manifest { pub gateway_urls: Vec, #[serde(default)] pub no_tee: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub networking: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub networks: Vec, +} + +impl<'de> Deserialize<'de> for Manifest { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct ManifestCompat { + id: String, + name: String, + app_id: String, + vcpu: u32, + memory: u32, + disk_size: u32, + image: String, + port_map: Vec, + created_at_ms: u64, + #[serde(default)] + hugepages: bool, + #[serde(default)] + pin_numa: bool, + #[serde(default)] + gpus: Option, + #[serde(default)] + kms_urls: Vec, + #[serde(default)] + gateway_urls: Vec, + #[serde(default)] + no_tee: bool, + #[serde(default)] + networks: Vec, + #[serde(default)] + networking: Option, + } + + let compat = ManifestCompat::deserialize(deserializer)?; + let networks = if compat.networks.is_empty() { + compat.networking.into_iter().collect() + } else { + compat.networks + }; + Ok(Self { + id: compat.id, + name: compat.name, + app_id: compat.app_id, + vcpu: compat.vcpu, + memory: compat.memory, + disk_size: compat.disk_size, + image: compat.image, + port_map: compat.port_map, + created_at_ms: compat.created_at_ms, + hugepages: compat.hugepages, + pin_numa: compat.pin_numa, + gpus: compat.gpus, + kms_urls: compat.kms_urls, + gateway_urls: compat.gateway_urls, + no_tee: compat.no_tee, + networks, + }) + } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -281,6 +337,13 @@ impl App { let image_path = self.config.image.path.join(&manifest.image); let image = Image::load(&image_path).context("Failed to load image")?; let vm_id = manifest.id.clone(); + let mut runtime_networks = vm_work_dir.runtime_networks(); + if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { + runtime_networks = crate::app::qemu::resolved_networks(&manifest, &self.config.cvm); + if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { + warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); + } + } let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; @@ -302,9 +365,12 @@ impl App { match states.get_mut(&vm_id) { Some(vm) => { vm.config = vm_config.into(); + vm.state.runtime_networks = runtime_networks; } None => { - states.add(VmState::new(vm_config)); + let mut vm_state = VmState::new(vm_config); + vm_state.state.runtime_networks = runtime_networks; + states.add(vm_state); } } }; @@ -363,9 +429,13 @@ impl App { .with_context(|| format!("Failed to start process {}", process.id))?; } + let runtime_networks = + crate::app::qemu::resolved_networks(&vm_config.manifest, &self.config.cvm); + work_dir.set_runtime_networks(&runtime_networks)?; let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; vm_state.state.devices = devices; + vm_state.state.runtime_networks = runtime_networks; } Ok(()) } @@ -517,25 +587,39 @@ impl App { /// Handle a DHCP lease notification: look up VM by MAC address, persist /// the guest IP, and reconfigure port forwarding. pub async fn report_dhcp_lease(&self, mac: &str, ip: &str) { - use crate::app::mr_config::mac_address_for_vm; + use crate::app::qemu::mac_address_for_vm_index; let vm_id = { let mut state = self.lock(); - let prefix = self.config.cvm.networking.mac_prefix_bytes(); - let found = state - .vms - .iter_mut() - .find(|(id, _)| mac_address_for_vm(id, &prefix) == mac); - let Some((id, vm)) = found else { + let found = state.vms.iter_mut().find_map(|(id, vm)| { + let networks = if vm.state.runtime_networks.is_empty() { + crate::app::qemu::resolved_networks(&vm.config.manifest, &self.config.cvm) + } else { + vm.state.runtime_networks.clone() + }; + networks.iter().enumerate().find_map(|(index, networking)| { + let prefix = networking.mac_prefix_bytes(); + let nic_mac = mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); + (nic_mac == mac).then(|| (id.clone(), nic_mac)) + }) + }); + let Some((vm_id, nic_mac)) = found else { debug!(mac, ip, "DHCP lease for unknown MAC, ignoring"); return; }; - let vm_id = id.clone(); + let Some(vm) = state.get_mut(&vm_id) else { + debug!(mac, ip, id = %vm_id, "DHCP lease for missing VM, ignoring"); + return; + }; let workdir = VmWorkDir::new(vm.config.workdir.clone()); if let Err(e) = workdir.set_guest_ip(ip) { error!(mac, ip, "failed to persist guest IP: {e}"); } + if let Err(e) = workdir.set_guest_ip_for_mac(&nic_mac, ip) { + error!(mac, ip, "failed to persist guest IP map: {e}"); + } vm.state.guest_ip = ip.to_string(); + vm.state.guest_ips.insert(nic_mac, ip.to_string()); info!(mac, ip, id = %vm_id, "DHCP lease updated"); vm_id }; @@ -553,16 +637,32 @@ impl App { let Some(vm) = state.get(id) else { return; }; - let networking = vm - .config - .manifest - .networking - .as_ref() - .unwrap_or(&self.config.cvm.networking); - if !networking.is_bridge() || !networking.forward_service_enabled { + let networks = if vm.state.runtime_networks.is_empty() { + crate::app::qemu::resolved_networks(&vm.config.manifest, &self.config.cvm) + } else { + vm.state.runtime_networks.clone() + }; + if networks + .iter() + .any(|networking| networking.mode == crate::config::NetworkingMode::User) + { return; } - let guest_ip = vm.state.guest_ip.clone(); + let Some((index, networking)) = networks.iter().enumerate().find(|(_, networking)| { + networking.is_bridge() && networking.forward_service_enabled + }) else { + return; + }; + let prefix = networking.mac_prefix_bytes(); + let mac = + crate::app::qemu::mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); + let guest_ip = vm.state.guest_ips.get(&mac).cloned().unwrap_or_else(|| { + if vm.state.guest_ips.is_empty() { + vm.state.guest_ip.clone() + } else { + String::new() + } + }); let port_map = vm.config.manifest.port_map.clone(); (guest_ip, port_map) }; @@ -702,11 +802,16 @@ impl App { let vm_ids: Vec = self.lock().vms.keys().cloned().collect(); for id in vm_ids { let workdir = self.work_dir(&id); - if let Some(ip) = workdir.guest_ip() { + let guest_ip = workdir.guest_ip(); + let guest_ips = workdir.guest_ips(); + if guest_ip.is_some() || !guest_ips.is_empty() { { let mut state = self.lock(); if let Some(vm) = state.get_mut(&id) { - vm.state.guest_ip = ip; + if let Some(ip) = guest_ip { + vm.state.guest_ip = ip; + } + vm.state.guest_ips = guest_ips; } } self.reconfigure_port_forward(&id).await; @@ -861,6 +966,13 @@ impl App { let image = Image::load(&image_path).context("Failed to load image")?; let vm_id = manifest.id.clone(); let already_running = cids_assigned.contains_key(&vm_id); + let mut runtime_networks = vm_work_dir.runtime_networks(); + if runtime_networks.is_empty() && already_running { + runtime_networks = crate::app::qemu::resolved_networks(&manifest, &self.config.cvm); + if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { + warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); + } + } let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; @@ -893,7 +1005,10 @@ impl App { match states.get_mut(&vm_id) { Some(vm) => { // Update existing VM but preserve statistics and CID - let old_state = vm.state.clone(); + let mut old_state = vm.state.clone(); + if old_state.runtime_networks.is_empty() { + old_state.runtime_networks = runtime_networks; + } vm.config = vm_config.into(); vm.state = old_state; // Preserve the existing state with statistics } @@ -902,7 +1017,9 @@ impl App { if !cids_assigned.contains_key(&vm_id) { states.cid_pool.occupy(cid)?; } - states.add(VmState::new(vm_config)); + let mut vm_state = VmState::new(vm_config); + vm_state.state.runtime_networks = runtime_networks; + states.add(vm_state); is_new = true; } } @@ -962,7 +1079,7 @@ impl App { &self.work_dir(&vm.config.manifest.id), ) }) - .map(|info| info.to_pb(&self.config.gateway, request.brief)) + .map(|info| info.to_pb(&self.config.gateway, &self.config.cvm, request.brief)) .collect::>(); Ok(StatusResponse { vms, @@ -991,7 +1108,7 @@ impl App { }; let info = vm_state .merged_info(proc_state.as_ref(), &self.work_dir(id)) - .to_pb(&self.config.gateway, false); + .to_pb(&self.config.gateway, &self.config.cvm, false); Ok(Some(info)) } @@ -1453,7 +1570,9 @@ fn make_vm_config( #[cfg(test)] mod tests { use super::*; - use crate::config::{load_config_figment, TdxAttestationVariantConfig, TeePlatform}; + use crate::config::{ + load_config_figment, Networking, NetworkingMode, TdxAttestationVariantConfig, TeePlatform, + }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, TdxOsImageMeasurementDocument, TdxTdvfMeasurement, @@ -1465,6 +1584,55 @@ mod tests { hex::encode(vec![byte; len]) } + #[test] + fn put_manifest_keeps_legacy_networking_for_rollback() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-manifest-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + let workdir = VmWorkDir::new(&temp); + let mut manifest = test_manifest(1024); + manifest.networks = vec![Networking { + mode: NetworkingMode::Bridge, + bridge: "dstack-br0".to_string(), + forward_service_enabled: true, + mac_prefix: String::new(), + net: String::new(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + }]; + + workdir.put_manifest(&manifest)?; + let raw = fs::read_to_string(workdir.manifest_path())?; + let value: serde_json::Value = serde_json::from_str(&raw)?; + assert_eq!(value["networks"].as_array().map(Vec::len), Some(1)); + assert_eq!(value["networking"]["mode"], "bridge"); + fs::remove_dir_all(temp)?; + Ok(()) + } + + #[test] + fn manifest_deserializes_legacy_singular_networking_as_networks() { + let manifest: Manifest = serde_json::from_value(serde_json::json!({ + "id": "vm-1", + "name": "vm-1", + "app_id": "app-1", + "vcpu": 1, + "memory": 1024, + "disk_size": 10, + "image": "dstack-test", + "port_map": [], + "created_at_ms": 0, + "networking": { "mode": "bridge", "bridge": "dstack-br0" } + })) + .unwrap(); + + assert_eq!(manifest.networks.len(), 1); + assert_eq!(manifest.networks[0].mode, NetworkingMode::Bridge); + assert_eq!(manifest.networks[0].bridge, "dstack-br0"); + } + fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); } @@ -1555,7 +1723,7 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: false, - networking: None, + networks: vec![], } } @@ -1635,6 +1803,31 @@ mod tests { assert_eq!(effective_vcpu_count(3, None), 3); } + #[test] + fn vm_measurement_config_ignores_networking_changes() -> Result<()> { + let config = test_tdx_config()?; + let mut bridge_manifest = test_manifest(2048); + bridge_manifest.networks = vec![Networking { + mode: NetworkingMode::Bridge, + bridge: "dstack-br0".to_string(), + forward_service_enabled: true, + mac_prefix: "02:aa:bb".to_string(), + net: String::new(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + }]; + let user_manifest = test_manifest(2048); + let image = test_tdx_image(true); + let compose_hash = hex_of(0x22, 32); + + let bridge_config = make_vm_config(&config, &bridge_manifest, &image, &compose_hash, None)?; + let user_config = make_vm_config(&config, &user_manifest, &image, &compose_hash, None)?; + + assert_eq!(bridge_config, user_config); + Ok(()) + } + #[test] fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; @@ -1789,7 +1982,7 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: false, - networking: None, + networks: vec![], }; let mr_config = MrConfigV3::new( @@ -1917,6 +2110,8 @@ struct VmStateMut { boot_error: String, shutdown_progress: String, guest_ip: String, + guest_ips: HashMap, + runtime_networks: Vec, devices: GpuConfig, events: VecDeque, /// True when the VM is being removed (cleanup in progress). diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 31bf3a882..048a97136 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -2,34 +2,176 @@ // // SPDX-License-Identifier: Apache-2.0 -//! QEMU launch preparation and command construction. +//! QEMU related code use crate::{ app::Manifest, - config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, + config::{ + CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform, + }, }; use std::collections::HashMap; use std::os::unix::fs::PermissionsExt; use std::{ fs::Permissions, io::Write, + ops::Deref, path::{Path, PathBuf}, process::{Command, Stdio}, + time::{Duration, SystemTime}, }; use super::{ - effective_vcpu_count, - host_share::create_shared_disk, - hugepage_numa_nodes, - image::Image, - mr_config::{mac_address_for_vm, snp_host_data, tdx_mr_config_id}, - pci_numa_node, round_up, GpuConfig, VmWorkDir, + effective_vcpu_count, hugepage_numa_nodes, image::Image, pci_numa_node, round_up, GpuConfig, + VmState, }; use anyhow::{bail, Context, Result}; +use base64::prelude::*; use bon::Builder; -use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; +use dstack_types::{ + mr_config::{MrConfig, MrConfigV3}, + shared_filenames::{ + APP_COMPOSE, ENCRYPTED_ENV, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, + }, + AppCompose, KeyProviderKind, SysConfig, +}; +use dstack_vmm_rpc as pb; +use sha2::{Digest, Sha256}; + +/// Derive a deterministic MAC address from a VM ID using SHA256. +/// Sets locally-administered + unicast bits (0x02) per IEEE 802. +/// Derive a deterministic MAC address from a VM ID. +/// +/// `prefix` may contain 0-3 fixed bytes. The first byte always has the +/// locally-administered + unicast bits set (0x02). Remaining bytes are +/// filled from SHA256(vm_id). +pub fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { + mac_address_for_vm_index(vm_id, prefix, 0) +} + +pub fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) -> String { + let hash_input = if index == 0 { + vm_id.to_string() + } else { + format!("{vm_id}:{index}") + }; + let hash = Sha256::digest(hash_input.as_bytes()); + let prefix_len = prefix.len().min(3); + let mut bytes = [0u8; 6]; + // Fill prefix bytes + bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); + // Fill remaining bytes from hash + for i in prefix_len..6 { + bytes[i] = hash[i - prefix_len]; + } + // Ensure locally-administered + unicast on first byte + bytes[0] = (bytes[0] & 0xfe) | 0x02; + format!( + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] + ) +} use fs_err as fs; -use serde::Serialize; -use supervisor_client::supervisor::ProcessConfig; +use serde::{Deserialize, Serialize}; +use serde_human_bytes as hex_bytes; +use supervisor_client::supervisor::{ProcessConfig, ProcessInfo}; + +pub fn networking_mode_name(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "bridge", + NetworkingMode::User => "user", + NetworkingMode::Custom => "custom", + } +} + +fn networking_backend_name(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "tap_bridge", + NetworkingMode::User => "slirp", + NetworkingMode::Custom => "custom", + } +} + +fn networking_to_proto(n: &Networking) -> pb::NetworkingConfig { + pb::NetworkingConfig { + mode: networking_mode_name(n.mode).into(), + bridge_name: if n.mode == NetworkingMode::Bridge { + n.bridge.clone() + } else { + String::new() + }, + } +} + +pub fn resolve_networking(vm_net: &Networking, cfg: &CvmConfig) -> Networking { + Networking { + mode: vm_net.mode, + bridge: if vm_net.bridge.is_empty() { + cfg.networking.bridge.clone() + } else { + vm_net.bridge.clone() + }, + forward_service_enabled: vm_net.forward_service_enabled, + mac_prefix: if vm_net.mac_prefix.is_empty() { + cfg.networking.mac_prefix.clone() + } else { + vm_net.mac_prefix.clone() + }, + net: if vm_net.net.is_empty() { + cfg.networking.net.clone() + } else { + vm_net.net.clone() + }, + dhcp_start: if vm_net.dhcp_start.is_empty() { + cfg.networking.dhcp_start.clone() + } else { + vm_net.dhcp_start.clone() + }, + restrict: cfg.networking.restrict || vm_net.restrict, + netdev: if vm_net.netdev.is_empty() { + cfg.networking.netdev.clone() + } else { + vm_net.netdev.clone() + }, + } +} + +pub fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { + if manifest.networks.is_empty() { + vec![cfg.networking.clone()] + } else { + manifest + .networks + .iter() + .map(|networking| resolve_networking(networking, cfg)) + .collect() + } +} + +pub fn validate_resolved_network(networking: &Networking) -> Result<()> { + if networking.mode == NetworkingMode::Bridge { + if networking.bridge.is_empty() { + bail!("bridge networking requested but no bridge is configured"); + } + if !Path::new("/sys/class/net") + .join(&networking.bridge) + .exists() + { + bail!("bridge interface '{}' does not exist", networking.bridge); + } + } + Ok(()) +} + +pub fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { + for networking in networks { + validate_resolved_network(networking)?; + } + Ok(()) +} + +fn sanitize_optional>(value: Option) -> Option { + value.filter(|value| !value.as_ref().trim().is_empty()) +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct AmdSevSnpLaunchParams { @@ -129,6 +271,34 @@ fn detect_amd_sev_snp_qemu_capabilities(qemu_path: &Path) -> Result, + #[serde(default, with = "hex_bytes")] + pub instance_id: Vec, + #[serde(default, with = "hex_bytes")] + pub app_id: Vec, +} + +pub struct VmInfo { + pub manifest: Manifest, + pub workdir: PathBuf, + pub status: &'static str, + pub uptime: String, + pub exited_at: Option, + pub instance_id: Option, + pub boot_progress: String, + pub boot_error: String, + pub shutdown_progress: String, + pub image_version: String, + pub gateway_enabled: bool, + pub events: Vec, + pub guest_ip: String, + pub guest_ips: HashMap, + pub runtime_networks: Vec, +} + #[derive(Debug, Builder)] pub struct VmConfig { pub manifest: Manifest, @@ -138,6 +308,11 @@ pub struct VmConfig { pub gateway_enabled: bool, } +#[derive(Deserialize, Serialize)] +pub struct State { + started: bool, +} + fn create_hd( image_file: impl AsRef, backing_file: Option>, @@ -163,154 +338,368 @@ fn create_hd( Ok(()) } -fn virtio_pci_device(device: &str, snp: bool) -> String { - if snp { - format!("{device},disable-legacy=on,iommu_platform=true") - } else { - device.to_string() +/// Create a FAT32 disk image from a directory +fn create_shared_disk(disk_path: impl AsRef, shared_dir: impl AsRef) -> Result<()> { + use fatfs::{FileSystem, FormatVolumeOptions, FsOptions}; + use std::io::{Seek, SeekFrom, Write}; + + let disk_path = disk_path.as_ref(); + let shared_dir = shared_dir.as_ref(); + + // Must be large enough to hold all host-shared files (app-compose.json and + // .user-config can each be up to 50 MiB, see HostShared::copy) plus FAT32 overhead. + const DISK_SIZE: u64 = 128 * 1024 * 1024; + + // Back the image by a file (sparse until written) and stream files into it so + // peak memory stays bounded regardless of DISK_SIZE or input file sizes. + let mut image = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(disk_path) + .with_context(|| format!("Failed to create disk image at {}", disk_path.display()))?; + image + .set_len(DISK_SIZE) + .context("Failed to size disk image")?; + + { + let mut label_bytes = [b' '; 11]; + let label_str = HOST_SHARED_DISK_LABEL.as_bytes(); + let copy_len = label_str.len().min(11); + label_bytes[..copy_len].copy_from_slice(&label_str[..copy_len]); + let format_opts = FormatVolumeOptions::new() + .fat_type(fatfs::FatType::Fat32) + .volume_label(label_bytes); + fatfs::format_volume(&mut image, format_opts).context("Failed to format disk as FAT32")?; } -} -struct PreparedQemuLaunch { - workdir: VmWorkDir, - platform: TeePlatform, - hugepage_numa_nodes: Option>, - gpu_numa_nodes: HashMap, - numa_cpus: Option, - tpm_path: Option<&'static str>, - tdx_mr_config_id: Option, - snp_host_data: Option, - snp_launch_params: Option, -} + // Open the formatted filesystem and stream files into it + { + image + .seek(SeekFrom::Start(0)) + .context("Failed to seek to start")?; + let fs = FileSystem::new(&mut image, FsOptions::new()) + .context("Failed to open FAT32 filesystem")?; + let root_dir = fs.root_dir(); -impl PreparedQemuLaunch { - fn prepare( - vm: &VmConfig, - workdir: impl AsRef, - cfg: &CvmConfig, - gpus: &GpuConfig, - ) -> Result { - let workdir = VmWorkDir::new(workdir); - prepare_data_disk(vm, &workdir, cfg)?; - prepare_shared_dir(&workdir)?; - let app_compose = workdir.app_compose().context("failed to get app compose")?; - let platform = cfg.resolved_platform(); + // Copy all files from shared_dir to the FAT32 root + for entry in fs::read_dir(shared_dir).context("Failed to read shared directory")? { + let entry = entry.context("Failed to read directory entry")?; + let path = entry.path(); - let hugepage_numa_nodes = if vm.manifest.hugepages { - Some(hugepage_numa_nodes(gpus)?) - } else { - None - }; - let gpu_numa_nodes = if vm.manifest.hugepages { - gpus.gpus - .iter() - .map(|gpu| Ok((gpu.slot.clone(), pci_numa_node(&gpu.slot)?))) - .collect::>()? - } else { - HashMap::new() - }; - let numa_cpus = if vm.manifest.pin_numa { - let device = gpus.gpus.first().map(|gpu| gpu.slot.clone()); - Some(find_numa(device)?.1) - } else { - None - }; - let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - Some(detect_tpm_device()?) + if path.is_file() { + let filename = entry.file_name(); + let filename_str = filename.to_string_lossy(); + + let mut src = fs::File::open(&path) + .with_context(|| format!("Failed to open file {}", path.display()))?; + let mut fat_file = root_dir + .create_file(&filename_str) + .with_context(|| format!("Failed to create file {filename_str} in FAT32"))?; + std::io::copy(&mut src, &mut fat_file) + .with_context(|| format!("Failed to write file {filename_str} to FAT32"))?; + fat_file.flush().context("Failed to flush FAT32 file")?; + } + } + } + + Ok(()) +} + +impl VmInfo { + pub fn to_pb(&self, gw: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { + let workdir = VmWorkDir::new(&self.workdir); + let vm_config = workdir.manifest(); + let custom_gateway_urls = vm_config + .as_ref() + .map(|c| c.gateway_urls.clone()) + .unwrap_or_default(); + let configured_networks = self + .manifest + .networks + .iter() + .map(networking_to_proto) + .collect::>(); + let configured_networking = configured_networks.first().cloned(); + let effective_networks = if self.runtime_networks.is_empty() { + resolved_networks(&self.manifest, cvm) } else { - None + self.runtime_networks.clone() }; - prepare_shared_disk(&workdir, cfg)?; + let ip_index = effective_networks + .iter() + .position(|networking| networking.mode == NetworkingMode::Bridge) + .unwrap_or(0); + let interfaces = effective_networks + .iter() + .enumerate() + .map(|(index, networking)| { + let prefix = networking.mac_prefix_bytes(); + let mac = mac_address_for_vm_index(&self.manifest.id, &prefix, index); + let ip = self.guest_ips.get(&mac).cloned().or_else(|| { + (self.guest_ips.is_empty() + && index == ip_index + && networking.mode == NetworkingMode::Bridge) + .then(|| self.guest_ip.clone()) + .filter(|ip| !ip.is_empty()) + }); + pb::NetworkInterfaceStatus { + mode: networking_mode_name(networking.mode).into(), + backend: networking_backend_name(networking.mode).into(), + mac: mac.clone(), + ip, + bridge_name: (networking.mode == NetworkingMode::Bridge) + .then(|| networking.bridge.clone()), + netdev_id: Some(format!("net{index}")), + if_name: None, + lowerdev: None, + pci_bdf: None, + } + }) + .collect(); - let tee_enabled = !vm.manifest.no_tee; - let tdx_mr_config_id = if tee_enabled - && platform == TeePlatform::Tdx - && cfg.use_mrconfigid - && vm.image.info.version_tuple().unwrap_or_default() >= (0, 5, 2) - { - Some(tdx_mr_config_id(&workdir, &app_compose)?) + pb::VmInfo { + id: self.manifest.id.clone(), + name: self.manifest.name.clone(), + status: self.status.into(), + uptime: self.uptime.clone(), + boot_progress: self.boot_progress.clone(), + boot_error: self.boot_error.clone(), + shutdown_progress: self.shutdown_progress.clone(), + image_version: self.image_version.clone(), + configuration: if brief { + None + } else { + let kms_urls = vm_config + .as_ref() + .map(|c| c.kms_urls.clone()) + .unwrap_or_default(); + let no_tee = vm_config + .as_ref() + .map(|c| c.no_tee) + .unwrap_or(self.manifest.no_tee); + let stopped = !workdir.started().unwrap_or(false); + + Some(pb::VmConfiguration { + name: self.manifest.name.clone(), + image: self.manifest.image.clone(), + compose_file: { + fs::read_to_string(workdir.app_compose_path()).unwrap_or_default() + }, + encrypted_env: { fs::read(workdir.encrypted_env_path()).unwrap_or_default() }, + user_config: { + fs::read_to_string(workdir.user_config_path()).unwrap_or_default() + }, + vcpu: self.manifest.vcpu, + memory: self.manifest.memory, + disk_size: self.manifest.disk_size, + ports: self + .manifest + .port_map + .iter() + .map(|pm| pb::PortMapping { + protocol: pm.protocol.as_str().into(), + host_address: pm.address.to_string(), + host_port: pm.from as u32, + vm_port: pm.to as u32, + }) + .collect(), + app_id: Some(self.manifest.app_id.clone()), + hugepages: self.manifest.hugepages, + pin_numa: self.manifest.pin_numa, + gpus: self.manifest.gpus.as_ref().map(|g| pb::GpuConfig { + attach_mode: g.attach_mode.to_string(), + gpus: g + .gpus + .iter() + .map(|gpu| pb::GpuSpec { + slot: gpu.slot.clone(), + }) + .collect(), + }), + kms_urls, + gateway_urls: custom_gateway_urls.clone(), + stopped, + no_tee, + networking: configured_networking, + networks: configured_networks, + }) + }, + app_url: self + .gateway_enabled + .then_some(self.instance_id.as_deref()) + .flatten() + .and_then(|id| sanitize_optional(Some(id))) + .map(|id| { + // Use custom gateway URL if available, otherwise fall back to global config + if let Some(custom_gw_url) = custom_gateway_urls.first() { + if let Ok(url) = url::Url::parse(custom_gw_url) { + let host = url.host_str().unwrap_or(&gw.base_domain); + let port = url.port().unwrap_or(443); + if port == 443 { + return format!("https://{id}-{}.{}", gw.agent_port, host); + } else { + return format!("https://{id}-{}.{}:{}", gw.agent_port, host, port); + } + } + } + // Fall back to global gateway config + if gw.port == 443 { + format!("https://{id}-{}.{}", gw.agent_port, gw.base_domain) + } else { + format!( + "https://{id}-{}.{}:{}", + gw.agent_port, gw.base_domain, gw.port + ) + } + }), + app_id: self.manifest.app_id.clone(), + instance_id: sanitize_optional(self.instance_id.clone()), + exited_at: self.exited_at.clone(), + events: self.events.clone(), + interfaces, + } + } +} + +impl VmState { + pub fn merged_info(&self, proc_state: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo { + fn truncate(d: Duration) -> Duration { + Duration::from_secs(d.as_secs()) + } + let is_running = match proc_state { + Some(info) => info.state.status.is_running(), + None => false, + }; + let started = workdir.started().unwrap_or(false); + let status = if self.state.removing { + "removing" } else { - None + match (started, is_running) { + (true, true) => "running", + (true, false) => "exited", + (false, true) => "stopping", + (false, false) => "stopped", + } }; - let (snp_host_data, snp_launch_params) = - if tee_enabled && platform == TeePlatform::AmdSevSnp { - ( - Some(snp_host_data(&workdir)?), - Some( - detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path).context( - "failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU", - )?, - ), - ) - } else { - (None, None) - }; - Ok(Self { - workdir, - platform, - hugepage_numa_nodes, - gpu_numa_nodes, - numa_cpus, - tpm_path, - tdx_mr_config_id, - snp_host_data, - snp_launch_params, - }) + fn display_ts(t: Option<&SystemTime>) -> String { + match t { + None => "never".into(), + Some(t) => { + let ts = t.elapsed().unwrap_or(Duration::MAX); + humantime::format_duration(truncate(ts)).to_string() + } + } + } + let uptime = display_ts(proc_state.and_then(|info| info.state.started_at.as_ref())); + let exited_at = display_ts(proc_state.and_then(|info| info.state.stopped_at.as_ref())); + let instance_id = sanitize_optional( + workdir + .instance_info() + .ok() + .map(|info| hex::encode(info.instance_id)), + ); + VmInfo { + manifest: self.config.manifest.clone(), + workdir: workdir.path().to_path_buf(), + instance_id, + status, + uptime, + exited_at: Some(exited_at), + boot_progress: self.state.boot_progress.clone(), + boot_error: self.state.boot_error.clone(), + shutdown_progress: self.state.shutdown_progress.clone(), + image_version: self.config.image.info.version.clone(), + gateway_enabled: self.config.gateway_enabled, + events: self.state.events.clone().into(), + guest_ip: self.state.guest_ip.clone(), + guest_ips: self.state.guest_ips.clone(), + runtime_networks: self.state.runtime_networks.clone(), + } } } -fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { - let hda_path = workdir.hda_path(); - if !hda_path.exists() { - create_hd( - &hda_path, - vm.image.hda.as_ref(), - &format!("{}G", vm.manifest.disk_size), - )?; +#[cfg(test)] +mod tests { + use super::{ + amd_sev_snp_memory_backend_arg, mac_address_for_vm, mac_address_for_vm_index, + parse_amd_sev_snp_qmp_capabilities, sanitize_optional, virtio_pci_device, + }; + + #[test] + fn sanitize_optional_filters_empty_owned_values() { + assert_eq!(sanitize_optional(Some(String::new())), None); + assert_eq!(sanitize_optional(Some(" ".to_string())), None); + assert_eq!( + sanitize_optional(Some("instance-123".to_string())), + Some("instance-123".to_string()) + ); } - if !cfg.user.is_empty() { - fs::set_permissions(&hda_path, Permissions::from_mode(0o660))?; + + #[test] + fn sanitize_optional_filters_empty_borrowed_values() { + assert_eq!(sanitize_optional(Some("")), None); + assert_eq!(sanitize_optional(Some(" ")), None); + assert_eq!( + sanitize_optional(Some("instance-123")), + Some("instance-123") + ); } - Ok(()) -} -fn prepare_shared_dir(workdir: &VmWorkDir) -> Result<()> { - let shared_dir = workdir.shared_dir(); - if !shared_dir.exists() { - fs::create_dir_all(&shared_dir)?; + #[test] + fn mac_address_index_zero_keeps_existing_derivation() { + assert_eq!(mac_address_for_vm("vm-123", &[]), "96:b1:d8:b9:08:e6"); + assert_eq!( + mac_address_for_vm_index("vm-123", &[], 0), + "96:b1:d8:b9:08:e6" + ); + assert_eq!( + mac_address_for_vm_index("vm-123", &[], 1), + "c6:74:2c:65:14:b9" + ); } - Ok(()) -} -fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { - if cfg.host_share_mode != "vhd" { - return Ok(()); + #[test] + fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { + assert_eq!( + amd_sev_snp_memory_backend_arg(4096), + "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" + ); } - let shared_dir = workdir.shared_dir(); - let shared_disk_path = workdir.shared_disk_path(); - if shared_disk_path.exists() { - fs::remove_file(&shared_disk_path).context("failed to remove shared disk")?; + #[test] + fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { + let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} +{"return":{}} +{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} +{"return":{}} +"#; + let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); + assert_eq!(params.cbitpos, 51); + assert_eq!(params.reduced_phys_bits, 1); } - create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk") -} -fn detect_tpm_device() -> Result<&'static str> { - if Path::new("/dev/tpmrm0").exists() { - Ok("/dev/tpmrm0") - } else if Path::new("/dev/tpm0").exists() { - Ok("/dev/tpm0") - } else { - bail!("tpm key provider requested but no TPM device found on host") + #[test] + fn amd_sev_snp_uses_confidential_virtio_pci_options() { + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", true), + "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" + ); + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", false), + "virtio-blk-pci,drive=hd0" + ); } } -struct QemuCommandBuilder<'a> { - vm: &'a VmConfig, - cfg: &'a CvmConfig, - gpus: &'a GpuConfig, - prepared: &'a PreparedQemuLaunch, +fn virtio_pci_device(device: &str, snp: bool) -> String { + if snp { + format!("{device},disable-legacy=on,iommu_platform=true") + } else { + device.to_string() + } } impl VmConfig { @@ -320,220 +709,201 @@ impl VmConfig { cfg: &CvmConfig, gpus: &GpuConfig, ) -> Result> { - let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; - let process = QemuCommandBuilder { - vm: self, - cfg, - gpus, - prepared: &prepared, + let workdir = VmWorkDir::new(workdir); + let serial_file = workdir.serial_file(); + let serial_pty = workdir.serial_pty(); + let shared_dir = workdir.shared_dir(); + let disk_size = format!("{}G", self.manifest.disk_size); + let hda_path = workdir.hda_path(); + if !hda_path.exists() { + create_hd(&hda_path, self.image.hda.as_ref(), &disk_size)?; } - .build()?; - Ok(vec![process]) - } -} - -impl QemuCommandBuilder<'_> { - fn build(&self) -> Result { - let mut command = self.base_command(); - self.configure_rootfs(&mut command)?; - self.configure_data_disk(&mut command); - self.configure_networking(&mut command); - self.vm.configure_smbios(&mut command, self.cfg); - self.configure_tpm_and_vsock(&mut command); - self.configure_host_share(&mut command)?; - - let (smp, mem) = self.configure_hugepage_memory(&mut command)?; - self.vm - .configure_machine(&mut command, self.cfg, self.prepared, mem)?; - self.configure_gpus(&mut command)?; - command.arg("-smp").arg(smp.to_string()); - command.arg("-m").arg(format!("{mem}M")); - - // SNP app identity is bound through HOST_DATA, so the measured cmdline - // remains the image-provided cmdline. - if let Some(cmdline) = &self.vm.image.info.cmdline { - command.arg("-append").arg(cmdline); + if !cfg.user.is_empty() { + fs_err::set_permissions(&hda_path, Permissions::from_mode(0o660))?; } - self.process_config(command) - } - - fn is_amd_sev_snp(&self) -> bool { - self.prepared.platform == TeePlatform::AmdSevSnp && !self.vm.manifest.no_tee - } - fn base_command(&self) -> Command { - let workdir = &self.prepared.workdir; - let mut command = Command::new(&self.cfg.qemu_path); - command.arg("-accel").arg("kvm"); - command.arg("-cpu").arg(if self.is_amd_sev_snp() { - "EPYC-v4" + if !shared_dir.exists() { + fs::create_dir_all(&shared_dir)?; + } + let app_compose = workdir.app_compose().context("Failed to get app compose")?; + let qemu = &cfg.qemu_path; + let is_amd_sev_snp = + cfg.resolved_platform() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; + let mut numa_nodes_for_hugepages = if self.manifest.hugepages { + Some(hugepage_numa_nodes(gpus)?) } else { - "host" - }); + None + }; + let smp = effective_vcpu_count( + self.manifest.vcpu, + numa_nodes_for_hugepages + .as_ref() + .map(|nodes| nodes.len() as u32), + ); + let mut mem = self.manifest.memory; + let mut command = Command::new(qemu); + command.arg("-accel").arg("kvm"); + let cpu = if is_amd_sev_snp { "EPYC-v4" } else { "host" }; + command.arg("-cpu").arg(cpu); command.arg("-nographic"); command.arg("-nodefaults"); command.arg("-chardev").arg(format!( "pty,id=com0,path={},logfile={}", - workdir.serial_pty().display(), - workdir.serial_file().display() + serial_pty.display(), + serial_file.display() )); command.arg("-serial").arg("chardev:com0"); - if self.cfg.qmp_socket { + if cfg.qmp_socket { command.arg("-qmp").arg(format!( "unix:{},server,wait=off", workdir.qmp_socket().display() )); } - if let Some(bios) = self.vm.image.firmware(self.is_amd_sev_snp()) { + if let Some(bios) = self.image.firmware(is_amd_sev_snp) { command.arg("-bios").arg(bios); } - command.arg("-kernel").arg(&self.vm.image.kernel); - command.arg("-initrd").arg(&self.vm.image.initrd); - if self.cfg.qemu_hotplug_off { + command.arg("-kernel").arg(&self.image.kernel); + command.arg("-initrd").arg(&self.image.initrd); + if cfg.qemu_hotplug_off { command.args([ "-global", "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off", ]); } - if self.cfg.qemu_pci_hole64_size > 0 { + if cfg.qemu_pci_hole64_size > 0 { command.args([ "-global", &format!( "q35-pcihost.pci-hole64-size=0x{:x}", - self.cfg.qemu_pci_hole64_size + cfg.qemu_pci_hole64_size ), ]); } - command - } - - fn configure_rootfs(&self, command: &mut Command) -> Result<()> { - let Some(rootfs) = &self.vm.image.rootfs else { - return Ok(()); - }; - let image_version = self.vm.image.info.version_tuple().unwrap_or_default(); - let extension = rootfs - .extension() - .unwrap_or_default() - .to_str() - .unwrap_or_default(); - match extension { - "iso" => { - if image_version >= (0, 5, 0) { - bail!( - "Unsupported rootfs type: {extension}. Image versions >= 0.5.0 must use verity rootfs" - ); + if let Some(rootfs) = &self.image.rootfs { + let img_ver = self.image.info.version_tuple().unwrap_or_default(); + let ext = rootfs + .extension() + .unwrap_or_default() + .to_str() + .unwrap_or_default(); + match ext { + "iso" => { + if img_ver >= (0, 5, 0) { + bail!( + "Unsupported rootfs type: {ext}. Image versions >= 0.5.0 must use verity rootfs" + ); + } + command.arg("-cdrom").arg(rootfs); + } + "verity" => { + command.arg("-drive").arg(format!( + "file={},if=none,id=hd0,format=raw,readonly=on", + rootfs.display() + )); + command.arg("-device").arg(virtio_pci_device( + "virtio-blk-pci,drive=hd0", + is_amd_sev_snp, + )); + } + _ => { + bail!("Unsupported rootfs type: {ext}"); } - command.arg("-cdrom").arg(rootfs); - } - "verity" => { - command.arg("-drive").arg(format!( - "file={},if=none,id=hd0,format=raw,readonly=on", - rootfs.display() - )); - command.arg("-device").arg(virtio_pci_device( - "virtio-blk-pci,drive=hd0", - self.is_amd_sev_snp(), - )); } - _ => bail!("Unsupported rootfs type: {extension}"), } - Ok(()) - } - - fn configure_data_disk(&self, command: &mut Command) { + let mut processes = vec![]; command .arg("-drive") - .arg(format!( - "file={},if=none,id=hd1", - self.prepared.workdir.hda_path().display() - )) + .arg(format!("file={},if=none,id=hd1", hda_path.display())) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=hd1", - self.is_amd_sev_snp(), + is_amd_sev_snp, )); - } - - fn configure_networking(&self, command: &mut Command) { - // Per-VM networking only overrides the mode and bridge name. All other - // fields continue to come from the global configuration. - let resolved_networking; - let networking = match self.vm.manifest.networking.as_ref() { - Some(vm_networking) => { - resolved_networking = Networking { - mode: vm_networking.mode, - bridge: if vm_networking.bridge.is_empty() { - self.cfg.networking.bridge.clone() - } else { - vm_networking.bridge.clone() - }, - ..self.cfg.networking.clone() - }; - &resolved_networking - } - None => &self.cfg.networking, - }; - let mac = mac_address_for_vm(&self.vm.manifest.id, &networking.mac_prefix_bytes()); - let net_device = virtio_pci_device( - &format!("virtio-net-pci,netdev=net0,mac={mac}"), - self.is_amd_sev_snp(), - ); - let netdev = match networking.mode { - NetworkingMode::User => { - let mut netdev = format!( - "user,id=net0,net={},dhcpstart={},restrict={}", - networking.net, - networking.dhcp_start, - if networking.restrict { "yes" } else { "no" } - ); - for mapping in &self.vm.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from, - mapping.to - )); + let networks = resolved_networks(&self.manifest, cfg); + validate_resolved_networks(&networks)?; + let hostfwd_index = networks + .iter() + .position(|networking| networking.mode == NetworkingMode::User); + for (index, networking) in networks.iter().enumerate() { + let net_id = format!("net{index}"); + let prefix = networking.mac_prefix_bytes(); + let mac = mac_address_for_vm_index(&self.manifest.id, &prefix, index); + let net_device = virtio_pci_device( + &format!("virtio-net-pci,netdev={net_id},mac={mac}"), + is_amd_sev_snp, + ); + let netdev = match networking.mode { + NetworkingMode::User => { + let mut netdev = format!( + "user,id={net_id},net={},dhcpstart={},restrict={}", + networking.net, + networking.dhcp_start, + if networking.restrict { "yes" } else { "no" } + ); + if hostfwd_index == Some(index) { + for pm in &self.manifest.port_map { + netdev.push_str(&format!( + ",hostfwd={}:{}:{}-:{}", + pm.protocol.as_str(), + pm.address, + pm.from, + pm.to + )); + } + } + netdev } - netdev - } - NetworkingMode::Bridge => { - tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - format!("bridge,id=net0,br={}", networking.bridge) - } - NetworkingMode::Custom => networking.netdev.clone(), - }; - command.arg("-netdev").arg(netdev); - command.arg("-device").arg(net_device); - } + NetworkingMode::Bridge => { + tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); + format!("bridge,id={net_id},br={}", networking.bridge) + } + NetworkingMode::Custom => { + if !networking.netdev.contains(&format!("id={net_id}")) { + bail!( + "custom networking netdev must contain id={net_id} for interface index {index}" + ); + } + networking.netdev.clone() + } + }; + command.arg("-netdev").arg(netdev); + command.arg("-device").arg(net_device); + } - fn configure_tpm_and_vsock(&self, command: &mut Command) { - if let Some(tpm_path) = self.prepared.tpm_path { + self.configure_smbios(&mut command, cfg); + + if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { + let tpm_path = if Path::new("/dev/tpmrm0").exists() { + "/dev/tpmrm0" + } else if Path::new("/dev/tpm0").exists() { + "/dev/tpm0" + } else { + bail!("TPM key provider requested but no TPM device found on host"); + }; command .arg("-tpmdev") .arg(format!("passthrough,id=tpm0,path={tpm_path}")) .arg("-device") .arg("tpm-tis,tpmdev=tpm0"); } + command.arg("-device").arg(virtio_pci_device( - &format!("vhost-vsock-pci,guest-cid={}", self.vm.cid), - self.is_amd_sev_snp(), + &format!("vhost-vsock-pci,guest-cid={}", self.cid), + is_amd_sev_snp, )); - } - fn configure_host_share(&self, command: &mut Command) -> Result<()> { - let workdir = &self.prepared.workdir; - match self.cfg.host_share_mode.as_str() { + // Configure shared files delivery: either via disk or 9p + match cfg.host_share_mode.as_str() { "9p" => { - let read_only = if self.vm.image.info.shared_ro { + // Use 9p virtfs (default) + let ro = if self.image.info.shared_ro { "on" } else { "off" }; command.arg("-virtfs").arg(format!( - "local,path={},mount_tag=host-shared,readonly={read_only},security_model=mapped,id=virtfs0", - workdir.shared_dir().display(), + "local,path={},mount_tag=host-shared,readonly={ro},security_model=mapped,id=virtfs0", + shared_dir.display(), )); } "vvfat" => { @@ -541,152 +911,214 @@ impl QemuCommandBuilder<'_> { .arg("-blockdev") .arg(format!( "driver=vvfat,node-name=vvfat0,read-only=on,dir={},label={}", - workdir.shared_dir().display(), + shared_dir.display(), HOST_SHARED_DISK_LABEL )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=vvfat0", - self.is_amd_sev_snp(), + is_amd_sev_snp, )); } "vhd" => { + // Use a second virtual disk (hd2) to share files + let shared_disk_path = workdir.shared_disk_path(); + if shared_disk_path.exists() { + fs::remove_file(&shared_disk_path).context("Failed to remove shared disk")?; + } + create_shared_disk(&shared_disk_path, &shared_dir) + .context("Failed to create shared disk")?; command .arg("-drive") .arg(format!( "file={},if=none,id=hd2,format=raw,readonly=on", - workdir.shared_disk_path().display() + shared_disk_path.display() )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=hd2", - self.is_amd_sev_snp(), + is_amd_sev_snp, )); } - _ => bail!("Invalid host sharing mode: {}", self.cfg.host_share_mode), + _ => { + bail!("Invalid host sharing mode: {}", cfg.host_share_mode); + } } - Ok(()) - } - fn configure_hugepage_memory(&self, command: &mut Command) -> Result<(u32, u32)> { - let numa_nodes = self.prepared.hugepage_numa_nodes.as_ref(); - let smp = effective_vcpu_count( - self.vm.manifest.vcpu, - numa_nodes.map(|nodes| nodes.len() as u32), - ); - if !self.vm.manifest.hugepages { - return Ok((smp, self.vm.manifest.memory)); - } - - let numa_nodes = numa_nodes - .context("hugepage NUMA nodes should be computed during launch preparation")?; - let numa_count = numa_nodes.len() as u32; - let memory_gib = round_up(self.vm.manifest.memory / 1024, numa_count); - let vcpus_per_node = smp / numa_count; - let memory_per_node = memory_gib / numa_count; - let mut bus_number = 5_u32; - for (index, (node, device_count)) in numa_nodes.iter().enumerate() { - let index = index as u32; - let cpu_start = index * vcpus_per_node; - let cpu_end = (index + 1) * vcpus_per_node - 1; - command.arg("-numa").arg(format!( - "node,nodeid={index},cpus={cpu_start}-{cpu_end},memdev=mem{index}", - )); - command.arg("-object").arg(format!( - "memory-backend-file,id=mem{index},size={memory_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind", - )); - let address = 0xa + index; - command.arg("-device").arg(format!( - "pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={address},numa_node={index},bus_nr={bus_number}", - )); - bus_number += device_count + 1; + let hugepages = self.manifest.hugepages; + let pin_numa = self.manifest.pin_numa; + // Handle GPU configuration + let mut dev_num = 1; + let memory = self.manifest.memory; + + // Handle hugepages configuration + if hugepages { + let numa_nodes = numa_nodes_for_hugepages + .take() + .context("hugepage NUMA nodes should be computed above")?; + let n_numa = numa_nodes.len() as u32; + + // Round up CPU cores and memory to multiple times of NUMA nodes. + // `smp` is already the shared effective vCPU count used by vm_config. + let vcpu_count = smp; + let mem_gb = round_up(memory / 1024, n_numa); + let vcpu_per_node = vcpu_count / n_numa; + let mem_per_node = mem_gb / n_numa; + + mem = mem_gb * 1024; + + let mut bus_nr = 5_u32; + + // Configure NUMA nodes + for (ind, (node, count)) in numa_nodes.into_iter().enumerate() { + let ind = ind as u32; + let cpu_start = ind * vcpu_per_node; + let cpu_end = (ind + 1) * vcpu_per_node - 1; + command.arg("-numa").arg(format!( + "node,nodeid={ind},cpus={cpu_start}-{cpu_end},memdev=mem{ind}", + )); + + command.arg("-object").arg(format!( + "memory-backend-file,id=mem{ind},size={mem_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind", + )); + + let addr = 0xa + ind; + command.arg("-device").arg(format!( + "pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={addr},numa_node={ind},bus_nr={bus_nr}", + )); + bus_nr += count + 1; + } } - Ok((smp, memory_gib * 1024)) - } - fn configure_gpus(&self, command: &mut Command) -> Result<()> { - if self.gpus.gpus.is_empty() { - return Ok(()); + self.configure_machine(&mut command, &workdir, cfg, &app_compose, mem)?; + + // Configure GPU devices + if !gpus.gpus.is_empty() { + // Add iommufd object + command.arg("-object").arg("iommufd,id=iommufd0"); + + if !hugepages { + // Add each GPU + for device in &gpus.gpus { + let slot = &device.slot; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", + )); + + dev_num += 1; + } + } else { + // Add each GPU with NUMA node awareness for hugepages configuration + for device in &gpus.gpus { + let slot = &device.slot; + let node = pci_numa_node(slot)?; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{dev_num},bus=pcie.node{node},chassis={dev_num}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", + )); + dev_num += 1; + } + } + + // Add bridges (NVSwitches) if any + if !gpus.bridges.is_empty() { + for bridge in &gpus.bridges { + let slot = &bridge.slot; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", + )); + dev_num += 1; + } + } } - command.arg("-object").arg("iommufd,id=iommufd0"); - let mut device_number = 1; - for device in &self.gpus.gpus { - let slot = &device.slot; - let bus = if self.vm.manifest.hugepages { - let node = self - .prepared - .gpu_numa_nodes - .get(slot) - .context("gpu NUMA node should be computed during launch preparation")?; - format!("pcie.node{node}") + command.arg("-smp").arg(smp.to_string()); + command.arg("-m").arg(format!("{}M", mem)); + + // NUMA pinning if requested + let mut numa_cpus = None; + if pin_numa { + if !gpus.gpus.is_empty() { + let (_, cpus) = find_numa(Some(gpus.gpus[0].slot.clone()))?; + numa_cpus = Some(cpus); } else { - "pcie.0".into() - }; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{device_number},bus={bus},chassis={device_number}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{device_number},iommufd=iommufd0", - )); - device_number += 1; + // Default to NUMA node 0 if no GPUs + let (_, cpus) = find_numa(None)?; + numa_cpus = Some(cpus); + } } - for bridge in &self.gpus.bridges { - let slot = &bridge.slot; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{device_number},bus=pcie.0,chassis={device_number}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{device_number},iommufd=iommufd0", - )); - device_number += 1; + + // SNP app identity is bound through HOST_DATA, so the measured cmdline + // remains the image-provided cmdline. + let cmdline = self.image.info.cmdline.clone(); + if let Some(cmdline) = cmdline { + command.arg("-append").arg(cmdline); } - Ok(()) - } - fn process_config(&self, command: Command) -> Result { - let workdir = &self.prepared.workdir; - let mut arguments = vec![self.cfg.qemu_path.to_string_lossy().to_string()]; - arguments.extend( - command - .get_args() - .map(|argument| argument.to_string_lossy().to_string()), - ); - if let Some(cpus) = &self.prepared.numa_cpus { - arguments.splice(0..0, ["taskset", "-c", cpus].into_iter().map(String::from)); + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + let pidfile_path = workdir.pid_file(); + let stdout_path = workdir.stdout_file(); + let stderr_path = workdir.stderr_file(); + + let workdir = workdir.path(); + + let mut cmd_args = vec![]; + cmd_args.push(qemu.to_string_lossy().to_string()); + cmd_args.extend(args); + + // If we have NUMA pinning, we'll need to wrap the command with taskset + if let Some(cpus) = numa_cpus { + cmd_args.splice(0..0, ["taskset", "-c", &cpus].into_iter().map(|s| s.into())); } - if !self.cfg.user.is_empty() { - arguments.splice( + + if !cfg.user.is_empty() { + cmd_args.splice( 0..0, - ["sudo", "-u", &self.cfg.user].into_iter().map(String::from), + ["sudo", "-u", &cfg.user].into_iter().map(|s| s.into()), ); } - let command = arguments.remove(0); - let note = serde_json::to_string(&ProcessAnnotation { + let command = cmd_args.remove(0); + let note = ProcessAnnotation { kind: "cvm".to_string(), live_for: None, - })?; - Ok(ProcessConfig { - id: self.vm.manifest.id.clone(), - args: arguments, - name: self.vm.manifest.name.clone(), + }; + let note = serde_json::to_string(¬e)?; + let process_config = ProcessConfig { + id: self.manifest.id.clone(), + args: cmd_args, + name: self.manifest.name.clone(), command, env: Default::default(), - cwd: workdir.path().to_string_lossy().to_string(), - stdout: workdir.stdout_file().to_string_lossy().to_string(), - stderr: workdir.stderr_file().to_string_lossy().to_string(), - pidfile: workdir.pid_file().to_string_lossy().to_string(), - cid: Some(self.vm.cid), + cwd: workdir.to_string_lossy().to_string(), + stdout: stdout_path.to_string_lossy().to_string(), + stderr: stderr_path.to_string_lossy().to_string(), + pidfile: pidfile_path.to_string_lossy().to_string(), + cid: Some(self.cid), note, - }) + }; + processes.push(process_config); + + Ok(processes) } -} -impl VmConfig { + fn configure_machine( &self, command: &mut Command, + workdir: &VmWorkDir, cfg: &CvmConfig, - prepared: &PreparedQemuLaunch, + app_compose: &AppCompose, mem: u32, ) -> Result<()> { if self.manifest.no_tee { @@ -696,22 +1128,15 @@ impl VmConfig { return Ok(()); } - match prepared.platform { + match cfg.resolved_platform() { TeePlatform::Tdx => { command .arg("-machine") .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); - self.configure_tdx_guest(command, cfg, prepared.tdx_mr_config_id.as_deref())?; + self.configure_tdx_guest(command, workdir, cfg, app_compose)?; } TeePlatform::AmdSevSnp => { - let host_data = prepared - .snp_host_data - .as_deref() - .context("snp host data should be computed during launch preparation")?; - let launch_params = prepared.snp_launch_params.context( - "snp launch parameters should be detected during launch preparation", - )?; - self.configure_amd_sev_snp_guest(command, cfg, mem, host_data, launch_params); + self.configure_amd_sev_snp_guest(command, workdir, cfg, mem)?; } } Ok(()) @@ -720,9 +1145,60 @@ impl VmConfig { fn configure_tdx_guest( &self, command: &mut Command, + workdir: &VmWorkDir, cfg: &CvmConfig, - mrconfigid: Option<&str>, + app_compose: &AppCompose, ) -> Result<()> { + let img_ver = self.image.info.version_tuple().unwrap_or_default(); + let support_mr_config_id = img_ver >= (0, 5, 2); + + // Compute mrconfigid if needed + let mrconfigid = if cfg.use_mrconfigid && support_mr_config_id { + if let Some(mr_config_document) = workdir + .sys_config() + .context("Failed to read sys config for tdx mrconfigid")? + .mr_config + { + MrConfigV3::from_document(&mr_config_document) + .context("Invalid mr_config document")?; + Some( + BASE64_STANDARD.encode(MrConfigV3::tdx_mr_config_id_from_document( + &mr_config_document, + )), + ) + } else { + let compose_hash = workdir + .app_compose_hash() + .context("Failed to get compose hash")?; + let mr_config = if app_compose.key_provider_id.is_empty() { + MrConfig::V1 { + compose_hash: &compose_hash, + } + } else { + let instance_info = workdir + .instance_info() + .context("Failed to get instance info")?; + let app_id = if instance_info.app_id.is_empty() { + &compose_hash[..20] + } else { + &instance_info.app_id + }; + + let key_provider = app_compose.key_provider(); + let key_provider_id = &app_compose.key_provider_id; + MrConfig::V2 { + compose_hash: &compose_hash, + app_id: &app_id.try_into().context("Invalid app ID")?, + key_provider, + key_provider_id, + } + }; + Some(BASE64_STANDARD.encode(mr_config.to_mr_config_id())) + } + } else { + None + }; + // Build tdx-guest object with optional quote-generation-socket for kernel-level TSM support #[derive(Serialize)] struct QgsSocket { @@ -748,7 +1224,7 @@ impl VmConfig { let tdx_object = TdxGuestObject { qom_type: "tdx-guest", id: "tdx", - mrconfigid: mrconfigid.map(str::to_string), + mrconfigid: mrconfigid.clone(), quote_generation_socket: cfg.qgs_port.map(|port| QgsSocket { r#type: "vsock", cid: "2", @@ -766,14 +1242,24 @@ impl VmConfig { fn configure_amd_sev_snp_guest( &self, command: &mut Command, + workdir: &VmWorkDir, cfg: &CvmConfig, mem: u32, - host_data: &str, - snp_params: AmdSevSnpLaunchParams, - ) { + ) -> Result<()> { + let mr_config_document = workdir + .sys_config() + .context("Failed to read sys config for amd sev-snp host-data")? + .mr_config + .context("mr_config is required for amd sev-snp host-data")?; + MrConfigV3::from_document(&mr_config_document).context("Invalid mr_config document")?; + let host_data = + BASE64_STANDARD.encode(MrConfigV3::snp_host_data_from_document(&mr_config_document)); + command .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); + let snp_params = detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path) + .context("failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU")?; command.arg("-object").arg(format!( "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos={},reduced-phys-bits={}", snp_params.cbitpos, snp_params.reduced_phys_bits @@ -784,6 +1270,7 @@ impl VmConfig { if cfg.qgs_port.is_some() { tracing::warn!("qgs_port is ignored for amd sev-snp guests"); } + Ok(()) } fn configure_smbios(&self, command: &mut Command, cfg: &CvmConfig) { @@ -849,146 +1336,294 @@ fn find_numa(device: Option) -> Result<(String, String)> { Ok((numa_node, cpus)) } -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::path::PathBuf; +pub struct VmWorkDir { + workdir: PathBuf, +} - use rocket::figment::{ - providers::{Format, Toml}, - Figment, - }; +impl Deref for VmWorkDir { + type Target = PathBuf; + fn deref(&self) -> &Self::Target { + &self.workdir + } +} - use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, QemuCommandBuilder, VmConfig, - }; - use crate::app::image::{Image, ImageInfo}; - use crate::app::{GpuConfig, Manifest, VmWorkDir}; - use crate::config::{Config, TeePlatform, DEFAULT_CONFIG}; +impl AsRef for &VmWorkDir { + fn as_ref(&self) -> &Path { + self.workdir.as_ref() + } +} - #[test] - fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { - assert_eq!( - amd_sev_snp_memory_backend_arg(4096), - "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" - ); +impl VmWorkDir { + pub fn new(workdir: impl AsRef) -> Self { + Self { + workdir: workdir.as_ref().to_path_buf(), + } } - #[test] - fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { - let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} -{"return":{}} -{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} -{"return":{}} -"#; - let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); - assert_eq!(params.cbitpos, 51); - assert_eq!(params.reduced_phys_bits, 1); + pub fn manifest_path(&self) -> PathBuf { + self.workdir.join("vm-manifest.json") } - #[test] - fn amd_sev_snp_uses_confidential_virtio_pci_options() { - assert_eq!( - virtio_pci_device("virtio-blk-pci,drive=hd0", true), - "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" - ); - assert_eq!( - virtio_pci_device("virtio-blk-pci,drive=hd0", false), - "virtio-blk-pci,drive=hd0" - ); + pub fn state_path(&self) -> PathBuf { + self.workdir.join("vm-state.json") } - #[test] - fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { - let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) - .extract() - .unwrap(); - config.cvm.platform = Some(TeePlatform::Tdx); - config.cvm.qemu_path = PathBuf::from("/not-installed/qemu-system-x86_64"); - config.cvm.qgs_port = None; - - let vm = VmConfig { - manifest: Manifest { - id: "vm-1".into(), - name: "test-vm".into(), - app_id: "app-1".into(), - vcpu: 2, - memory: 2048, - disk_size: 10, - image: "test-image".into(), - port_map: vec![], - created_at_ms: 0, - hugepages: false, - pin_numa: false, - gpus: None, - kms_urls: vec![], - gateway_urls: vec![], - no_tee: true, - networking: None, - }, - image: Image { - info: ImageInfo { - cmdline: Some("console=hvc0".into()), - kernel: "kernel".into(), - initrd: "initrd".into(), - hda: None, - rootfs: None, - bios: None, - bios_sev: None, - rootfs_hash: None, - shared_ro: false, - version: "0.5.4".into(), - is_dev: false, - ovmf_variant: None, - }, - initrd: PathBuf::from("/does-not-exist/initrd"), - kernel: PathBuf::from("/does-not-exist/kernel"), - hda: None, - rootfs: None, - bios: None, - bios_sev: None, - digest: None, - tdx_measurement: None, - sev_measurement: None, + pub fn manifest(&self) -> Result { + let manifest_path = self.manifest_path(); + let manifest = fs::read_to_string(manifest_path).context("Failed to read manifest")?; + let manifest: Manifest = + serde_json::from_str(&manifest).context("Failed to parse manifest")?; + Ok(manifest) + } + + pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { + fs::create_dir_all(&self.workdir).context("Failed to create workdir")?; + let manifest_path = self.manifest_path(); + let mut value = serde_json::to_value(manifest)?; + if let Some(networking) = manifest.networks.first() { + value["networking"] = serde_json::to_value(networking)?; + } + fs::write(manifest_path, serde_json::to_string(&value)?).context("Failed to write manifest") + } + + pub fn started(&self) -> Result { + let state_path = self.state_path(); + if !state_path.exists() { + return Ok(false); + } + let state: State = + serde_json::from_str(&fs::read_to_string(state_path).context("Failed to read state")?) + .context("Failed to parse state")?; + Ok(state.started) + } + + pub fn set_started(&self, started: bool) -> Result<()> { + let state_path = self.state_path(); + fs::write(state_path, serde_json::to_string(&State { started })?) + .context("Failed to write state") + } + + pub fn shared_dir(&self) -> PathBuf { + self.workdir.join("shared") + } + + pub fn app_compose_path(&self) -> PathBuf { + self.shared_dir().join(APP_COMPOSE) + } + + pub fn app_compose_hash(&self) -> Result<[u8; 32]> { + use sha2::Digest; + let compose_path = self.app_compose_path(); + let compose = fs::read(compose_path).context("Failed to read compose")?; + Ok(sha2::Sha256::new_with_prefix(&compose).finalize().into()) + } + + pub fn user_config_path(&self) -> PathBuf { + self.shared_dir().join(USER_CONFIG) + } + + pub fn encrypted_env_path(&self) -> PathBuf { + self.shared_dir().join(ENCRYPTED_ENV) + } + + pub fn instance_info_path(&self) -> PathBuf { + self.shared_dir().join(INSTANCE_INFO) + } + + pub fn guest_ip_path(&self) -> PathBuf { + self.workdir.join("guest-ip") + } + + pub fn guest_ips_path(&self) -> PathBuf { + self.workdir.join("guest-ips.json") + } + + pub fn runtime_networks_path(&self) -> PathBuf { + self.workdir.join("runtime-networks.json") + } + + pub fn guest_ip(&self) -> Option { + fs::read_to_string(self.guest_ip_path()) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + } + + pub fn set_guest_ip(&self, ip: &str) -> Result<()> { + fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") + } + + pub fn guest_ips(&self) -> HashMap { + fs::read_to_string(self.guest_ips_path()) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() + } + + pub fn set_guest_ip_for_mac(&self, mac: &str, ip: &str) -> Result<()> { + let mut guest_ips = self.guest_ips(); + guest_ips.insert(mac.to_string(), ip.to_string()); + fs::write(self.guest_ips_path(), serde_json::to_string(&guest_ips)?) + .context("failed to write guest IP map") + } + + pub fn runtime_networks(&self) -> Vec { + fs::read_to_string(self.runtime_networks_path()) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() + } + + pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { + fs::write( + self.runtime_networks_path(), + serde_json::to_string(networks)?, + ) + .context("failed to write runtime networks") + } + + pub fn clear_runtime_networks(&self) -> Result<()> { + let path = self.runtime_networks_path(); + if path.exists() { + fs::remove_file(path).context("failed to clear runtime networks")?; + } + Ok(()) + } + + pub fn serial_file(&self) -> PathBuf { + self.workdir.join("serial.log") + } + + pub fn serial_history_file(&self) -> PathBuf { + self.workdir.join("serial.history.log") + } + + pub fn serial_pty(&self) -> PathBuf { + self.workdir.join("serial.pty") + } + + pub fn stdout_file(&self) -> PathBuf { + self.workdir.join("stdout.log") + } + + pub fn stderr_file(&self) -> PathBuf { + self.workdir.join("stderr.log") + } + + pub fn pid_file(&self) -> PathBuf { + self.workdir.join("qemu.pid") + } + + pub fn hda_path(&self) -> PathBuf { + self.workdir.join("hda.img") + } + + pub fn shared_disk_path(&self) -> PathBuf { + self.workdir.join("shared.img") + } + + pub fn qmp_socket(&self) -> PathBuf { + self.workdir.join("qmp.sock") + } + + pub fn removing_marker(&self) -> PathBuf { + self.workdir.join(".removing") + } + + pub fn is_removing(&self) -> bool { + self.removing_marker().exists() + } + + pub fn set_removing(&self) -> Result<()> { + fs::write(self.removing_marker(), "").context("Failed to write .removing marker") + } + + pub fn path(&self) -> &Path { + &self.workdir + } +} + +impl VmWorkDir { + pub fn instance_info(&self) -> Result { + let info_file = self.instance_info_path(); + let info: InstanceInfo = serde_json::from_slice(&fs::read(&info_file)?)?; + Ok(info) + } + + pub fn instance_info_or_default(&self) -> Result { + match self.instance_info() { + Ok(info) => Ok(info), + Err(err) => match err.downcast_ref::() { + Some(io_err) if io_err.kind() == std::io::ErrorKind::NotFound => { + Ok(InstanceInfo::default()) + } + _ => Err(err), }, - cid: 100, - workdir: PathBuf::from("/does-not-exist/vm-1"), - gateway_enabled: false, + } + } + + pub fn sys_config(&self) -> Result { + let sys_config_file = self.shared_dir().join(SYS_CONFIG); + let sys_config: SysConfig = serde_json::from_slice(&fs::read(sys_config_file)?)?; + Ok(sys_config) + } + + pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose) -> Result { + let compose_hash = self + .app_compose_hash() + .context("Failed to get compose hash")?; + let mut instance_info = self + .instance_info_or_default() + .context("Failed to get instance info")?; + let app_id = if instance_info.app_id.is_empty() { + compose_hash[..20].to_vec() + } else { + instance_info.app_id.clone() }; - let prepared = PreparedQemuLaunch { - workdir: VmWorkDir::new("/does-not-exist/vm-1"), - platform: TeePlatform::Tdx, - hugepage_numa_nodes: None, - gpu_numa_nodes: HashMap::new(), - numa_cpus: None, - tpm_path: None, - tdx_mr_config_id: None, - snp_host_data: None, - snp_launch_params: None, + if app_id.len() != 20 { + bail!( + "Invalid app ID length: expected 20 bytes, got {}", + app_id.len() + ); + } + + let disk_reusable = !app_compose.key_provider().is_none(); + if !disk_reusable || instance_info.instance_id_seed.is_empty() { + instance_info.instance_id_seed = { + let mut seed = vec![0u8; 20]; + getrandom::fill(&mut seed).context("Failed to generate instance id seed")?; + seed + }; + } + + let instance_id = if app_compose.no_instance_id { + Vec::new() + } else { + let mut id_path = instance_info.instance_id_seed.clone(); + id_path.extend_from_slice(&app_id); + Sha256::digest(id_path)[..20].to_vec() }; + instance_info.app_id = app_id.clone(); + instance_info.instance_id = instance_id.clone(); + fs::write( + self.instance_info_path(), + serde_json::to_string(&instance_info).context("Failed to serialize instance info")?, + ) + .context("Failed to write instance info")?; + + Ok(MrConfigV3::new( + app_id, + compose_hash.to_vec(), + app_compose.key_provider(), + app_compose.key_provider_id.clone(), + instance_id, + ) + .to_canonical_json()) + } - let process = QemuCommandBuilder { - vm: &vm, - cfg: &config.cvm, - gpus: &GpuConfig::default(), - prepared: &prepared, - } - .build() - .unwrap(); - - assert_eq!(process.command, "/not-installed/qemu-system-x86_64"); - assert!(process - .args - .windows(2) - .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"])); - assert!(process - .args - .windows(2) - .any(|args| args == ["-kernel", "/does-not-exist/kernel"])); - assert!(process - .args - .windows(2) - .any(|args| args == ["-append", "console=hvc0"])); + pub fn app_compose(&self) -> Result { + let compose_file = self.app_compose_path(); + let compose: AppCompose = serde_json::from_str(&fs::read_to_string(compose_file)?)?; + Ok(compose) } } diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 88aa884c1..e5d4482a1 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -23,6 +23,7 @@ use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; use crate::app::{App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir}; +use crate::config::{CvmConfig, Networking, NetworkingMode}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -196,34 +197,85 @@ pub fn create_manifest_from_vm_config( kms_urls: request.kms_urls.clone(), gateway_urls: request.gateway_urls.clone(), no_tee: request.no_tee, - networking: request.networking.as_ref().and_then(networking_from_proto), + networks: networks_from_vm_config(&request, cvm_config)?, }) } -fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Option { - use crate::config::NetworkingMode; +fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result> { + let bridge = proto.bridge_name.trim().to_string(); let mode = match proto.mode.as_str() { "bridge" => NetworkingMode::Bridge, - "user" => NetworkingMode::User, - "custom" => NetworkingMode::Custom, - "" => return None, // not set, use global default - other => { - tracing::warn!("unsupported per-VM networking mode '{other}', using global default"); - return None; - } + "" if bridge.is_empty() => return Ok(None), + "" => bail!("networking mode is required when bridge is set"), + "custom" => bail!("custom networking mode is manifest-only"), + other => bail!("unsupported networking mode '{other}'"), }; - // Only set mode; other fields will be merged from global config at runtime - Some(crate::config::Networking { + if mode != NetworkingMode::Bridge && !bridge.is_empty() { + bail!("bridge_name is only valid for bridge networking mode"); + } + Ok(Some(Networking { mode, - bridge: String::new(), + bridge, mac_prefix: String::new(), net: String::new(), dhcp_start: String::new(), restrict: false, netdev: String::new(), forward_service_enabled: false, - }) + })) +} + +fn network_from_required_proto(proto: &rpc::NetworkingConfig) -> Result { + networking_from_proto(proto)?.context("networking mode is required") +} + +fn networks_from_proto(networks: &[rpc::NetworkingConfig]) -> Result> { + networks.iter().map(network_from_required_proto).collect() +} + +fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { + crate::app::qemu::validate_resolved_network(&cvm_config.networking) +} + +fn resolve_requested_networks( + mut networks: Vec, + cvm_config: &CvmConfig, +) -> Result> { + for networking in &mut networks { + networking.forward_service_enabled = cvm_config.networking.forward_service_enabled; + } + let resolved = networks + .iter() + .map(|networking| crate::app::qemu::resolve_networking(networking, cvm_config)) + .collect::>(); + crate::app::qemu::validate_resolved_networks(&resolved)?; + Ok(resolved) +} + +fn has_host_bridge_interface() -> bool { + let Ok(entries) = fs::read_dir("/sys/class/net") else { + return false; + }; + entries + .filter_map(|entry| entry.ok()) + .any(|entry| entry.path().join("bridge").exists()) +} + +fn networks_from_vm_config( + request: &VmConfiguration, + cvm_config: &CvmConfig, +) -> Result> { + if !request.networks.is_empty() { + resolve_requested_networks(networks_from_proto(&request.networks)?, cvm_config) + } else if let Some(networking) = request.networking.as_ref() { + match networking_from_proto(networking)? { + Some(networking) => resolve_requested_networks(vec![networking], cvm_config), + None => Ok(vec![]), + } + } else { + Ok(vec![]) + } } impl RpcHandler { @@ -438,6 +490,26 @@ impl VmmRpc for RpcHandler { if request.update_gateway_urls { manifest.gateway_urls = request.gateway_urls.clone(); } + if request.update_networking { + manifest.networks = if request.networks.is_empty() { + validate_default_network(&self.app.config.cvm)?; + vec![] + } else { + resolve_requested_networks( + networks_from_proto(&request.networks)?, + &self.app.config.cvm, + )? + }; + let is_running = self + .app + .supervisor + .info(&request.id) + .await? + .is_some_and(|info| info.state.status.is_running()); + if !is_running { + vm_work_dir.clear_runtime_networks()?; + } + } vm_work_dir .put_manifest(&manifest) .context("Failed to put manifest")?; @@ -447,7 +519,7 @@ impl VmmRpc for RpcHandler { .await .context("Failed to load VM")?; if request.update_ports { - self.app.reconfigure_port_forward(&new_id).await; + self.app.reconfigure_port_forward(&request.id).await; } Ok(Id { id: new_id }) } @@ -519,6 +591,15 @@ impl VmmRpc for RpcHandler { } async fn get_meta(self) -> Result { + let mut supported_modes = vec!["user".to_string()]; + let default_networking = &self.app.config.cvm.networking; + let mut bridge_networking = default_networking.clone(); + bridge_networking.mode = NetworkingMode::Bridge; + if crate::app::qemu::validate_resolved_network(&bridge_networking).is_ok() + || has_host_bridge_interface() + { + supported_modes.push("bridge".to_string()); + } Ok(GetMetaResponse { kms: Some(KmsSettings { url: self @@ -550,6 +631,15 @@ impl VmmRpc for RpcHandler { max_allocable_vcpu: self.app.config.cvm.max_allocable_vcpu, max_allocable_memory_in_mb: self.app.config.cvm.max_allocable_memory_in_mb, }), + networking: Some(rpc::NetworkingCapabilities { + supported_modes, + default_mode: match default_networking.mode { + NetworkingMode::User => "user".to_string(), + NetworkingMode::Bridge => "bridge".to_string(), + NetworkingMode::Custom => String::new(), + }, + forward_service_enabled: default_networking.forward_service_enabled, + }), }) } @@ -740,6 +830,99 @@ impl VmmRpc for RpcHandler { } } +#[cfg(test)] +mod tests { + use super::*; + use rocket::figment::Figment; + + fn test_cvm_config() -> CvmConfig { + let config: crate::config::Config = Figment::from(crate::config::load_config_figment(None)) + .extract() + .unwrap(); + config.cvm + } + + fn test_vm_configuration() -> VmConfiguration { + VmConfiguration { + name: "vm-test".to_string(), + image: "dstack-test".to_string(), + compose_file: "{}".to_string(), + vcpu: 1, + memory: 1024, + disk_size: 10, + ports: vec![], + encrypted_env: vec![], + app_id: None, + user_config: String::new(), + hugepages: false, + pin_numa: false, + gpus: None, + kms_urls: vec![], + gateway_urls: vec![], + stopped: false, + no_tee: false, + networking: None, + networks: vec![], + } + } + + #[test] + fn create_without_networking_persists_following_default() { + let manifest = + create_manifest_from_vm_config(test_vm_configuration(), &test_cvm_config()).unwrap(); + + assert!(manifest.networks.is_empty()); + } + + #[test] + fn explicit_user_networking_is_resolved_before_persist() { + let mut request = test_vm_configuration(); + request.networks = vec![rpc::NetworkingConfig { + mode: "user".to_string(), + bridge_name: String::new(), + }]; + + let manifest = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap(); + + assert_eq!(manifest.networks.len(), 1); + assert_eq!(manifest.networks[0].mode, NetworkingMode::User); + assert!(!manifest.networks[0].net.is_empty()); + } + + #[test] + fn bridge_name_is_rejected_for_user_mode() { + let err = networks_from_proto(&[rpc::NetworkingConfig { + mode: "user".to_string(), + bridge_name: "dstack-br0".to_string(), + }]) + .unwrap_err(); + + assert!(err.to_string().contains("bridge_name is only valid")); + } + + #[test] + fn repeated_networks_rejects_empty_entries() { + let err = networks_from_proto(&[rpc::NetworkingConfig { + mode: String::new(), + bridge_name: String::new(), + }]) + .unwrap_err(); + + assert!(err.to_string().contains("networking mode is required")); + } + + #[test] + fn repeated_networks_rejects_custom_entries() { + let err = networks_from_proto(&[rpc::NetworkingConfig { + mode: "custom".to_string(), + bridge_name: String::new(), + }]) + .unwrap_err(); + + assert!(err.to_string().contains("custom networking mode")); + } +} + impl RpcCall for RpcHandler { type PrpcService = VmmServer; From f1e78f2c2393a63d3c7826368378127b47ba1f34 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Wed, 8 Jul 2026 06:53:01 +0800 Subject: [PATCH 02/10] refactor(vmm): simplify networking code following upstream conventions - Replace hand-written Manifest Deserialize impl with derive + Manifest::from_json() for legacy "networking" field migration - Simplify resolve_networking to clone global config as base then selectively override non-empty per-VM fields - Extract effective_networks() on VmState and VmInfo to eliminate four duplicated runtime_networks fallback blocks - Remove redundant forward_service_enabled pre-assignment in resolve_requested_networks (now inherited from global config clone) --- dstack/vmm/src/app.rs | 95 +++++++++------------------------- dstack/vmm/src/app/qemu.rs | 69 +++++++++++------------- dstack/vmm/src/main_service.rs | 16 +++--- 3 files changed, 62 insertions(+), 118 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ff78a46ad..d58d2809b 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -20,7 +20,7 @@ use guest_api::client::DefaultClient as GuestClient; use id_pool::IdPool; use or_panic::ResultOrPanic; use ra_rpc::client::RaClient; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; @@ -47,7 +47,7 @@ pub struct PortMapping { pub to: u16, } -#[derive(Serialize, Clone, Builder, Debug)] +#[derive(Deserialize, Serialize, Clone, Builder, Debug)] pub struct Manifest { pub id: String, pub name: String, @@ -74,64 +74,19 @@ pub struct Manifest { pub networks: Vec, } -impl<'de> Deserialize<'de> for Manifest { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - struct ManifestCompat { - id: String, - name: String, - app_id: String, - vcpu: u32, - memory: u32, - disk_size: u32, - image: String, - port_map: Vec, - created_at_ms: u64, - #[serde(default)] - hugepages: bool, - #[serde(default)] - pin_numa: bool, - #[serde(default)] - gpus: Option, - #[serde(default)] - kms_urls: Vec, - #[serde(default)] - gateway_urls: Vec, - #[serde(default)] - no_tee: bool, - #[serde(default)] - networks: Vec, - #[serde(default)] - networking: Option, +impl Manifest { + pub fn from_json(value: serde_json::Value) -> serde_json::Result { + let mut map = value; + if let Some(obj) = map.as_object_mut() { + if !obj.contains_key("networks") + || obj["networks"].as_array().is_some_and(|a| a.is_empty()) + { + if let Some(legacy) = obj.remove("networking") { + obj.insert("networks".into(), serde_json::Value::Array(vec![legacy])); + } + } } - - let compat = ManifestCompat::deserialize(deserializer)?; - let networks = if compat.networks.is_empty() { - compat.networking.into_iter().collect() - } else { - compat.networks - }; - Ok(Self { - id: compat.id, - name: compat.name, - app_id: compat.app_id, - vcpu: compat.vcpu, - memory: compat.memory, - disk_size: compat.disk_size, - image: compat.image, - port_map: compat.port_map, - created_at_ms: compat.created_at_ms, - hugepages: compat.hugepages, - pin_numa: compat.pin_numa, - gpus: compat.gpus, - kms_urls: compat.kms_urls, - gateway_urls: compat.gateway_urls, - no_tee: compat.no_tee, - networks, - }) + serde_json::from_value(map) } } @@ -592,11 +547,7 @@ impl App { let vm_id = { let mut state = self.lock(); let found = state.vms.iter_mut().find_map(|(id, vm)| { - let networks = if vm.state.runtime_networks.is_empty() { - crate::app::qemu::resolved_networks(&vm.config.manifest, &self.config.cvm) - } else { - vm.state.runtime_networks.clone() - }; + let networks = vm.effective_networks(&self.config.cvm); networks.iter().enumerate().find_map(|(index, networking)| { let prefix = networking.mac_prefix_bytes(); let nic_mac = mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); @@ -637,11 +588,7 @@ impl App { let Some(vm) = state.get(id) else { return; }; - let networks = if vm.state.runtime_networks.is_empty() { - crate::app::qemu::resolved_networks(&vm.config.manifest, &self.config.cvm) - } else { - vm.state.runtime_networks.clone() - }; + let networks = vm.effective_networks(&self.config.cvm); if networks .iter() .any(|networking| networking.mode == crate::config::NetworkingMode::User) @@ -1614,7 +1561,7 @@ mod tests { #[test] fn manifest_deserializes_legacy_singular_networking_as_networks() { - let manifest: Manifest = serde_json::from_value(serde_json::json!({ + let manifest: Manifest = Manifest::from_json(serde_json::json!({ "id": "vm-1", "name": "vm-1", "app_id": "app-1", @@ -2143,6 +2090,14 @@ impl VmState { state: VmStateMut::default(), } } + + pub fn effective_networks(&self, cvm: &crate::config::CvmConfig) -> Vec { + if self.state.runtime_networks.is_empty() { + crate::app::qemu::resolved_networks(&self.config.manifest, cvm) + } else { + self.state.runtime_networks.clone() + } + } } pub(crate) struct AppState { diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 048a97136..86672cca5 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -103,36 +103,25 @@ fn networking_to_proto(n: &Networking) -> pb::NetworkingConfig { } pub fn resolve_networking(vm_net: &Networking, cfg: &CvmConfig) -> Networking { - Networking { - mode: vm_net.mode, - bridge: if vm_net.bridge.is_empty() { - cfg.networking.bridge.clone() - } else { - vm_net.bridge.clone() - }, - forward_service_enabled: vm_net.forward_service_enabled, - mac_prefix: if vm_net.mac_prefix.is_empty() { - cfg.networking.mac_prefix.clone() - } else { - vm_net.mac_prefix.clone() - }, - net: if vm_net.net.is_empty() { - cfg.networking.net.clone() - } else { - vm_net.net.clone() - }, - dhcp_start: if vm_net.dhcp_start.is_empty() { - cfg.networking.dhcp_start.clone() - } else { - vm_net.dhcp_start.clone() - }, - restrict: cfg.networking.restrict || vm_net.restrict, - netdev: if vm_net.netdev.is_empty() { - cfg.networking.netdev.clone() - } else { - vm_net.netdev.clone() - }, + let mut resolved = cfg.networking.clone(); + resolved.mode = vm_net.mode; + resolved.restrict = cfg.networking.restrict || vm_net.restrict; + if !vm_net.bridge.is_empty() { + resolved.bridge = vm_net.bridge.clone(); + } + if !vm_net.mac_prefix.is_empty() { + resolved.mac_prefix = vm_net.mac_prefix.clone(); + } + if !vm_net.net.is_empty() { + resolved.net = vm_net.net.clone(); } + if !vm_net.dhcp_start.is_empty() { + resolved.dhcp_start = vm_net.dhcp_start.clone(); + } + if !vm_net.netdev.is_empty() { + resolved.netdev = vm_net.netdev.clone(); + } + resolved } pub fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { @@ -408,6 +397,14 @@ fn create_shared_disk(disk_path: impl AsRef, shared_dir: impl AsRef) } impl VmInfo { + pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { + if self.runtime_networks.is_empty() { + resolved_networks(&self.manifest, cvm) + } else { + self.runtime_networks.clone() + } + } + pub fn to_pb(&self, gw: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); @@ -422,11 +419,7 @@ impl VmInfo { .map(networking_to_proto) .collect::>(); let configured_networking = configured_networks.first().cloned(); - let effective_networks = if self.runtime_networks.is_empty() { - resolved_networks(&self.manifest, cvm) - } else { - self.runtime_networks.clone() - }; + let effective_networks = self.effective_networks(cvm); let ip_index = effective_networks .iter() .position(|networking| networking.mode == NetworkingMode::Bridge) @@ -1370,10 +1363,10 @@ impl VmWorkDir { pub fn manifest(&self) -> Result { let manifest_path = self.manifest_path(); - let manifest = fs::read_to_string(manifest_path).context("Failed to read manifest")?; - let manifest: Manifest = - serde_json::from_str(&manifest).context("Failed to parse manifest")?; - Ok(manifest) + let raw = fs::read_to_string(manifest_path).context("Failed to read manifest")?; + let value: serde_json::Value = + serde_json::from_str(&raw).context("Failed to parse manifest JSON")?; + Manifest::from_json(value).context("Failed to deserialize manifest") } pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index e5d4482a1..c3626ecc4 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -239,12 +239,9 @@ fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { } fn resolve_requested_networks( - mut networks: Vec, + networks: &[Networking], cvm_config: &CvmConfig, ) -> Result> { - for networking in &mut networks { - networking.forward_service_enabled = cvm_config.networking.forward_service_enabled; - } let resolved = networks .iter() .map(|networking| crate::app::qemu::resolve_networking(networking, cvm_config)) @@ -267,10 +264,11 @@ fn networks_from_vm_config( cvm_config: &CvmConfig, ) -> Result> { if !request.networks.is_empty() { - resolve_requested_networks(networks_from_proto(&request.networks)?, cvm_config) + let networks = networks_from_proto(&request.networks)?; + resolve_requested_networks(&networks, cvm_config) } else if let Some(networking) = request.networking.as_ref() { match networking_from_proto(networking)? { - Some(networking) => resolve_requested_networks(vec![networking], cvm_config), + Some(networking) => resolve_requested_networks(&[networking], cvm_config), None => Ok(vec![]), } } else { @@ -495,10 +493,8 @@ impl VmmRpc for RpcHandler { validate_default_network(&self.app.config.cvm)?; vec![] } else { - resolve_requested_networks( - networks_from_proto(&request.networks)?, - &self.app.config.cvm, - )? + let networks = networks_from_proto(&request.networks)?; + resolve_requested_networks(&networks, &self.app.config.cvm)? }; let is_running = self .app From 033c80e045e968dcd0beb676b8efd73c04dd7fc8 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Wed, 8 Jul 2026 06:58:34 +0800 Subject: [PATCH 03/10] refactor(vmm): remove unused proto fields from NetworkInterfaceStatus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop if_name, lowerdev, and pci_bdf — all three were always set to None in the only construction site. Add them back when a backend actually populates them. --- dstack/vmm/rpc/proto/vmm_rpc.proto | 6 ------ dstack/vmm/src/app/qemu.rs | 3 --- 2 files changed, 9 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 619dcdc9c..5bfcee6cf 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -53,12 +53,6 @@ message NetworkInterfaceStatus { optional string bridge_name = 5; // QEMU netdev id, e.g. "net0". optional string netdev_id = 6; - // Host-side interface name when known, e.g. tap/macvtap device. - optional string if_name = 7; - // Lower device for macvtap-like backends. - optional string lowerdev = 8; - // PCI BDF for passthrough-like backends. - optional string pci_bdf = 9; } // Structured log or lifecycle event emitted by the guest or runtime. diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 86672cca5..67444d1b9 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -445,9 +445,6 @@ impl VmInfo { bridge_name: (networking.mode == NetworkingMode::Bridge) .then(|| networking.bridge.clone()), netdev_id: Some(format!("net{index}")), - if_name: None, - lowerdev: None, - pci_bdf: None, } }) .collect(); From 66a5fc7184aa76f2924340ca896041c0ab563dc5 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Wed, 8 Jul 2026 17:43:48 +0800 Subject: [PATCH 04/10] review(vmm): align networking backend API --- dstack/vmm/rpc/proto/vmm_rpc.proto | 2 ++ dstack/vmm/src/app.rs | 1 + dstack/vmm/src/app/qemu.rs | 23 ++++++++++++----------- dstack/vmm/src/main_service.rs | 16 +++++++++------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 5bfcee6cf..ff6956d1b 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -290,6 +290,8 @@ message NetworkingCapabilities { repeated string supported_modes = 1; string default_mode = 2; bool forward_service_enabled = 3; + // Default bridge configured in vmm.toml [cvm.networking].bridge. + string default_bridge = 4; } // Aggregated metadata exposed through GetMeta. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index d58d2809b..84487c2e6 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -32,6 +32,7 @@ use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; +pub(crate) use qemu::{resolve_networking, validate_resolved_network, validate_resolved_networks}; pub use qemu::{VmConfig, VmWorkDir}; mod id_pool; diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 67444d1b9..751784b97 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -44,11 +44,12 @@ use sha2::{Digest, Sha256}; /// `prefix` may contain 0-3 fixed bytes. The first byte always has the /// locally-administered + unicast bits set (0x02). Remaining bytes are /// filled from SHA256(vm_id). -pub fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { +#[cfg(test)] +fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { mac_address_for_vm_index(vm_id, prefix, 0) } -pub fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) -> String { +pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) -> String { let hash_input = if index == 0 { vm_id.to_string() } else { @@ -75,7 +76,7 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use supervisor_client::supervisor::{ProcessConfig, ProcessInfo}; -pub fn networking_mode_name(mode: NetworkingMode) -> &'static str { +fn networking_mode_name(mode: NetworkingMode) -> &'static str { match mode { NetworkingMode::Bridge => "bridge", NetworkingMode::User => "user", @@ -102,7 +103,7 @@ fn networking_to_proto(n: &Networking) -> pb::NetworkingConfig { } } -pub fn resolve_networking(vm_net: &Networking, cfg: &CvmConfig) -> Networking { +pub(crate) fn resolve_networking(vm_net: &Networking, cfg: &CvmConfig) -> Networking { let mut resolved = cfg.networking.clone(); resolved.mode = vm_net.mode; resolved.restrict = cfg.networking.restrict || vm_net.restrict; @@ -124,7 +125,7 @@ pub fn resolve_networking(vm_net: &Networking, cfg: &CvmConfig) -> Networking { resolved } -pub fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { +pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { if manifest.networks.is_empty() { vec![cfg.networking.clone()] } else { @@ -136,7 +137,7 @@ pub fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec Result<()> { +pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { if networking.mode == NetworkingMode::Bridge { if networking.bridge.is_empty() { bail!("bridge networking requested but no bridge is configured"); @@ -151,7 +152,7 @@ pub fn validate_resolved_network(networking: &Networking) -> Result<()> { Ok(()) } -pub fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { +pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { for networking in networks { validate_resolved_network(networking)?; } @@ -1360,10 +1361,10 @@ impl VmWorkDir { pub fn manifest(&self) -> Result { let manifest_path = self.manifest_path(); - let raw = fs::read_to_string(manifest_path).context("Failed to read manifest")?; + let raw = fs::read_to_string(manifest_path).context("failed to read manifest")?; let value: serde_json::Value = - serde_json::from_str(&raw).context("Failed to parse manifest JSON")?; - Manifest::from_json(value).context("Failed to deserialize manifest") + serde_json::from_str(&raw).context("failed to parse manifest json")?; + Manifest::from_json(value).context("failed to deserialize manifest") } pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { @@ -1373,7 +1374,7 @@ impl VmWorkDir { if let Some(networking) = manifest.networks.first() { value["networking"] = serde_json::to_value(networking)?; } - fs::write(manifest_path, serde_json::to_string(&value)?).context("Failed to write manifest") + fs::write(manifest_path, serde_json::to_string(&value)?).context("failed to write manifest") } pub fn started(&self) -> Result { diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index c3626ecc4..29d0ddeb5 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -22,7 +22,10 @@ use or_panic::ResultOrPanic; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; -use crate::app::{App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir}; +use crate::app::{ + resolve_networking, validate_resolved_network, validate_resolved_networks, App, AttachMode, + GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, +}; use crate::config::{CvmConfig, Networking, NetworkingMode}; fn hex_sha256(data: &str) -> String { @@ -235,7 +238,7 @@ fn networks_from_proto(networks: &[rpc::NetworkingConfig]) -> Result Result<()> { - crate::app::qemu::validate_resolved_network(&cvm_config.networking) + validate_resolved_network(&cvm_config.networking) } fn resolve_requested_networks( @@ -244,9 +247,9 @@ fn resolve_requested_networks( ) -> Result> { let resolved = networks .iter() - .map(|networking| crate::app::qemu::resolve_networking(networking, cvm_config)) + .map(|networking| resolve_networking(networking, cvm_config)) .collect::>(); - crate::app::qemu::validate_resolved_networks(&resolved)?; + validate_resolved_networks(&resolved)?; Ok(resolved) } @@ -591,9 +594,7 @@ impl VmmRpc for RpcHandler { let default_networking = &self.app.config.cvm.networking; let mut bridge_networking = default_networking.clone(); bridge_networking.mode = NetworkingMode::Bridge; - if crate::app::qemu::validate_resolved_network(&bridge_networking).is_ok() - || has_host_bridge_interface() - { + if validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface() { supported_modes.push("bridge".to_string()); } Ok(GetMetaResponse { @@ -635,6 +636,7 @@ impl VmmRpc for RpcHandler { NetworkingMode::Custom => String::new(), }, forward_service_enabled: default_networking.forward_service_enabled, + default_bridge: default_networking.bridge.clone(), }), }) } From 56811418e821a5a299d26c61d3e155c37def9328 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Wed, 8 Jul 2026 17:43:48 +0800 Subject: [PATCH 05/10] feat(vmm-ui): add VM networking controls --- .../vmm/ui/src/components/CreateVmDialog.ts | 36 +++++-- .../vmm/ui/src/components/UpdateVmDialog.ts | 32 ++++++ dstack/vmm/ui/src/composables/useVmManager.ts | 102 ++++++++++++++++-- dstack/vmm/ui/src/styles/main.css | 53 +++++++-- dstack/vmm/ui/src/templates/app.html | 17 +++ 5 files changed, 217 insertions(+), 23 deletions(-) diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 22f473435..c0e59eb20 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -20,6 +20,9 @@ const CreateVmDialogComponent = { allowAttachAllGpus: { type: Boolean, required: true }, kmsAvailable: { type: Boolean, required: true }, portMappingEnabled: { type: Boolean, required: true }, + networkingModes: { type: Array, required: true }, + defaultBridge: { type: String, default: '' }, + defaultNetworkingLabel: { type: String, required: true }, }, emits: ['close', 'submit', 'load-compose'], template: /* html */ ` @@ -147,14 +150,31 @@ const CreateVmDialogComponent = { -
- - +
+ +
+
{{ defaultNetworkingLabel }}
+
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} + Guest IP is assigned by host DHCP on that bridge and reported after boot. + +
+ +
diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index c30ef894f..f994f0487 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -19,6 +19,9 @@ const UpdateVmDialogComponent = { availableGpus: { type: Array, required: true }, allowAttachAllGpus: { type: Boolean, required: true }, portMappingEnabled: { type: Boolean, required: true }, + networkingModes: { type: Array, required: true }, + defaultBridge: { type: String, default: '' }, + defaultNetworkingLabel: { type: String, required: true }, kmsEnabled: { type: Boolean, required: true }, composeHashPreview: { type: String, required: true }, }, @@ -134,6 +137,35 @@ const UpdateVmDialogComponent = {
+
+
+ +
+
+
{{ defaultNetworkingLabel }}
+
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} + Guest IP is assigned by host DHCP on that bridge and reported after boot. + +
+ +
+
+
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index a994ee1d8..c6c60c741 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -73,6 +73,7 @@ type VmListItem = { boot_progress?: string; shutdown_progress?: string; image_version?: string; + interfaces?: VmmTypes.INetworkInterfaceStatus[]; configuration?: VmConfiguration; appCompose?: AppCompose; }; @@ -89,6 +90,11 @@ type PortFormEntry = { vm_port?: number | null; }; +type NetworkFormEntry = { + mode: string; + bridge_name?: string; +}; + type VmFormState = { name: string; image: string; @@ -118,7 +124,7 @@ type VmFormState = { no_tee: boolean; pin_numa: boolean; hugepages: boolean; - net_mode: string; + networks: NetworkFormEntry[]; user_config: string; kms_urls: string[]; gateway_urls: string[]; @@ -147,6 +153,8 @@ type UpdateDialogState = { attachAllGpus: boolean; selectedGpus: string[]; updateGpuConfig: boolean; + updateNetworking: boolean; + networks: NetworkFormEntry[]; user_config: string; }; @@ -163,6 +171,7 @@ type CloneConfigDialogState = { gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; gateway_urls?: string[]; + networks?: NetworkFormEntry[]; hugepages: boolean; pin_numa: boolean; no_tee: boolean; @@ -201,7 +210,7 @@ function createVmFormState(preLaunchScript: string): VmFormState { no_tee: false, pin_numa: false, hugepages: false, - net_mode: '', + networks: [], user_config: '', kms_urls: [], gateway_urls: [], @@ -232,6 +241,8 @@ function createUpdateDialogState(): UpdateDialogState { attachAllGpus: false, selectedGpus: [], updateGpuConfig: false, + updateNetworking: false, + networks: [], user_config: '', }; } @@ -250,6 +261,7 @@ function createCloneConfigDialogState(): CloneConfigDialogState { gpus: undefined, kms_urls: undefined, gateway_urls: undefined, + networks: undefined, hugepages: false, pin_numa: false, no_tee: false, @@ -305,7 +317,28 @@ fi const cloneConfigDialog: Ref = ref(createCloneConfigDialogState()); const showCreateDialog = ref(false); - const config = ref({ portMappingEnabled: false }); + const config = ref({ + portMappingEnabled: false, + networking: null as VmmTypes.INetworkingCapabilities | null, + }); + const networkingModes = computed(() => { + const supported = config.value.networking?.supported_modes || []; + const fallback = supported.length > 0 ? supported : ['user']; + return Array.from(new Set(fallback)); + }); + const defaultBridge = computed(() => config.value.networking?.default_bridge || ''); + const defaultNetworkingLabel = computed(() => { + const mode = config.value.networking?.default_mode || ''; + if (mode === 'bridge') { + return defaultBridge.value + ? `Default: Bridge (${defaultBridge.value})` + : 'Default: Bridge (no default bridge configured)'; + } + if (mode === 'user') { + return 'Default: User networking'; + } + return mode ? `Default: ${networkModeLabel(mode)}` : 'Default: node configuration'; + }); const composeHashPreview = ref(''); const updateComposeHashPreview = ref(''); @@ -386,6 +419,31 @@ fi vm_port: port.vm_port, })); + const cloneNetworks = (configuration?: VmConfiguration | null): NetworkFormEntry[] => { + const configured = configuration?.networks && configuration.networks.length > 0 + ? configuration.networks + : (configuration?.networking ? [configuration.networking] : []); + return configured.map((network) => ({ + mode: network.mode || '', + bridge_name: network.bridge_name || '', + })); + }; + + const normalizeNetworks = (networks: NetworkFormEntry[] = []): VmmTypes.INetworkingConfig[] => + networks + .map((network) => ({ + mode: (network.mode || '').trim(), + bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', + })) + .filter((network) => network.mode.length > 0); + + function networkModeLabel(mode?: string | null) { + if (!mode) { + return 'Default'; + } + return mode.charAt(0).toUpperCase() + mode.slice(1); + } + function deriveGpuSelection(gpuConfig?: VmmTypes.IGpuConfig) { if (!gpuConfig) { return { attachAll: false, selected: [] as string[] }; @@ -444,7 +502,7 @@ type CreateVmPayloadSource = { hugepages?: boolean; pin_numa?: boolean; no_tee?: boolean; - net_mode?: string; + networks?: NetworkFormEntry[]; gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; gateway_urls?: string[]; @@ -453,6 +511,7 @@ type CreateVmPayloadSource = { function buildCreateVmPayload(source: CreateVmPayloadSource): VmmTypes.IVmConfiguration { const normalizedPorts = normalizePorts(source.ports); + const normalizedNetworks = normalizeNetworks(source.networks || []); return { name: source.name.trim(), image: source.image.trim(), @@ -467,7 +526,7 @@ type CreateVmPayloadSource = { hugepages: !!source.hugepages, pin_numa: !!source.pin_numa, no_tee: source.no_tee ?? false, - networking: source.net_mode ? { mode: source.net_mode } : undefined, + networks: normalizedNetworks, gpus: source.gpus, kms_urls: source.kms_urls?.filter((url) => url && url.trim().length) ?? [], gateway_urls: source.gateway_urls?.filter((url) => url && url.trim().length) ?? [], @@ -551,7 +610,10 @@ type CreateVmPayloadSource = { return vm; }); - config.value = { portMappingEnabled: data.port_mapping_enabled }; + config.value = { + ...config.value, + portMappingEnabled: data.port_mapping_enabled, + }; if (expandedVMs.value.size > 0) { await refreshExpandedVMs(); @@ -608,6 +670,18 @@ type CreateVmPayloadSource = { return vms.value.find((item) => item.id === vm.id) || null; } + async function loadMeta() { + try { + const data = await vmmRpc.getMeta({}); + config.value = { + ...config.value, + networking: data.networking || null, + }; + } catch (error) { + recordError('error loading VMM metadata', error); + } + } + async function loadImages() { try { const data = await vmmRpc.listImages({}); @@ -912,6 +986,8 @@ type CreateVmPayloadSource = { attachAllGpus: gpuSelection.attachAll, selectedGpus: gpuSelection.selected, updateGpuConfig: false, + updateNetworking: false, + networks: cloneNetworks(config), user_config: config.user_config || '', }; } @@ -1004,7 +1080,7 @@ type CreateVmPayloadSource = { hugepages: vmForm.value.hugepages, pin_numa: vmForm.value.pin_numa, no_tee: vmForm.value.no_tee, - net_mode: vmForm.value.net_mode, + networks: vmForm.value.networks, gpus: configGpu(vmForm.value) || undefined, kms_urls: vmForm.value.kms_urls, gateway_urls: vmForm.value.gateway_urls, @@ -1088,6 +1164,10 @@ type CreateVmPayloadSource = { body.update_ports = true; body.ports = normalizePorts(updated.ports); body.gpus = updateDialog.value.updateGpuConfig ? configGpu(updated, true) : undefined; + if (updated.updateNetworking) { + body.update_networking = true; + body.networks = normalizeNetworks(updated.networks); + } await vmmRpc.updateVm(body); updateDialog.value.encryptedEnvs = []; @@ -1149,13 +1229,13 @@ type CreateVmPayloadSource = { key_provider_id: theVm.appCompose?.key_provider_id || '', gateway_enabled: !!theVm.appCompose?.gateway_enabled, gateway_urls: config.gateway_urls || [], + networks: cloneNetworks(config), public_logs: !!theVm.appCompose?.public_logs, public_sysinfo: !!theVm.appCompose?.public_sysinfo, public_tcbinfo: !!theVm.appCompose?.public_tcbinfo, pin_numa: !!config.pin_numa, hugepages: !!config.hugepages, no_tee: !!config.no_tee, - net_mode: config.networking?.mode || '', user_config: config.user_config || '', stopped: !!config.stopped, }; @@ -1185,6 +1265,7 @@ type CreateVmPayloadSource = { hugepages: source.hugepages, pin_numa: source.pin_numa, no_tee: source.no_tee, + networks: source.networks, gpus: source.gpus, kms_urls: source.kms_urls, gateway_urls: source.gateway_urls, @@ -1653,6 +1734,7 @@ type CreateVmPayloadSource = { onMounted(() => { watchVmList(); + loadMeta(); loadImages(); loadGpus(); loadVersion(); @@ -1683,6 +1765,9 @@ type CreateVmPayloadSource = { cloneConfigDialog, showCreateDialog, config, + networkingModes, + defaultBridge, + defaultNetworkingLabel, composeHashPreview, updateComposeHashPreview, showDeployDialog, @@ -1717,6 +1802,7 @@ type CreateVmPayloadSource = { downloadAppCompose, downloadUserConfig, getVmFeatures, + networkModeLabel, systemMenu, toggleSystemMenu, closeSystemMenu, diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index 2fd1d0100..af6594747 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -587,31 +587,44 @@ h1, h2, h3, h4, h5, h6 { border-style: dashed; } -.port-mappings { +.port-mappings, +.runtime-network-section { display: flex; flex-direction: column; gap: 12px; } -.port-mappings h4 { +.port-mappings h4, +.runtime-network-section h4 { font-size: 14px; font-weight: 600; color: var(--color-text-primary); margin-bottom: 4px; } -.port-item { +.port-item, +.runtime-network-item { padding: 12px 16px; background: var(--color-bg-primary); border: 1px solid var(--color-border); border-radius: var(--radius-md); - display: flex; - justify-content: space-between; align-items: center; font-size: 14px; font-family: 'SF Mono', 'Monaco', 'Cascadia Code', monospace; } +.port-item { + display: flex; + justify-content: space-between; +} + +.runtime-network-item { + display: grid; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1fr 1.4fr; + gap: 12px; + overflow-wrap: anywhere; +} + .features-section { margin-top: 24px; } @@ -1385,17 +1398,43 @@ h1, h2, h3, h4, h5, h6 { .encrypted-env-editor, .port-mapping-editor, -.gpu-config-editor { +.gpu-config-editor, +.network-config-editor { display: flex; flex-direction: column; gap: 16px; } -.port-mapping-editor > button { +.port-mapping-editor > button, +.network-config-editor > button { width: fit-content; min-width: 120px; } +.network-config-empty { + padding: 12px 16px; + background: var(--color-bg-secondary); + border: 1px dashed var(--color-border); + border-radius: var(--radius-md); + color: var(--color-text-secondary); + font-size: 14px; +} + +.network-config-row { + display: grid; + grid-template-columns: 160px minmax(180px, 1fr) 100px; + gap: 12px; + align-items: center; +} + +.network-config-placeholder { + min-height: 1px; +} + +.network-config-hint { + grid-column: 1 / -1; +} + .gpu-section-label { font-size: 14px; font-weight: 600; diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 7b6a63bdc..cc9bfdaaa 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -85,6 +85,9 @@

dstack-vmm

:allow-attach-all-gpus="allowAttachAllGpus" :kms-available="vmForm.kms_enabled" :port-mapping-enabled="config.portMappingEnabled" + :networking-modes="networkingModes" + :default-bridge="defaultBridge" + :default-networking-label="defaultNetworkingLabel" @close="showCreateDialog = false" @submit="createVm" @load-compose="loadComposeFile" @@ -97,6 +100,9 @@

dstack-vmm

:available-gpus="availableGpus" :allow-attach-all-gpus="allowAttachAllGpus" :port-mapping-enabled="config.portMappingEnabled" + :networking-modes="networkingModes" + :default-bridge="defaultBridge" + :default-networking-label="defaultNetworkingLabel" :kms-enabled="kmsEnabled(updateDialog.vm || {})" :compose-hash-preview="updateComposeHashPreview" @close="updateDialog.show = false" @@ -344,6 +350,17 @@

Port Mappings

+
+

VMM Network Interfaces

+
+ {{ networkModeLabel(iface.mode) }} / {{ iface.backend || '-' }} + {{ iface.netdev_id || '-' }} + {{ iface.bridge_name || '-' }} + {{ iface.ip || '-' }} + {{ iface.mac || '-' }} +
+
+

Features

{{ getVmFeatures(vm) }} From e8c71c103acc8fb31664434b38a641f5b2719d88 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 8 Jul 2026 05:56:02 -0700 Subject: [PATCH 06/10] feat(mr): measure virtio-net NIC count in RTMR0 Each virtio-net-pci device changes the guest ACPI/DSDT layout and thus RTMR0. Before this change dstack-mr hardcoded exactly one NIC, so a CVM booted with the new multi-network support (repeated `networks`) would produce an RTMR0 the verifier could never reconstruct, and attestation would fail closed. Dropping ACPI from the measurement instead would let an untrusted host silently attach un-attested devices under an unchanged quote, so the NIC count must be measured. Mirror the existing `num_gpus` plumbing: add `num_nics` to `dstack_types::VmConfig` and `dstack_mr::Machine`, emit one `-netdev/-device virtio-net-pci` pair per NIC in the ACPI generator, and thread the resolved interface count from the VMM through to the verifier. Backward compatibility: - `num_nics` defaults to 1 and is omitted from the serialized VmConfig when equal to 1, so existing configs, their measurement cache keys, and RTMR0 values are byte-for-byte unchanged (verified against the previous hardcoded layout). - legacy configs without the field deserialize to 1. Verified: RTMR0(num_nics=1) == RTMR0(previous hardcoded layout); RTMR0(num_nics=2) differs. --- dstack/dstack-mr/cli/src/main.rs | 5 +++ dstack/dstack-mr/src/acpi.rs | 16 ++++++-- dstack/dstack-mr/src/machine.rs | 5 +++ dstack/dstack-mr/src/tdx.rs | 3 ++ dstack/dstack-types/src/lib.rs | 59 +++++++++++++++++++++++++++++ dstack/verifier/src/verification.rs | 1 + dstack/vmm/src/app.rs | 6 +++ 7 files changed, 91 insertions(+), 4 deletions(-) diff --git a/dstack/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs index fb78808e2..146112b3d 100644 --- a/dstack/dstack-mr/cli/src/main.rs +++ b/dstack/dstack-mr/cli/src/main.rs @@ -66,6 +66,10 @@ struct MachineConfig { #[arg(long, default_value = "0")] num_nvswitches: u32, + /// Number of virtio-net NICs + #[arg(long, default_value = "1")] + num_nics: u32, + /// Disable hotplug #[arg(long, default_value = "false")] hotplug_off: Bool, @@ -133,6 +137,7 @@ fn main() -> Result<()> { .hugepages(config.hugepages) .num_gpus(config.num_gpus) .num_nvswitches(config.num_nvswitches) + .num_nics(config.num_nics) .hotplug_off(config.hotplug_off) .root_verity(config.root_verity) .maybe_qemu_version(config.qemu_version.clone()) diff --git a/dstack/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs index a39759e83..db8f52845 100644 --- a/dstack/dstack-mr/src/acpi.rs +++ b/dstack/dstack-mr/src/acpi.rs @@ -55,10 +55,18 @@ impl Machine<'_> { &format!("file={dummy_disk},if=none,id=hd1,format=raw,readonly=on"), "-device", "virtio-blk-pci,drive=hd1", - "-netdev", - "user,id=net0", - "-device", - "virtio-net-pci,netdev=net0", + ]); + + // One virtio-net-pci per NIC. Emitted at the same position and, for the + // default single-NIC case, with the exact same args as the previous + // hardcoded layout so RTMR0 stays byte-for-byte unchanged. + for i in 0..self.num_nics { + cmd.arg("-netdev").arg(format!("user,id=net{i}")); + cmd.arg("-device") + .arg(format!("virtio-net-pci,netdev=net{i}")); + } + + cmd.args([ "-object", "tdx-guest,id=tdx", "-device", diff --git a/dstack/dstack-mr/src/machine.rs b/dstack/dstack-mr/src/machine.rs index 823470d23..59d061084 100644 --- a/dstack/dstack-mr/src/machine.rs +++ b/dstack/dstack-mr/src/machine.rs @@ -28,6 +28,11 @@ pub struct Machine<'a> { pub hugepages: bool, pub num_gpus: u32, pub num_nvswitches: u32, + /// Number of virtio-net NICs. Each NIC contributes a PCI device to the + /// ACPI/DSDT layout measured into RTMR0. Defaults to 1 so callers that + /// predate this field keep the historical single-NIC layout. + #[builder(default = 1)] + pub num_nics: u32, pub hotplug_off: bool, pub root_verity: bool, #[builder(default)] diff --git a/dstack/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs index 087556567..abd6c7f99 100644 --- a/dstack/dstack-mr/src/tdx.rs +++ b/dstack/dstack-mr/src/tdx.rs @@ -82,6 +82,7 @@ fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Res }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(measurement.tdvf.ovmf_variant) @@ -472,6 +473,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -567,6 +569,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 31ec16ba9..b871be957 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -643,6 +643,14 @@ impl SysConfig { } } +fn default_num_nics() -> u32 { + 1 +} + +fn is_default_num_nics(n: &u32) -> bool { + *n == default_num_nics() +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmConfig { #[serde(with = "hex_bytes", default)] @@ -666,6 +674,16 @@ pub struct VmConfig { pub num_gpus: u32, #[serde(default)] pub num_nvswitches: u32, + /// Number of virtio-net NICs attached to the guest. Each NIC adds a PCI + /// device to the ACPI/DSDT layout and therefore changes RTMR0, so it must + /// be measured. Defaults to 1 and is omitted from the serialized form when + /// equal to 1, keeping configs (and their cache keys / hashes) produced + /// before this field existed byte-for-byte stable. + #[serde( + default = "default_num_nics", + skip_serializing_if = "is_default_num_nics" + )] + pub num_nics: u32, #[serde(default)] pub hotplug_off: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1529,3 +1547,44 @@ impl Platform { } } } + +#[cfg(test)] +mod vm_config_num_nics_tests { + use super::VmConfig; + + fn legacy_json() -> serde_json::Value { + serde_json::json!({ + "cpu_count": 4, + "memory_size": 4294967296u64, + "num_gpus": 0, + "num_nvswitches": 0, + }) + } + + #[test] + fn legacy_config_without_num_nics_defaults_to_one() { + let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + assert_eq!(cfg.num_nics, 1); + } + + #[test] + fn single_nic_is_omitted_to_keep_cache_key_stable() { + // A config with the default single NIC must serialize identically to a + // legacy config, so the verifier's measurement cache key (a hash of the + // serialized VmConfig) is unchanged for existing deployments. + let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + let serialized = serde_json::to_value(&cfg).unwrap(); + assert!( + serialized.get("num_nics").is_none(), + "num_nics must be omitted when equal to 1, got {serialized}" + ); + } + + #[test] + fn multi_nic_is_serialized() { + let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + cfg.num_nics = 2; + let serialized = serde_json::to_value(&cfg).unwrap(); + assert_eq!(serialized.get("num_nics").and_then(|v| v.as_u64()), Some(2)); + } +} diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 9faf03459..ad3ece1d0 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -312,6 +312,7 @@ impl CvmVerifier { }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 84487c2e6..d9292ed7a 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1471,6 +1471,11 @@ fn make_vm_config( GpuConfig::default() }; let effective_vcpus = effective_vcpu_count_for_manifest(manifest, &gpus)?; + // Each resolved network interface becomes one virtio-net-pci device in the + // QEMU command (see `VmConfig::config_qemu`), which changes the guest's + // ACPI/DSDT layout and therefore RTMR0. Measure the interface count so the + // verifier reconstructs the exact device layout. + let num_nics = crate::app::qemu::resolved_networks(manifest, &cfg.cvm).len() as u32; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, @@ -1482,6 +1487,7 @@ fn make_vm_config( hugepages: manifest.hugepages, num_gpus: gpus.gpus.len() as u32, num_nvswitches: gpus.bridges.len() as u32, + num_nics, host_share_mode: cfg.cvm.host_share_mode.clone(), hotplug_off: cfg.cvm.qemu_hotplug_off, image: Some(manifest.image.clone()), From 40d5df2bc1c8f298b865a523eaef878d7aba6d1d Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Thu, 9 Jul 2026 23:58:47 +0800 Subject: [PATCH 07/10] revert(vmm): drop per-MAC guest_ips extension Per PR #756 review: the guest_ip machinery exists only to serve VMM's built-in port-forward, a subsystem already slated for deprecation. It learns the guest address out-of-band from an external DHCP server via an unauthenticated ReportDhcpLease RPC, so the stored value is stale-prone and empty for slirp NICs. Extending it into a per-MAC map doubled down on that subsystem: a second on-disk file with the same staleness/trust caveats plus a wider public API surface. Roll back the extension rather than build on top of it: - proto: remove NetworkInterfaceStatus.ip, renumber trailing fields - app.rs: drop guest_ips map from state and DHCP-lease path; port forwarding uses the single guest_ip again - qemu.rs: remove guest_ips field, guest-ips.json persistence (guest_ips_path/guest_ips/set_guest_ip_for_mac), per-NIC ip population in VmInfo::interfaces, and the now-dead VmInfo.guest_ip field that only fed that population - ui: drop the per-interface IP column and its grid track Multi-NIC status reporting can be revisited once the fate of port-forward is settled; if it stays, the DHCP-report path needs authentication and a release/cleanup path before persisting and exposing guest IPs more widely. --- dstack/vmm/rpc/proto/vmm_rpc.proto | 5 ++-- dstack/vmm/src/app.rs | 38 ++++++++-------------------- dstack/vmm/src/app/qemu.rs | 37 +-------------------------- dstack/vmm/ui/src/styles/main.css | 2 +- dstack/vmm/ui/src/templates/app.html | 1 - 5 files changed, 14 insertions(+), 69 deletions(-) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index ff6956d1b..3a4e53e0d 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -48,11 +48,10 @@ message NetworkInterfaceStatus { // QEMU/network backend shape: "slirp", "tap_bridge", or "custom". string backend = 2; string mac = 3; - optional string ip = 4; // Linux bridge name for tap_bridge backend. - optional string bridge_name = 5; + optional string bridge_name = 4; // QEMU netdev id, e.g. "net0". - optional string netdev_id = 6; + optional string netdev_id = 5; } // Structured log or lifecycle event emitted by the guest or runtime. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index d9292ed7a..4d73f04f8 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -552,10 +552,10 @@ impl App { networks.iter().enumerate().find_map(|(index, networking)| { let prefix = networking.mac_prefix_bytes(); let nic_mac = mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); - (nic_mac == mac).then(|| (id.clone(), nic_mac)) + (nic_mac == mac).then(|| id.clone()) }) }); - let Some((vm_id, nic_mac)) = found else { + let Some(vm_id) = found else { debug!(mac, ip, "DHCP lease for unknown MAC, ignoring"); return; }; @@ -567,11 +567,7 @@ impl App { if let Err(e) = workdir.set_guest_ip(ip) { error!(mac, ip, "failed to persist guest IP: {e}"); } - if let Err(e) = workdir.set_guest_ip_for_mac(&nic_mac, ip) { - error!(mac, ip, "failed to persist guest IP map: {e}"); - } vm.state.guest_ip = ip.to_string(); - vm.state.guest_ips.insert(nic_mac, ip.to_string()); info!(mac, ip, id = %vm_id, "DHCP lease updated"); vm_id }; @@ -596,21 +592,13 @@ impl App { { return; } - let Some((index, networking)) = networks.iter().enumerate().find(|(_, networking)| { - networking.is_bridge() && networking.forward_service_enabled - }) else { + if !networks + .iter() + .any(|networking| networking.is_bridge() && networking.forward_service_enabled) + { return; - }; - let prefix = networking.mac_prefix_bytes(); - let mac = - crate::app::qemu::mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); - let guest_ip = vm.state.guest_ips.get(&mac).cloned().unwrap_or_else(|| { - if vm.state.guest_ips.is_empty() { - vm.state.guest_ip.clone() - } else { - String::new() - } - }); + } + let guest_ip = vm.state.guest_ip.clone(); let port_map = vm.config.manifest.port_map.clone(); (guest_ip, port_map) }; @@ -750,16 +738,11 @@ impl App { let vm_ids: Vec = self.lock().vms.keys().cloned().collect(); for id in vm_ids { let workdir = self.work_dir(&id); - let guest_ip = workdir.guest_ip(); - let guest_ips = workdir.guest_ips(); - if guest_ip.is_some() || !guest_ips.is_empty() { + if let Some(ip) = workdir.guest_ip() { { let mut state = self.lock(); if let Some(vm) = state.get_mut(&id) { - if let Some(ip) = guest_ip { - vm.state.guest_ip = ip; - } - vm.state.guest_ips = guest_ips; + vm.state.guest_ip = ip; } } self.reconfigure_port_forward(&id).await; @@ -2064,7 +2047,6 @@ struct VmStateMut { boot_error: String, shutdown_progress: String, guest_ip: String, - guest_ips: HashMap, runtime_networks: Vec, devices: GpuConfig, events: VecDeque, diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 751784b97..515b98a24 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -9,7 +9,6 @@ use crate::{ CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform, }, }; -use std::collections::HashMap; use std::os::unix::fs::PermissionsExt; use std::{ fs::Permissions, @@ -284,8 +283,6 @@ pub struct VmInfo { pub image_version: String, pub gateway_enabled: bool, pub events: Vec, - pub guest_ip: String, - pub guest_ips: HashMap, pub runtime_networks: Vec, } @@ -421,28 +418,16 @@ impl VmInfo { .collect::>(); let configured_networking = configured_networks.first().cloned(); let effective_networks = self.effective_networks(cvm); - let ip_index = effective_networks - .iter() - .position(|networking| networking.mode == NetworkingMode::Bridge) - .unwrap_or(0); let interfaces = effective_networks .iter() .enumerate() .map(|(index, networking)| { let prefix = networking.mac_prefix_bytes(); let mac = mac_address_for_vm_index(&self.manifest.id, &prefix, index); - let ip = self.guest_ips.get(&mac).cloned().or_else(|| { - (self.guest_ips.is_empty() - && index == ip_index - && networking.mode == NetworkingMode::Bridge) - .then(|| self.guest_ip.clone()) - .filter(|ip| !ip.is_empty()) - }); pb::NetworkInterfaceStatus { mode: networking_mode_name(networking.mode).into(), backend: networking_backend_name(networking.mode).into(), - mac: mac.clone(), - ip, + mac, bridge_name: (networking.mode == NetworkingMode::Bridge) .then(|| networking.bridge.clone()), netdev_id: Some(format!("net{index}")), @@ -605,8 +590,6 @@ impl VmState { image_version: self.config.image.info.version.clone(), gateway_enabled: self.config.gateway_enabled, events: self.state.events.clone().into(), - guest_ip: self.state.guest_ip.clone(), - guest_ips: self.state.guest_ips.clone(), runtime_networks: self.state.runtime_networks.clone(), } } @@ -1425,10 +1408,6 @@ impl VmWorkDir { self.workdir.join("guest-ip") } - pub fn guest_ips_path(&self) -> PathBuf { - self.workdir.join("guest-ips.json") - } - pub fn runtime_networks_path(&self) -> PathBuf { self.workdir.join("runtime-networks.json") } @@ -1444,20 +1423,6 @@ impl VmWorkDir { fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") } - pub fn guest_ips(&self) -> HashMap { - fs::read_to_string(self.guest_ips_path()) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default() - } - - pub fn set_guest_ip_for_mac(&self, mac: &str, ip: &str) -> Result<()> { - let mut guest_ips = self.guest_ips(); - guest_ips.insert(mac.to_string(), ip.to_string()); - fs::write(self.guest_ips_path(), serde_json::to_string(&guest_ips)?) - .context("failed to write guest IP map") - } - pub fn runtime_networks(&self) -> Vec { fs::read_to_string(self.runtime_networks_path()) .ok() diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index af6594747..7a7b0ee72 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -620,7 +620,7 @@ h1, h2, h3, h4, h5, h6 { .runtime-network-item { display: grid; - grid-template-columns: 1.1fr 0.6fr 0.8fr 1fr 1.4fr; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr; gap: 12px; overflow-wrap: anywhere; } diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index cc9bfdaaa..7e38806c7 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -356,7 +356,6 @@

VMM Network Interfaces

{{ networkModeLabel(iface.mode) }} / {{ iface.backend || '-' }} {{ iface.netdev_id || '-' }} {{ iface.bridge_name || '-' }} - {{ iface.ip || '-' }} {{ iface.mac || '-' }}
From 305ccd10af1e7b121fea64dbb80d27ea1977cd47 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Fri, 10 Jul 2026 00:34:04 +0800 Subject: [PATCH 08/10] chore(vmm): adapt make_vm_config test callers to requirements param Rebasing onto master pulled in the new make_vm_config(.., requirements: Option<&Requirements>) parameter added by the launch-token-requirement work (#763). The multi-NIC bridge/user config-equality test still called the 5-arg form; pass None for the new argument so the crate builds against the rebased signature. --- dstack/vmm/src/app.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 4d73f04f8..99b2309ed 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1758,8 +1758,10 @@ mod tests { let image = test_tdx_image(true); let compose_hash = hex_of(0x22, 32); - let bridge_config = make_vm_config(&config, &bridge_manifest, &image, &compose_hash, None)?; - let user_config = make_vm_config(&config, &user_manifest, &image, &compose_hash, None)?; + let bridge_config = + make_vm_config(&config, &bridge_manifest, &image, &compose_hash, None, None)?; + let user_config = + make_vm_config(&config, &user_manifest, &image, &compose_hash, None, None)?; assert_eq!(bridge_config, user_config); Ok(()) From aa70691982dc5027cc45ad5781f9d8a25c4b8a54 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Fri, 10 Jul 2026 07:32:53 +0800 Subject: [PATCH 09/10] fix(vmm): make runtime networking state consistent Reject configurations with multiple forwarding-enabled bridges so DHCP lease updates cannot redirect the VM-wide forwarding target between interfaces.\n\nPersist runtime network snapshots atomically before launching QEMU, publish the snapshot to in-memory state before DHCP can arrive, and clear both copies when process startup fails. Add regression coverage for forwarding validation and atomic snapshot replacement. --- dstack/vmm/src/app.rs | 29 ++++++++++++++++------- dstack/vmm/src/app/qemu.rs | 43 +++++++++++++++++++++++++++++----- dstack/vmm/src/main_service.rs | 23 ++++++++++++++++++ 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 99b2309ed..ebc01e2cf 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -378,20 +378,33 @@ impl App { let devices = self.try_allocate_gpus(&vm_config.manifest)?; let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; - for process in processes { - self.supervisor - .deploy(&process) - .await - .with_context(|| format!("Failed to start process {}", process.id))?; - } - let runtime_networks = crate::app::qemu::resolved_networks(&vm_config.manifest, &self.config.cvm); work_dir.set_runtime_networks(&runtime_networks)?; + { + let mut state = self.lock(); + let vm_state = state.get_mut(id).context("VM not found")?; + vm_state.state.runtime_networks = runtime_networks; + } + for process in processes { + if let Err(err) = self.supervisor.deploy(&process).await { + if let Err(clear_err) = work_dir.clear_runtime_networks() { + warn!( + id, + "failed to clear runtime networks after start failure: {clear_err}" + ); + } + if let Some(vm_state) = self.lock().get_mut(id) { + vm_state.state.runtime_networks.clear(); + } + return Err(err) + .with_context(|| format!("failed to start process {}", process.id)); + } + } + let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; vm_state.state.devices = devices; - vm_state.state.runtime_networks = runtime_networks; } Ok(()) } diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 515b98a24..db9593a93 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -152,6 +152,13 @@ pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { } pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { + let forwarding_bridges = networks + .iter() + .filter(|networking| networking.is_bridge() && networking.forward_service_enabled) + .count(); + if forwarding_bridges > 1 { + bail!("built-in port forwarding supports only one bridge"); + } for networking in networks { validate_resolved_network(networking)?; } @@ -599,7 +606,12 @@ impl VmState { mod tests { use super::{ amd_sev_snp_memory_backend_arg, mac_address_for_vm, mac_address_for_vm_index, - parse_amd_sev_snp_qmp_capabilities, sanitize_optional, virtio_pci_device, + parse_amd_sev_snp_qmp_capabilities, sanitize_optional, virtio_pci_device, VmWorkDir, + }; + use std::{ + fs, + os::unix::fs::symlink, + time::{SystemTime, UNIX_EPOCH}, }; #[test] @@ -635,6 +647,27 @@ mod tests { ); } + #[test] + fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> anyhow::Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-runtime-networks-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + fs::create_dir_all(&temp)?; + let external = temp.join("external.json"); + fs::write(&external, "sentinel")?; + let workdir = VmWorkDir::new(temp.join("vm")); + fs::create_dir_all(workdir.path())?; + symlink(&external, workdir.runtime_networks_path())?; + + workdir.set_runtime_networks(&[])?; + + assert_eq!(fs::read_to_string(&external)?, "sentinel"); + assert_eq!(fs::read_to_string(workdir.runtime_networks_path())?, "[]"); + fs::remove_dir_all(temp)?; + Ok(()) + } + #[test] fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { assert_eq!( @@ -1431,11 +1464,9 @@ impl VmWorkDir { } pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { - fs::write( - self.runtime_networks_path(), - serde_json::to_string(networks)?, - ) - .context("failed to write runtime networks") + let serialized = serde_json::to_vec(networks)?; + safe_write::safe_write(self.runtime_networks_path(), serialized) + .context("failed to write runtime networks") } pub fn clear_runtime_networks(&self) -> Result<()> { diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 29d0ddeb5..7290565d3 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -919,6 +919,29 @@ mod tests { assert!(err.to_string().contains("custom networking mode")); } + + #[test] + fn multiple_bridges_are_rejected_when_builtin_forwarding_is_enabled() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.forward_service_enabled = true; + let mut request = test_vm_configuration(); + request.networks = vec![ + rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "lo".to_string(), + }, + rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "lo".to_string(), + }, + ]; + + let err = create_manifest_from_vm_config(request, &cvm_config).unwrap_err(); + + assert!(err + .to_string() + .contains("built-in port forwarding supports only one bridge")); + } } impl RpcCall for RpcHandler { From 1e329bf9328859631a6c9529f001cf93e026d1b9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 16 Jul 2026 01:02:09 -0700 Subject: [PATCH 10/10] refactor(vmm): align networking with split qemu modules --- dstack/vmm/src/app.rs | 25 +- dstack/vmm/src/app/mr_config.rs | 33 - dstack/vmm/src/app/network.rs | 118 +++ dstack/vmm/src/app/qemu.rs | 1664 ++++++++++--------------------- dstack/vmm/src/app/vm_info.rs | 76 +- dstack/vmm/src/app/workdir.rs | 76 +- dstack/vmm/src/main_service.rs | 20 +- 7 files changed, 819 insertions(+), 1193 deletions(-) create mode 100644 dstack/vmm/src/app/network.rs diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index ebc01e2cf..be2096e13 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -32,13 +32,21 @@ use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; -pub(crate) use qemu::{resolve_networking, validate_resolved_network, validate_resolved_networks}; -pub use qemu::{VmConfig, VmWorkDir}; +pub(crate) use network::{ + resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, +}; +pub use qemu::VmConfig; +pub use workdir::VmWorkDir; +mod host_share; mod id_pool; mod image; +mod mr_config; +mod network; mod qemu; pub(crate) mod registry; +mod vm_info; +mod workdir; #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { @@ -295,7 +303,7 @@ impl App { let vm_id = manifest.id.clone(); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { - runtime_networks = crate::app::qemu::resolved_networks(&manifest, &self.config.cvm); + runtime_networks = resolved_networks(&manifest, &self.config.cvm); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -378,8 +386,7 @@ impl App { let devices = self.try_allocate_gpus(&vm_config.manifest)?; let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; - let runtime_networks = - crate::app::qemu::resolved_networks(&vm_config.manifest, &self.config.cvm); + let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); work_dir.set_runtime_networks(&runtime_networks)?; { let mut state = self.lock(); @@ -556,7 +563,7 @@ impl App { /// Handle a DHCP lease notification: look up VM by MAC address, persist /// the guest IP, and reconfigure port forwarding. pub async fn report_dhcp_lease(&self, mac: &str, ip: &str) { - use crate::app::qemu::mac_address_for_vm_index; + use crate::app::network::mac_address_for_vm_index; let vm_id = { let mut state = self.lock(); @@ -912,7 +919,7 @@ impl App { let already_running = cids_assigned.contains_key(&vm_id); let mut runtime_networks = vm_work_dir.runtime_networks(); if runtime_networks.is_empty() && already_running { - runtime_networks = crate::app::qemu::resolved_networks(&manifest, &self.config.cvm); + runtime_networks = resolved_networks(&manifest, &self.config.cvm); if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); } @@ -1471,7 +1478,7 @@ fn make_vm_config( // QEMU command (see `VmConfig::config_qemu`), which changes the guest's // ACPI/DSDT layout and therefore RTMR0. Measure the interface count so the // verifier reconstructs the exact device layout. - let num_nics = crate::app::qemu::resolved_networks(manifest, &cfg.cvm).len() as u32; + let num_nics = resolved_networks(manifest, &cfg.cvm).len() as u32; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, @@ -2097,7 +2104,7 @@ impl VmState { pub fn effective_networks(&self, cvm: &crate::config::CvmConfig) -> Vec { if self.state.runtime_networks.is_empty() { - crate::app::qemu::resolved_networks(&self.config.manifest, cvm) + resolved_networks(&self.config.manifest, cvm) } else { self.state.runtime_networks.clone() } diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index 71c8a427f..f1b908e4f 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -13,25 +13,6 @@ use sha2::{Digest, Sha256}; use super::VmWorkDir; -/// Derives a deterministic, locally administered unicast MAC address from a VM ID. -/// -/// `prefix` may contain zero to three fixed bytes. Remaining bytes are filled -/// from SHA-256 of `vm_id`. -pub(super) fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { - let hash = Sha256::digest(vm_id.as_bytes()); - let prefix_len = prefix.len().min(3); - let mut bytes = [0_u8; 6]; - bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); - for index in prefix_len..bytes.len() { - bytes[index] = hash[index - prefix_len]; - } - bytes[0] = (bytes[0] & 0xfe) | 0x02; - format!( - "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] - ) -} - pub(super) fn tdx_mr_config_id(workdir: &VmWorkDir, app_compose: &AppCompose) -> Result { if let Some(document) = workdir .sys_config() @@ -132,17 +113,3 @@ impl VmWorkDir { .to_canonical_json()) } } - -#[cfg(test)] -mod tests { - use super::mac_address_for_vm; - - #[test] - fn mac_address_is_deterministic_and_locally_administered() { - let mac = mac_address_for_vm("vm-1", &[0xaa, 0xbb, 0xcc]); - assert_eq!(&mac[..8], "aa:bb:cc"); - assert_eq!(mac, mac_address_for_vm("vm-1", &[0xaa, 0xbb, 0xcc])); - let first = u8::from_str_radix(&mac[..2], 16).unwrap(); - assert_eq!(first & 0x03, 0x02); - } -} diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs new file mode 100644 index 000000000..2ed379423 --- /dev/null +++ b/dstack/vmm/src/app/network.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! VM network resolution, validation, and interface identity. + +use std::path::Path; + +use anyhow::{bail, Result}; +use sha2::{Digest, Sha256}; + +use super::Manifest; +use crate::config::{CvmConfig, Networking, NetworkingMode}; + +pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking { + let mut resolved = cfg.networking.clone(); + resolved.mode = networking.mode; + resolved.restrict = cfg.networking.restrict || networking.restrict; + if !networking.bridge.is_empty() { + resolved.bridge = networking.bridge.clone(); + } + if !networking.mac_prefix.is_empty() { + resolved.mac_prefix = networking.mac_prefix.clone(); + } + if !networking.net.is_empty() { + resolved.net = networking.net.clone(); + } + if !networking.dhcp_start.is_empty() { + resolved.dhcp_start = networking.dhcp_start.clone(); + } + if !networking.netdev.is_empty() { + resolved.netdev = networking.netdev.clone(); + } + resolved +} + +pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { + if manifest.networks.is_empty() { + vec![cfg.networking.clone()] + } else { + manifest + .networks + .iter() + .map(|networking| resolve_networking(networking, cfg)) + .collect() + } +} + +pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { + if networking.mode != NetworkingMode::Bridge { + return Ok(()); + } + if networking.bridge.is_empty() { + bail!("bridge networking requested but no bridge is configured"); + } + if !Path::new("/sys/class/net") + .join(&networking.bridge) + .exists() + { + bail!("bridge interface '{}' does not exist", networking.bridge); + } + Ok(()) +} + +pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { + let forwarding_bridges = networks + .iter() + .filter(|networking| networking.is_bridge() && networking.forward_service_enabled) + .count(); + if forwarding_bridges > 1 { + bail!("built-in port forwarding supports only one bridge"); + } + for networking in networks { + validate_resolved_network(networking)?; + } + Ok(()) +} + +/// Derives a deterministic, locally administered unicast MAC address. +/// +/// Index zero preserves the legacy single-NIC derivation. Later interfaces +/// hash `vm_id:index`, preventing an upgrade from changing the primary NIC. +pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) -> String { + let hash_input = if index == 0 { + vm_id.to_string() + } else { + format!("{vm_id}:{index}") + }; + let hash = Sha256::digest(hash_input.as_bytes()); + let prefix_len = prefix.len().min(3); + let mut bytes = [0_u8; 6]; + bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); + for index in prefix_len..bytes.len() { + bytes[index] = hash[index - prefix_len]; + } + bytes[0] = (bytes[0] & 0xfe) | 0x02; + format!( + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] + ) +} + +#[cfg(test)] +mod tests { + use super::mac_address_for_vm_index; + + #[test] + fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { + assert_eq!( + mac_address_for_vm_index("vm-123", &[], 0), + "96:b1:d8:b9:08:e6" + ); + assert_eq!( + mac_address_for_vm_index("vm-123", &[], 1), + "c6:74:2c:65:14:b9" + ); + } +} diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index db9593a93..6850bb653 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -2,172 +2,35 @@ // // SPDX-License-Identifier: Apache-2.0 -//! QEMU related code +//! QEMU launch preparation and command construction. use crate::{ app::Manifest, - config::{ - CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform, - }, + config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, }; +use std::collections::HashMap; use std::os::unix::fs::PermissionsExt; use std::{ fs::Permissions, io::Write, - ops::Deref, path::{Path, PathBuf}, process::{Command, Stdio}, - time::{Duration, SystemTime}, }; use super::{ - effective_vcpu_count, hugepage_numa_nodes, image::Image, pci_numa_node, round_up, GpuConfig, - VmState, + effective_vcpu_count, + host_share::create_shared_disk, + hugepage_numa_nodes, + image::Image, + mr_config::{snp_host_data, tdx_mr_config_id}, + network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, + pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use anyhow::{bail, Context, Result}; -use base64::prelude::*; use bon::Builder; -use dstack_types::{ - mr_config::{MrConfig, MrConfigV3}, - shared_filenames::{ - APP_COMPOSE, ENCRYPTED_ENV, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, - }, - AppCompose, KeyProviderKind, SysConfig, -}; -use dstack_vmm_rpc as pb; -use sha2::{Digest, Sha256}; - -/// Derive a deterministic MAC address from a VM ID using SHA256. -/// Sets locally-administered + unicast bits (0x02) per IEEE 802. -/// Derive a deterministic MAC address from a VM ID. -/// -/// `prefix` may contain 0-3 fixed bytes. The first byte always has the -/// locally-administered + unicast bits set (0x02). Remaining bytes are -/// filled from SHA256(vm_id). -#[cfg(test)] -fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { - mac_address_for_vm_index(vm_id, prefix, 0) -} - -pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) -> String { - let hash_input = if index == 0 { - vm_id.to_string() - } else { - format!("{vm_id}:{index}") - }; - let hash = Sha256::digest(hash_input.as_bytes()); - let prefix_len = prefix.len().min(3); - let mut bytes = [0u8; 6]; - // Fill prefix bytes - bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); - // Fill remaining bytes from hash - for i in prefix_len..6 { - bytes[i] = hash[i - prefix_len]; - } - // Ensure locally-administered + unicast on first byte - bytes[0] = (bytes[0] & 0xfe) | 0x02; - format!( - "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] - ) -} +use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; use fs_err as fs; -use serde::{Deserialize, Serialize}; -use serde_human_bytes as hex_bytes; -use supervisor_client::supervisor::{ProcessConfig, ProcessInfo}; - -fn networking_mode_name(mode: NetworkingMode) -> &'static str { - match mode { - NetworkingMode::Bridge => "bridge", - NetworkingMode::User => "user", - NetworkingMode::Custom => "custom", - } -} - -fn networking_backend_name(mode: NetworkingMode) -> &'static str { - match mode { - NetworkingMode::Bridge => "tap_bridge", - NetworkingMode::User => "slirp", - NetworkingMode::Custom => "custom", - } -} - -fn networking_to_proto(n: &Networking) -> pb::NetworkingConfig { - pb::NetworkingConfig { - mode: networking_mode_name(n.mode).into(), - bridge_name: if n.mode == NetworkingMode::Bridge { - n.bridge.clone() - } else { - String::new() - }, - } -} - -pub(crate) fn resolve_networking(vm_net: &Networking, cfg: &CvmConfig) -> Networking { - let mut resolved = cfg.networking.clone(); - resolved.mode = vm_net.mode; - resolved.restrict = cfg.networking.restrict || vm_net.restrict; - if !vm_net.bridge.is_empty() { - resolved.bridge = vm_net.bridge.clone(); - } - if !vm_net.mac_prefix.is_empty() { - resolved.mac_prefix = vm_net.mac_prefix.clone(); - } - if !vm_net.net.is_empty() { - resolved.net = vm_net.net.clone(); - } - if !vm_net.dhcp_start.is_empty() { - resolved.dhcp_start = vm_net.dhcp_start.clone(); - } - if !vm_net.netdev.is_empty() { - resolved.netdev = vm_net.netdev.clone(); - } - resolved -} - -pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { - if manifest.networks.is_empty() { - vec![cfg.networking.clone()] - } else { - manifest - .networks - .iter() - .map(|networking| resolve_networking(networking, cfg)) - .collect() - } -} - -pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { - if networking.mode == NetworkingMode::Bridge { - if networking.bridge.is_empty() { - bail!("bridge networking requested but no bridge is configured"); - } - if !Path::new("/sys/class/net") - .join(&networking.bridge) - .exists() - { - bail!("bridge interface '{}' does not exist", networking.bridge); - } - } - Ok(()) -} - -pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { - let forwarding_bridges = networks - .iter() - .filter(|networking| networking.is_bridge() && networking.forward_service_enabled) - .count(); - if forwarding_bridges > 1 { - bail!("built-in port forwarding supports only one bridge"); - } - for networking in networks { - validate_resolved_network(networking)?; - } - Ok(()) -} - -fn sanitize_optional>(value: Option) -> Option { - value.filter(|value| !value.as_ref().trim().is_empty()) -} +use serde::Serialize; +use supervisor_client::supervisor::ProcessConfig; #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct AmdSevSnpLaunchParams { @@ -267,32 +130,6 @@ fn detect_amd_sev_snp_qemu_capabilities(qemu_path: &Path) -> Result, - #[serde(default, with = "hex_bytes")] - pub instance_id: Vec, - #[serde(default, with = "hex_bytes")] - pub app_id: Vec, -} - -pub struct VmInfo { - pub manifest: Manifest, - pub workdir: PathBuf, - pub status: &'static str, - pub uptime: String, - pub exited_at: Option, - pub instance_id: Option, - pub boot_progress: String, - pub boot_error: String, - pub shutdown_progress: String, - pub image_version: String, - pub gateway_enabled: bool, - pub events: Vec, - pub runtime_networks: Vec, -} - #[derive(Debug, Builder)] pub struct VmConfig { pub manifest: Manifest, @@ -302,11 +139,6 @@ pub struct VmConfig { pub gateway_enabled: bool, } -#[derive(Deserialize, Serialize)] -pub struct State { - started: bool, -} - fn create_hd( image_file: impl AsRef, backing_file: Option>, @@ -332,383 +164,160 @@ fn create_hd( Ok(()) } -/// Create a FAT32 disk image from a directory -fn create_shared_disk(disk_path: impl AsRef, shared_dir: impl AsRef) -> Result<()> { - use fatfs::{FileSystem, FormatVolumeOptions, FsOptions}; - use std::io::{Seek, SeekFrom, Write}; - - let disk_path = disk_path.as_ref(); - let shared_dir = shared_dir.as_ref(); - - // Must be large enough to hold all host-shared files (app-compose.json and - // .user-config can each be up to 50 MiB, see HostShared::copy) plus FAT32 overhead. - const DISK_SIZE: u64 = 128 * 1024 * 1024; - - // Back the image by a file (sparse until written) and stream files into it so - // peak memory stays bounded regardless of DISK_SIZE or input file sizes. - let mut image = fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(disk_path) - .with_context(|| format!("Failed to create disk image at {}", disk_path.display()))?; - image - .set_len(DISK_SIZE) - .context("Failed to size disk image")?; - - { - let mut label_bytes = [b' '; 11]; - let label_str = HOST_SHARED_DISK_LABEL.as_bytes(); - let copy_len = label_str.len().min(11); - label_bytes[..copy_len].copy_from_slice(&label_str[..copy_len]); - let format_opts = FormatVolumeOptions::new() - .fat_type(fatfs::FatType::Fat32) - .volume_label(label_bytes); - fatfs::format_volume(&mut image, format_opts).context("Failed to format disk as FAT32")?; - } - - // Open the formatted filesystem and stream files into it - { - image - .seek(SeekFrom::Start(0)) - .context("Failed to seek to start")?; - let fs = FileSystem::new(&mut image, FsOptions::new()) - .context("Failed to open FAT32 filesystem")?; - let root_dir = fs.root_dir(); - - // Copy all files from shared_dir to the FAT32 root - for entry in fs::read_dir(shared_dir).context("Failed to read shared directory")? { - let entry = entry.context("Failed to read directory entry")?; - let path = entry.path(); - - if path.is_file() { - let filename = entry.file_name(); - let filename_str = filename.to_string_lossy(); - - let mut src = fs::File::open(&path) - .with_context(|| format!("Failed to open file {}", path.display()))?; - let mut fat_file = root_dir - .create_file(&filename_str) - .with_context(|| format!("Failed to create file {filename_str} in FAT32"))?; - std::io::copy(&mut src, &mut fat_file) - .with_context(|| format!("Failed to write file {filename_str} to FAT32"))?; - fat_file.flush().context("Failed to flush FAT32 file")?; - } - } +fn virtio_pci_device(device: &str, snp: bool) -> String { + if snp { + format!("{device},disable-legacy=on,iommu_platform=true") + } else { + device.to_string() } - - Ok(()) } -impl VmInfo { - pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { - if self.runtime_networks.is_empty() { - resolved_networks(&self.manifest, cvm) - } else { - self.runtime_networks.clone() - } - } - - pub fn to_pb(&self, gw: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { - let workdir = VmWorkDir::new(&self.workdir); - let vm_config = workdir.manifest(); - let custom_gateway_urls = vm_config - .as_ref() - .map(|c| c.gateway_urls.clone()) - .unwrap_or_default(); - let configured_networks = self - .manifest - .networks - .iter() - .map(networking_to_proto) - .collect::>(); - let configured_networking = configured_networks.first().cloned(); - let effective_networks = self.effective_networks(cvm); - let interfaces = effective_networks - .iter() - .enumerate() - .map(|(index, networking)| { - let prefix = networking.mac_prefix_bytes(); - let mac = mac_address_for_vm_index(&self.manifest.id, &prefix, index); - pb::NetworkInterfaceStatus { - mode: networking_mode_name(networking.mode).into(), - backend: networking_backend_name(networking.mode).into(), - mac, - bridge_name: (networking.mode == NetworkingMode::Bridge) - .then(|| networking.bridge.clone()), - netdev_id: Some(format!("net{index}")), - } - }) - .collect(); - - pb::VmInfo { - id: self.manifest.id.clone(), - name: self.manifest.name.clone(), - status: self.status.into(), - uptime: self.uptime.clone(), - boot_progress: self.boot_progress.clone(), - boot_error: self.boot_error.clone(), - shutdown_progress: self.shutdown_progress.clone(), - image_version: self.image_version.clone(), - configuration: if brief { - None - } else { - let kms_urls = vm_config - .as_ref() - .map(|c| c.kms_urls.clone()) - .unwrap_or_default(); - let no_tee = vm_config - .as_ref() - .map(|c| c.no_tee) - .unwrap_or(self.manifest.no_tee); - let stopped = !workdir.started().unwrap_or(false); - - Some(pb::VmConfiguration { - name: self.manifest.name.clone(), - image: self.manifest.image.clone(), - compose_file: { - fs::read_to_string(workdir.app_compose_path()).unwrap_or_default() - }, - encrypted_env: { fs::read(workdir.encrypted_env_path()).unwrap_or_default() }, - user_config: { - fs::read_to_string(workdir.user_config_path()).unwrap_or_default() - }, - vcpu: self.manifest.vcpu, - memory: self.manifest.memory, - disk_size: self.manifest.disk_size, - ports: self - .manifest - .port_map - .iter() - .map(|pm| pb::PortMapping { - protocol: pm.protocol.as_str().into(), - host_address: pm.address.to_string(), - host_port: pm.from as u32, - vm_port: pm.to as u32, - }) - .collect(), - app_id: Some(self.manifest.app_id.clone()), - hugepages: self.manifest.hugepages, - pin_numa: self.manifest.pin_numa, - gpus: self.manifest.gpus.as_ref().map(|g| pb::GpuConfig { - attach_mode: g.attach_mode.to_string(), - gpus: g - .gpus - .iter() - .map(|gpu| pb::GpuSpec { - slot: gpu.slot.clone(), - }) - .collect(), - }), - kms_urls, - gateway_urls: custom_gateway_urls.clone(), - stopped, - no_tee, - networking: configured_networking, - networks: configured_networks, - }) - }, - app_url: self - .gateway_enabled - .then_some(self.instance_id.as_deref()) - .flatten() - .and_then(|id| sanitize_optional(Some(id))) - .map(|id| { - // Use custom gateway URL if available, otherwise fall back to global config - if let Some(custom_gw_url) = custom_gateway_urls.first() { - if let Ok(url) = url::Url::parse(custom_gw_url) { - let host = url.host_str().unwrap_or(&gw.base_domain); - let port = url.port().unwrap_or(443); - if port == 443 { - return format!("https://{id}-{}.{}", gw.agent_port, host); - } else { - return format!("https://{id}-{}.{}:{}", gw.agent_port, host, port); - } - } - } - // Fall back to global gateway config - if gw.port == 443 { - format!("https://{id}-{}.{}", gw.agent_port, gw.base_domain) - } else { - format!( - "https://{id}-{}.{}:{}", - gw.agent_port, gw.base_domain, gw.port - ) - } - }), - app_id: self.manifest.app_id.clone(), - instance_id: sanitize_optional(self.instance_id.clone()), - exited_at: self.exited_at.clone(), - events: self.events.clone(), - interfaces, - } - } +struct PreparedQemuLaunch { + workdir: VmWorkDir, + platform: TeePlatform, + networks: Vec, + hugepage_numa_nodes: Option>, + gpu_numa_nodes: HashMap, + numa_cpus: Option, + tpm_path: Option<&'static str>, + tdx_mr_config_id: Option, + snp_host_data: Option, + snp_launch_params: Option, } -impl VmState { - pub fn merged_info(&self, proc_state: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo { - fn truncate(d: Duration) -> Duration { - Duration::from_secs(d.as_secs()) - } - let is_running = match proc_state { - Some(info) => info.state.status.is_running(), - None => false, +impl PreparedQemuLaunch { + fn prepare( + vm: &VmConfig, + workdir: impl AsRef, + cfg: &CvmConfig, + gpus: &GpuConfig, + ) -> Result { + let workdir = VmWorkDir::new(workdir); + prepare_data_disk(vm, &workdir, cfg)?; + prepare_shared_dir(&workdir)?; + let app_compose = workdir.app_compose().context("failed to get app compose")?; + let platform = cfg.resolved_platform(); + let networks = resolved_networks(&vm.manifest, cfg); + validate_resolved_networks(&networks)?; + + let hugepage_numa_nodes = if vm.manifest.hugepages { + Some(hugepage_numa_nodes(gpus)?) + } else { + None }; - let started = workdir.started().unwrap_or(false); - let status = if self.state.removing { - "removing" + let gpu_numa_nodes = if vm.manifest.hugepages { + gpus.gpus + .iter() + .map(|gpu| Ok((gpu.slot.clone(), pci_numa_node(&gpu.slot)?))) + .collect::>()? } else { - match (started, is_running) { - (true, true) => "running", - (true, false) => "exited", - (false, true) => "stopping", - (false, false) => "stopped", - } + HashMap::new() }; + let numa_cpus = if vm.manifest.pin_numa { + let device = gpus.gpus.first().map(|gpu| gpu.slot.clone()); + Some(find_numa(device)?.1) + } else { + None + }; + let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { + Some(detect_tpm_device()?) + } else { + None + }; + prepare_shared_disk(&workdir, cfg)?; - fn display_ts(t: Option<&SystemTime>) -> String { - match t { - None => "never".into(), - Some(t) => { - let ts = t.elapsed().unwrap_or(Duration::MAX); - humantime::format_duration(truncate(ts)).to_string() - } - } - } - let uptime = display_ts(proc_state.and_then(|info| info.state.started_at.as_ref())); - let exited_at = display_ts(proc_state.and_then(|info| info.state.stopped_at.as_ref())); - let instance_id = sanitize_optional( - workdir - .instance_info() - .ok() - .map(|info| hex::encode(info.instance_id)), - ); - VmInfo { - manifest: self.config.manifest.clone(), - workdir: workdir.path().to_path_buf(), - instance_id, - status, - uptime, - exited_at: Some(exited_at), - boot_progress: self.state.boot_progress.clone(), - boot_error: self.state.boot_error.clone(), - shutdown_progress: self.state.shutdown_progress.clone(), - image_version: self.config.image.info.version.clone(), - gateway_enabled: self.config.gateway_enabled, - events: self.state.events.clone().into(), - runtime_networks: self.state.runtime_networks.clone(), - } - } -} - -#[cfg(test)] -mod tests { - use super::{ - amd_sev_snp_memory_backend_arg, mac_address_for_vm, mac_address_for_vm_index, - parse_amd_sev_snp_qmp_capabilities, sanitize_optional, virtio_pci_device, VmWorkDir, - }; - use std::{ - fs, - os::unix::fs::symlink, - time::{SystemTime, UNIX_EPOCH}, - }; - - #[test] - fn sanitize_optional_filters_empty_owned_values() { - assert_eq!(sanitize_optional(Some(String::new())), None); - assert_eq!(sanitize_optional(Some(" ".to_string())), None); - assert_eq!( - sanitize_optional(Some("instance-123".to_string())), - Some("instance-123".to_string()) - ); - } + let tee_enabled = !vm.manifest.no_tee; + let tdx_mr_config_id = if tee_enabled + && platform == TeePlatform::Tdx + && cfg.use_mrconfigid + && vm.image.info.version_tuple().unwrap_or_default() >= (0, 5, 2) + { + Some(tdx_mr_config_id(&workdir, &app_compose)?) + } else { + None + }; + let (snp_host_data, snp_launch_params) = + if tee_enabled && platform == TeePlatform::AmdSevSnp { + ( + Some(snp_host_data(&workdir)?), + Some( + detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path).context( + "failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU", + )?, + ), + ) + } else { + (None, None) + }; - #[test] - fn sanitize_optional_filters_empty_borrowed_values() { - assert_eq!(sanitize_optional(Some("")), None); - assert_eq!(sanitize_optional(Some(" ")), None); - assert_eq!( - sanitize_optional(Some("instance-123")), - Some("instance-123") - ); + Ok(Self { + workdir, + platform, + networks, + hugepage_numa_nodes, + gpu_numa_nodes, + numa_cpus, + tpm_path, + tdx_mr_config_id, + snp_host_data, + snp_launch_params, + }) } +} - #[test] - fn mac_address_index_zero_keeps_existing_derivation() { - assert_eq!(mac_address_for_vm("vm-123", &[]), "96:b1:d8:b9:08:e6"); - assert_eq!( - mac_address_for_vm_index("vm-123", &[], 0), - "96:b1:d8:b9:08:e6" - ); - assert_eq!( - mac_address_for_vm_index("vm-123", &[], 1), - "c6:74:2c:65:14:b9" - ); +fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { + let hda_path = workdir.hda_path(); + if !hda_path.exists() { + create_hd( + &hda_path, + vm.image.hda.as_ref(), + &format!("{}G", vm.manifest.disk_size), + )?; } - - #[test] - fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> anyhow::Result<()> { - let temp = std::env::temp_dir().join(format!( - "dstack-vmm-runtime-networks-test-{}", - SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() - )); - fs::create_dir_all(&temp)?; - let external = temp.join("external.json"); - fs::write(&external, "sentinel")?; - let workdir = VmWorkDir::new(temp.join("vm")); - fs::create_dir_all(workdir.path())?; - symlink(&external, workdir.runtime_networks_path())?; - - workdir.set_runtime_networks(&[])?; - - assert_eq!(fs::read_to_string(&external)?, "sentinel"); - assert_eq!(fs::read_to_string(workdir.runtime_networks_path())?, "[]"); - fs::remove_dir_all(temp)?; - Ok(()) + if !cfg.user.is_empty() { + fs::set_permissions(&hda_path, Permissions::from_mode(0o660))?; } + Ok(()) +} - #[test] - fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { - assert_eq!( - amd_sev_snp_memory_backend_arg(4096), - "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" - ); +fn prepare_shared_dir(workdir: &VmWorkDir) -> Result<()> { + let shared_dir = workdir.shared_dir(); + if !shared_dir.exists() { + fs::create_dir_all(&shared_dir)?; } + Ok(()) +} - #[test] - fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { - let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} -{"return":{}} -{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} -{"return":{}} -"#; - let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); - assert_eq!(params.cbitpos, 51); - assert_eq!(params.reduced_phys_bits, 1); +fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { + if cfg.host_share_mode != "vhd" { + return Ok(()); } - #[test] - fn amd_sev_snp_uses_confidential_virtio_pci_options() { - assert_eq!( - virtio_pci_device("virtio-blk-pci,drive=hd0", true), - "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" - ); - assert_eq!( - virtio_pci_device("virtio-blk-pci,drive=hd0", false), - "virtio-blk-pci,drive=hd0" - ); + let shared_dir = workdir.shared_dir(); + let shared_disk_path = workdir.shared_disk_path(); + if shared_disk_path.exists() { + fs::remove_file(&shared_disk_path).context("failed to remove shared disk")?; } + create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk") } -fn virtio_pci_device(device: &str, snp: bool) -> String { - if snp { - format!("{device},disable-legacy=on,iommu_platform=true") +fn detect_tpm_device() -> Result<&'static str> { + if Path::new("/dev/tpmrm0").exists() { + Ok("/dev/tpmrm0") + } else if Path::new("/dev/tpm0").exists() { + Ok("/dev/tpm0") } else { - device.to_string() + bail!("tpm key provider requested but no TPM device found on host") } } +struct QemuCommandBuilder<'a> { + vm: &'a VmConfig, + cfg: &'a CvmConfig, + gpus: &'a GpuConfig, + prepared: &'a PreparedQemuLaunch, +} + impl VmConfig { pub fn config_qemu( &self, @@ -716,128 +325,157 @@ impl VmConfig { cfg: &CvmConfig, gpus: &GpuConfig, ) -> Result> { - let workdir = VmWorkDir::new(workdir); - let serial_file = workdir.serial_file(); - let serial_pty = workdir.serial_pty(); - let shared_dir = workdir.shared_dir(); - let disk_size = format!("{}G", self.manifest.disk_size); - let hda_path = workdir.hda_path(); - if !hda_path.exists() { - create_hd(&hda_path, self.image.hda.as_ref(), &disk_size)?; - } - if !cfg.user.is_empty() { - fs_err::set_permissions(&hda_path, Permissions::from_mode(0o660))?; + let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; + let process = QemuCommandBuilder { + vm: self, + cfg, + gpus, + prepared: &prepared, } + .build()?; + Ok(vec![process]) + } +} + +impl QemuCommandBuilder<'_> { + fn build(&self) -> Result { + let mut command = self.base_command(); + self.configure_rootfs(&mut command)?; + self.configure_data_disk(&mut command); + self.configure_networking(&mut command)?; + self.vm.configure_smbios(&mut command, self.cfg); + self.configure_tpm_and_vsock(&mut command); + self.configure_host_share(&mut command)?; + + let (smp, mem) = self.configure_hugepage_memory(&mut command)?; + self.vm + .configure_machine(&mut command, self.cfg, self.prepared, mem)?; + self.configure_gpus(&mut command)?; + command.arg("-smp").arg(smp.to_string()); + command.arg("-m").arg(format!("{mem}M")); - if !shared_dir.exists() { - fs::create_dir_all(&shared_dir)?; + // SNP app identity is bound through HOST_DATA, so the measured cmdline + // remains the image-provided cmdline. + if let Some(cmdline) = &self.vm.image.info.cmdline { + command.arg("-append").arg(cmdline); } - let app_compose = workdir.app_compose().context("Failed to get app compose")?; - let qemu = &cfg.qemu_path; - let is_amd_sev_snp = - cfg.resolved_platform() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; - let mut numa_nodes_for_hugepages = if self.manifest.hugepages { - Some(hugepage_numa_nodes(gpus)?) - } else { - None - }; - let smp = effective_vcpu_count( - self.manifest.vcpu, - numa_nodes_for_hugepages - .as_ref() - .map(|nodes| nodes.len() as u32), - ); - let mut mem = self.manifest.memory; - let mut command = Command::new(qemu); + self.process_config(command) + } + + fn is_amd_sev_snp(&self) -> bool { + self.prepared.platform == TeePlatform::AmdSevSnp && !self.vm.manifest.no_tee + } + + fn base_command(&self) -> Command { + let workdir = &self.prepared.workdir; + let mut command = Command::new(&self.cfg.qemu_path); command.arg("-accel").arg("kvm"); - let cpu = if is_amd_sev_snp { "EPYC-v4" } else { "host" }; - command.arg("-cpu").arg(cpu); + command.arg("-cpu").arg(if self.is_amd_sev_snp() { + "EPYC-v4" + } else { + "host" + }); command.arg("-nographic"); command.arg("-nodefaults"); command.arg("-chardev").arg(format!( "pty,id=com0,path={},logfile={}", - serial_pty.display(), - serial_file.display() + workdir.serial_pty().display(), + workdir.serial_file().display() )); command.arg("-serial").arg("chardev:com0"); - if cfg.qmp_socket { + if self.cfg.qmp_socket { command.arg("-qmp").arg(format!( "unix:{},server,wait=off", workdir.qmp_socket().display() )); } - if let Some(bios) = self.image.firmware(is_amd_sev_snp) { + if let Some(bios) = self.vm.image.firmware(self.is_amd_sev_snp()) { command.arg("-bios").arg(bios); } - command.arg("-kernel").arg(&self.image.kernel); - command.arg("-initrd").arg(&self.image.initrd); - if cfg.qemu_hotplug_off { + command.arg("-kernel").arg(&self.vm.image.kernel); + command.arg("-initrd").arg(&self.vm.image.initrd); + if self.cfg.qemu_hotplug_off { command.args([ "-global", "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off", ]); } - if cfg.qemu_pci_hole64_size > 0 { + if self.cfg.qemu_pci_hole64_size > 0 { command.args([ "-global", &format!( "q35-pcihost.pci-hole64-size=0x{:x}", - cfg.qemu_pci_hole64_size + self.cfg.qemu_pci_hole64_size ), ]); } - if let Some(rootfs) = &self.image.rootfs { - let img_ver = self.image.info.version_tuple().unwrap_or_default(); - let ext = rootfs - .extension() - .unwrap_or_default() - .to_str() - .unwrap_or_default(); - match ext { - "iso" => { - if img_ver >= (0, 5, 0) { - bail!( - "Unsupported rootfs type: {ext}. Image versions >= 0.5.0 must use verity rootfs" - ); - } - command.arg("-cdrom").arg(rootfs); - } - "verity" => { - command.arg("-drive").arg(format!( - "file={},if=none,id=hd0,format=raw,readonly=on", - rootfs.display() - )); - command.arg("-device").arg(virtio_pci_device( - "virtio-blk-pci,drive=hd0", - is_amd_sev_snp, - )); - } - _ => { - bail!("Unsupported rootfs type: {ext}"); + command + } + + fn configure_rootfs(&self, command: &mut Command) -> Result<()> { + let Some(rootfs) = &self.vm.image.rootfs else { + return Ok(()); + }; + let image_version = self.vm.image.info.version_tuple().unwrap_or_default(); + let extension = rootfs + .extension() + .unwrap_or_default() + .to_str() + .unwrap_or_default(); + match extension { + "iso" => { + if image_version >= (0, 5, 0) { + bail!( + "Unsupported rootfs type: {extension}. Image versions >= 0.5.0 must use verity rootfs" + ); } + command.arg("-cdrom").arg(rootfs); } + "verity" => { + command.arg("-drive").arg(format!( + "file={},if=none,id=hd0,format=raw,readonly=on", + rootfs.display() + )); + command.arg("-device").arg(virtio_pci_device( + "virtio-blk-pci,drive=hd0", + self.is_amd_sev_snp(), + )); + } + _ => bail!("Unsupported rootfs type: {extension}"), } - let mut processes = vec![]; + Ok(()) + } + + fn configure_data_disk(&self, command: &mut Command) { command .arg("-drive") - .arg(format!("file={},if=none,id=hd1", hda_path.display())) + .arg(format!( + "file={},if=none,id=hd1", + self.prepared.workdir.hda_path().display() + )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=hd1", - is_amd_sev_snp, + self.is_amd_sev_snp(), )); - let networks = resolved_networks(&self.manifest, cfg); - validate_resolved_networks(&networks)?; - let hostfwd_index = networks + } + + fn configure_networking(&self, command: &mut Command) -> Result<()> { + let hostfwd_index = self + .prepared + .networks .iter() .position(|networking| networking.mode == NetworkingMode::User); - for (index, networking) in networks.iter().enumerate() { + for (index, networking) in self.prepared.networks.iter().enumerate() { let net_id = format!("net{index}"); - let prefix = networking.mac_prefix_bytes(); - let mac = mac_address_for_vm_index(&self.manifest.id, &prefix, index); + let mac = mac_address_for_vm_index( + &self.vm.manifest.id, + &networking.mac_prefix_bytes(), + index, + ); let net_device = virtio_pci_device( &format!("virtio-net-pci,netdev={net_id},mac={mac}"), - is_amd_sev_snp, + self.is_amd_sev_snp(), ); let netdev = match networking.mode { NetworkingMode::User => { @@ -848,13 +486,13 @@ impl VmConfig { if networking.restrict { "yes" } else { "no" } ); if hostfwd_index == Some(index) { - for pm in &self.manifest.port_map { + for mapping in &self.vm.manifest.port_map { netdev.push_str(&format!( ",hostfwd={}:{}:{}-:{}", - pm.protocol.as_str(), - pm.address, - pm.from, - pm.to + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to )); } } @@ -876,41 +514,35 @@ impl VmConfig { command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); } + Ok(()) + } - self.configure_smbios(&mut command, cfg); - - if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - let tpm_path = if Path::new("/dev/tpmrm0").exists() { - "/dev/tpmrm0" - } else if Path::new("/dev/tpm0").exists() { - "/dev/tpm0" - } else { - bail!("TPM key provider requested but no TPM device found on host"); - }; + fn configure_tpm_and_vsock(&self, command: &mut Command) { + if let Some(tpm_path) = self.prepared.tpm_path { command .arg("-tpmdev") .arg(format!("passthrough,id=tpm0,path={tpm_path}")) .arg("-device") .arg("tpm-tis,tpmdev=tpm0"); } - command.arg("-device").arg(virtio_pci_device( - &format!("vhost-vsock-pci,guest-cid={}", self.cid), - is_amd_sev_snp, + &format!("vhost-vsock-pci,guest-cid={}", self.vm.cid), + self.is_amd_sev_snp(), )); + } - // Configure shared files delivery: either via disk or 9p - match cfg.host_share_mode.as_str() { + fn configure_host_share(&self, command: &mut Command) -> Result<()> { + let workdir = &self.prepared.workdir; + match self.cfg.host_share_mode.as_str() { "9p" => { - // Use 9p virtfs (default) - let ro = if self.image.info.shared_ro { + let read_only = if self.vm.image.info.shared_ro { "on" } else { "off" }; command.arg("-virtfs").arg(format!( - "local,path={},mount_tag=host-shared,readonly={ro},security_model=mapped,id=virtfs0", - shared_dir.display(), + "local,path={},mount_tag=host-shared,readonly={read_only},security_model=mapped,id=virtfs0", + workdir.shared_dir().display(), )); } "vvfat" => { @@ -918,214 +550,152 @@ impl VmConfig { .arg("-blockdev") .arg(format!( "driver=vvfat,node-name=vvfat0,read-only=on,dir={},label={}", - shared_dir.display(), + workdir.shared_dir().display(), HOST_SHARED_DISK_LABEL )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=vvfat0", - is_amd_sev_snp, + self.is_amd_sev_snp(), )); } "vhd" => { - // Use a second virtual disk (hd2) to share files - let shared_disk_path = workdir.shared_disk_path(); - if shared_disk_path.exists() { - fs::remove_file(&shared_disk_path).context("Failed to remove shared disk")?; - } - create_shared_disk(&shared_disk_path, &shared_dir) - .context("Failed to create shared disk")?; command .arg("-drive") .arg(format!( "file={},if=none,id=hd2,format=raw,readonly=on", - shared_disk_path.display() + workdir.shared_disk_path().display() )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=hd2", - is_amd_sev_snp, + self.is_amd_sev_snp(), )); } - _ => { - bail!("Invalid host sharing mode: {}", cfg.host_share_mode); - } + _ => bail!("Invalid host sharing mode: {}", self.cfg.host_share_mode), } + Ok(()) + } - let hugepages = self.manifest.hugepages; - let pin_numa = self.manifest.pin_numa; - // Handle GPU configuration - let mut dev_num = 1; - let memory = self.manifest.memory; - - // Handle hugepages configuration - if hugepages { - let numa_nodes = numa_nodes_for_hugepages - .take() - .context("hugepage NUMA nodes should be computed above")?; - let n_numa = numa_nodes.len() as u32; - - // Round up CPU cores and memory to multiple times of NUMA nodes. - // `smp` is already the shared effective vCPU count used by vm_config. - let vcpu_count = smp; - let mem_gb = round_up(memory / 1024, n_numa); - let vcpu_per_node = vcpu_count / n_numa; - let mem_per_node = mem_gb / n_numa; - - mem = mem_gb * 1024; - - let mut bus_nr = 5_u32; - - // Configure NUMA nodes - for (ind, (node, count)) in numa_nodes.into_iter().enumerate() { - let ind = ind as u32; - let cpu_start = ind * vcpu_per_node; - let cpu_end = (ind + 1) * vcpu_per_node - 1; - command.arg("-numa").arg(format!( - "node,nodeid={ind},cpus={cpu_start}-{cpu_end},memdev=mem{ind}", - )); - - command.arg("-object").arg(format!( - "memory-backend-file,id=mem{ind},size={mem_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind", - )); - - let addr = 0xa + ind; - command.arg("-device").arg(format!( - "pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={addr},numa_node={ind},bus_nr={bus_nr}", - )); - bus_nr += count + 1; - } + fn configure_hugepage_memory(&self, command: &mut Command) -> Result<(u32, u32)> { + let numa_nodes = self.prepared.hugepage_numa_nodes.as_ref(); + let smp = effective_vcpu_count( + self.vm.manifest.vcpu, + numa_nodes.map(|nodes| nodes.len() as u32), + ); + if !self.vm.manifest.hugepages { + return Ok((smp, self.vm.manifest.memory)); } - self.configure_machine(&mut command, &workdir, cfg, &app_compose, mem)?; - - // Configure GPU devices - if !gpus.gpus.is_empty() { - // Add iommufd object - command.arg("-object").arg("iommufd,id=iommufd0"); - - if !hugepages { - // Add each GPU - for device in &gpus.gpus { - let slot = &device.slot; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", - )); - - dev_num += 1; - } - } else { - // Add each GPU with NUMA node awareness for hugepages configuration - for device in &gpus.gpus { - let slot = &device.slot; - let node = pci_numa_node(slot)?; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{dev_num},bus=pcie.node{node},chassis={dev_num}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", - )); - dev_num += 1; - } - } - - // Add bridges (NVSwitches) if any - if !gpus.bridges.is_empty() { - for bridge in &gpus.bridges { - let slot = &bridge.slot; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", - )); - dev_num += 1; - } - } + let numa_nodes = numa_nodes + .context("hugepage NUMA nodes should be computed during launch preparation")?; + let numa_count = numa_nodes.len() as u32; + let memory_gib = round_up(self.vm.manifest.memory / 1024, numa_count); + let vcpus_per_node = smp / numa_count; + let memory_per_node = memory_gib / numa_count; + let mut bus_number = 5_u32; + for (index, (node, device_count)) in numa_nodes.iter().enumerate() { + let index = index as u32; + let cpu_start = index * vcpus_per_node; + let cpu_end = (index + 1) * vcpus_per_node - 1; + command.arg("-numa").arg(format!( + "node,nodeid={index},cpus={cpu_start}-{cpu_end},memdev=mem{index}", + )); + command.arg("-object").arg(format!( + "memory-backend-file,id=mem{index},size={memory_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind", + )); + let address = 0xa + index; + command.arg("-device").arg(format!( + "pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={address},numa_node={index},bus_nr={bus_number}", + )); + bus_number += device_count + 1; } - command.arg("-smp").arg(smp.to_string()); - command.arg("-m").arg(format!("{}M", mem)); + Ok((smp, memory_gib * 1024)) + } - // NUMA pinning if requested - let mut numa_cpus = None; - if pin_numa { - if !gpus.gpus.is_empty() { - let (_, cpus) = find_numa(Some(gpus.gpus[0].slot.clone()))?; - numa_cpus = Some(cpus); + fn configure_gpus(&self, command: &mut Command) -> Result<()> { + if self.gpus.gpus.is_empty() { + return Ok(()); + } + command.arg("-object").arg("iommufd,id=iommufd0"); + let mut device_number = 1; + for device in &self.gpus.gpus { + let slot = &device.slot; + let bus = if self.vm.manifest.hugepages { + let node = self + .prepared + .gpu_numa_nodes + .get(slot) + .context("gpu NUMA node should be computed during launch preparation")?; + format!("pcie.node{node}") } else { - // Default to NUMA node 0 if no GPUs - let (_, cpus) = find_numa(None)?; - numa_cpus = Some(cpus); - } + "pcie.0".into() + }; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{device_number},bus={bus},chassis={device_number}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{device_number},iommufd=iommufd0", + )); + device_number += 1; } - - // SNP app identity is bound through HOST_DATA, so the measured cmdline - // remains the image-provided cmdline. - let cmdline = self.image.info.cmdline.clone(); - if let Some(cmdline) = cmdline { - command.arg("-append").arg(cmdline); + for bridge in &self.gpus.bridges { + let slot = &bridge.slot; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{device_number},bus=pcie.0,chassis={device_number}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{device_number},iommufd=iommufd0", + )); + device_number += 1; } + Ok(()) + } - let args = command - .get_args() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - let pidfile_path = workdir.pid_file(); - let stdout_path = workdir.stdout_file(); - let stderr_path = workdir.stderr_file(); - - let workdir = workdir.path(); - - let mut cmd_args = vec![]; - cmd_args.push(qemu.to_string_lossy().to_string()); - cmd_args.extend(args); - - // If we have NUMA pinning, we'll need to wrap the command with taskset - if let Some(cpus) = numa_cpus { - cmd_args.splice(0..0, ["taskset", "-c", &cpus].into_iter().map(|s| s.into())); + fn process_config(&self, command: Command) -> Result { + let workdir = &self.prepared.workdir; + let mut arguments = vec![self.cfg.qemu_path.to_string_lossy().to_string()]; + arguments.extend( + command + .get_args() + .map(|argument| argument.to_string_lossy().to_string()), + ); + if let Some(cpus) = &self.prepared.numa_cpus { + arguments.splice(0..0, ["taskset", "-c", cpus].into_iter().map(String::from)); } - - if !cfg.user.is_empty() { - cmd_args.splice( + if !self.cfg.user.is_empty() { + arguments.splice( 0..0, - ["sudo", "-u", &cfg.user].into_iter().map(|s| s.into()), + ["sudo", "-u", &self.cfg.user].into_iter().map(String::from), ); } - let command = cmd_args.remove(0); - let note = ProcessAnnotation { + let command = arguments.remove(0); + let note = serde_json::to_string(&ProcessAnnotation { kind: "cvm".to_string(), live_for: None, - }; - let note = serde_json::to_string(¬e)?; - let process_config = ProcessConfig { - id: self.manifest.id.clone(), - args: cmd_args, - name: self.manifest.name.clone(), + })?; + Ok(ProcessConfig { + id: self.vm.manifest.id.clone(), + args: arguments, + name: self.vm.manifest.name.clone(), command, env: Default::default(), - cwd: workdir.to_string_lossy().to_string(), - stdout: stdout_path.to_string_lossy().to_string(), - stderr: stderr_path.to_string_lossy().to_string(), - pidfile: pidfile_path.to_string_lossy().to_string(), - cid: Some(self.cid), + cwd: workdir.path().to_string_lossy().to_string(), + stdout: workdir.stdout_file().to_string_lossy().to_string(), + stderr: workdir.stderr_file().to_string_lossy().to_string(), + pidfile: workdir.pid_file().to_string_lossy().to_string(), + cid: Some(self.vm.cid), note, - }; - processes.push(process_config); - - Ok(processes) + }) } - +} +impl VmConfig { fn configure_machine( &self, command: &mut Command, - workdir: &VmWorkDir, cfg: &CvmConfig, - app_compose: &AppCompose, + prepared: &PreparedQemuLaunch, mem: u32, ) -> Result<()> { if self.manifest.no_tee { @@ -1135,15 +705,22 @@ impl VmConfig { return Ok(()); } - match cfg.resolved_platform() { + match prepared.platform { TeePlatform::Tdx => { command .arg("-machine") .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); - self.configure_tdx_guest(command, workdir, cfg, app_compose)?; + self.configure_tdx_guest(command, cfg, prepared.tdx_mr_config_id.as_deref())?; } TeePlatform::AmdSevSnp => { - self.configure_amd_sev_snp_guest(command, workdir, cfg, mem)?; + let host_data = prepared + .snp_host_data + .as_deref() + .context("snp host data should be computed during launch preparation")?; + let launch_params = prepared.snp_launch_params.context( + "snp launch parameters should be detected during launch preparation", + )?; + self.configure_amd_sev_snp_guest(command, cfg, mem, host_data, launch_params); } } Ok(()) @@ -1152,60 +729,9 @@ impl VmConfig { fn configure_tdx_guest( &self, command: &mut Command, - workdir: &VmWorkDir, cfg: &CvmConfig, - app_compose: &AppCompose, + mrconfigid: Option<&str>, ) -> Result<()> { - let img_ver = self.image.info.version_tuple().unwrap_or_default(); - let support_mr_config_id = img_ver >= (0, 5, 2); - - // Compute mrconfigid if needed - let mrconfigid = if cfg.use_mrconfigid && support_mr_config_id { - if let Some(mr_config_document) = workdir - .sys_config() - .context("Failed to read sys config for tdx mrconfigid")? - .mr_config - { - MrConfigV3::from_document(&mr_config_document) - .context("Invalid mr_config document")?; - Some( - BASE64_STANDARD.encode(MrConfigV3::tdx_mr_config_id_from_document( - &mr_config_document, - )), - ) - } else { - let compose_hash = workdir - .app_compose_hash() - .context("Failed to get compose hash")?; - let mr_config = if app_compose.key_provider_id.is_empty() { - MrConfig::V1 { - compose_hash: &compose_hash, - } - } else { - let instance_info = workdir - .instance_info() - .context("Failed to get instance info")?; - let app_id = if instance_info.app_id.is_empty() { - &compose_hash[..20] - } else { - &instance_info.app_id - }; - - let key_provider = app_compose.key_provider(); - let key_provider_id = &app_compose.key_provider_id; - MrConfig::V2 { - compose_hash: &compose_hash, - app_id: &app_id.try_into().context("Invalid app ID")?, - key_provider, - key_provider_id, - } - }; - Some(BASE64_STANDARD.encode(mr_config.to_mr_config_id())) - } - } else { - None - }; - // Build tdx-guest object with optional quote-generation-socket for kernel-level TSM support #[derive(Serialize)] struct QgsSocket { @@ -1231,7 +757,7 @@ impl VmConfig { let tdx_object = TdxGuestObject { qom_type: "tdx-guest", id: "tdx", - mrconfigid: mrconfigid.clone(), + mrconfigid: mrconfigid.map(str::to_string), quote_generation_socket: cfg.qgs_port.map(|port| QgsSocket { r#type: "vsock", cid: "2", @@ -1249,24 +775,14 @@ impl VmConfig { fn configure_amd_sev_snp_guest( &self, command: &mut Command, - workdir: &VmWorkDir, cfg: &CvmConfig, mem: u32, - ) -> Result<()> { - let mr_config_document = workdir - .sys_config() - .context("Failed to read sys config for amd sev-snp host-data")? - .mr_config - .context("mr_config is required for amd sev-snp host-data")?; - MrConfigV3::from_document(&mr_config_document).context("Invalid mr_config document")?; - let host_data = - BASE64_STANDARD.encode(MrConfigV3::snp_host_data_from_document(&mr_config_document)); - + host_data: &str, + snp_params: AmdSevSnpLaunchParams, + ) { command .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); - let snp_params = detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path) - .context("failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU")?; command.arg("-object").arg(format!( "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos={},reduced-phys-bits={}", snp_params.cbitpos, snp_params.reduced_phys_bits @@ -1277,7 +793,6 @@ impl VmConfig { if cfg.qgs_port.is_some() { tracing::warn!("qgs_port is ignored for amd sev-snp guests"); } - Ok(()) } fn configure_smbios(&self, command: &mut Command, cfg: &CvmConfig) { @@ -1343,274 +858,171 @@ fn find_numa(device: Option) -> Result<(String, String)> { Ok((numa_node, cpus)) } -pub struct VmWorkDir { - workdir: PathBuf, -} - -impl Deref for VmWorkDir { - type Target = PathBuf; - fn deref(&self) -> &Self::Target { - &self.workdir - } -} - -impl AsRef for &VmWorkDir { - fn as_ref(&self) -> &Path { - self.workdir.as_ref() - } -} - -impl VmWorkDir { - pub fn new(workdir: impl AsRef) -> Self { - Self { - workdir: workdir.as_ref().to_path_buf(), - } - } - - pub fn manifest_path(&self) -> PathBuf { - self.workdir.join("vm-manifest.json") - } - - pub fn state_path(&self) -> PathBuf { - self.workdir.join("vm-state.json") - } - - pub fn manifest(&self) -> Result { - let manifest_path = self.manifest_path(); - let raw = fs::read_to_string(manifest_path).context("failed to read manifest")?; - let value: serde_json::Value = - serde_json::from_str(&raw).context("failed to parse manifest json")?; - Manifest::from_json(value).context("failed to deserialize manifest") - } - - pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { - fs::create_dir_all(&self.workdir).context("Failed to create workdir")?; - let manifest_path = self.manifest_path(); - let mut value = serde_json::to_value(manifest)?; - if let Some(networking) = manifest.networks.first() { - value["networking"] = serde_json::to_value(networking)?; - } - fs::write(manifest_path, serde_json::to_string(&value)?).context("failed to write manifest") - } - - pub fn started(&self) -> Result { - let state_path = self.state_path(); - if !state_path.exists() { - return Ok(false); - } - let state: State = - serde_json::from_str(&fs::read_to_string(state_path).context("Failed to read state")?) - .context("Failed to parse state")?; - Ok(state.started) - } - - pub fn set_started(&self, started: bool) -> Result<()> { - let state_path = self.state_path(); - fs::write(state_path, serde_json::to_string(&State { started })?) - .context("Failed to write state") - } - - pub fn shared_dir(&self) -> PathBuf { - self.workdir.join("shared") - } - - pub fn app_compose_path(&self) -> PathBuf { - self.shared_dir().join(APP_COMPOSE) - } - - pub fn app_compose_hash(&self) -> Result<[u8; 32]> { - use sha2::Digest; - let compose_path = self.app_compose_path(); - let compose = fs::read(compose_path).context("Failed to read compose")?; - Ok(sha2::Sha256::new_with_prefix(&compose).finalize().into()) - } - - pub fn user_config_path(&self) -> PathBuf { - self.shared_dir().join(USER_CONFIG) - } - - pub fn encrypted_env_path(&self) -> PathBuf { - self.shared_dir().join(ENCRYPTED_ENV) - } - - pub fn instance_info_path(&self) -> PathBuf { - self.shared_dir().join(INSTANCE_INFO) - } - - pub fn guest_ip_path(&self) -> PathBuf { - self.workdir.join("guest-ip") - } - - pub fn runtime_networks_path(&self) -> PathBuf { - self.workdir.join("runtime-networks.json") - } - - pub fn guest_ip(&self) -> Option { - fs::read_to_string(self.guest_ip_path()) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - } - - pub fn set_guest_ip(&self, ip: &str) -> Result<()> { - fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") - } - - pub fn runtime_networks(&self) -> Vec { - fs::read_to_string(self.runtime_networks_path()) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default() - } - - pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { - let serialized = serde_json::to_vec(networks)?; - safe_write::safe_write(self.runtime_networks_path(), serialized) - .context("failed to write runtime networks") - } - - pub fn clear_runtime_networks(&self) -> Result<()> { - let path = self.runtime_networks_path(); - if path.exists() { - fs::remove_file(path).context("failed to clear runtime networks")?; - } - Ok(()) - } - - pub fn serial_file(&self) -> PathBuf { - self.workdir.join("serial.log") - } - - pub fn serial_history_file(&self) -> PathBuf { - self.workdir.join("serial.history.log") - } - - pub fn serial_pty(&self) -> PathBuf { - self.workdir.join("serial.pty") - } - - pub fn stdout_file(&self) -> PathBuf { - self.workdir.join("stdout.log") - } - - pub fn stderr_file(&self) -> PathBuf { - self.workdir.join("stderr.log") - } - - pub fn pid_file(&self) -> PathBuf { - self.workdir.join("qemu.pid") - } - - pub fn hda_path(&self) -> PathBuf { - self.workdir.join("hda.img") - } - - pub fn shared_disk_path(&self) -> PathBuf { - self.workdir.join("shared.img") - } - - pub fn qmp_socket(&self) -> PathBuf { - self.workdir.join("qmp.sock") - } +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::path::PathBuf; - pub fn removing_marker(&self) -> PathBuf { - self.workdir.join(".removing") - } + use rocket::figment::{ + providers::{Format, Toml}, + Figment, + }; - pub fn is_removing(&self) -> bool { - self.removing_marker().exists() - } + use super::{ + amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, + PreparedQemuLaunch, QemuCommandBuilder, VmConfig, + }; + use crate::app::image::{Image, ImageInfo}; + use crate::app::{GpuConfig, Manifest, PortMapping, VmWorkDir}; + use crate::config::{Config, Protocol, TeePlatform, DEFAULT_CONFIG}; - pub fn set_removing(&self) -> Result<()> { - fs::write(self.removing_marker(), "").context("Failed to write .removing marker") + #[test] + fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { + assert_eq!( + amd_sev_snp_memory_backend_arg(4096), + "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" + ); } - pub fn path(&self) -> &Path { - &self.workdir + #[test] + fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { + let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} +{"return":{}} +{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} +{"return":{}} +"#; + let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); + assert_eq!(params.cbitpos, 51); + assert_eq!(params.reduced_phys_bits, 1); } -} -impl VmWorkDir { - pub fn instance_info(&self) -> Result { - let info_file = self.instance_info_path(); - let info: InstanceInfo = serde_json::from_slice(&fs::read(&info_file)?)?; - Ok(info) + #[test] + fn amd_sev_snp_uses_confidential_virtio_pci_options() { + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", true), + "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" + ); + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", false), + "virtio-blk-pci,drive=hd0" + ); } - pub fn instance_info_or_default(&self) -> Result { - match self.instance_info() { - Ok(info) => Ok(info), - Err(err) => match err.downcast_ref::() { - Some(io_err) if io_err.kind() == std::io::ErrorKind::NotFound => { - Ok(InstanceInfo::default()) - } - _ => Err(err), + #[test] + fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) + .extract() + .unwrap(); + config.cvm.platform = Some(TeePlatform::Tdx); + config.cvm.qemu_path = PathBuf::from("/not-installed/qemu-system-x86_64"); + config.cvm.qgs_port = None; + + let vm = VmConfig { + manifest: Manifest { + id: "vm-1".into(), + name: "test-vm".into(), + app_id: "app-1".into(), + vcpu: 2, + memory: 2048, + disk_size: 10, + image: "test-image".into(), + port_map: vec![PortMapping { + address: "127.0.0.1".parse().unwrap(), + protocol: Protocol::Tcp, + from: 18080, + to: 8080, + }], + created_at_ms: 0, + hugepages: false, + pin_numa: false, + gpus: None, + kms_urls: vec![], + gateway_urls: vec![], + no_tee: true, + networks: vec![], }, - } - } - - pub fn sys_config(&self) -> Result { - let sys_config_file = self.shared_dir().join(SYS_CONFIG); - let sys_config: SysConfig = serde_json::from_slice(&fs::read(sys_config_file)?)?; - Ok(sys_config) - } - - pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose) -> Result { - let compose_hash = self - .app_compose_hash() - .context("Failed to get compose hash")?; - let mut instance_info = self - .instance_info_or_default() - .context("Failed to get instance info")?; - let app_id = if instance_info.app_id.is_empty() { - compose_hash[..20].to_vec() - } else { - instance_info.app_id.clone() + image: Image { + info: ImageInfo { + cmdline: Some("console=hvc0".into()), + kernel: "kernel".into(), + initrd: "initrd".into(), + hda: None, + rootfs: None, + bios: None, + bios_sev: None, + rootfs_hash: None, + shared_ro: false, + version: "0.5.4".into(), + is_dev: false, + ovmf_variant: None, + }, + initrd: PathBuf::from("/does-not-exist/initrd"), + kernel: PathBuf::from("/does-not-exist/kernel"), + hda: None, + rootfs: None, + bios: None, + bios_sev: None, + digest: None, + tdx_measurement: None, + sev_measurement: None, + }, + cid: 100, + workdir: PathBuf::from("/does-not-exist/vm-1"), + gateway_enabled: false, }; - if app_id.len() != 20 { - bail!( - "Invalid app ID length: expected 20 bytes, got {}", - app_id.len() - ); - } - - let disk_reusable = !app_compose.key_provider().is_none(); - if !disk_reusable || instance_info.instance_id_seed.is_empty() { - instance_info.instance_id_seed = { - let mut seed = vec![0u8; 20]; - getrandom::fill(&mut seed).context("Failed to generate instance id seed")?; - seed - }; - } - - let instance_id = if app_compose.no_instance_id { - Vec::new() - } else { - let mut id_path = instance_info.instance_id_seed.clone(); - id_path.extend_from_slice(&app_id); - Sha256::digest(id_path)[..20].to_vec() + let prepared = PreparedQemuLaunch { + workdir: VmWorkDir::new("/does-not-exist/vm-1"), + platform: TeePlatform::Tdx, + networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], + hugepage_numa_nodes: None, + gpu_numa_nodes: HashMap::new(), + numa_cpus: None, + tpm_path: None, + tdx_mr_config_id: None, + snp_host_data: None, + snp_launch_params: None, }; - instance_info.app_id = app_id.clone(); - instance_info.instance_id = instance_id.clone(); - fs::write( - self.instance_info_path(), - serde_json::to_string(&instance_info).context("Failed to serialize instance info")?, - ) - .context("Failed to write instance info")?; - Ok(MrConfigV3::new( - app_id, - compose_hash.to_vec(), - app_compose.key_provider(), - app_compose.key_provider_id.clone(), - instance_id, - ) - .to_canonical_json()) - } - - pub fn app_compose(&self) -> Result { - let compose_file = self.app_compose_path(); - let compose: AppCompose = serde_json::from_str(&fs::read_to_string(compose_file)?)?; - Ok(compose) + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + + assert_eq!(process.command, "/not-installed/qemu-system-x86_64"); + assert!(process + .args + .windows(2) + .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"])); + assert!(process + .args + .windows(2) + .any(|args| args == ["-kernel", "/does-not-exist/kernel"])); + assert!(process + .args + .windows(2) + .any(|args| args == ["-append", "console=hvc0"])); + let netdevs = process + .args + .windows(2) + .filter(|args| args[0] == "-netdev") + .map(|args| args[1].as_str()) + .collect::>(); + assert_eq!(netdevs.len(), 2); + assert!(netdevs[0].contains("user,id=net0")); + assert!(netdevs[0].contains("hostfwd=tcp:127.0.0.1:18080-:8080")); + assert!(netdevs[1].contains("user,id=net1")); + assert!(!netdevs[1].contains("hostfwd=")); + assert!(process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net0"))); + assert!(process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); } } diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 801c40cd5..d039418c8 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -11,8 +11,11 @@ use dstack_vmm_rpc as pb; use fs_err as fs; use supervisor_client::supervisor::ProcessInfo; -use super::{Manifest, VmState, VmWorkDir}; -use crate::config::{GatewayConfig, Networking, NetworkingMode}; +use super::{ + network::{mac_address_for_vm_index, resolved_networks}, + Manifest, VmState, VmWorkDir, +}; +use crate::config::{CvmConfig, GatewayConfig, Networking, NetworkingMode}; pub(crate) struct VmInfo { pub manifest: Manifest, @@ -27,15 +30,34 @@ pub(crate) struct VmInfo { pub image_version: String, pub gateway_enabled: bool, pub events: Vec, + pub runtime_networks: Vec, } -fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { - let mode = match networking.mode { +fn networking_mode_name(mode: NetworkingMode) -> &'static str { + match mode { NetworkingMode::Bridge => "bridge", NetworkingMode::User => "user", NetworkingMode::Custom => "custom", - }; - pb::NetworkingConfig { mode: mode.into() } + } +} + +fn networking_backend_name(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "tap_bridge", + NetworkingMode::User => "slirp", + NetworkingMode::Custom => "custom", + } +} + +fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { + pb::NetworkingConfig { + mode: networking_mode_name(networking.mode).into(), + bridge_name: if networking.mode == NetworkingMode::Bridge { + networking.bridge.clone() + } else { + String::new() + }, + } } fn sanitize_optional>(value: Option) -> Option { @@ -43,13 +65,48 @@ fn sanitize_optional>(value: Option) -> Option { } impl VmInfo { - pub fn to_pb(&self, gateway: &GatewayConfig, brief: bool) -> pb::VmInfo { + pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { + if self.runtime_networks.is_empty() { + resolved_networks(&self.manifest, cvm) + } else { + self.runtime_networks.clone() + } + } + + pub fn to_pb(&self, gateway: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); let custom_gateway_urls = vm_config .as_ref() .map(|config| config.gateway_urls.clone()) .unwrap_or_default(); + let configured_networks = self + .manifest + .networks + .iter() + .map(networking_to_proto) + .collect::>(); + let configured_networking = configured_networks.first().cloned(); + let interfaces = self + .effective_networks(cvm) + .iter() + .enumerate() + .map(|(index, networking)| { + let mac = mac_address_for_vm_index( + &self.manifest.id, + &networking.mac_prefix_bytes(), + index, + ); + pb::NetworkInterfaceStatus { + mode: networking_mode_name(networking.mode).into(), + backend: networking_backend_name(networking.mode).into(), + mac, + bridge_name: (networking.mode == NetworkingMode::Bridge) + .then(|| networking.bridge.clone()), + netdev_id: Some(format!("net{index}")), + } + }) + .collect(); pb::VmInfo { id: self.manifest.id.clone(), name: self.manifest.name.clone(), @@ -110,7 +167,8 @@ impl VmInfo { gateway_urls: custom_gateway_urls.clone(), stopped, no_tee, - networking: self.manifest.networking.as_ref().map(networking_to_proto), + networking: configured_networking, + networks: configured_networks, }) }, app_url: self @@ -123,6 +181,7 @@ impl VmInfo { instance_id: sanitize_optional(self.instance_id.clone()), exited_at: self.exited_at.clone(), events: self.events.clone(), + interfaces, } } } @@ -202,6 +261,7 @@ impl VmState { image_version: self.config.image.info.version.clone(), gateway_enabled: self.config.gateway_enabled, events: self.state.events.clone().into(), + runtime_networks: self.state.runtime_networks.clone(), } } } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index e7a28ee7b..6984cd4b5 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -16,7 +16,7 @@ use fs_err as fs; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; -use crate::app::Manifest; +use crate::{app::Manifest, config::Networking}; #[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct InstanceInfo { @@ -67,17 +67,20 @@ impl VmWorkDir { pub fn manifest(&self) -> Result { let manifest_path = self.manifest_path(); - let manifest = fs::read_to_string(manifest_path).context("failed to read manifest")?; - let manifest: Manifest = - serde_json::from_str(&manifest).context("failed to parse manifest")?; - Ok(manifest) + let raw = fs::read_to_string(manifest_path).context("failed to read manifest")?; + let value: serde_json::Value = + serde_json::from_str(&raw).context("failed to parse manifest json")?; + Manifest::from_json(value).context("failed to deserialize manifest") } pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { fs::create_dir_all(&self.workdir).context("failed to create workdir")?; let manifest_path = self.manifest_path(); - fs::write(manifest_path, serde_json::to_string(manifest)?) - .context("failed to write manifest") + let mut value = serde_json::to_value(manifest)?; + if let Some(networking) = manifest.networks.first() { + value["networking"] = serde_json::to_value(networking)?; + } + fs::write(manifest_path, serde_json::to_string(&value)?).context("failed to write manifest") } pub fn started(&self) -> Result { @@ -128,6 +131,10 @@ impl VmWorkDir { self.workdir.join("guest-ip") } + pub fn runtime_networks_path(&self) -> PathBuf { + self.workdir.join("runtime-networks.json") + } + pub fn guest_ip(&self) -> Option { fs::read_to_string(self.guest_ip_path()) .ok() @@ -139,6 +146,27 @@ impl VmWorkDir { fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") } + pub fn runtime_networks(&self) -> Vec { + fs::read_to_string(self.runtime_networks_path()) + .ok() + .and_then(|contents| serde_json::from_str(&contents).ok()) + .unwrap_or_default() + } + + pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { + let serialized = serde_json::to_vec(networks)?; + safe_write::safe_write(self.runtime_networks_path(), serialized) + .context("failed to write runtime networks") + } + + pub fn clear_runtime_networks(&self) -> Result<()> { + let path = self.runtime_networks_path(); + if path.exists() { + fs::remove_file(path).context("failed to clear runtime networks")?; + } + Ok(()) + } + pub fn serial_file(&self) -> PathBuf { self.workdir.join("serial.log") } @@ -221,3 +249,37 @@ impl VmWorkDir { Ok(compose) } } + +#[cfg(test)] +mod tests { + use std::{ + os::unix::fs::symlink, + time::{SystemTime, UNIX_EPOCH}, + }; + + use anyhow::Result; + use fs_err as fs; + + use super::VmWorkDir; + + #[test] + fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-runtime-networks-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + fs::create_dir_all(&temp)?; + let external = temp.join("external.json"); + fs::write(&external, "sentinel")?; + let workdir = VmWorkDir::new(temp.join("vm")); + fs::create_dir_all(workdir.path())?; + symlink(&external, workdir.runtime_networks_path())?; + + workdir.set_runtime_networks(&[])?; + + assert_eq!(fs::read_to_string(&external)?, "sentinel"); + assert_eq!(fs::read_to_string(workdir.runtime_networks_path())?, "[]"); + fs::remove_dir_all(temp)?; + Ok(()) + } +} diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 7290565d3..932c4c17d 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -828,6 +828,16 @@ impl VmmRpc for RpcHandler { } } +impl RpcCall for RpcHandler { + type PrpcService = VmmServer; + + fn construct(context: CallContext<'_, App>) -> Result { + Ok(RpcHandler { + app: context.state.clone(), + }) + } +} + #[cfg(test)] mod tests { use super::*; @@ -943,13 +953,3 @@ mod tests { .contains("built-in port forwarding supports only one bridge")); } } - -impl RpcCall for RpcHandler { - type PrpcService = VmmServer; - - fn construct(context: CallContext<'_, App>) -> Result { - Ok(RpcHandler { - app: context.state.clone(), - }) - } -}