diff --git a/dstack/dstack-mr/cli/src/main.rs b/dstack/dstack-mr/cli/src/main.rs index fb78808e2..146112b3d 100644 --- a/dstack/dstack-mr/cli/src/main.rs +++ b/dstack/dstack-mr/cli/src/main.rs @@ -66,6 +66,10 @@ struct MachineConfig { #[arg(long, default_value = "0")] num_nvswitches: u32, + /// Number of virtio-net NICs + #[arg(long, default_value = "1")] + num_nics: u32, + /// Disable hotplug #[arg(long, default_value = "false")] hotplug_off: Bool, @@ -133,6 +137,7 @@ fn main() -> Result<()> { .hugepages(config.hugepages) .num_gpus(config.num_gpus) .num_nvswitches(config.num_nvswitches) + .num_nics(config.num_nics) .hotplug_off(config.hotplug_off) .root_verity(config.root_verity) .maybe_qemu_version(config.qemu_version.clone()) diff --git a/dstack/dstack-mr/src/acpi.rs b/dstack/dstack-mr/src/acpi.rs index a39759e83..db8f52845 100644 --- a/dstack/dstack-mr/src/acpi.rs +++ b/dstack/dstack-mr/src/acpi.rs @@ -55,10 +55,18 @@ impl Machine<'_> { &format!("file={dummy_disk},if=none,id=hd1,format=raw,readonly=on"), "-device", "virtio-blk-pci,drive=hd1", - "-netdev", - "user,id=net0", - "-device", - "virtio-net-pci,netdev=net0", + ]); + + // One virtio-net-pci per NIC. Emitted at the same position and, for the + // default single-NIC case, with the exact same args as the previous + // hardcoded layout so RTMR0 stays byte-for-byte unchanged. + for i in 0..self.num_nics { + cmd.arg("-netdev").arg(format!("user,id=net{i}")); + cmd.arg("-device") + .arg(format!("virtio-net-pci,netdev=net{i}")); + } + + cmd.args([ "-object", "tdx-guest,id=tdx", "-device", diff --git a/dstack/dstack-mr/src/machine.rs b/dstack/dstack-mr/src/machine.rs index 823470d23..59d061084 100644 --- a/dstack/dstack-mr/src/machine.rs +++ b/dstack/dstack-mr/src/machine.rs @@ -28,6 +28,11 @@ pub struct Machine<'a> { pub hugepages: bool, pub num_gpus: u32, pub num_nvswitches: u32, + /// Number of virtio-net NICs. Each NIC contributes a PCI device to the + /// ACPI/DSDT layout measured into RTMR0. Defaults to 1 so callers that + /// predate this field keep the historical single-NIC layout. + #[builder(default = 1)] + pub num_nics: u32, pub hotplug_off: bool, pub root_verity: bool, #[builder(default)] diff --git a/dstack/dstack-mr/src/tdx.rs b/dstack/dstack-mr/src/tdx.rs index 087556567..abd6c7f99 100644 --- a/dstack/dstack-mr/src/tdx.rs +++ b/dstack/dstack-mr/src/tdx.rs @@ -82,6 +82,7 @@ fn select_mrtd(measurement: &TdxOsImageMeasurement, vm_config: &VmConfig) -> Res }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(measurement.tdvf.ovmf_variant) @@ -472,6 +473,7 @@ pub fn tdx_measurements_for_image_dir_without_rtmr0( }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) @@ -567,6 +569,7 @@ pub fn tdx_measurements_for_image_dir_with_acpi_hashes( }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 31ec16ba9..b871be957 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -643,6 +643,14 @@ impl SysConfig { } } +fn default_num_nics() -> u32 { + 1 +} + +fn is_default_num_nics(n: &u32) -> bool { + *n == default_num_nics() +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VmConfig { #[serde(with = "hex_bytes", default)] @@ -666,6 +674,16 @@ pub struct VmConfig { pub num_gpus: u32, #[serde(default)] pub num_nvswitches: u32, + /// Number of virtio-net NICs attached to the guest. Each NIC adds a PCI + /// device to the ACPI/DSDT layout and therefore changes RTMR0, so it must + /// be measured. Defaults to 1 and is omitted from the serialized form when + /// equal to 1, keeping configs (and their cache keys / hashes) produced + /// before this field existed byte-for-byte stable. + #[serde( + default = "default_num_nics", + skip_serializing_if = "is_default_num_nics" + )] + pub num_nics: u32, #[serde(default)] pub hotplug_off: bool, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1529,3 +1547,44 @@ impl Platform { } } } + +#[cfg(test)] +mod vm_config_num_nics_tests { + use super::VmConfig; + + fn legacy_json() -> serde_json::Value { + serde_json::json!({ + "cpu_count": 4, + "memory_size": 4294967296u64, + "num_gpus": 0, + "num_nvswitches": 0, + }) + } + + #[test] + fn legacy_config_without_num_nics_defaults_to_one() { + let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + assert_eq!(cfg.num_nics, 1); + } + + #[test] + fn single_nic_is_omitted_to_keep_cache_key_stable() { + // A config with the default single NIC must serialize identically to a + // legacy config, so the verifier's measurement cache key (a hash of the + // serialized VmConfig) is unchanged for existing deployments. + let cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + let serialized = serde_json::to_value(&cfg).unwrap(); + assert!( + serialized.get("num_nics").is_none(), + "num_nics must be omitted when equal to 1, got {serialized}" + ); + } + + #[test] + fn multi_nic_is_serialized() { + let mut cfg: VmConfig = serde_json::from_value(legacy_json()).unwrap(); + cfg.num_nics = 2; + let serialized = serde_json::to_value(&cfg).unwrap(); + assert_eq!(serialized.get("num_nics").and_then(|v| v.as_u64()), Some(2)); + } +} diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 9faf03459..ad3ece1d0 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -312,6 +312,7 @@ impl CvmVerifier { }) .hugepages(vm_config.hugepages) .num_gpus(vm_config.num_gpus) + .num_nics(vm_config.num_nics) .num_nvswitches(vm_config.num_nvswitches) .host_share_mode(vm_config.host_share_mode.clone()) .ovmf_variant(ovmf_variant) diff --git a/dstack/vmm/rpc/proto/vmm_rpc.proto b/dstack/vmm/rpc/proto/vmm_rpc.proto index 8d79fea66..3a4e53e0d 100644 --- a/dstack/vmm/rpc/proto/vmm_rpc.proto +++ b/dstack/vmm/rpc/proto/vmm_rpc.proto @@ -37,6 +37,21 @@ message VmInfo { string image_version = 13; // Events repeated GuestEvent events = 14; + // Effective network interfaces resolved against node config. + repeated NetworkInterfaceStatus interfaces = 15; +} + +// Runtime status of a resolved VM network interface. +message NetworkInterfaceStatus { + // Product-facing mode: "user", "bridge", or "custom". + string mode = 1; + // QEMU/network backend shape: "slirp", "tap_bridge", or "custom". + string backend = 2; + string mac = 3; + // Linux bridge name for tap_bridge backend. + optional string bridge_name = 4; + // QEMU netdev id, e.g. "net0". + optional string netdev_id = 5; } // Structured log or lifecycle event emitted by the guest or runtime. @@ -101,12 +116,16 @@ message VmConfiguration { bool no_tee = 17; // Per-VM networking mode override (if unset, uses global cvm.networking). optional NetworkingConfig networking = 18; + // Per-VM networking overrides. If set, wins over singular networking. + repeated NetworkingConfig networks = 19; } // Per-VM networking configuration. message NetworkingConfig { // Networking mode: "bridge", "user" string mode = 1; + // Per-VM bridge interface name. Empty = node default bridge. + string bridge_name = 2; } // Requested GPU layout for a CVM. @@ -163,6 +182,10 @@ message UpdateVmRequest { optional string image = 17; // Disable or re-enable TEE for an existing VM. optional bool no_tee = 18; + // Optional update networking. + bool update_networking = 19; + // Networking list. Empty + update_networking=true resets to node default. + repeated NetworkingConfig networks = 20; } // Parameters for listing CVMs with pagination and keyword filtering. @@ -262,11 +285,20 @@ message ResourcesSettings { uint32 max_allocable_memory_in_mb = 3; // in MB. } +message NetworkingCapabilities { + repeated string supported_modes = 1; + string default_mode = 2; + bool forward_service_enabled = 3; + // Default bridge configured in vmm.toml [cvm.networking].bridge. + string default_bridge = 4; +} + // Aggregated metadata exposed through GetMeta. message GetMetaResponse { KmsSettings kms = 1; GatewaySettings gateway = 2; ResourcesSettings resources = 3; + NetworkingCapabilities networking = 4; } // Build information for the running VMM binary. diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index dd4b527d0..be2096e13 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -32,6 +32,9 @@ use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; +pub(crate) use network::{ + resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, +}; pub use qemu::VmConfig; pub use workdir::VmWorkDir; @@ -39,6 +42,7 @@ mod host_share; mod id_pool; mod image; mod mr_config; +mod network; mod qemu; pub(crate) mod registry; mod vm_info; @@ -75,8 +79,24 @@ pub struct Manifest { pub gateway_urls: Vec, #[serde(default)] pub no_tee: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub networking: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub networks: Vec, +} + +impl Manifest { + pub fn from_json(value: serde_json::Value) -> serde_json::Result { + let mut map = value; + if let Some(obj) = map.as_object_mut() { + if !obj.contains_key("networks") + || obj["networks"].as_array().is_some_and(|a| a.is_empty()) + { + if let Some(legacy) = obj.remove("networking") { + obj.insert("networks".into(), serde_json::Value::Array(vec![legacy])); + } + } + } + serde_json::from_value(map) + } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -281,6 +301,13 @@ impl App { let image_path = self.config.image.path.join(&manifest.image); let image = Image::load(&image_path).context("Failed to load image")?; let vm_id = manifest.id.clone(); + let mut runtime_networks = vm_work_dir.runtime_networks(); + if runtime_networks.is_empty() && cids_assigned.contains_key(&vm_id) { + runtime_networks = resolved_networks(&manifest, &self.config.cvm); + if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { + warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); + } + } let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; @@ -302,9 +329,12 @@ impl App { match states.get_mut(&vm_id) { Some(vm) => { vm.config = vm_config.into(); + vm.state.runtime_networks = runtime_networks; } None => { - states.add(VmState::new(vm_config)); + let mut vm_state = VmState::new(vm_config); + vm_state.state.runtime_networks = runtime_networks; + states.add(vm_state); } } }; @@ -356,11 +386,27 @@ impl App { let devices = self.try_allocate_gpus(&vm_config.manifest)?; let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; + let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + work_dir.set_runtime_networks(&runtime_networks)?; + { + let mut state = self.lock(); + let vm_state = state.get_mut(id).context("VM not found")?; + vm_state.state.runtime_networks = runtime_networks; + } for process in processes { - self.supervisor - .deploy(&process) - .await - .with_context(|| format!("Failed to start process {}", process.id))?; + if let Err(err) = self.supervisor.deploy(&process).await { + if let Err(clear_err) = work_dir.clear_runtime_networks() { + warn!( + id, + "failed to clear runtime networks after start failure: {clear_err}" + ); + } + if let Some(vm_state) = self.lock().get_mut(id) { + vm_state.state.runtime_networks.clear(); + } + return Err(err) + .with_context(|| format!("failed to start process {}", process.id)); + } } let mut state = self.lock(); @@ -517,20 +563,26 @@ impl App { /// Handle a DHCP lease notification: look up VM by MAC address, persist /// the guest IP, and reconfigure port forwarding. pub async fn report_dhcp_lease(&self, mac: &str, ip: &str) { - use crate::app::mr_config::mac_address_for_vm; + use crate::app::network::mac_address_for_vm_index; let vm_id = { let mut state = self.lock(); - let prefix = self.config.cvm.networking.mac_prefix_bytes(); - let found = state - .vms - .iter_mut() - .find(|(id, _)| mac_address_for_vm(id, &prefix) == mac); - let Some((id, vm)) = found else { + let found = state.vms.iter_mut().find_map(|(id, vm)| { + let networks = vm.effective_networks(&self.config.cvm); + networks.iter().enumerate().find_map(|(index, networking)| { + let prefix = networking.mac_prefix_bytes(); + let nic_mac = mac_address_for_vm_index(&vm.config.manifest.id, &prefix, index); + (nic_mac == mac).then(|| id.clone()) + }) + }); + let Some(vm_id) = found else { debug!(mac, ip, "DHCP lease for unknown MAC, ignoring"); return; }; - let vm_id = id.clone(); + let Some(vm) = state.get_mut(&vm_id) else { + debug!(mac, ip, id = %vm_id, "DHCP lease for missing VM, ignoring"); + return; + }; let workdir = VmWorkDir::new(vm.config.workdir.clone()); if let Err(e) = workdir.set_guest_ip(ip) { error!(mac, ip, "failed to persist guest IP: {e}"); @@ -553,13 +605,17 @@ impl App { let Some(vm) = state.get(id) else { return; }; - let networking = vm - .config - .manifest - .networking - .as_ref() - .unwrap_or(&self.config.cvm.networking); - if !networking.is_bridge() || !networking.forward_service_enabled { + let networks = vm.effective_networks(&self.config.cvm); + if networks + .iter() + .any(|networking| networking.mode == crate::config::NetworkingMode::User) + { + return; + } + if !networks + .iter() + .any(|networking| networking.is_bridge() && networking.forward_service_enabled) + { return; } let guest_ip = vm.state.guest_ip.clone(); @@ -861,6 +917,13 @@ impl App { let image = Image::load(&image_path).context("Failed to load image")?; let vm_id = manifest.id.clone(); let already_running = cids_assigned.contains_key(&vm_id); + let mut runtime_networks = vm_work_dir.runtime_networks(); + if runtime_networks.is_empty() && already_running { + runtime_networks = resolved_networks(&manifest, &self.config.cvm); + if let Err(err) = vm_work_dir.set_runtime_networks(&runtime_networks) { + warn!(id = %vm_id, "failed to persist inferred runtime networks: {err}"); + } + } let app_compose = vm_work_dir .app_compose() .context("Failed to read compose file")?; @@ -893,7 +956,10 @@ impl App { match states.get_mut(&vm_id) { Some(vm) => { // Update existing VM but preserve statistics and CID - let old_state = vm.state.clone(); + let mut old_state = vm.state.clone(); + if old_state.runtime_networks.is_empty() { + old_state.runtime_networks = runtime_networks; + } vm.config = vm_config.into(); vm.state = old_state; // Preserve the existing state with statistics } @@ -902,7 +968,9 @@ impl App { if !cids_assigned.contains_key(&vm_id) { states.cid_pool.occupy(cid)?; } - states.add(VmState::new(vm_config)); + let mut vm_state = VmState::new(vm_config); + vm_state.state.runtime_networks = runtime_networks; + states.add(vm_state); is_new = true; } } @@ -962,7 +1030,7 @@ impl App { &self.work_dir(&vm.config.manifest.id), ) }) - .map(|info| info.to_pb(&self.config.gateway, request.brief)) + .map(|info| info.to_pb(&self.config.gateway, &self.config.cvm, request.brief)) .collect::>(); Ok(StatusResponse { vms, @@ -991,7 +1059,7 @@ impl App { }; let info = vm_state .merged_info(proc_state.as_ref(), &self.work_dir(id)) - .to_pb(&self.config.gateway, false); + .to_pb(&self.config.gateway, &self.config.cvm, false); Ok(Some(info)) } @@ -1406,6 +1474,11 @@ fn make_vm_config( GpuConfig::default() }; let effective_vcpus = effective_vcpu_count_for_manifest(manifest, &gpus)?; + // Each resolved network interface becomes one virtio-net-pci device in the + // QEMU command (see `VmConfig::config_qemu`), which changes the guest's + // ACPI/DSDT layout and therefore RTMR0. Measure the interface count so the + // verifier reconstructs the exact device layout. + let num_nics = resolved_networks(manifest, &cfg.cvm).len() as u32; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, @@ -1417,6 +1490,7 @@ fn make_vm_config( hugepages: manifest.hugepages, num_gpus: gpus.gpus.len() as u32, num_nvswitches: gpus.bridges.len() as u32, + num_nics, host_share_mode: cfg.cvm.host_share_mode.clone(), hotplug_off: cfg.cvm.qemu_hotplug_off, image: Some(manifest.image.clone()), @@ -1453,7 +1527,9 @@ fn make_vm_config( #[cfg(test)] mod tests { use super::*; - use crate::config::{load_config_figment, TdxAttestationVariantConfig, TeePlatform}; + use crate::config::{ + load_config_figment, Networking, NetworkingMode, TdxAttestationVariantConfig, TeePlatform, + }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, TdxOsImageMeasurementDocument, TdxTdvfMeasurement, @@ -1465,6 +1541,55 @@ mod tests { hex::encode(vec![byte; len]) } + #[test] + fn put_manifest_keeps_legacy_networking_for_rollback() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-manifest-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + let workdir = VmWorkDir::new(&temp); + let mut manifest = test_manifest(1024); + manifest.networks = vec![Networking { + mode: NetworkingMode::Bridge, + bridge: "dstack-br0".to_string(), + forward_service_enabled: true, + mac_prefix: String::new(), + net: String::new(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + }]; + + workdir.put_manifest(&manifest)?; + let raw = fs::read_to_string(workdir.manifest_path())?; + let value: serde_json::Value = serde_json::from_str(&raw)?; + assert_eq!(value["networks"].as_array().map(Vec::len), Some(1)); + assert_eq!(value["networking"]["mode"], "bridge"); + fs::remove_dir_all(temp)?; + Ok(()) + } + + #[test] + fn manifest_deserializes_legacy_singular_networking_as_networks() { + let manifest: Manifest = Manifest::from_json(serde_json::json!({ + "id": "vm-1", + "name": "vm-1", + "app_id": "app-1", + "vcpu": 1, + "memory": 1024, + "disk_size": 10, + "image": "dstack-test", + "port_map": [], + "created_at_ms": 0, + "networking": { "mode": "bridge", "bridge": "dstack-br0" } + })) + .unwrap(); + + assert_eq!(manifest.networks.len(), 1); + assert_eq!(manifest.networks[0].mode, NetworkingMode::Bridge); + assert_eq!(manifest.networks[0].bridge, "dstack-br0"); + } + fn write_u16_le_at(buf: &mut [u8], off: usize, value: u16) { buf[off..off + 2].copy_from_slice(&value.to_le_bytes()); } @@ -1555,7 +1680,7 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: false, - networking: None, + networks: vec![], } } @@ -1635,6 +1760,33 @@ mod tests { assert_eq!(effective_vcpu_count(3, None), 3); } + #[test] + fn vm_measurement_config_ignores_networking_changes() -> Result<()> { + let config = test_tdx_config()?; + let mut bridge_manifest = test_manifest(2048); + bridge_manifest.networks = vec![Networking { + mode: NetworkingMode::Bridge, + bridge: "dstack-br0".to_string(), + forward_service_enabled: true, + mac_prefix: "02:aa:bb".to_string(), + net: String::new(), + dhcp_start: String::new(), + restrict: false, + netdev: String::new(), + }]; + let user_manifest = test_manifest(2048); + let image = test_tdx_image(true); + let compose_hash = hex_of(0x22, 32); + + let bridge_config = + make_vm_config(&config, &bridge_manifest, &image, &compose_hash, None, None)?; + let user_config = + make_vm_config(&config, &user_manifest, &image, &compose_hash, None, None)?; + + assert_eq!(bridge_config, user_config); + Ok(()) + } + #[test] fn tdx_auto_variant_uses_legacy_for_low_non_2g_memory() -> Result<()> { let config = test_tdx_config()?; @@ -1789,7 +1941,7 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: false, - networking: None, + networks: vec![], }; let mr_config = MrConfigV3::new( @@ -1917,6 +2069,7 @@ struct VmStateMut { boot_error: String, shutdown_progress: String, guest_ip: String, + runtime_networks: Vec, devices: GpuConfig, events: VecDeque, /// True when the VM is being removed (cleanup in progress). @@ -1948,6 +2101,14 @@ impl VmState { state: VmStateMut::default(), } } + + pub fn effective_networks(&self, cvm: &crate::config::CvmConfig) -> Vec { + if self.state.runtime_networks.is_empty() { + resolved_networks(&self.config.manifest, cvm) + } else { + self.state.runtime_networks.clone() + } + } } pub(crate) struct AppState { diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index 71c8a427f..f1b908e4f 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -13,25 +13,6 @@ use sha2::{Digest, Sha256}; use super::VmWorkDir; -/// Derives a deterministic, locally administered unicast MAC address from a VM ID. -/// -/// `prefix` may contain zero to three fixed bytes. Remaining bytes are filled -/// from SHA-256 of `vm_id`. -pub(super) fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { - let hash = Sha256::digest(vm_id.as_bytes()); - let prefix_len = prefix.len().min(3); - let mut bytes = [0_u8; 6]; - bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); - for index in prefix_len..bytes.len() { - bytes[index] = hash[index - prefix_len]; - } - bytes[0] = (bytes[0] & 0xfe) | 0x02; - format!( - "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] - ) -} - pub(super) fn tdx_mr_config_id(workdir: &VmWorkDir, app_compose: &AppCompose) -> Result { if let Some(document) = workdir .sys_config() @@ -132,17 +113,3 @@ impl VmWorkDir { .to_canonical_json()) } } - -#[cfg(test)] -mod tests { - use super::mac_address_for_vm; - - #[test] - fn mac_address_is_deterministic_and_locally_administered() { - let mac = mac_address_for_vm("vm-1", &[0xaa, 0xbb, 0xcc]); - assert_eq!(&mac[..8], "aa:bb:cc"); - assert_eq!(mac, mac_address_for_vm("vm-1", &[0xaa, 0xbb, 0xcc])); - let first = u8::from_str_radix(&mac[..2], 16).unwrap(); - assert_eq!(first & 0x03, 0x02); - } -} diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs new file mode 100644 index 000000000..2ed379423 --- /dev/null +++ b/dstack/vmm/src/app/network.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! VM network resolution, validation, and interface identity. + +use std::path::Path; + +use anyhow::{bail, Result}; +use sha2::{Digest, Sha256}; + +use super::Manifest; +use crate::config::{CvmConfig, Networking, NetworkingMode}; + +pub(crate) fn resolve_networking(networking: &Networking, cfg: &CvmConfig) -> Networking { + let mut resolved = cfg.networking.clone(); + resolved.mode = networking.mode; + resolved.restrict = cfg.networking.restrict || networking.restrict; + if !networking.bridge.is_empty() { + resolved.bridge = networking.bridge.clone(); + } + if !networking.mac_prefix.is_empty() { + resolved.mac_prefix = networking.mac_prefix.clone(); + } + if !networking.net.is_empty() { + resolved.net = networking.net.clone(); + } + if !networking.dhcp_start.is_empty() { + resolved.dhcp_start = networking.dhcp_start.clone(); + } + if !networking.netdev.is_empty() { + resolved.netdev = networking.netdev.clone(); + } + resolved +} + +pub(crate) fn resolved_networks(manifest: &Manifest, cfg: &CvmConfig) -> Vec { + if manifest.networks.is_empty() { + vec![cfg.networking.clone()] + } else { + manifest + .networks + .iter() + .map(|networking| resolve_networking(networking, cfg)) + .collect() + } +} + +pub(crate) fn validate_resolved_network(networking: &Networking) -> Result<()> { + if networking.mode != NetworkingMode::Bridge { + return Ok(()); + } + if networking.bridge.is_empty() { + bail!("bridge networking requested but no bridge is configured"); + } + if !Path::new("/sys/class/net") + .join(&networking.bridge) + .exists() + { + bail!("bridge interface '{}' does not exist", networking.bridge); + } + Ok(()) +} + +pub(crate) fn validate_resolved_networks(networks: &[Networking]) -> Result<()> { + let forwarding_bridges = networks + .iter() + .filter(|networking| networking.is_bridge() && networking.forward_service_enabled) + .count(); + if forwarding_bridges > 1 { + bail!("built-in port forwarding supports only one bridge"); + } + for networking in networks { + validate_resolved_network(networking)?; + } + Ok(()) +} + +/// Derives a deterministic, locally administered unicast MAC address. +/// +/// Index zero preserves the legacy single-NIC derivation. Later interfaces +/// hash `vm_id:index`, preventing an upgrade from changing the primary NIC. +pub(crate) fn mac_address_for_vm_index(vm_id: &str, prefix: &[u8], index: usize) -> String { + let hash_input = if index == 0 { + vm_id.to_string() + } else { + format!("{vm_id}:{index}") + }; + let hash = Sha256::digest(hash_input.as_bytes()); + let prefix_len = prefix.len().min(3); + let mut bytes = [0_u8; 6]; + bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); + for index in prefix_len..bytes.len() { + bytes[index] = hash[index - prefix_len]; + } + bytes[0] = (bytes[0] & 0xfe) | 0x02; + format!( + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] + ) +} + +#[cfg(test)] +mod tests { + use super::mac_address_for_vm_index; + + #[test] + fn primary_mac_keeps_legacy_derivation_and_later_nics_are_distinct() { + assert_eq!( + mac_address_for_vm_index("vm-123", &[], 0), + "96:b1:d8:b9:08:e6" + ); + assert_eq!( + mac_address_for_vm_index("vm-123", &[], 1), + "c6:74:2c:65:14:b9" + ); + } +} diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 31bf3a882..6850bb653 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -21,7 +21,8 @@ use super::{ host_share::create_shared_disk, hugepage_numa_nodes, image::Image, - mr_config::{mac_address_for_vm, snp_host_data, tdx_mr_config_id}, + mr_config::{snp_host_data, tdx_mr_config_id}, + network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use anyhow::{bail, Context, Result}; @@ -174,6 +175,7 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { struct PreparedQemuLaunch { workdir: VmWorkDir, platform: TeePlatform, + networks: Vec, hugepage_numa_nodes: Option>, gpu_numa_nodes: HashMap, numa_cpus: Option, @@ -195,6 +197,8 @@ impl PreparedQemuLaunch { prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); + let networks = resolved_networks(&vm.manifest, cfg); + validate_resolved_networks(&networks)?; let hugepage_numa_nodes = if vm.manifest.hugepages { Some(hugepage_numa_nodes(gpus)?) @@ -249,6 +253,7 @@ impl PreparedQemuLaunch { Ok(Self { workdir, platform, + networks, hugepage_numa_nodes, gpu_numa_nodes, numa_cpus, @@ -337,7 +342,7 @@ impl QemuCommandBuilder<'_> { let mut command = self.base_command(); self.configure_rootfs(&mut command)?; self.configure_data_disk(&mut command); - self.configure_networking(&mut command); + self.configure_networking(&mut command)?; self.vm.configure_smbios(&mut command, self.cfg); self.configure_tpm_and_vsock(&mut command); self.configure_host_share(&mut command)?; @@ -455,57 +460,61 @@ impl QemuCommandBuilder<'_> { )); } - fn configure_networking(&self, command: &mut Command) { - // Per-VM networking only overrides the mode and bridge name. All other - // fields continue to come from the global configuration. - let resolved_networking; - let networking = match self.vm.manifest.networking.as_ref() { - Some(vm_networking) => { - resolved_networking = Networking { - mode: vm_networking.mode, - bridge: if vm_networking.bridge.is_empty() { - self.cfg.networking.bridge.clone() - } else { - vm_networking.bridge.clone() - }, - ..self.cfg.networking.clone() - }; - &resolved_networking - } - None => &self.cfg.networking, - }; - let mac = mac_address_for_vm(&self.vm.manifest.id, &networking.mac_prefix_bytes()); - let net_device = virtio_pci_device( - &format!("virtio-net-pci,netdev=net0,mac={mac}"), - self.is_amd_sev_snp(), - ); - let netdev = match networking.mode { - NetworkingMode::User => { - let mut netdev = format!( - "user,id=net0,net={},dhcpstart={},restrict={}", - networking.net, - networking.dhcp_start, - if networking.restrict { "yes" } else { "no" } - ); - for mapping in &self.vm.manifest.port_map { - netdev.push_str(&format!( - ",hostfwd={}:{}:{}-:{}", - mapping.protocol.as_str(), - mapping.address, - mapping.from, - mapping.to - )); + fn configure_networking(&self, command: &mut Command) -> Result<()> { + let hostfwd_index = self + .prepared + .networks + .iter() + .position(|networking| networking.mode == NetworkingMode::User); + for (index, networking) in self.prepared.networks.iter().enumerate() { + let net_id = format!("net{index}"); + let mac = mac_address_for_vm_index( + &self.vm.manifest.id, + &networking.mac_prefix_bytes(), + index, + ); + let net_device = virtio_pci_device( + &format!("virtio-net-pci,netdev={net_id},mac={mac}"), + self.is_amd_sev_snp(), + ); + let netdev = match networking.mode { + NetworkingMode::User => { + let mut netdev = format!( + "user,id={net_id},net={},dhcpstart={},restrict={}", + networking.net, + networking.dhcp_start, + if networking.restrict { "yes" } else { "no" } + ); + if hostfwd_index == Some(index) { + for mapping in &self.vm.manifest.port_map { + netdev.push_str(&format!( + ",hostfwd={}:{}:{}-:{}", + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to + )); + } + } + netdev } - netdev - } - NetworkingMode::Bridge => { - tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - format!("bridge,id=net0,br={}", networking.bridge) - } - NetworkingMode::Custom => networking.netdev.clone(), - }; - command.arg("-netdev").arg(netdev); - command.arg("-device").arg(net_device); + NetworkingMode::Bridge => { + tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); + format!("bridge,id={net_id},br={}", networking.bridge) + } + NetworkingMode::Custom => { + if !networking.netdev.contains(&format!("id={net_id}")) { + bail!( + "custom networking netdev must contain id={net_id} for interface index {index}" + ); + } + networking.netdev.clone() + } + }; + command.arg("-netdev").arg(netdev); + command.arg("-device").arg(net_device); + } + Ok(()) } fn configure_tpm_and_vsock(&self, command: &mut Command) { @@ -864,8 +873,8 @@ mod tests { PreparedQemuLaunch, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; - use crate::app::{GpuConfig, Manifest, VmWorkDir}; - use crate::config::{Config, TeePlatform, DEFAULT_CONFIG}; + use crate::app::{GpuConfig, Manifest, PortMapping, VmWorkDir}; + use crate::config::{Config, Protocol, TeePlatform, DEFAULT_CONFIG}; #[test] fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { @@ -917,7 +926,12 @@ mod tests { memory: 2048, disk_size: 10, image: "test-image".into(), - port_map: vec![], + port_map: vec![PortMapping { + address: "127.0.0.1".parse().unwrap(), + protocol: Protocol::Tcp, + from: 18080, + to: 8080, + }], created_at_ms: 0, hugepages: false, pin_numa: false, @@ -925,7 +939,7 @@ mod tests { kms_urls: vec![], gateway_urls: vec![], no_tee: true, - networking: None, + networks: vec![], }, image: Image { info: ImageInfo { @@ -959,6 +973,7 @@ mod tests { let prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), platform: TeePlatform::Tdx, + networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), numa_cpus: None, @@ -990,5 +1005,24 @@ mod tests { .args .windows(2) .any(|args| args == ["-append", "console=hvc0"])); + let netdevs = process + .args + .windows(2) + .filter(|args| args[0] == "-netdev") + .map(|args| args[1].as_str()) + .collect::>(); + assert_eq!(netdevs.len(), 2); + assert!(netdevs[0].contains("user,id=net0")); + assert!(netdevs[0].contains("hostfwd=tcp:127.0.0.1:18080-:8080")); + assert!(netdevs[1].contains("user,id=net1")); + assert!(!netdevs[1].contains("hostfwd=")); + assert!(process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net0"))); + assert!(process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); } } diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs index 801c40cd5..d039418c8 100644 --- a/dstack/vmm/src/app/vm_info.rs +++ b/dstack/vmm/src/app/vm_info.rs @@ -11,8 +11,11 @@ use dstack_vmm_rpc as pb; use fs_err as fs; use supervisor_client::supervisor::ProcessInfo; -use super::{Manifest, VmState, VmWorkDir}; -use crate::config::{GatewayConfig, Networking, NetworkingMode}; +use super::{ + network::{mac_address_for_vm_index, resolved_networks}, + Manifest, VmState, VmWorkDir, +}; +use crate::config::{CvmConfig, GatewayConfig, Networking, NetworkingMode}; pub(crate) struct VmInfo { pub manifest: Manifest, @@ -27,15 +30,34 @@ pub(crate) struct VmInfo { pub image_version: String, pub gateway_enabled: bool, pub events: Vec, + pub runtime_networks: Vec, } -fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { - let mode = match networking.mode { +fn networking_mode_name(mode: NetworkingMode) -> &'static str { + match mode { NetworkingMode::Bridge => "bridge", NetworkingMode::User => "user", NetworkingMode::Custom => "custom", - }; - pb::NetworkingConfig { mode: mode.into() } + } +} + +fn networking_backend_name(mode: NetworkingMode) -> &'static str { + match mode { + NetworkingMode::Bridge => "tap_bridge", + NetworkingMode::User => "slirp", + NetworkingMode::Custom => "custom", + } +} + +fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { + pb::NetworkingConfig { + mode: networking_mode_name(networking.mode).into(), + bridge_name: if networking.mode == NetworkingMode::Bridge { + networking.bridge.clone() + } else { + String::new() + }, + } } fn sanitize_optional>(value: Option) -> Option { @@ -43,13 +65,48 @@ fn sanitize_optional>(value: Option) -> Option { } impl VmInfo { - pub fn to_pb(&self, gateway: &GatewayConfig, brief: bool) -> pb::VmInfo { + pub fn effective_networks(&self, cvm: &CvmConfig) -> Vec { + if self.runtime_networks.is_empty() { + resolved_networks(&self.manifest, cvm) + } else { + self.runtime_networks.clone() + } + } + + pub fn to_pb(&self, gateway: &GatewayConfig, cvm: &CvmConfig, brief: bool) -> pb::VmInfo { let workdir = VmWorkDir::new(&self.workdir); let vm_config = workdir.manifest(); let custom_gateway_urls = vm_config .as_ref() .map(|config| config.gateway_urls.clone()) .unwrap_or_default(); + let configured_networks = self + .manifest + .networks + .iter() + .map(networking_to_proto) + .collect::>(); + let configured_networking = configured_networks.first().cloned(); + let interfaces = self + .effective_networks(cvm) + .iter() + .enumerate() + .map(|(index, networking)| { + let mac = mac_address_for_vm_index( + &self.manifest.id, + &networking.mac_prefix_bytes(), + index, + ); + pb::NetworkInterfaceStatus { + mode: networking_mode_name(networking.mode).into(), + backend: networking_backend_name(networking.mode).into(), + mac, + bridge_name: (networking.mode == NetworkingMode::Bridge) + .then(|| networking.bridge.clone()), + netdev_id: Some(format!("net{index}")), + } + }) + .collect(); pb::VmInfo { id: self.manifest.id.clone(), name: self.manifest.name.clone(), @@ -110,7 +167,8 @@ impl VmInfo { gateway_urls: custom_gateway_urls.clone(), stopped, no_tee, - networking: self.manifest.networking.as_ref().map(networking_to_proto), + networking: configured_networking, + networks: configured_networks, }) }, app_url: self @@ -123,6 +181,7 @@ impl VmInfo { instance_id: sanitize_optional(self.instance_id.clone()), exited_at: self.exited_at.clone(), events: self.events.clone(), + interfaces, } } } @@ -202,6 +261,7 @@ impl VmState { image_version: self.config.image.info.version.clone(), gateway_enabled: self.config.gateway_enabled, events: self.state.events.clone().into(), + runtime_networks: self.state.runtime_networks.clone(), } } } diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs index e7a28ee7b..6984cd4b5 100644 --- a/dstack/vmm/src/app/workdir.rs +++ b/dstack/vmm/src/app/workdir.rs @@ -16,7 +16,7 @@ use fs_err as fs; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; -use crate::app::Manifest; +use crate::{app::Manifest, config::Networking}; #[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct InstanceInfo { @@ -67,17 +67,20 @@ impl VmWorkDir { pub fn manifest(&self) -> Result { let manifest_path = self.manifest_path(); - let manifest = fs::read_to_string(manifest_path).context("failed to read manifest")?; - let manifest: Manifest = - serde_json::from_str(&manifest).context("failed to parse manifest")?; - Ok(manifest) + let raw = fs::read_to_string(manifest_path).context("failed to read manifest")?; + let value: serde_json::Value = + serde_json::from_str(&raw).context("failed to parse manifest json")?; + Manifest::from_json(value).context("failed to deserialize manifest") } pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { fs::create_dir_all(&self.workdir).context("failed to create workdir")?; let manifest_path = self.manifest_path(); - fs::write(manifest_path, serde_json::to_string(manifest)?) - .context("failed to write manifest") + let mut value = serde_json::to_value(manifest)?; + if let Some(networking) = manifest.networks.first() { + value["networking"] = serde_json::to_value(networking)?; + } + fs::write(manifest_path, serde_json::to_string(&value)?).context("failed to write manifest") } pub fn started(&self) -> Result { @@ -128,6 +131,10 @@ impl VmWorkDir { self.workdir.join("guest-ip") } + pub fn runtime_networks_path(&self) -> PathBuf { + self.workdir.join("runtime-networks.json") + } + pub fn guest_ip(&self) -> Option { fs::read_to_string(self.guest_ip_path()) .ok() @@ -139,6 +146,27 @@ impl VmWorkDir { fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") } + pub fn runtime_networks(&self) -> Vec { + fs::read_to_string(self.runtime_networks_path()) + .ok() + .and_then(|contents| serde_json::from_str(&contents).ok()) + .unwrap_or_default() + } + + pub fn set_runtime_networks(&self, networks: &[Networking]) -> Result<()> { + let serialized = serde_json::to_vec(networks)?; + safe_write::safe_write(self.runtime_networks_path(), serialized) + .context("failed to write runtime networks") + } + + pub fn clear_runtime_networks(&self) -> Result<()> { + let path = self.runtime_networks_path(); + if path.exists() { + fs::remove_file(path).context("failed to clear runtime networks")?; + } + Ok(()) + } + pub fn serial_file(&self) -> PathBuf { self.workdir.join("serial.log") } @@ -221,3 +249,37 @@ impl VmWorkDir { Ok(compose) } } + +#[cfg(test)] +mod tests { + use std::{ + os::unix::fs::symlink, + time::{SystemTime, UNIX_EPOCH}, + }; + + use anyhow::Result; + use fs_err as fs; + + use super::VmWorkDir; + + #[test] + fn runtime_networks_snapshot_replaces_target_instead_of_following_it() -> Result<()> { + let temp = std::env::temp_dir().join(format!( + "dstack-vmm-runtime-networks-test-{}", + SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos() + )); + fs::create_dir_all(&temp)?; + let external = temp.join("external.json"); + fs::write(&external, "sentinel")?; + let workdir = VmWorkDir::new(temp.join("vm")); + fs::create_dir_all(workdir.path())?; + symlink(&external, workdir.runtime_networks_path())?; + + workdir.set_runtime_networks(&[])?; + + assert_eq!(fs::read_to_string(&external)?, "sentinel"); + assert_eq!(fs::read_to_string(workdir.runtime_networks_path())?, "[]"); + fs::remove_dir_all(temp)?; + Ok(()) + } +} diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index 88aa884c1..932c4c17d 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -22,7 +22,11 @@ use or_panic::ResultOrPanic; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; -use crate::app::{App, AttachMode, GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir}; +use crate::app::{ + resolve_networking, validate_resolved_network, validate_resolved_networks, App, AttachMode, + GpuConfig, GpuSpec, Manifest, PortMapping, VmWorkDir, +}; +use crate::config::{CvmConfig, Networking, NetworkingMode}; fn hex_sha256(data: &str) -> String { use sha2::Digest; @@ -196,34 +200,83 @@ pub fn create_manifest_from_vm_config( kms_urls: request.kms_urls.clone(), gateway_urls: request.gateway_urls.clone(), no_tee: request.no_tee, - networking: request.networking.as_ref().and_then(networking_from_proto), + networks: networks_from_vm_config(&request, cvm_config)?, }) } -fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Option { - use crate::config::NetworkingMode; +fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result> { + let bridge = proto.bridge_name.trim().to_string(); let mode = match proto.mode.as_str() { "bridge" => NetworkingMode::Bridge, - "user" => NetworkingMode::User, - "custom" => NetworkingMode::Custom, - "" => return None, // not set, use global default - other => { - tracing::warn!("unsupported per-VM networking mode '{other}', using global default"); - return None; - } + "" if bridge.is_empty() => return Ok(None), + "" => bail!("networking mode is required when bridge is set"), + "custom" => bail!("custom networking mode is manifest-only"), + other => bail!("unsupported networking mode '{other}'"), }; - // Only set mode; other fields will be merged from global config at runtime - Some(crate::config::Networking { + if mode != NetworkingMode::Bridge && !bridge.is_empty() { + bail!("bridge_name is only valid for bridge networking mode"); + } + Ok(Some(Networking { mode, - bridge: String::new(), + bridge, mac_prefix: String::new(), net: String::new(), dhcp_start: String::new(), restrict: false, netdev: String::new(), forward_service_enabled: false, - }) + })) +} + +fn network_from_required_proto(proto: &rpc::NetworkingConfig) -> Result { + networking_from_proto(proto)?.context("networking mode is required") +} + +fn networks_from_proto(networks: &[rpc::NetworkingConfig]) -> Result> { + networks.iter().map(network_from_required_proto).collect() +} + +fn validate_default_network(cvm_config: &CvmConfig) -> Result<()> { + validate_resolved_network(&cvm_config.networking) +} + +fn resolve_requested_networks( + networks: &[Networking], + cvm_config: &CvmConfig, +) -> Result> { + let resolved = networks + .iter() + .map(|networking| resolve_networking(networking, cvm_config)) + .collect::>(); + validate_resolved_networks(&resolved)?; + Ok(resolved) +} + +fn has_host_bridge_interface() -> bool { + let Ok(entries) = fs::read_dir("/sys/class/net") else { + return false; + }; + entries + .filter_map(|entry| entry.ok()) + .any(|entry| entry.path().join("bridge").exists()) +} + +fn networks_from_vm_config( + request: &VmConfiguration, + cvm_config: &CvmConfig, +) -> Result> { + if !request.networks.is_empty() { + let networks = networks_from_proto(&request.networks)?; + resolve_requested_networks(&networks, cvm_config) + } else if let Some(networking) = request.networking.as_ref() { + match networking_from_proto(networking)? { + Some(networking) => resolve_requested_networks(&[networking], cvm_config), + None => Ok(vec![]), + } + } else { + Ok(vec![]) + } } impl RpcHandler { @@ -438,6 +491,24 @@ impl VmmRpc for RpcHandler { if request.update_gateway_urls { manifest.gateway_urls = request.gateway_urls.clone(); } + if request.update_networking { + manifest.networks = if request.networks.is_empty() { + validate_default_network(&self.app.config.cvm)?; + vec![] + } else { + let networks = networks_from_proto(&request.networks)?; + resolve_requested_networks(&networks, &self.app.config.cvm)? + }; + let is_running = self + .app + .supervisor + .info(&request.id) + .await? + .is_some_and(|info| info.state.status.is_running()); + if !is_running { + vm_work_dir.clear_runtime_networks()?; + } + } vm_work_dir .put_manifest(&manifest) .context("Failed to put manifest")?; @@ -447,7 +518,7 @@ impl VmmRpc for RpcHandler { .await .context("Failed to load VM")?; if request.update_ports { - self.app.reconfigure_port_forward(&new_id).await; + self.app.reconfigure_port_forward(&request.id).await; } Ok(Id { id: new_id }) } @@ -519,6 +590,13 @@ impl VmmRpc for RpcHandler { } async fn get_meta(self) -> Result { + let mut supported_modes = vec!["user".to_string()]; + let default_networking = &self.app.config.cvm.networking; + let mut bridge_networking = default_networking.clone(); + bridge_networking.mode = NetworkingMode::Bridge; + if validate_resolved_network(&bridge_networking).is_ok() || has_host_bridge_interface() { + supported_modes.push("bridge".to_string()); + } Ok(GetMetaResponse { kms: Some(KmsSettings { url: self @@ -550,6 +628,16 @@ impl VmmRpc for RpcHandler { max_allocable_vcpu: self.app.config.cvm.max_allocable_vcpu, max_allocable_memory_in_mb: self.app.config.cvm.max_allocable_memory_in_mb, }), + networking: Some(rpc::NetworkingCapabilities { + supported_modes, + default_mode: match default_networking.mode { + NetworkingMode::User => "user".to_string(), + NetworkingMode::Bridge => "bridge".to_string(), + NetworkingMode::Custom => String::new(), + }, + forward_service_enabled: default_networking.forward_service_enabled, + default_bridge: default_networking.bridge.clone(), + }), }) } @@ -749,3 +837,119 @@ impl RpcCall for RpcHandler { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use rocket::figment::Figment; + + fn test_cvm_config() -> CvmConfig { + let config: crate::config::Config = Figment::from(crate::config::load_config_figment(None)) + .extract() + .unwrap(); + config.cvm + } + + fn test_vm_configuration() -> VmConfiguration { + VmConfiguration { + name: "vm-test".to_string(), + image: "dstack-test".to_string(), + compose_file: "{}".to_string(), + vcpu: 1, + memory: 1024, + disk_size: 10, + ports: vec![], + encrypted_env: vec![], + app_id: None, + user_config: String::new(), + hugepages: false, + pin_numa: false, + gpus: None, + kms_urls: vec![], + gateway_urls: vec![], + stopped: false, + no_tee: false, + networking: None, + networks: vec![], + } + } + + #[test] + fn create_without_networking_persists_following_default() { + let manifest = + create_manifest_from_vm_config(test_vm_configuration(), &test_cvm_config()).unwrap(); + + assert!(manifest.networks.is_empty()); + } + + #[test] + fn explicit_user_networking_is_resolved_before_persist() { + let mut request = test_vm_configuration(); + request.networks = vec![rpc::NetworkingConfig { + mode: "user".to_string(), + bridge_name: String::new(), + }]; + + let manifest = create_manifest_from_vm_config(request, &test_cvm_config()).unwrap(); + + assert_eq!(manifest.networks.len(), 1); + assert_eq!(manifest.networks[0].mode, NetworkingMode::User); + assert!(!manifest.networks[0].net.is_empty()); + } + + #[test] + fn bridge_name_is_rejected_for_user_mode() { + let err = networks_from_proto(&[rpc::NetworkingConfig { + mode: "user".to_string(), + bridge_name: "dstack-br0".to_string(), + }]) + .unwrap_err(); + + assert!(err.to_string().contains("bridge_name is only valid")); + } + + #[test] + fn repeated_networks_rejects_empty_entries() { + let err = networks_from_proto(&[rpc::NetworkingConfig { + mode: String::new(), + bridge_name: String::new(), + }]) + .unwrap_err(); + + assert!(err.to_string().contains("networking mode is required")); + } + + #[test] + fn repeated_networks_rejects_custom_entries() { + let err = networks_from_proto(&[rpc::NetworkingConfig { + mode: "custom".to_string(), + bridge_name: String::new(), + }]) + .unwrap_err(); + + assert!(err.to_string().contains("custom networking mode")); + } + + #[test] + fn multiple_bridges_are_rejected_when_builtin_forwarding_is_enabled() { + let mut cvm_config = test_cvm_config(); + cvm_config.networking.forward_service_enabled = true; + let mut request = test_vm_configuration(); + request.networks = vec![ + rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "lo".to_string(), + }, + rpc::NetworkingConfig { + mode: "bridge".to_string(), + bridge_name: "lo".to_string(), + }, + ]; + + let err = create_manifest_from_vm_config(request, &cvm_config).unwrap_err(); + + assert!(err + .to_string() + .contains("built-in port forwarding supports only one bridge")); + } +} diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index 22f473435..c0e59eb20 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -20,6 +20,9 @@ const CreateVmDialogComponent = { allowAttachAllGpus: { type: Boolean, required: true }, kmsAvailable: { type: Boolean, required: true }, portMappingEnabled: { type: Boolean, required: true }, + networkingModes: { type: Array, required: true }, + defaultBridge: { type: String, default: '' }, + defaultNetworkingLabel: { type: String, required: true }, }, emits: ['close', 'submit', 'load-compose'], template: /* html */ ` @@ -147,14 +150,31 @@ const CreateVmDialogComponent = { -
- - +
+ +
+
{{ defaultNetworkingLabel }}
+
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} + Guest IP is assigned by host DHCP on that bridge and reported after boot. + +
+ +
diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index c30ef894f..f994f0487 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -19,6 +19,9 @@ const UpdateVmDialogComponent = { availableGpus: { type: Array, required: true }, allowAttachAllGpus: { type: Boolean, required: true }, portMappingEnabled: { type: Boolean, required: true }, + networkingModes: { type: Array, required: true }, + defaultBridge: { type: String, default: '' }, + defaultNetworkingLabel: { type: String, required: true }, kmsEnabled: { type: Boolean, required: true }, composeHashPreview: { type: String, required: true }, }, @@ -134,6 +137,35 @@ const UpdateVmDialogComponent = {
+
+
+ +
+
+
{{ defaultNetworkingLabel }}
+
+ + + + + + {{ defaultBridge ? 'Leave empty to use the VMM default bridge from vmm.toml: ' + defaultBridge + '.' : 'No default bridge is configured in vmm.toml; enter a bridge interface name.' }} + Guest IP is assigned by host DHCP on that bridge and reported after boot. + +
+ +
+
+
diff --git a/dstack/vmm/ui/src/composables/useVmManager.ts b/dstack/vmm/ui/src/composables/useVmManager.ts index a994ee1d8..c6c60c741 100644 --- a/dstack/vmm/ui/src/composables/useVmManager.ts +++ b/dstack/vmm/ui/src/composables/useVmManager.ts @@ -73,6 +73,7 @@ type VmListItem = { boot_progress?: string; shutdown_progress?: string; image_version?: string; + interfaces?: VmmTypes.INetworkInterfaceStatus[]; configuration?: VmConfiguration; appCompose?: AppCompose; }; @@ -89,6 +90,11 @@ type PortFormEntry = { vm_port?: number | null; }; +type NetworkFormEntry = { + mode: string; + bridge_name?: string; +}; + type VmFormState = { name: string; image: string; @@ -118,7 +124,7 @@ type VmFormState = { no_tee: boolean; pin_numa: boolean; hugepages: boolean; - net_mode: string; + networks: NetworkFormEntry[]; user_config: string; kms_urls: string[]; gateway_urls: string[]; @@ -147,6 +153,8 @@ type UpdateDialogState = { attachAllGpus: boolean; selectedGpus: string[]; updateGpuConfig: boolean; + updateNetworking: boolean; + networks: NetworkFormEntry[]; user_config: string; }; @@ -163,6 +171,7 @@ type CloneConfigDialogState = { gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; gateway_urls?: string[]; + networks?: NetworkFormEntry[]; hugepages: boolean; pin_numa: boolean; no_tee: boolean; @@ -201,7 +210,7 @@ function createVmFormState(preLaunchScript: string): VmFormState { no_tee: false, pin_numa: false, hugepages: false, - net_mode: '', + networks: [], user_config: '', kms_urls: [], gateway_urls: [], @@ -232,6 +241,8 @@ function createUpdateDialogState(): UpdateDialogState { attachAllGpus: false, selectedGpus: [], updateGpuConfig: false, + updateNetworking: false, + networks: [], user_config: '', }; } @@ -250,6 +261,7 @@ function createCloneConfigDialogState(): CloneConfigDialogState { gpus: undefined, kms_urls: undefined, gateway_urls: undefined, + networks: undefined, hugepages: false, pin_numa: false, no_tee: false, @@ -305,7 +317,28 @@ fi const cloneConfigDialog: Ref = ref(createCloneConfigDialogState()); const showCreateDialog = ref(false); - const config = ref({ portMappingEnabled: false }); + const config = ref({ + portMappingEnabled: false, + networking: null as VmmTypes.INetworkingCapabilities | null, + }); + const networkingModes = computed(() => { + const supported = config.value.networking?.supported_modes || []; + const fallback = supported.length > 0 ? supported : ['user']; + return Array.from(new Set(fallback)); + }); + const defaultBridge = computed(() => config.value.networking?.default_bridge || ''); + const defaultNetworkingLabel = computed(() => { + const mode = config.value.networking?.default_mode || ''; + if (mode === 'bridge') { + return defaultBridge.value + ? `Default: Bridge (${defaultBridge.value})` + : 'Default: Bridge (no default bridge configured)'; + } + if (mode === 'user') { + return 'Default: User networking'; + } + return mode ? `Default: ${networkModeLabel(mode)}` : 'Default: node configuration'; + }); const composeHashPreview = ref(''); const updateComposeHashPreview = ref(''); @@ -386,6 +419,31 @@ fi vm_port: port.vm_port, })); + const cloneNetworks = (configuration?: VmConfiguration | null): NetworkFormEntry[] => { + const configured = configuration?.networks && configuration.networks.length > 0 + ? configuration.networks + : (configuration?.networking ? [configuration.networking] : []); + return configured.map((network) => ({ + mode: network.mode || '', + bridge_name: network.bridge_name || '', + })); + }; + + const normalizeNetworks = (networks: NetworkFormEntry[] = []): VmmTypes.INetworkingConfig[] => + networks + .map((network) => ({ + mode: (network.mode || '').trim(), + bridge_name: network.mode === 'bridge' ? (network.bridge_name || '').trim() : '', + })) + .filter((network) => network.mode.length > 0); + + function networkModeLabel(mode?: string | null) { + if (!mode) { + return 'Default'; + } + return mode.charAt(0).toUpperCase() + mode.slice(1); + } + function deriveGpuSelection(gpuConfig?: VmmTypes.IGpuConfig) { if (!gpuConfig) { return { attachAll: false, selected: [] as string[] }; @@ -444,7 +502,7 @@ type CreateVmPayloadSource = { hugepages?: boolean; pin_numa?: boolean; no_tee?: boolean; - net_mode?: string; + networks?: NetworkFormEntry[]; gpus?: VmmTypes.IGpuConfig; kms_urls?: string[]; gateway_urls?: string[]; @@ -453,6 +511,7 @@ type CreateVmPayloadSource = { function buildCreateVmPayload(source: CreateVmPayloadSource): VmmTypes.IVmConfiguration { const normalizedPorts = normalizePorts(source.ports); + const normalizedNetworks = normalizeNetworks(source.networks || []); return { name: source.name.trim(), image: source.image.trim(), @@ -467,7 +526,7 @@ type CreateVmPayloadSource = { hugepages: !!source.hugepages, pin_numa: !!source.pin_numa, no_tee: source.no_tee ?? false, - networking: source.net_mode ? { mode: source.net_mode } : undefined, + networks: normalizedNetworks, gpus: source.gpus, kms_urls: source.kms_urls?.filter((url) => url && url.trim().length) ?? [], gateway_urls: source.gateway_urls?.filter((url) => url && url.trim().length) ?? [], @@ -551,7 +610,10 @@ type CreateVmPayloadSource = { return vm; }); - config.value = { portMappingEnabled: data.port_mapping_enabled }; + config.value = { + ...config.value, + portMappingEnabled: data.port_mapping_enabled, + }; if (expandedVMs.value.size > 0) { await refreshExpandedVMs(); @@ -608,6 +670,18 @@ type CreateVmPayloadSource = { return vms.value.find((item) => item.id === vm.id) || null; } + async function loadMeta() { + try { + const data = await vmmRpc.getMeta({}); + config.value = { + ...config.value, + networking: data.networking || null, + }; + } catch (error) { + recordError('error loading VMM metadata', error); + } + } + async function loadImages() { try { const data = await vmmRpc.listImages({}); @@ -912,6 +986,8 @@ type CreateVmPayloadSource = { attachAllGpus: gpuSelection.attachAll, selectedGpus: gpuSelection.selected, updateGpuConfig: false, + updateNetworking: false, + networks: cloneNetworks(config), user_config: config.user_config || '', }; } @@ -1004,7 +1080,7 @@ type CreateVmPayloadSource = { hugepages: vmForm.value.hugepages, pin_numa: vmForm.value.pin_numa, no_tee: vmForm.value.no_tee, - net_mode: vmForm.value.net_mode, + networks: vmForm.value.networks, gpus: configGpu(vmForm.value) || undefined, kms_urls: vmForm.value.kms_urls, gateway_urls: vmForm.value.gateway_urls, @@ -1088,6 +1164,10 @@ type CreateVmPayloadSource = { body.update_ports = true; body.ports = normalizePorts(updated.ports); body.gpus = updateDialog.value.updateGpuConfig ? configGpu(updated, true) : undefined; + if (updated.updateNetworking) { + body.update_networking = true; + body.networks = normalizeNetworks(updated.networks); + } await vmmRpc.updateVm(body); updateDialog.value.encryptedEnvs = []; @@ -1149,13 +1229,13 @@ type CreateVmPayloadSource = { key_provider_id: theVm.appCompose?.key_provider_id || '', gateway_enabled: !!theVm.appCompose?.gateway_enabled, gateway_urls: config.gateway_urls || [], + networks: cloneNetworks(config), public_logs: !!theVm.appCompose?.public_logs, public_sysinfo: !!theVm.appCompose?.public_sysinfo, public_tcbinfo: !!theVm.appCompose?.public_tcbinfo, pin_numa: !!config.pin_numa, hugepages: !!config.hugepages, no_tee: !!config.no_tee, - net_mode: config.networking?.mode || '', user_config: config.user_config || '', stopped: !!config.stopped, }; @@ -1185,6 +1265,7 @@ type CreateVmPayloadSource = { hugepages: source.hugepages, pin_numa: source.pin_numa, no_tee: source.no_tee, + networks: source.networks, gpus: source.gpus, kms_urls: source.kms_urls, gateway_urls: source.gateway_urls, @@ -1653,6 +1734,7 @@ type CreateVmPayloadSource = { onMounted(() => { watchVmList(); + loadMeta(); loadImages(); loadGpus(); loadVersion(); @@ -1683,6 +1765,9 @@ type CreateVmPayloadSource = { cloneConfigDialog, showCreateDialog, config, + networkingModes, + defaultBridge, + defaultNetworkingLabel, composeHashPreview, updateComposeHashPreview, showDeployDialog, @@ -1717,6 +1802,7 @@ type CreateVmPayloadSource = { downloadAppCompose, downloadUserConfig, getVmFeatures, + networkModeLabel, systemMenu, toggleSystemMenu, closeSystemMenu, diff --git a/dstack/vmm/ui/src/styles/main.css b/dstack/vmm/ui/src/styles/main.css index 2fd1d0100..7a7b0ee72 100644 --- a/dstack/vmm/ui/src/styles/main.css +++ b/dstack/vmm/ui/src/styles/main.css @@ -587,31 +587,44 @@ h1, h2, h3, h4, h5, h6 { border-style: dashed; } -.port-mappings { +.port-mappings, +.runtime-network-section { display: flex; flex-direction: column; gap: 12px; } -.port-mappings h4 { +.port-mappings h4, +.runtime-network-section h4 { font-size: 14px; font-weight: 600; color: var(--color-text-primary); margin-bottom: 4px; } -.port-item { +.port-item, +.runtime-network-item { padding: 12px 16px; background: var(--color-bg-primary); border: 1px solid var(--color-border); border-radius: var(--radius-md); - display: flex; - justify-content: space-between; align-items: center; font-size: 14px; font-family: 'SF Mono', 'Monaco', 'Cascadia Code', monospace; } +.port-item { + display: flex; + justify-content: space-between; +} + +.runtime-network-item { + display: grid; + grid-template-columns: 1.1fr 0.6fr 0.8fr 1.4fr; + gap: 12px; + overflow-wrap: anywhere; +} + .features-section { margin-top: 24px; } @@ -1385,17 +1398,43 @@ h1, h2, h3, h4, h5, h6 { .encrypted-env-editor, .port-mapping-editor, -.gpu-config-editor { +.gpu-config-editor, +.network-config-editor { display: flex; flex-direction: column; gap: 16px; } -.port-mapping-editor > button { +.port-mapping-editor > button, +.network-config-editor > button { width: fit-content; min-width: 120px; } +.network-config-empty { + padding: 12px 16px; + background: var(--color-bg-secondary); + border: 1px dashed var(--color-border); + border-radius: var(--radius-md); + color: var(--color-text-secondary); + font-size: 14px; +} + +.network-config-row { + display: grid; + grid-template-columns: 160px minmax(180px, 1fr) 100px; + gap: 12px; + align-items: center; +} + +.network-config-placeholder { + min-height: 1px; +} + +.network-config-hint { + grid-column: 1 / -1; +} + .gpu-section-label { font-size: 14px; font-weight: 600; diff --git a/dstack/vmm/ui/src/templates/app.html b/dstack/vmm/ui/src/templates/app.html index 7b6a63bdc..7e38806c7 100644 --- a/dstack/vmm/ui/src/templates/app.html +++ b/dstack/vmm/ui/src/templates/app.html @@ -85,6 +85,9 @@

dstack-vmm

:allow-attach-all-gpus="allowAttachAllGpus" :kms-available="vmForm.kms_enabled" :port-mapping-enabled="config.portMappingEnabled" + :networking-modes="networkingModes" + :default-bridge="defaultBridge" + :default-networking-label="defaultNetworkingLabel" @close="showCreateDialog = false" @submit="createVm" @load-compose="loadComposeFile" @@ -97,6 +100,9 @@

dstack-vmm

:available-gpus="availableGpus" :allow-attach-all-gpus="allowAttachAllGpus" :port-mapping-enabled="config.portMappingEnabled" + :networking-modes="networkingModes" + :default-bridge="defaultBridge" + :default-networking-label="defaultNetworkingLabel" :kms-enabled="kmsEnabled(updateDialog.vm || {})" :compose-hash-preview="updateComposeHashPreview" @close="updateDialog.show = false" @@ -344,6 +350,16 @@

Port Mappings

+
+

VMM Network Interfaces

+
+ {{ networkModeLabel(iface.mode) }} / {{ iface.backend || '-' }} + {{ iface.netdev_id || '-' }} + {{ iface.bridge_name || '-' }} + {{ iface.mac || '-' }} +
+
+

Features

{{ getVmFeatures(vm) }}