diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 0ce99b437..dd4b527d0 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -32,12 +32,17 @@ use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; pub use image::{Image, ImageInfo}; -pub use qemu::{VmConfig, VmWorkDir}; +pub use qemu::VmConfig; +pub use workdir::VmWorkDir; +mod host_share; mod id_pool; mod image; +mod mr_config; mod qemu; pub(crate) mod registry; +mod vm_info; +mod workdir; #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { @@ -512,7 +517,7 @@ impl App { /// Handle a DHCP lease notification: look up VM by MAC address, persist /// the guest IP, and reconfigure port forwarding. pub async fn report_dhcp_lease(&self, mac: &str, ip: &str) { - use crate::app::qemu::mac_address_for_vm; + use crate::app::mr_config::mac_address_for_vm; let vm_id = { let mut state = self.lock(); diff --git a/dstack/vmm/src/app/host_share.rs b/dstack/vmm/src/app/host_share.rs new file mode 100644 index 000000000..df5b1b536 --- /dev/null +++ b/dstack/vmm/src/app/host_share.rs @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Host-to-guest shared-file disk creation. + +use std::io::{Seek, SeekFrom, Write}; +use std::path::Path; + +use anyhow::{Context, Result}; +use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; +use fatfs::{FileSystem, FormatVolumeOptions, FsOptions}; +use fs_err as fs; + +/// Creates a FAT32 disk image containing the files in `shared_dir`. +pub(super) fn create_shared_disk( + disk_path: impl AsRef, + shared_dir: impl AsRef, +) -> Result<()> { + let disk_path = disk_path.as_ref(); + let shared_dir = shared_dir.as_ref(); + + // Must be large enough to hold all host-shared files (app-compose.json and + // .user-config can each be up to 50 MiB, see HostShared::copy) plus FAT32 overhead. + const DISK_SIZE: u64 = 128 * 1024 * 1024; + + // Back the image by a file (sparse until written) and stream files into it so + // peak memory stays bounded regardless of DISK_SIZE or input file sizes. + let mut image = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(disk_path) + .with_context(|| format!("failed to create disk image at {}", disk_path.display()))?; + image + .set_len(DISK_SIZE) + .context("failed to size disk image")?; + + { + let mut label_bytes = [b' '; 11]; + let label = HOST_SHARED_DISK_LABEL.as_bytes(); + let copy_len = label.len().min(label_bytes.len()); + label_bytes[..copy_len].copy_from_slice(&label[..copy_len]); + let format_opts = FormatVolumeOptions::new() + .fat_type(fatfs::FatType::Fat32) + .volume_label(label_bytes); + fatfs::format_volume(&mut image, format_opts).context("failed to format disk as FAT32")?; + } + + image + .seek(SeekFrom::Start(0)) + .context("failed to seek to start")?; + let filesystem = + FileSystem::new(&mut image, FsOptions::new()).context("failed to open FAT32 filesystem")?; + let root_dir = filesystem.root_dir(); + + for entry in fs::read_dir(shared_dir).context("failed to read shared directory")? { + let entry = entry.context("failed to read directory entry")?; + let path = entry.path(); + if !path.is_file() { + continue; + } + + let filename = entry.file_name(); + let filename = filename.to_string_lossy(); + let mut source = fs::File::open(&path) + .with_context(|| format!("failed to open file {}", path.display()))?; + let mut destination = root_dir + .create_file(&filename) + .with_context(|| format!("failed to create file {filename} in FAT32"))?; + std::io::copy(&mut source, &mut destination) + .with_context(|| format!("failed to write file {filename} to FAT32"))?; + destination.flush().context("failed to flush FAT32 file")?; + } + + Ok(()) +} diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs new file mode 100644 index 000000000..71c8a427f --- /dev/null +++ b/dstack/vmm/src/app/mr_config.rs @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Application identity and VM measurement-configuration derivation. + +use anyhow::{bail, Context, Result}; +use base64::prelude::*; +use dstack_types::mr_config::{MrConfig, MrConfigV3}; +use dstack_types::AppCompose; +use fs_err as fs; +use sha2::{Digest, Sha256}; + +use super::VmWorkDir; + +/// Derives a deterministic, locally administered unicast MAC address from a VM ID. +/// +/// `prefix` may contain zero to three fixed bytes. Remaining bytes are filled +/// from SHA-256 of `vm_id`. +pub(super) fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { + let hash = Sha256::digest(vm_id.as_bytes()); + let prefix_len = prefix.len().min(3); + let mut bytes = [0_u8; 6]; + bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); + for index in prefix_len..bytes.len() { + bytes[index] = hash[index - prefix_len]; + } + bytes[0] = (bytes[0] & 0xfe) | 0x02; + format!( + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] + ) +} + +pub(super) fn tdx_mr_config_id(workdir: &VmWorkDir, app_compose: &AppCompose) -> Result { + if let Some(document) = workdir + .sys_config() + .context("failed to read sys config for tdx mrconfigid")? + .mr_config + { + MrConfigV3::from_document(&document).context("invalid mr_config document")?; + return Ok(BASE64_STANDARD.encode(MrConfigV3::tdx_mr_config_id_from_document(&document))); + } + + let compose_hash = workdir + .app_compose_hash() + .context("failed to get compose hash")?; + let mr_config = if app_compose.key_provider_id.is_empty() { + MrConfig::V1 { + compose_hash: &compose_hash, + } + } else { + let instance_info = workdir + .instance_info() + .context("failed to get instance info")?; + let app_id = if instance_info.app_id.is_empty() { + &compose_hash[..20] + } else { + &instance_info.app_id + }; + MrConfig::V2 { + compose_hash: &compose_hash, + app_id: &app_id.try_into().context("invalid app ID")?, + key_provider: app_compose.key_provider(), + key_provider_id: &app_compose.key_provider_id, + } + }; + Ok(BASE64_STANDARD.encode(mr_config.to_mr_config_id())) +} + +pub(super) fn snp_host_data(workdir: &VmWorkDir) -> Result { + let document = workdir + .sys_config() + .context("failed to read sys config for amd sev-snp host-data")? + .mr_config + .context("mr_config is required for amd sev-snp host-data")?; + MrConfigV3::from_document(&document).context("invalid mr_config document")?; + Ok(BASE64_STANDARD.encode(MrConfigV3::snp_host_data_from_document(&document))) +} + +impl VmWorkDir { + pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose) -> Result { + let compose_hash = self + .app_compose_hash() + .context("failed to get compose hash")?; + let mut instance_info = self + .instance_info_or_default() + .context("failed to get instance info")?; + let app_id = if instance_info.app_id.is_empty() { + compose_hash[..20].to_vec() + } else { + instance_info.app_id.clone() + }; + if app_id.len() != 20 { + bail!( + "invalid app ID length: expected 20 bytes, got {}", + app_id.len() + ); + } + + let disk_reusable = !app_compose.key_provider().is_none(); + if !disk_reusable || instance_info.instance_id_seed.is_empty() { + instance_info.instance_id_seed = { + let mut seed = vec![0_u8; 20]; + getrandom::fill(&mut seed).context("failed to generate instance ID seed")?; + seed + }; + } + + let instance_id = if app_compose.no_instance_id { + Vec::new() + } else { + let mut id_path = instance_info.instance_id_seed.clone(); + id_path.extend_from_slice(&app_id); + Sha256::digest(id_path)[..20].to_vec() + }; + instance_info.app_id = app_id.clone(); + instance_info.instance_id = instance_id.clone(); + fs::write( + self.instance_info_path(), + serde_json::to_string(&instance_info).context("failed to serialize instance info")?, + ) + .context("failed to write instance info")?; + + Ok(MrConfigV3::new( + app_id, + compose_hash.to_vec(), + app_compose.key_provider(), + app_compose.key_provider_id.clone(), + instance_id, + ) + .to_canonical_json()) + } +} + +#[cfg(test)] +mod tests { + use super::mac_address_for_vm; + + #[test] + fn mac_address_is_deterministic_and_locally_administered() { + let mac = mac_address_for_vm("vm-1", &[0xaa, 0xbb, 0xcc]); + assert_eq!(&mac[..8], "aa:bb:cc"); + assert_eq!(mac, mac_address_for_vm("vm-1", &[0xaa, 0xbb, 0xcc])); + let first = u8::from_str_radix(&mac[..2], 16).unwrap(); + assert_eq!(first & 0x03, 0x02); + } +} diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 0749fc95b..31bf3a882 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -2,82 +2,34 @@ // // SPDX-License-Identifier: Apache-2.0 -//! QEMU related code +//! QEMU launch preparation and command construction. use crate::{ app::Manifest, - config::{ - CvmConfig, GatewayConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform, - }, + config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, }; +use std::collections::HashMap; use std::os::unix::fs::PermissionsExt; use std::{ fs::Permissions, io::Write, - ops::Deref, path::{Path, PathBuf}, process::{Command, Stdio}, - time::{Duration, SystemTime}, }; use super::{ - effective_vcpu_count, hugepage_numa_nodes, image::Image, pci_numa_node, round_up, GpuConfig, - VmState, + effective_vcpu_count, + host_share::create_shared_disk, + hugepage_numa_nodes, + image::Image, + mr_config::{mac_address_for_vm, snp_host_data, tdx_mr_config_id}, + pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use anyhow::{bail, Context, Result}; -use base64::prelude::*; use bon::Builder; -use dstack_types::{ - mr_config::{MrConfig, MrConfigV3}, - shared_filenames::{ - APP_COMPOSE, ENCRYPTED_ENV, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, - }, - AppCompose, KeyProviderKind, SysConfig, -}; -use dstack_vmm_rpc as pb; -use sha2::{Digest, Sha256}; - -/// Derive a deterministic MAC address from a VM ID using SHA256. -/// Sets locally-administered + unicast bits (0x02) per IEEE 802. -/// Derive a deterministic MAC address from a VM ID. -/// -/// `prefix` may contain 0-3 fixed bytes. The first byte always has the -/// locally-administered + unicast bits set (0x02). Remaining bytes are -/// filled from SHA256(vm_id). -pub fn mac_address_for_vm(vm_id: &str, prefix: &[u8]) -> String { - let hash = Sha256::digest(vm_id.as_bytes()); - let prefix_len = prefix.len().min(3); - let mut bytes = [0u8; 6]; - // Fill prefix bytes - bytes[..prefix_len].copy_from_slice(&prefix[..prefix_len]); - // Fill remaining bytes from hash - for i in prefix_len..6 { - bytes[i] = hash[i - prefix_len]; - } - // Ensure locally-administered + unicast on first byte - bytes[0] = (bytes[0] & 0xfe) | 0x02; - format!( - "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", - bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5] - ) -} +use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; use fs_err as fs; -use serde::{Deserialize, Serialize}; -use serde_human_bytes as hex_bytes; -use supervisor_client::supervisor::{ProcessConfig, ProcessInfo}; - -fn networking_to_proto(n: &Networking) -> pb::NetworkingConfig { - let mode = match n.mode { - NetworkingMode::Bridge => "bridge", - NetworkingMode::User => "user", - - NetworkingMode::Custom => "custom", - }; - pb::NetworkingConfig { mode: mode.into() } -} - -fn sanitize_optional>(value: Option) -> Option { - value.filter(|value| !value.as_ref().trim().is_empty()) -} +use serde::Serialize; +use supervisor_client::supervisor::ProcessConfig; #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct AmdSevSnpLaunchParams { @@ -177,31 +129,6 @@ fn detect_amd_sev_snp_qemu_capabilities(qemu_path: &Path) -> Result, - #[serde(default, with = "hex_bytes")] - pub instance_id: Vec, - #[serde(default, with = "hex_bytes")] - pub app_id: Vec, -} - -pub struct VmInfo { - pub manifest: Manifest, - pub workdir: PathBuf, - pub status: &'static str, - pub uptime: String, - pub exited_at: Option, - pub instance_id: Option, - pub boot_progress: String, - pub boot_error: String, - pub shutdown_progress: String, - pub image_version: String, - pub gateway_enabled: bool, - pub events: Vec, -} - #[derive(Debug, Builder)] pub struct VmConfig { pub manifest: Manifest, @@ -211,11 +138,6 @@ pub struct VmConfig { pub gateway_enabled: bool, } -#[derive(Deserialize, Serialize)] -pub struct State { - started: bool, -} - fn create_hd( image_file: impl AsRef, backing_file: Option>, @@ -241,308 +163,156 @@ fn create_hd( Ok(()) } -/// Create a FAT32 disk image from a directory -fn create_shared_disk(disk_path: impl AsRef, shared_dir: impl AsRef) -> Result<()> { - use fatfs::{FileSystem, FormatVolumeOptions, FsOptions}; - use std::io::{Seek, SeekFrom, Write}; - - let disk_path = disk_path.as_ref(); - let shared_dir = shared_dir.as_ref(); - - // Must be large enough to hold all host-shared files (app-compose.json and - // .user-config can each be up to 50 MiB, see HostShared::copy) plus FAT32 overhead. - const DISK_SIZE: u64 = 128 * 1024 * 1024; - - // Back the image by a file (sparse until written) and stream files into it so - // peak memory stays bounded regardless of DISK_SIZE or input file sizes. - let mut image = fs::OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(disk_path) - .with_context(|| format!("Failed to create disk image at {}", disk_path.display()))?; - image - .set_len(DISK_SIZE) - .context("Failed to size disk image")?; - - { - let mut label_bytes = [b' '; 11]; - let label_str = HOST_SHARED_DISK_LABEL.as_bytes(); - let copy_len = label_str.len().min(11); - label_bytes[..copy_len].copy_from_slice(&label_str[..copy_len]); - let format_opts = FormatVolumeOptions::new() - .fat_type(fatfs::FatType::Fat32) - .volume_label(label_bytes); - fatfs::format_volume(&mut image, format_opts).context("Failed to format disk as FAT32")?; - } - - // Open the formatted filesystem and stream files into it - { - image - .seek(SeekFrom::Start(0)) - .context("Failed to seek to start")?; - let fs = FileSystem::new(&mut image, FsOptions::new()) - .context("Failed to open FAT32 filesystem")?; - let root_dir = fs.root_dir(); - - // Copy all files from shared_dir to the FAT32 root - for entry in fs::read_dir(shared_dir).context("Failed to read shared directory")? { - let entry = entry.context("Failed to read directory entry")?; - let path = entry.path(); - - if path.is_file() { - let filename = entry.file_name(); - let filename_str = filename.to_string_lossy(); - - let mut src = fs::File::open(&path) - .with_context(|| format!("Failed to open file {}", path.display()))?; - let mut fat_file = root_dir - .create_file(&filename_str) - .with_context(|| format!("Failed to create file {filename_str} in FAT32"))?; - std::io::copy(&mut src, &mut fat_file) - .with_context(|| format!("Failed to write file {filename_str} to FAT32"))?; - fat_file.flush().context("Failed to flush FAT32 file")?; - } - } +fn virtio_pci_device(device: &str, snp: bool) -> String { + if snp { + format!("{device},disable-legacy=on,iommu_platform=true") + } else { + device.to_string() } - - Ok(()) } -impl VmInfo { - pub fn to_pb(&self, gw: &GatewayConfig, brief: bool) -> pb::VmInfo { - let workdir = VmWorkDir::new(&self.workdir); - let vm_config = workdir.manifest(); - let custom_gateway_urls = vm_config - .as_ref() - .map(|c| c.gateway_urls.clone()) - .unwrap_or_default(); - pb::VmInfo { - id: self.manifest.id.clone(), - name: self.manifest.name.clone(), - status: self.status.into(), - uptime: self.uptime.clone(), - boot_progress: self.boot_progress.clone(), - boot_error: self.boot_error.clone(), - shutdown_progress: self.shutdown_progress.clone(), - image_version: self.image_version.clone(), - configuration: if brief { - None - } else { - let kms_urls = vm_config - .as_ref() - .map(|c| c.kms_urls.clone()) - .unwrap_or_default(); - let no_tee = vm_config - .as_ref() - .map(|c| c.no_tee) - .unwrap_or(self.manifest.no_tee); - let stopped = !workdir.started().unwrap_or(false); - - Some(pb::VmConfiguration { - name: self.manifest.name.clone(), - image: self.manifest.image.clone(), - compose_file: { - fs::read_to_string(workdir.app_compose_path()).unwrap_or_default() - }, - encrypted_env: { fs::read(workdir.encrypted_env_path()).unwrap_or_default() }, - user_config: { - fs::read_to_string(workdir.user_config_path()).unwrap_or_default() - }, - vcpu: self.manifest.vcpu, - memory: self.manifest.memory, - disk_size: self.manifest.disk_size, - ports: self - .manifest - .port_map - .iter() - .map(|pm| pb::PortMapping { - protocol: pm.protocol.as_str().into(), - host_address: pm.address.to_string(), - host_port: pm.from as u32, - vm_port: pm.to as u32, - }) - .collect(), - app_id: Some(self.manifest.app_id.clone()), - hugepages: self.manifest.hugepages, - pin_numa: self.manifest.pin_numa, - gpus: self.manifest.gpus.as_ref().map(|g| pb::GpuConfig { - attach_mode: g.attach_mode.to_string(), - gpus: g - .gpus - .iter() - .map(|gpu| pb::GpuSpec { - slot: gpu.slot.clone(), - }) - .collect(), - }), - kms_urls, - gateway_urls: custom_gateway_urls.clone(), - stopped, - no_tee, - networking: self.manifest.networking.as_ref().map(networking_to_proto), - }) - }, - app_url: self - .gateway_enabled - .then_some(self.instance_id.as_deref()) - .flatten() - .and_then(|id| sanitize_optional(Some(id))) - .map(|id| { - // Use custom gateway URL if available, otherwise fall back to global config - if let Some(custom_gw_url) = custom_gateway_urls.first() { - if let Ok(url) = url::Url::parse(custom_gw_url) { - let host = url.host_str().unwrap_or(&gw.base_domain); - let port = url.port().unwrap_or(443); - if port == 443 { - return format!("https://{id}-{}.{}", gw.agent_port, host); - } else { - return format!("https://{id}-{}.{}:{}", gw.agent_port, host, port); - } - } - } - // Fall back to global gateway config - if gw.port == 443 { - format!("https://{id}-{}.{}", gw.agent_port, gw.base_domain) - } else { - format!( - "https://{id}-{}.{}:{}", - gw.agent_port, gw.base_domain, gw.port - ) - } - }), - app_id: self.manifest.app_id.clone(), - instance_id: sanitize_optional(self.instance_id.clone()), - exited_at: self.exited_at.clone(), - events: self.events.clone(), - } - } +struct PreparedQemuLaunch { + workdir: VmWorkDir, + platform: TeePlatform, + hugepage_numa_nodes: Option>, + gpu_numa_nodes: HashMap, + numa_cpus: Option, + tpm_path: Option<&'static str>, + tdx_mr_config_id: Option, + snp_host_data: Option, + snp_launch_params: Option, } -impl VmState { - pub fn merged_info(&self, proc_state: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo { - fn truncate(d: Duration) -> Duration { - Duration::from_secs(d.as_secs()) - } - let is_running = match proc_state { - Some(info) => info.state.status.is_running(), - None => false, +impl PreparedQemuLaunch { + fn prepare( + vm: &VmConfig, + workdir: impl AsRef, + cfg: &CvmConfig, + gpus: &GpuConfig, + ) -> Result { + let workdir = VmWorkDir::new(workdir); + prepare_data_disk(vm, &workdir, cfg)?; + prepare_shared_dir(&workdir)?; + let app_compose = workdir.app_compose().context("failed to get app compose")?; + let platform = cfg.resolved_platform(); + + let hugepage_numa_nodes = if vm.manifest.hugepages { + Some(hugepage_numa_nodes(gpus)?) + } else { + None }; - let started = workdir.started().unwrap_or(false); - let status = if self.state.removing { - "removing" + let gpu_numa_nodes = if vm.manifest.hugepages { + gpus.gpus + .iter() + .map(|gpu| Ok((gpu.slot.clone(), pci_numa_node(&gpu.slot)?))) + .collect::>()? } else { - match (started, is_running) { - (true, true) => "running", - (true, false) => "exited", - (false, true) => "stopping", - (false, false) => "stopped", - } + HashMap::new() }; + let numa_cpus = if vm.manifest.pin_numa { + let device = gpus.gpus.first().map(|gpu| gpu.slot.clone()); + Some(find_numa(device)?.1) + } else { + None + }; + let tpm_path = if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { + Some(detect_tpm_device()?) + } else { + None + }; + prepare_shared_disk(&workdir, cfg)?; + + let tee_enabled = !vm.manifest.no_tee; + let tdx_mr_config_id = if tee_enabled + && platform == TeePlatform::Tdx + && cfg.use_mrconfigid + && vm.image.info.version_tuple().unwrap_or_default() >= (0, 5, 2) + { + Some(tdx_mr_config_id(&workdir, &app_compose)?) + } else { + None + }; + let (snp_host_data, snp_launch_params) = + if tee_enabled && platform == TeePlatform::AmdSevSnp { + ( + Some(snp_host_data(&workdir)?), + Some( + detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path).context( + "failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU", + )?, + ), + ) + } else { + (None, None) + }; - fn display_ts(t: Option<&SystemTime>) -> String { - match t { - None => "never".into(), - Some(t) => { - let ts = t.elapsed().unwrap_or(Duration::MAX); - humantime::format_duration(truncate(ts)).to_string() - } - } - } - let uptime = display_ts(proc_state.and_then(|info| info.state.started_at.as_ref())); - let exited_at = display_ts(proc_state.and_then(|info| info.state.stopped_at.as_ref())); - let instance_id = sanitize_optional( - workdir - .instance_info() - .ok() - .map(|info| hex::encode(info.instance_id)), - ); - VmInfo { - manifest: self.config.manifest.clone(), - workdir: workdir.path().to_path_buf(), - instance_id, - status, - uptime, - exited_at: Some(exited_at), - boot_progress: self.state.boot_progress.clone(), - boot_error: self.state.boot_error.clone(), - shutdown_progress: self.state.shutdown_progress.clone(), - image_version: self.config.image.info.version.clone(), - gateway_enabled: self.config.gateway_enabled, - events: self.state.events.clone().into(), - } + Ok(Self { + workdir, + platform, + hugepage_numa_nodes, + gpu_numa_nodes, + numa_cpus, + tpm_path, + tdx_mr_config_id, + snp_host_data, + snp_launch_params, + }) } } -#[cfg(test)] -mod tests { - use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, sanitize_optional, - virtio_pci_device, - }; - - #[test] - fn sanitize_optional_filters_empty_owned_values() { - assert_eq!(sanitize_optional(Some(String::new())), None); - assert_eq!(sanitize_optional(Some(" ".to_string())), None); - assert_eq!( - sanitize_optional(Some("instance-123".to_string())), - Some("instance-123".to_string()) - ); +fn prepare_data_disk(vm: &VmConfig, workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { + let hda_path = workdir.hda_path(); + if !hda_path.exists() { + create_hd( + &hda_path, + vm.image.hda.as_ref(), + &format!("{}G", vm.manifest.disk_size), + )?; } - - #[test] - fn sanitize_optional_filters_empty_borrowed_values() { - assert_eq!(sanitize_optional(Some("")), None); - assert_eq!(sanitize_optional(Some(" ")), None); - assert_eq!( - sanitize_optional(Some("instance-123")), - Some("instance-123") - ); + if !cfg.user.is_empty() { + fs::set_permissions(&hda_path, Permissions::from_mode(0o660))?; } + Ok(()) +} - #[test] - fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { - assert_eq!( - amd_sev_snp_memory_backend_arg(4096), - "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" - ); +fn prepare_shared_dir(workdir: &VmWorkDir) -> Result<()> { + let shared_dir = workdir.shared_dir(); + if !shared_dir.exists() { + fs::create_dir_all(&shared_dir)?; } + Ok(()) +} - #[test] - fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { - let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} -{"return":{}} -{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} -{"return":{}} -"#; - let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); - assert_eq!(params.cbitpos, 51); - assert_eq!(params.reduced_phys_bits, 1); +fn prepare_shared_disk(workdir: &VmWorkDir, cfg: &CvmConfig) -> Result<()> { + if cfg.host_share_mode != "vhd" { + return Ok(()); } - #[test] - fn amd_sev_snp_uses_confidential_virtio_pci_options() { - assert_eq!( - virtio_pci_device("virtio-blk-pci,drive=hd0", true), - "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" - ); - assert_eq!( - virtio_pci_device("virtio-blk-pci,drive=hd0", false), - "virtio-blk-pci,drive=hd0" - ); + let shared_dir = workdir.shared_dir(); + let shared_disk_path = workdir.shared_disk_path(); + if shared_disk_path.exists() { + fs::remove_file(&shared_disk_path).context("failed to remove shared disk")?; } + create_shared_disk(&shared_disk_path, shared_dir).context("failed to create shared disk") } -fn virtio_pci_device(device: &str, snp: bool) -> String { - if snp { - format!("{device},disable-legacy=on,iommu_platform=true") +fn detect_tpm_device() -> Result<&'static str> { + if Path::new("/dev/tpmrm0").exists() { + Ok("/dev/tpmrm0") + } else if Path::new("/dev/tpm0").exists() { + Ok("/dev/tpm0") } else { - device.to_string() + bail!("tpm key provider requested but no TPM device found on host") } } +struct QemuCommandBuilder<'a> { + vm: &'a VmConfig, + cfg: &'a CvmConfig, + gpus: &'a GpuConfig, + prepared: &'a PreparedQemuLaunch, +} + impl VmConfig { pub fn config_qemu( &self, @@ -550,142 +320,164 @@ impl VmConfig { cfg: &CvmConfig, gpus: &GpuConfig, ) -> Result> { - let workdir = VmWorkDir::new(workdir); - let serial_file = workdir.serial_file(); - let serial_pty = workdir.serial_pty(); - let shared_dir = workdir.shared_dir(); - let disk_size = format!("{}G", self.manifest.disk_size); - let hda_path = workdir.hda_path(); - if !hda_path.exists() { - create_hd(&hda_path, self.image.hda.as_ref(), &disk_size)?; - } - if !cfg.user.is_empty() { - fs_err::set_permissions(&hda_path, Permissions::from_mode(0o660))?; + let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; + let process = QemuCommandBuilder { + vm: self, + cfg, + gpus, + prepared: &prepared, } + .build()?; + Ok(vec![process]) + } +} + +impl QemuCommandBuilder<'_> { + fn build(&self) -> Result { + let mut command = self.base_command(); + self.configure_rootfs(&mut command)?; + self.configure_data_disk(&mut command); + self.configure_networking(&mut command); + self.vm.configure_smbios(&mut command, self.cfg); + self.configure_tpm_and_vsock(&mut command); + self.configure_host_share(&mut command)?; + + let (smp, mem) = self.configure_hugepage_memory(&mut command)?; + self.vm + .configure_machine(&mut command, self.cfg, self.prepared, mem)?; + self.configure_gpus(&mut command)?; + command.arg("-smp").arg(smp.to_string()); + command.arg("-m").arg(format!("{mem}M")); - if !shared_dir.exists() { - fs::create_dir_all(&shared_dir)?; + // SNP app identity is bound through HOST_DATA, so the measured cmdline + // remains the image-provided cmdline. + if let Some(cmdline) = &self.vm.image.info.cmdline { + command.arg("-append").arg(cmdline); } - let app_compose = workdir.app_compose().context("Failed to get app compose")?; - let qemu = &cfg.qemu_path; - let is_amd_sev_snp = - cfg.resolved_platform() == TeePlatform::AmdSevSnp && !self.manifest.no_tee; - let mut numa_nodes_for_hugepages = if self.manifest.hugepages { - Some(hugepage_numa_nodes(gpus)?) - } else { - None - }; - let smp = effective_vcpu_count( - self.manifest.vcpu, - numa_nodes_for_hugepages - .as_ref() - .map(|nodes| nodes.len() as u32), - ); - let mut mem = self.manifest.memory; - let mut command = Command::new(qemu); + self.process_config(command) + } + + fn is_amd_sev_snp(&self) -> bool { + self.prepared.platform == TeePlatform::AmdSevSnp && !self.vm.manifest.no_tee + } + + fn base_command(&self) -> Command { + let workdir = &self.prepared.workdir; + let mut command = Command::new(&self.cfg.qemu_path); command.arg("-accel").arg("kvm"); - let cpu = if is_amd_sev_snp { "EPYC-v4" } else { "host" }; - command.arg("-cpu").arg(cpu); + command.arg("-cpu").arg(if self.is_amd_sev_snp() { + "EPYC-v4" + } else { + "host" + }); command.arg("-nographic"); command.arg("-nodefaults"); command.arg("-chardev").arg(format!( "pty,id=com0,path={},logfile={}", - serial_pty.display(), - serial_file.display() + workdir.serial_pty().display(), + workdir.serial_file().display() )); command.arg("-serial").arg("chardev:com0"); - if cfg.qmp_socket { + if self.cfg.qmp_socket { command.arg("-qmp").arg(format!( "unix:{},server,wait=off", workdir.qmp_socket().display() )); } - if let Some(bios) = self.image.firmware(is_amd_sev_snp) { + if let Some(bios) = self.vm.image.firmware(self.is_amd_sev_snp()) { command.arg("-bios").arg(bios); } - command.arg("-kernel").arg(&self.image.kernel); - command.arg("-initrd").arg(&self.image.initrd); - if cfg.qemu_hotplug_off { + command.arg("-kernel").arg(&self.vm.image.kernel); + command.arg("-initrd").arg(&self.vm.image.initrd); + if self.cfg.qemu_hotplug_off { command.args([ "-global", "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off", ]); } - if cfg.qemu_pci_hole64_size > 0 { + if self.cfg.qemu_pci_hole64_size > 0 { command.args([ "-global", &format!( "q35-pcihost.pci-hole64-size=0x{:x}", - cfg.qemu_pci_hole64_size + self.cfg.qemu_pci_hole64_size ), ]); } - if let Some(rootfs) = &self.image.rootfs { - let img_ver = self.image.info.version_tuple().unwrap_or_default(); - let ext = rootfs - .extension() - .unwrap_or_default() - .to_str() - .unwrap_or_default(); - match ext { - "iso" => { - if img_ver >= (0, 5, 0) { - bail!( - "Unsupported rootfs type: {ext}. Image versions >= 0.5.0 must use verity rootfs" - ); - } - command.arg("-cdrom").arg(rootfs); - } - "verity" => { - command.arg("-drive").arg(format!( - "file={},if=none,id=hd0,format=raw,readonly=on", - rootfs.display() - )); - command.arg("-device").arg(virtio_pci_device( - "virtio-blk-pci,drive=hd0", - is_amd_sev_snp, - )); - } - _ => { - bail!("Unsupported rootfs type: {ext}"); + command + } + + fn configure_rootfs(&self, command: &mut Command) -> Result<()> { + let Some(rootfs) = &self.vm.image.rootfs else { + return Ok(()); + }; + let image_version = self.vm.image.info.version_tuple().unwrap_or_default(); + let extension = rootfs + .extension() + .unwrap_or_default() + .to_str() + .unwrap_or_default(); + match extension { + "iso" => { + if image_version >= (0, 5, 0) { + bail!( + "Unsupported rootfs type: {extension}. Image versions >= 0.5.0 must use verity rootfs" + ); } + command.arg("-cdrom").arg(rootfs); } + "verity" => { + command.arg("-drive").arg(format!( + "file={},if=none,id=hd0,format=raw,readonly=on", + rootfs.display() + )); + command.arg("-device").arg(virtio_pci_device( + "virtio-blk-pci,drive=hd0", + self.is_amd_sev_snp(), + )); + } + _ => bail!("Unsupported rootfs type: {extension}"), } - let mut processes = vec![]; + Ok(()) + } + + fn configure_data_disk(&self, command: &mut Command) { command .arg("-drive") - .arg(format!("file={},if=none,id=hd1", hda_path.display())) + .arg(format!( + "file={},if=none,id=hd1", + self.prepared.workdir.hda_path().display() + )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=hd1", - is_amd_sev_snp, + self.is_amd_sev_snp(), )); - // Resolve per-VM networking override against global config. - // Per-VM only sets mode; shared fields (bridge name, mac_prefix, etc.) - // are merged from global config. + } + + fn configure_networking(&self, command: &mut Command) { + // Per-VM networking only overrides the mode and bridge name. All other + // fields continue to come from the global configuration. let resolved_networking; - let networking = match self.manifest.networking.as_ref() { - Some(vm_net) => { - // Per-VM override: take mode from VM, fill other fields from global + let networking = match self.vm.manifest.networking.as_ref() { + Some(vm_networking) => { resolved_networking = Networking { - mode: vm_net.mode, - bridge: if vm_net.bridge.is_empty() { - cfg.networking.bridge.clone() + mode: vm_networking.mode, + bridge: if vm_networking.bridge.is_empty() { + self.cfg.networking.bridge.clone() } else { - vm_net.bridge.clone() + vm_networking.bridge.clone() }, - ..cfg.networking.clone() + ..self.cfg.networking.clone() }; &resolved_networking } - None => &cfg.networking, + None => &self.cfg.networking, }; - // Generate deterministic MAC for all networking modes - let prefix = networking.mac_prefix_bytes(); - let mac = mac_address_for_vm(&self.manifest.id, &prefix); + let mac = mac_address_for_vm(&self.vm.manifest.id, &networking.mac_prefix_bytes()); let net_device = virtio_pci_device( &format!("virtio-net-pci,netdev=net0,mac={mac}"), - is_amd_sev_snp, + self.is_amd_sev_snp(), ); let netdev = match networking.mode { NetworkingMode::User => { @@ -695,13 +487,13 @@ impl VmConfig { networking.dhcp_start, if networking.restrict { "yes" } else { "no" } ); - for pm in &self.manifest.port_map { + for mapping in &self.vm.manifest.port_map { netdev.push_str(&format!( ",hostfwd={}:{}:{}-:{}", - pm.protocol.as_str(), - pm.address, - pm.from, - pm.to + mapping.protocol.as_str(), + mapping.address, + mapping.from, + mapping.to )); } netdev @@ -714,41 +506,34 @@ impl VmConfig { }; command.arg("-netdev").arg(netdev); command.arg("-device").arg(net_device); + } - self.configure_smbios(&mut command, cfg); - - if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { - let tpm_path = if Path::new("/dev/tpmrm0").exists() { - "/dev/tpmrm0" - } else if Path::new("/dev/tpm0").exists() { - "/dev/tpm0" - } else { - bail!("TPM key provider requested but no TPM device found on host"); - }; + fn configure_tpm_and_vsock(&self, command: &mut Command) { + if let Some(tpm_path) = self.prepared.tpm_path { command .arg("-tpmdev") .arg(format!("passthrough,id=tpm0,path={tpm_path}")) .arg("-device") .arg("tpm-tis,tpmdev=tpm0"); } - command.arg("-device").arg(virtio_pci_device( - &format!("vhost-vsock-pci,guest-cid={}", self.cid), - is_amd_sev_snp, + &format!("vhost-vsock-pci,guest-cid={}", self.vm.cid), + self.is_amd_sev_snp(), )); + } - // Configure shared files delivery: either via disk or 9p - match cfg.host_share_mode.as_str() { + fn configure_host_share(&self, command: &mut Command) -> Result<()> { + let workdir = &self.prepared.workdir; + match self.cfg.host_share_mode.as_str() { "9p" => { - // Use 9p virtfs (default) - let ro = if self.image.info.shared_ro { + let read_only = if self.vm.image.info.shared_ro { "on" } else { "off" }; command.arg("-virtfs").arg(format!( - "local,path={},mount_tag=host-shared,readonly={ro},security_model=mapped,id=virtfs0", - shared_dir.display(), + "local,path={},mount_tag=host-shared,readonly={read_only},security_model=mapped,id=virtfs0", + workdir.shared_dir().display(), )); } "vvfat" => { @@ -756,214 +541,152 @@ impl VmConfig { .arg("-blockdev") .arg(format!( "driver=vvfat,node-name=vvfat0,read-only=on,dir={},label={}", - shared_dir.display(), + workdir.shared_dir().display(), HOST_SHARED_DISK_LABEL )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=vvfat0", - is_amd_sev_snp, + self.is_amd_sev_snp(), )); } "vhd" => { - // Use a second virtual disk (hd2) to share files - let shared_disk_path = workdir.shared_disk_path(); - if shared_disk_path.exists() { - fs::remove_file(&shared_disk_path).context("Failed to remove shared disk")?; - } - create_shared_disk(&shared_disk_path, &shared_dir) - .context("Failed to create shared disk")?; command .arg("-drive") .arg(format!( "file={},if=none,id=hd2,format=raw,readonly=on", - shared_disk_path.display() + workdir.shared_disk_path().display() )) .arg("-device") .arg(virtio_pci_device( "virtio-blk-pci,drive=hd2", - is_amd_sev_snp, + self.is_amd_sev_snp(), )); } - _ => { - bail!("Invalid host sharing mode: {}", cfg.host_share_mode); - } + _ => bail!("Invalid host sharing mode: {}", self.cfg.host_share_mode), } + Ok(()) + } - let hugepages = self.manifest.hugepages; - let pin_numa = self.manifest.pin_numa; - // Handle GPU configuration - let mut dev_num = 1; - let memory = self.manifest.memory; - - // Handle hugepages configuration - if hugepages { - let numa_nodes = numa_nodes_for_hugepages - .take() - .context("hugepage NUMA nodes should be computed above")?; - let n_numa = numa_nodes.len() as u32; - - // Round up CPU cores and memory to multiple times of NUMA nodes. - // `smp` is already the shared effective vCPU count used by vm_config. - let vcpu_count = smp; - let mem_gb = round_up(memory / 1024, n_numa); - let vcpu_per_node = vcpu_count / n_numa; - let mem_per_node = mem_gb / n_numa; - - mem = mem_gb * 1024; - - let mut bus_nr = 5_u32; - - // Configure NUMA nodes - for (ind, (node, count)) in numa_nodes.into_iter().enumerate() { - let ind = ind as u32; - let cpu_start = ind * vcpu_per_node; - let cpu_end = (ind + 1) * vcpu_per_node - 1; - command.arg("-numa").arg(format!( - "node,nodeid={ind},cpus={cpu_start}-{cpu_end},memdev=mem{ind}", - )); - - command.arg("-object").arg(format!( - "memory-backend-file,id=mem{ind},size={mem_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind", - )); - - let addr = 0xa + ind; - command.arg("-device").arg(format!( - "pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={addr},numa_node={ind},bus_nr={bus_nr}", - )); - bus_nr += count + 1; - } + fn configure_hugepage_memory(&self, command: &mut Command) -> Result<(u32, u32)> { + let numa_nodes = self.prepared.hugepage_numa_nodes.as_ref(); + let smp = effective_vcpu_count( + self.vm.manifest.vcpu, + numa_nodes.map(|nodes| nodes.len() as u32), + ); + if !self.vm.manifest.hugepages { + return Ok((smp, self.vm.manifest.memory)); + } + + let numa_nodes = numa_nodes + .context("hugepage NUMA nodes should be computed during launch preparation")?; + let numa_count = numa_nodes.len() as u32; + let memory_gib = round_up(self.vm.manifest.memory / 1024, numa_count); + let vcpus_per_node = smp / numa_count; + let memory_per_node = memory_gib / numa_count; + let mut bus_number = 5_u32; + for (index, (node, device_count)) in numa_nodes.iter().enumerate() { + let index = index as u32; + let cpu_start = index * vcpus_per_node; + let cpu_end = (index + 1) * vcpus_per_node - 1; + command.arg("-numa").arg(format!( + "node,nodeid={index},cpus={cpu_start}-{cpu_end},memdev=mem{index}", + )); + command.arg("-object").arg(format!( + "memory-backend-file,id=mem{index},size={memory_per_node}G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes={node},policy=bind", + )); + let address = 0xa + index; + command.arg("-device").arg(format!( + "pxb-pcie,id=pcie.node{node},bus=pcie.0,addr={address},numa_node={index},bus_nr={bus_number}", + )); + bus_number += device_count + 1; } + Ok((smp, memory_gib * 1024)) + } - self.configure_machine(&mut command, &workdir, cfg, &app_compose, mem)?; - - // Configure GPU devices - if !gpus.gpus.is_empty() { - // Add iommufd object - command.arg("-object").arg("iommufd,id=iommufd0"); - - if !hugepages { - // Add each GPU - for device in &gpus.gpus { - let slot = &device.slot; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", - )); - - dev_num += 1; - } - } else { - // Add each GPU with NUMA node awareness for hugepages configuration - for device in &gpus.gpus { - let slot = &device.slot; - let node = pci_numa_node(slot)?; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{dev_num},bus=pcie.node{node},chassis={dev_num}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", - )); - dev_num += 1; - } - } - - // Add bridges (NVSwitches) if any - if !gpus.bridges.is_empty() { - for bridge in &gpus.bridges { - let slot = &bridge.slot; - command.arg("-device").arg(format!( - "pcie-root-port,id=pci.{dev_num},bus=pcie.0,chassis={dev_num}", - )); - command.arg("-device").arg(format!( - "vfio-pci,host={slot},bus=pci.{dev_num},iommufd=iommufd0", - )); - dev_num += 1; - } - } + fn configure_gpus(&self, command: &mut Command) -> Result<()> { + if self.gpus.gpus.is_empty() { + return Ok(()); } - command.arg("-smp").arg(smp.to_string()); - command.arg("-m").arg(format!("{}M", mem)); - - // NUMA pinning if requested - let mut numa_cpus = None; - if pin_numa { - if !gpus.gpus.is_empty() { - let (_, cpus) = find_numa(Some(gpus.gpus[0].slot.clone()))?; - numa_cpus = Some(cpus); + command.arg("-object").arg("iommufd,id=iommufd0"); + let mut device_number = 1; + for device in &self.gpus.gpus { + let slot = &device.slot; + let bus = if self.vm.manifest.hugepages { + let node = self + .prepared + .gpu_numa_nodes + .get(slot) + .context("gpu NUMA node should be computed during launch preparation")?; + format!("pcie.node{node}") } else { - // Default to NUMA node 0 if no GPUs - let (_, cpus) = find_numa(None)?; - numa_cpus = Some(cpus); - } + "pcie.0".into() + }; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{device_number},bus={bus},chassis={device_number}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{device_number},iommufd=iommufd0", + )); + device_number += 1; } - - // SNP app identity is bound through HOST_DATA, so the measured cmdline - // remains the image-provided cmdline. - let cmdline = self.image.info.cmdline.clone(); - if let Some(cmdline) = cmdline { - command.arg("-append").arg(cmdline); + for bridge in &self.gpus.bridges { + let slot = &bridge.slot; + command.arg("-device").arg(format!( + "pcie-root-port,id=pci.{device_number},bus=pcie.0,chassis={device_number}", + )); + command.arg("-device").arg(format!( + "vfio-pci,host={slot},bus=pci.{device_number},iommufd=iommufd0", + )); + device_number += 1; } + Ok(()) + } - let args = command - .get_args() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - let pidfile_path = workdir.pid_file(); - let stdout_path = workdir.stdout_file(); - let stderr_path = workdir.stderr_file(); - - let workdir = workdir.path(); - - let mut cmd_args = vec![]; - cmd_args.push(qemu.to_string_lossy().to_string()); - cmd_args.extend(args); - - // If we have NUMA pinning, we'll need to wrap the command with taskset - if let Some(cpus) = numa_cpus { - cmd_args.splice(0..0, ["taskset", "-c", &cpus].into_iter().map(|s| s.into())); + fn process_config(&self, command: Command) -> Result { + let workdir = &self.prepared.workdir; + let mut arguments = vec![self.cfg.qemu_path.to_string_lossy().to_string()]; + arguments.extend( + command + .get_args() + .map(|argument| argument.to_string_lossy().to_string()), + ); + if let Some(cpus) = &self.prepared.numa_cpus { + arguments.splice(0..0, ["taskset", "-c", cpus].into_iter().map(String::from)); } - - if !cfg.user.is_empty() { - cmd_args.splice( + if !self.cfg.user.is_empty() { + arguments.splice( 0..0, - ["sudo", "-u", &cfg.user].into_iter().map(|s| s.into()), + ["sudo", "-u", &self.cfg.user].into_iter().map(String::from), ); } - let command = cmd_args.remove(0); - let note = ProcessAnnotation { + let command = arguments.remove(0); + let note = serde_json::to_string(&ProcessAnnotation { kind: "cvm".to_string(), live_for: None, - }; - let note = serde_json::to_string(¬e)?; - let process_config = ProcessConfig { - id: self.manifest.id.clone(), - args: cmd_args, - name: self.manifest.name.clone(), + })?; + Ok(ProcessConfig { + id: self.vm.manifest.id.clone(), + args: arguments, + name: self.vm.manifest.name.clone(), command, env: Default::default(), - cwd: workdir.to_string_lossy().to_string(), - stdout: stdout_path.to_string_lossy().to_string(), - stderr: stderr_path.to_string_lossy().to_string(), - pidfile: pidfile_path.to_string_lossy().to_string(), - cid: Some(self.cid), + cwd: workdir.path().to_string_lossy().to_string(), + stdout: workdir.stdout_file().to_string_lossy().to_string(), + stderr: workdir.stderr_file().to_string_lossy().to_string(), + pidfile: workdir.pid_file().to_string_lossy().to_string(), + cid: Some(self.vm.cid), note, - }; - processes.push(process_config); - - Ok(processes) + }) } - +} +impl VmConfig { fn configure_machine( &self, command: &mut Command, - workdir: &VmWorkDir, cfg: &CvmConfig, - app_compose: &AppCompose, + prepared: &PreparedQemuLaunch, mem: u32, ) -> Result<()> { if self.manifest.no_tee { @@ -973,15 +696,22 @@ impl VmConfig { return Ok(()); } - match cfg.resolved_platform() { + match prepared.platform { TeePlatform::Tdx => { command .arg("-machine") .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); - self.configure_tdx_guest(command, workdir, cfg, app_compose)?; + self.configure_tdx_guest(command, cfg, prepared.tdx_mr_config_id.as_deref())?; } TeePlatform::AmdSevSnp => { - self.configure_amd_sev_snp_guest(command, workdir, cfg, mem)?; + let host_data = prepared + .snp_host_data + .as_deref() + .context("snp host data should be computed during launch preparation")?; + let launch_params = prepared.snp_launch_params.context( + "snp launch parameters should be detected during launch preparation", + )?; + self.configure_amd_sev_snp_guest(command, cfg, mem, host_data, launch_params); } } Ok(()) @@ -990,60 +720,9 @@ impl VmConfig { fn configure_tdx_guest( &self, command: &mut Command, - workdir: &VmWorkDir, cfg: &CvmConfig, - app_compose: &AppCompose, + mrconfigid: Option<&str>, ) -> Result<()> { - let img_ver = self.image.info.version_tuple().unwrap_or_default(); - let support_mr_config_id = img_ver >= (0, 5, 2); - - // Compute mrconfigid if needed - let mrconfigid = if cfg.use_mrconfigid && support_mr_config_id { - if let Some(mr_config_document) = workdir - .sys_config() - .context("Failed to read sys config for tdx mrconfigid")? - .mr_config - { - MrConfigV3::from_document(&mr_config_document) - .context("Invalid mr_config document")?; - Some( - BASE64_STANDARD.encode(MrConfigV3::tdx_mr_config_id_from_document( - &mr_config_document, - )), - ) - } else { - let compose_hash = workdir - .app_compose_hash() - .context("Failed to get compose hash")?; - let mr_config = if app_compose.key_provider_id.is_empty() { - MrConfig::V1 { - compose_hash: &compose_hash, - } - } else { - let instance_info = workdir - .instance_info() - .context("Failed to get instance info")?; - let app_id = if instance_info.app_id.is_empty() { - &compose_hash[..20] - } else { - &instance_info.app_id - }; - - let key_provider = app_compose.key_provider(); - let key_provider_id = &app_compose.key_provider_id; - MrConfig::V2 { - compose_hash: &compose_hash, - app_id: &app_id.try_into().context("Invalid app ID")?, - key_provider, - key_provider_id, - } - }; - Some(BASE64_STANDARD.encode(mr_config.to_mr_config_id())) - } - } else { - None - }; - // Build tdx-guest object with optional quote-generation-socket for kernel-level TSM support #[derive(Serialize)] struct QgsSocket { @@ -1069,7 +748,7 @@ impl VmConfig { let tdx_object = TdxGuestObject { qom_type: "tdx-guest", id: "tdx", - mrconfigid: mrconfigid.clone(), + mrconfigid: mrconfigid.map(str::to_string), quote_generation_socket: cfg.qgs_port.map(|port| QgsSocket { r#type: "vsock", cid: "2", @@ -1087,24 +766,14 @@ impl VmConfig { fn configure_amd_sev_snp_guest( &self, command: &mut Command, - workdir: &VmWorkDir, cfg: &CvmConfig, mem: u32, - ) -> Result<()> { - let mr_config_document = workdir - .sys_config() - .context("Failed to read sys config for amd sev-snp host-data")? - .mr_config - .context("mr_config is required for amd sev-snp host-data")?; - MrConfigV3::from_document(&mr_config_document).context("Invalid mr_config document")?; - let host_data = - BASE64_STANDARD.encode(MrConfigV3::snp_host_data_from_document(&mr_config_document)); - + host_data: &str, + snp_params: AmdSevSnpLaunchParams, + ) { command .arg("-object") .arg(amd_sev_snp_memory_backend_arg(mem)); - let snp_params = detect_amd_sev_snp_qemu_capabilities(&cfg.qemu_path) - .context("failed to detect AMD SEV-SNP cbitpos/reduced-phys-bits from QEMU")?; command.arg("-object").arg(format!( "sev-snp-guest,id=sev0,policy=0x30000,sev-device=/dev/sev,kernel-hashes=on,host-data={host_data},cbitpos={},reduced-phys-bits={}", snp_params.cbitpos, snp_params.reduced_phys_bits @@ -1115,7 +784,6 @@ impl VmConfig { if cfg.qgs_port.is_some() { tracing::warn!("qgs_port is ignored for amd sev-snp guests"); } - Ok(()) } fn configure_smbios(&self, command: &mut Command, cfg: &CvmConfig) { @@ -1181,246 +849,146 @@ fn find_numa(device: Option) -> Result<(String, String)> { Ok((numa_node, cpus)) } -pub struct VmWorkDir { - workdir: PathBuf, -} - -impl Deref for VmWorkDir { - type Target = PathBuf; - fn deref(&self) -> &Self::Target { - &self.workdir - } -} - -impl AsRef for &VmWorkDir { - fn as_ref(&self) -> &Path { - self.workdir.as_ref() - } -} - -impl VmWorkDir { - pub fn new(workdir: impl AsRef) -> Self { - Self { - workdir: workdir.as_ref().to_path_buf(), - } - } - - pub fn manifest_path(&self) -> PathBuf { - self.workdir.join("vm-manifest.json") - } - - pub fn state_path(&self) -> PathBuf { - self.workdir.join("vm-state.json") - } - - pub fn manifest(&self) -> Result { - let manifest_path = self.manifest_path(); - let manifest = fs::read_to_string(manifest_path).context("Failed to read manifest")?; - let manifest: Manifest = - serde_json::from_str(&manifest).context("Failed to parse manifest")?; - Ok(manifest) - } - - pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { - fs::create_dir_all(&self.workdir).context("Failed to create workdir")?; - let manifest_path = self.manifest_path(); - fs::write(manifest_path, serde_json::to_string(manifest)?) - .context("Failed to write manifest") - } - - pub fn started(&self) -> Result { - let state_path = self.state_path(); - if !state_path.exists() { - return Ok(false); - } - let state: State = - serde_json::from_str(&fs::read_to_string(state_path).context("Failed to read state")?) - .context("Failed to parse state")?; - Ok(state.started) - } - - pub fn set_started(&self, started: bool) -> Result<()> { - let state_path = self.state_path(); - fs::write(state_path, serde_json::to_string(&State { started })?) - .context("Failed to write state") - } - - pub fn shared_dir(&self) -> PathBuf { - self.workdir.join("shared") - } - - pub fn app_compose_path(&self) -> PathBuf { - self.shared_dir().join(APP_COMPOSE) - } - - pub fn app_compose_hash(&self) -> Result<[u8; 32]> { - use sha2::Digest; - let compose_path = self.app_compose_path(); - let compose = fs::read(compose_path).context("Failed to read compose")?; - Ok(sha2::Sha256::new_with_prefix(&compose).finalize().into()) - } - - pub fn user_config_path(&self) -> PathBuf { - self.shared_dir().join(USER_CONFIG) - } - - pub fn encrypted_env_path(&self) -> PathBuf { - self.shared_dir().join(ENCRYPTED_ENV) - } - - pub fn instance_info_path(&self) -> PathBuf { - self.shared_dir().join(INSTANCE_INFO) - } - - pub fn guest_ip_path(&self) -> PathBuf { - self.workdir.join("guest-ip") - } - - pub fn guest_ip(&self) -> Option { - fs::read_to_string(self.guest_ip_path()) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - } - - pub fn set_guest_ip(&self, ip: &str) -> Result<()> { - fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") - } - - pub fn serial_file(&self) -> PathBuf { - self.workdir.join("serial.log") - } - - pub fn serial_history_file(&self) -> PathBuf { - self.workdir.join("serial.history.log") - } - - pub fn serial_pty(&self) -> PathBuf { - self.workdir.join("serial.pty") - } - - pub fn stdout_file(&self) -> PathBuf { - self.workdir.join("stdout.log") - } - - pub fn stderr_file(&self) -> PathBuf { - self.workdir.join("stderr.log") - } - - pub fn pid_file(&self) -> PathBuf { - self.workdir.join("qemu.pid") - } - - pub fn hda_path(&self) -> PathBuf { - self.workdir.join("hda.img") - } - - pub fn shared_disk_path(&self) -> PathBuf { - self.workdir.join("shared.img") - } - - pub fn qmp_socket(&self) -> PathBuf { - self.workdir.join("qmp.sock") - } +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::path::PathBuf; - pub fn removing_marker(&self) -> PathBuf { - self.workdir.join(".removing") - } + use rocket::figment::{ + providers::{Format, Toml}, + Figment, + }; - pub fn is_removing(&self) -> bool { - self.removing_marker().exists() - } + use super::{ + amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, + PreparedQemuLaunch, QemuCommandBuilder, VmConfig, + }; + use crate::app::image::{Image, ImageInfo}; + use crate::app::{GpuConfig, Manifest, VmWorkDir}; + use crate::config::{Config, TeePlatform, DEFAULT_CONFIG}; - pub fn set_removing(&self) -> Result<()> { - fs::write(self.removing_marker(), "").context("Failed to write .removing marker") + #[test] + fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { + assert_eq!( + amd_sev_snp_memory_backend_arg(4096), + "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false" + ); } - pub fn path(&self) -> &Path { - &self.workdir + #[test] + fn amd_sev_snp_qmp_capabilities_extracts_launch_params() { + let stdout = br#"{"QMP":{"version":{"qemu":{"major":10,"minor":0,"micro":2}}}} +{"return":{}} +{"return":{"reduced-phys-bits":1,"cbitpos":51,"cert-chain":"ignored","pdh":"ignored","cpu0-id":"ignored"}} +{"return":{}} +"#; + let params = parse_amd_sev_snp_qmp_capabilities(stdout).unwrap(); + assert_eq!(params.cbitpos, 51); + assert_eq!(params.reduced_phys_bits, 1); } -} -impl VmWorkDir { - pub fn instance_info(&self) -> Result { - let info_file = self.instance_info_path(); - let info: InstanceInfo = serde_json::from_slice(&fs::read(&info_file)?)?; - Ok(info) + #[test] + fn amd_sev_snp_uses_confidential_virtio_pci_options() { + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", true), + "virtio-blk-pci,drive=hd0,disable-legacy=on,iommu_platform=true" + ); + assert_eq!( + virtio_pci_device("virtio-blk-pci,drive=hd0", false), + "virtio-blk-pci,drive=hd0" + ); } - pub fn instance_info_or_default(&self) -> Result { - match self.instance_info() { - Ok(info) => Ok(info), - Err(err) => match err.downcast_ref::() { - Some(io_err) if io_err.kind() == std::io::ErrorKind::NotFound => { - Ok(InstanceInfo::default()) - } - _ => Err(err), + #[test] + fn qemu_command_builder_does_not_require_prepared_paths_to_exist() { + let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) + .extract() + .unwrap(); + config.cvm.platform = Some(TeePlatform::Tdx); + config.cvm.qemu_path = PathBuf::from("/not-installed/qemu-system-x86_64"); + config.cvm.qgs_port = None; + + let vm = VmConfig { + manifest: Manifest { + id: "vm-1".into(), + name: "test-vm".into(), + app_id: "app-1".into(), + vcpu: 2, + memory: 2048, + disk_size: 10, + image: "test-image".into(), + port_map: vec![], + created_at_ms: 0, + hugepages: false, + pin_numa: false, + gpus: None, + kms_urls: vec![], + gateway_urls: vec![], + no_tee: true, + networking: None, }, - } - } - - pub fn sys_config(&self) -> Result { - let sys_config_file = self.shared_dir().join(SYS_CONFIG); - let sys_config: SysConfig = serde_json::from_slice(&fs::read(sys_config_file)?)?; - Ok(sys_config) - } - - pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose) -> Result { - let compose_hash = self - .app_compose_hash() - .context("Failed to get compose hash")?; - let mut instance_info = self - .instance_info_or_default() - .context("Failed to get instance info")?; - let app_id = if instance_info.app_id.is_empty() { - compose_hash[..20].to_vec() - } else { - instance_info.app_id.clone() + image: Image { + info: ImageInfo { + cmdline: Some("console=hvc0".into()), + kernel: "kernel".into(), + initrd: "initrd".into(), + hda: None, + rootfs: None, + bios: None, + bios_sev: None, + rootfs_hash: None, + shared_ro: false, + version: "0.5.4".into(), + is_dev: false, + ovmf_variant: None, + }, + initrd: PathBuf::from("/does-not-exist/initrd"), + kernel: PathBuf::from("/does-not-exist/kernel"), + hda: None, + rootfs: None, + bios: None, + bios_sev: None, + digest: None, + tdx_measurement: None, + sev_measurement: None, + }, + cid: 100, + workdir: PathBuf::from("/does-not-exist/vm-1"), + gateway_enabled: false, }; - if app_id.len() != 20 { - bail!( - "Invalid app ID length: expected 20 bytes, got {}", - app_id.len() - ); - } - - let disk_reusable = !app_compose.key_provider().is_none(); - if !disk_reusable || instance_info.instance_id_seed.is_empty() { - instance_info.instance_id_seed = { - let mut seed = vec![0u8; 20]; - getrandom::fill(&mut seed).context("Failed to generate instance id seed")?; - seed - }; - } - - let instance_id = if app_compose.no_instance_id { - Vec::new() - } else { - let mut id_path = instance_info.instance_id_seed.clone(); - id_path.extend_from_slice(&app_id); - Sha256::digest(id_path)[..20].to_vec() + let prepared = PreparedQemuLaunch { + workdir: VmWorkDir::new("/does-not-exist/vm-1"), + platform: TeePlatform::Tdx, + hugepage_numa_nodes: None, + gpu_numa_nodes: HashMap::new(), + numa_cpus: None, + tpm_path: None, + tdx_mr_config_id: None, + snp_host_data: None, + snp_launch_params: None, }; - instance_info.app_id = app_id.clone(); - instance_info.instance_id = instance_id.clone(); - fs::write( - self.instance_info_path(), - serde_json::to_string(&instance_info).context("Failed to serialize instance info")?, - ) - .context("Failed to write instance info")?; - - Ok(MrConfigV3::new( - app_id, - compose_hash.to_vec(), - app_compose.key_provider(), - app_compose.key_provider_id.clone(), - instance_id, - ) - .to_canonical_json()) - } - pub fn app_compose(&self) -> Result { - let compose_file = self.app_compose_path(); - let compose: AppCompose = serde_json::from_str(&fs::read_to_string(compose_file)?)?; - Ok(compose) + let process = QemuCommandBuilder { + vm: &vm, + cfg: &config.cvm, + gpus: &GpuConfig::default(), + prepared: &prepared, + } + .build() + .unwrap(); + + assert_eq!(process.command, "/not-installed/qemu-system-x86_64"); + assert!(process + .args + .windows(2) + .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"])); + assert!(process + .args + .windows(2) + .any(|args| args == ["-kernel", "/does-not-exist/kernel"])); + assert!(process + .args + .windows(2) + .any(|args| args == ["-append", "console=hvc0"])); } } diff --git a/dstack/vmm/src/app/vm_info.rs b/dstack/vmm/src/app/vm_info.rs new file mode 100644 index 000000000..801c40cd5 --- /dev/null +++ b/dstack/vmm/src/app/vm_info.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! VM runtime-state aggregation and RPC presentation. + +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use dstack_vmm_rpc as pb; +use fs_err as fs; +use supervisor_client::supervisor::ProcessInfo; + +use super::{Manifest, VmState, VmWorkDir}; +use crate::config::{GatewayConfig, Networking, NetworkingMode}; + +pub(crate) struct VmInfo { + pub manifest: Manifest, + pub workdir: PathBuf, + pub status: &'static str, + pub uptime: String, + pub exited_at: Option, + pub instance_id: Option, + pub boot_progress: String, + pub boot_error: String, + pub shutdown_progress: String, + pub image_version: String, + pub gateway_enabled: bool, + pub events: Vec, +} + +fn networking_to_proto(networking: &Networking) -> pb::NetworkingConfig { + let mode = match networking.mode { + NetworkingMode::Bridge => "bridge", + NetworkingMode::User => "user", + NetworkingMode::Custom => "custom", + }; + pb::NetworkingConfig { mode: mode.into() } +} + +fn sanitize_optional>(value: Option) -> Option { + value.filter(|value| !value.as_ref().trim().is_empty()) +} + +impl VmInfo { + pub fn to_pb(&self, gateway: &GatewayConfig, brief: bool) -> pb::VmInfo { + let workdir = VmWorkDir::new(&self.workdir); + let vm_config = workdir.manifest(); + let custom_gateway_urls = vm_config + .as_ref() + .map(|config| config.gateway_urls.clone()) + .unwrap_or_default(); + pb::VmInfo { + id: self.manifest.id.clone(), + name: self.manifest.name.clone(), + status: self.status.into(), + uptime: self.uptime.clone(), + boot_progress: self.boot_progress.clone(), + boot_error: self.boot_error.clone(), + shutdown_progress: self.shutdown_progress.clone(), + image_version: self.image_version.clone(), + configuration: if brief { + None + } else { + let kms_urls = vm_config + .as_ref() + .map(|config| config.kms_urls.clone()) + .unwrap_or_default(); + let no_tee = vm_config + .as_ref() + .map(|config| config.no_tee) + .unwrap_or(self.manifest.no_tee); + let stopped = !workdir.started().unwrap_or(false); + + Some(pb::VmConfiguration { + name: self.manifest.name.clone(), + image: self.manifest.image.clone(), + compose_file: fs::read_to_string(workdir.app_compose_path()) + .unwrap_or_default(), + encrypted_env: fs::read(workdir.encrypted_env_path()).unwrap_or_default(), + user_config: fs::read_to_string(workdir.user_config_path()).unwrap_or_default(), + vcpu: self.manifest.vcpu, + memory: self.manifest.memory, + disk_size: self.manifest.disk_size, + ports: self + .manifest + .port_map + .iter() + .map(|mapping| pb::PortMapping { + protocol: mapping.protocol.as_str().into(), + host_address: mapping.address.to_string(), + host_port: mapping.from as u32, + vm_port: mapping.to as u32, + }) + .collect(), + app_id: Some(self.manifest.app_id.clone()), + hugepages: self.manifest.hugepages, + pin_numa: self.manifest.pin_numa, + gpus: self.manifest.gpus.as_ref().map(|config| pb::GpuConfig { + attach_mode: config.attach_mode.to_string(), + gpus: config + .gpus + .iter() + .map(|gpu| pb::GpuSpec { + slot: gpu.slot.clone(), + }) + .collect(), + }), + kms_urls, + gateway_urls: custom_gateway_urls.clone(), + stopped, + no_tee, + networking: self.manifest.networking.as_ref().map(networking_to_proto), + }) + }, + app_url: self + .gateway_enabled + .then_some(self.instance_id.as_deref()) + .flatten() + .and_then(|id| sanitize_optional(Some(id))) + .map(|id| app_url(id, &custom_gateway_urls, gateway)), + app_id: self.manifest.app_id.clone(), + instance_id: sanitize_optional(self.instance_id.clone()), + exited_at: self.exited_at.clone(), + events: self.events.clone(), + } + } +} + +fn app_url(id: &str, custom_gateway_urls: &[String], gateway: &GatewayConfig) -> String { + if let Some(custom_gateway_url) = custom_gateway_urls.first() { + if let Ok(url) = url::Url::parse(custom_gateway_url) { + let host = url.host_str().unwrap_or(&gateway.base_domain); + let port = url.port().unwrap_or(443); + if port == 443 { + return format!("https://{id}-{}.{}", gateway.agent_port, host); + } + return format!("https://{id}-{}.{}:{port}", gateway.agent_port, host); + } + } + + if gateway.port == 443 { + format!( + "https://{id}-{}.{}", + gateway.agent_port, gateway.base_domain + ) + } else { + format!( + "https://{id}-{}.{}:{}", + gateway.agent_port, gateway.base_domain, gateway.port + ) + } +} + +impl VmState { + pub fn merged_info(&self, process: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo { + fn truncate(duration: Duration) -> Duration { + Duration::from_secs(duration.as_secs()) + } + + fn display_timestamp(timestamp: Option<&SystemTime>) -> String { + match timestamp { + None => "never".into(), + Some(timestamp) => { + let elapsed = timestamp.elapsed().unwrap_or(Duration::MAX); + humantime::format_duration(truncate(elapsed)).to_string() + } + } + } + + let is_running = process.is_some_and(|info| info.state.status.is_running()); + let started = workdir.started().unwrap_or(false); + let status = if self.state.removing { + "removing" + } else { + match (started, is_running) { + (true, true) => "running", + (true, false) => "exited", + (false, true) => "stopping", + (false, false) => "stopped", + } + }; + let uptime = display_timestamp(process.and_then(|info| info.state.started_at.as_ref())); + let exited_at = display_timestamp(process.and_then(|info| info.state.stopped_at.as_ref())); + let instance_id = sanitize_optional( + workdir + .instance_info() + .ok() + .map(|info| hex::encode(info.instance_id)), + ); + + VmInfo { + manifest: self.config.manifest.clone(), + workdir: workdir.path().to_path_buf(), + instance_id, + status, + uptime, + exited_at: Some(exited_at), + boot_progress: self.state.boot_progress.clone(), + boot_error: self.state.boot_error.clone(), + shutdown_progress: self.state.shutdown_progress.clone(), + image_version: self.config.image.info.version.clone(), + gateway_enabled: self.config.gateway_enabled, + events: self.state.events.clone().into(), + } + } +} + +#[cfg(test)] +mod tests { + use super::sanitize_optional; + + #[test] + fn sanitize_optional_filters_empty_owned_values() { + assert_eq!(sanitize_optional(Some(String::new())), None); + assert_eq!(sanitize_optional(Some(" ".to_string())), None); + assert_eq!( + sanitize_optional(Some("instance-123".to_string())), + Some("instance-123".to_string()) + ); + } + + #[test] + fn sanitize_optional_filters_empty_borrowed_values() { + assert_eq!(sanitize_optional(Some("")), None); + assert_eq!(sanitize_optional(Some(" ")), None); + assert_eq!( + sanitize_optional(Some("instance-123")), + Some("instance-123") + ); + } +} diff --git a/dstack/vmm/src/app/workdir.rs b/dstack/vmm/src/app/workdir.rs new file mode 100644 index 000000000..e7a28ee7b --- /dev/null +++ b/dstack/vmm/src/app/workdir.rs @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: © 2024-2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! VM working directory layout and persisted state. + +use std::ops::Deref; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use dstack_types::{ + shared_filenames::{APP_COMPOSE, ENCRYPTED_ENV, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG}, + AppCompose, SysConfig, +}; +use fs_err as fs; +use serde::{Deserialize, Serialize}; +use serde_human_bytes as hex_bytes; + +use crate::app::Manifest; + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct InstanceInfo { + #[serde(default, with = "hex_bytes")] + pub instance_id_seed: Vec, + #[serde(default, with = "hex_bytes")] + pub instance_id: Vec, + #[serde(default, with = "hex_bytes")] + pub app_id: Vec, +} + +#[derive(Deserialize, Serialize)] +struct State { + started: bool, +} + +pub struct VmWorkDir { + workdir: PathBuf, +} + +impl Deref for VmWorkDir { + type Target = PathBuf; + fn deref(&self) -> &Self::Target { + &self.workdir + } +} + +impl AsRef for &VmWorkDir { + fn as_ref(&self) -> &Path { + self.workdir.as_ref() + } +} + +impl VmWorkDir { + pub fn new(workdir: impl AsRef) -> Self { + Self { + workdir: workdir.as_ref().to_path_buf(), + } + } + + pub fn manifest_path(&self) -> PathBuf { + self.workdir.join("vm-manifest.json") + } + + pub fn state_path(&self) -> PathBuf { + self.workdir.join("vm-state.json") + } + + pub fn manifest(&self) -> Result { + let manifest_path = self.manifest_path(); + let manifest = fs::read_to_string(manifest_path).context("failed to read manifest")?; + let manifest: Manifest = + serde_json::from_str(&manifest).context("failed to parse manifest")?; + Ok(manifest) + } + + pub fn put_manifest(&self, manifest: &Manifest) -> Result<()> { + fs::create_dir_all(&self.workdir).context("failed to create workdir")?; + let manifest_path = self.manifest_path(); + fs::write(manifest_path, serde_json::to_string(manifest)?) + .context("failed to write manifest") + } + + pub fn started(&self) -> Result { + let state_path = self.state_path(); + if !state_path.exists() { + return Ok(false); + } + let state: State = + serde_json::from_str(&fs::read_to_string(state_path).context("failed to read state")?) + .context("failed to parse state")?; + Ok(state.started) + } + + pub fn set_started(&self, started: bool) -> Result<()> { + let state_path = self.state_path(); + fs::write(state_path, serde_json::to_string(&State { started })?) + .context("failed to write state") + } + + pub fn shared_dir(&self) -> PathBuf { + self.workdir.join("shared") + } + + pub fn app_compose_path(&self) -> PathBuf { + self.shared_dir().join(APP_COMPOSE) + } + + pub fn app_compose_hash(&self) -> Result<[u8; 32]> { + use sha2::Digest; + let compose_path = self.app_compose_path(); + let compose = fs::read(compose_path).context("failed to read compose")?; + Ok(sha2::Sha256::new_with_prefix(&compose).finalize().into()) + } + + pub fn user_config_path(&self) -> PathBuf { + self.shared_dir().join(USER_CONFIG) + } + + pub fn encrypted_env_path(&self) -> PathBuf { + self.shared_dir().join(ENCRYPTED_ENV) + } + + pub fn instance_info_path(&self) -> PathBuf { + self.shared_dir().join(INSTANCE_INFO) + } + + pub fn guest_ip_path(&self) -> PathBuf { + self.workdir.join("guest-ip") + } + + pub fn guest_ip(&self) -> Option { + fs::read_to_string(self.guest_ip_path()) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + } + + pub fn set_guest_ip(&self, ip: &str) -> Result<()> { + fs::write(self.guest_ip_path(), ip).context("failed to write guest IP") + } + + pub fn serial_file(&self) -> PathBuf { + self.workdir.join("serial.log") + } + + pub fn serial_history_file(&self) -> PathBuf { + self.workdir.join("serial.history.log") + } + + pub fn serial_pty(&self) -> PathBuf { + self.workdir.join("serial.pty") + } + + pub fn stdout_file(&self) -> PathBuf { + self.workdir.join("stdout.log") + } + + pub fn stderr_file(&self) -> PathBuf { + self.workdir.join("stderr.log") + } + + pub fn pid_file(&self) -> PathBuf { + self.workdir.join("qemu.pid") + } + + pub fn hda_path(&self) -> PathBuf { + self.workdir.join("hda.img") + } + + pub fn shared_disk_path(&self) -> PathBuf { + self.workdir.join("shared.img") + } + + pub fn qmp_socket(&self) -> PathBuf { + self.workdir.join("qmp.sock") + } + + pub fn removing_marker(&self) -> PathBuf { + self.workdir.join(".removing") + } + + pub fn is_removing(&self) -> bool { + self.removing_marker().exists() + } + + pub fn set_removing(&self) -> Result<()> { + fs::write(self.removing_marker(), "").context("failed to write .removing marker") + } + + pub fn path(&self) -> &Path { + &self.workdir + } + + pub fn instance_info(&self) -> Result { + let info_file = self.instance_info_path(); + let info: InstanceInfo = serde_json::from_slice(&fs::read(&info_file)?)?; + Ok(info) + } + + pub fn instance_info_or_default(&self) -> Result { + match self.instance_info() { + Ok(info) => Ok(info), + Err(err) => match err.downcast_ref::() { + Some(io_err) if io_err.kind() == std::io::ErrorKind::NotFound => { + Ok(InstanceInfo::default()) + } + _ => Err(err), + }, + } + } + + pub fn sys_config(&self) -> Result { + let sys_config_file = self.shared_dir().join(SYS_CONFIG); + let sys_config: SysConfig = serde_json::from_slice(&fs::read(sys_config_file)?)?; + Ok(sys_config) + } + + pub fn app_compose(&self) -> Result { + let compose_file = self.app_compose_path(); + let compose: AppCompose = serde_json::from_str(&fs::read_to_string(compose_file)?)?; + Ok(compose) + } +}