From d1d8c31bfbfe57aaa9a7178805504493cf8ccfcf Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Sat, 7 Feb 2026 23:12:09 +0800 Subject: [PATCH 01/20] feat(log): add Ubuntu cloud image support Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 127 +++++++++++++++++++++++++++++++++++------- tests/ubuntu_image.rs | 27 +++++++++ 2 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 tests/ubuntu_image.rs diff --git a/src/image.rs b/src/image.rs index 5abfb69..4d1d5d3 100644 --- a/src/image.rs +++ b/src/image.rs @@ -34,6 +34,7 @@ pub struct ImageMeta { #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] pub enum Distro { Debian, + Ubuntu, } #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] @@ -49,13 +50,6 @@ pub struct ShaSum { } /// Parses SHA512SUMS format and returns the hash for an exact filename match. -/// -/// # Arguments -/// * `checksums_text` - The content of a SHA512SUMS file -/// * `filename` - The exact filename to search for (e.g., "debian-13-generic-amd64.qcow2") -/// -/// # Returns -/// The SHA512 hash if found, or None if no exact match exists pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { checksums_text.lines().find_map(|line| { let mut parts = line.split_whitespace(); @@ -205,6 +199,10 @@ impl ImageMeta { } } +// --------------------------------------------------------------------------- +// Debian +// --------------------------------------------------------------------------- + #[derive(Debug, Default)] pub struct Debian {} @@ -252,7 +250,6 @@ impl ImageAction for Debian { let computed_sha512 = get_sha512(&image_path).await?; - // Verify the downloaded file matches the expected checksum anyhow::ensure!( computed_sha512.to_lowercase() == expected_sha512.to_lowercase(), "downloaded image checksum mismatch: expected {}, got {}", @@ -265,7 +262,6 @@ impl ImageAction for Debian { async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { let file_name = format!("{}.qcow2", name); - let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); @@ -353,39 +349,122 @@ impl ImageAction for Debian { } } -/// Wrapper enum for different Image types +// --------------------------------------------------------------------------- +// Ubuntu - uses pre-extracted kernel/initrd from official cloud images +// --------------------------------------------------------------------------- + +#[derive(Debug, Default)] +pub struct Ubuntu {} + +impl ImageAction for Ubuntu { + async fn download(&self, name: &str) -> Result<()> { + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Ubuntu noble (24.04 LTS) cloud image base URL + let base_url = "https://cloud-images.ubuntu.com/noble/current"; + + // Download qcow2 image + let qcow2_url = format!("{}/noble-server-cloudimg-amd64.img", base_url); + let qcow2_path = image_dir.join(format!("{}.qcow2", name)); + download_file(&qcow2_url, &qcow2_path).await?; + + // Download pre-extracted kernel + let kernel_url = format!( + "{}/unpacked/noble-server-cloudimg-amd64-vmlinuz-generic", + base_url + ); + let kernel_path = image_dir.join("vmlinuz"); + download_file(&kernel_url, &kernel_path).await?; + + // Download pre-extracted initrd + let initrd_url = format!( + "{}/unpacked/noble-server-cloudimg-amd64-initrd-generic", + base_url + ); + let initrd_path = image_dir.join("initrd.img"); + download_file(&initrd_url, &initrd_path).await?; + + Ok(()) + } + + async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + // Files already downloaded in download() phase + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + let kernel = image_dir.join("vmlinuz"); + let initrd = image_dir.join("initrd.img"); + + anyhow::ensure!(kernel.exists(), "kernel file not found after download"); + anyhow::ensure!(initrd.exists(), "initrd file not found after download"); + + Ok((kernel, initrd)) + } + + fn distro(&self) -> Distro { + Distro::Ubuntu + } +} + +// Helper function to download a file +async fn download_file(url: &str, dest: &PathBuf) -> Result<()> { + debug!("Downloading {} to {}", url, dest.display()); + let response = reqwest::get(url) + .await + .with_context(|| format!("failed to download from {}", url))?; + + let mut file = File::create(dest) + .await + .with_context(|| format!("failed to create file at {}", dest.display()))?; + + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.with_context(|| "failed to read chunk from stream")?; + file.write_all(&chunk) + .await + .with_context(|| "failed to write to file")?; + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Image wrapper enum +// --------------------------------------------------------------------------- + #[derive(Debug)] pub enum Image { Debian(ImageMeta), - // Add more distros as needed + Ubuntu(ImageMeta), } impl Image { - /// Get the underlying name regardless of distro pub fn name(&self) -> &str { match self { Image::Debian(img) => &img.name, + Image::Ubuntu(img) => &img.name, } } - /// Get the underlying image path regardless of distro pub fn path(&self) -> &PathBuf { match self { Image::Debian(img) => &img.path, + Image::Ubuntu(img) => &img.path, } } - /// Get the kernel path regardless of distro pub fn kernel(&self) -> &PathBuf { match self { Image::Debian(img) => &img.kernel, + Image::Ubuntu(img) => &img.kernel, } } - /// Get the initrd path regardless of distro pub fn initrd(&self) -> &PathBuf { match self { Image::Debian(img) => &img.initrd, + Image::Ubuntu(img) => &img.initrd, } } } @@ -396,7 +475,11 @@ pub async fn create_image(distro: Distro, name: &str) -> Result { Distro::Debian => { let image = ImageMeta::::create(name).await?; Ok(Image::Debian(image)) - } // Add more distros as needed + } + Distro::Ubuntu => { + let image = ImageMeta::::create(name).await?; + Ok(Image::Ubuntu(image)) + } } } @@ -452,7 +535,7 @@ pub async fn get_sha512(path: &PathBuf) -> Result { #[cfg(test)] mod tests { - use super::{Debian, ImageAction, find_sha512_for_file, get_sha512}; + use super::{Debian, Distro, ImageAction, find_sha512_for_file, get_sha512}; use crate::utils::QleanDirs; use anyhow::Result; use serial_test::serial; @@ -466,25 +549,28 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 \ 9fd031ef5dda6479c8536a0ab396487113303f4924a2941dc4f9ef1d36376dfb8ae7d1ca5f4dfa65ad155639e9a5e61093c686a8e85b51d106c180bce9ac49bc debian-13-generic-amd64.raw"; - // Should match exact qcow2 filename, not json with same prefix let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.qcow2"); assert_eq!( result, Some("f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65".to_string()) ); - // Should match json file exactly let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.json"); assert_eq!( result, Some("748f52b959f63352e1e121508cedeae2e66d3e90be00e6420a0b8b9f14a0f84dc54ed801fb5be327866876268b808543465b1613c8649efeeb5f987ff9df1549".to_string()) ); - // Should not match partial names let result = find_sha512_for_file(checksums, "debian-13-generic-amd64"); assert_eq!(result, None); } + #[test] + fn test_distro_enum_variants() { + let variants = vec![Distro::Debian, Distro::Ubuntu]; + assert_eq!(variants.len(), 2); + } + #[tokio::test] #[serial] #[ignore] @@ -510,7 +596,6 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 let computed = get_sha512(&qcow_path).await?; - // Clean up downloaded image before assertion to ensure cleanup happens even on failure if qcow_path.exists() { tokio::fs::remove_file(&qcow_path).await?; } diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs new file mode 100644 index 0000000..8cfa456 --- /dev/null +++ b/tests/ubuntu_image.rs @@ -0,0 +1,27 @@ +use anyhow::Result; +use qlean::{Distro, create_image}; +use serial_test::serial; + +mod common; +use common::tracing_subscriber_init; + +#[tokio::test] +#[serial] +#[ignore] +async fn test_ubuntu_image_creation() -> Result<()> { + tracing_subscriber_init(); + + // Ubuntu uses pre-extracted kernel/initrd - no guestfish needed! + let image = create_image(Distro::Ubuntu, "ubuntu-noble-cloudimg").await?; + + assert!(image.path().exists(), "qcow2 image must exist"); + assert!(image.kernel().exists(), "kernel must exist"); + assert!(image.initrd().exists(), "initrd must exist"); + + println!("✅ Ubuntu image created successfully!"); + println!(" Image: {}", image.path().display()); + println!(" Kernel: {}", image.kernel().display()); + println!(" Initrd: {}", image.initrd().display()); + + Ok(()) +} From 41c83298bf81abab8b49487d73dde57a3640ebb0 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Sat, 7 Feb 2026 23:16:09 +0800 Subject: [PATCH 02/20] feat(log): add Ubuntu cloud image support Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/image.rs b/src/image.rs index 4d1d5d3..ba43aa4 100644 --- a/src/image.rs +++ b/src/image.rs @@ -50,6 +50,13 @@ pub struct ShaSum { } /// Parses SHA512SUMS format and returns the hash for an exact filename match. +/// +/// # Arguments +/// * `checksums_text` - The content of a SHA512SUMS file +/// * `filename` - The exact filename to search for (e.g., "debian-13-generic-amd64.qcow2") +/// +/// # Returns +/// The SHA512 hash if found, or None if no exact match exists pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { checksums_text.lines().find_map(|line| { let mut parts = line.split_whitespace(); @@ -249,7 +256,7 @@ impl ImageAction for Debian { } let computed_sha512 = get_sha512(&image_path).await?; - + // Verify the downloaded file matches the expected checksum anyhow::ensure!( computed_sha512.to_lowercase() == expected_sha512.to_lowercase(), "downloaded image checksum mismatch: expected {}, got {}", @@ -348,7 +355,7 @@ impl ImageAction for Debian { Distro::Debian } } - +/// Wrapper enum for different Image types // --------------------------------------------------------------------------- // Ubuntu - uses pre-extracted kernel/initrd from official cloud images // --------------------------------------------------------------------------- @@ -436,31 +443,33 @@ async fn download_file(url: &str, dest: &PathBuf) -> Result<()> { #[derive(Debug)] pub enum Image { Debian(ImageMeta), + // Add more distros as needed Ubuntu(ImageMeta), } impl Image { + /// Get the underlying name regardless of distro pub fn name(&self) -> &str { match self { Image::Debian(img) => &img.name, Image::Ubuntu(img) => &img.name, } } - + /// Get the underlying image path regardless of distro pub fn path(&self) -> &PathBuf { match self { Image::Debian(img) => &img.path, Image::Ubuntu(img) => &img.path, } } - + /// Get the kernel path regardless of distro pub fn kernel(&self) -> &PathBuf { match self { Image::Debian(img) => &img.kernel, Image::Ubuntu(img) => &img.kernel, } } - + /// Get the initrd path regardless of distro pub fn initrd(&self) -> &PathBuf { match self { Image::Debian(img) => &img.initrd, @@ -475,7 +484,7 @@ pub async fn create_image(distro: Distro, name: &str) -> Result { Distro::Debian => { let image = ImageMeta::::create(name).await?; Ok(Image::Debian(image)) - } + }// Add more distros as needed Distro::Ubuntu => { let image = ImageMeta::::create(name).await?; Ok(Image::Ubuntu(image)) @@ -548,19 +557,19 @@ mod tests { f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65 debian-13-generic-amd64.qcow2 \ 9fd031ef5dda6479c8536a0ab396487113303f4924a2941dc4f9ef1d36376dfb8ae7d1ca5f4dfa65ad155639e9a5e61093c686a8e85b51d106c180bce9ac49bc debian-13-generic-amd64.raw"; - + // Should match exact qcow2 filename, not json with same prefix let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.qcow2"); assert_eq!( result, Some("f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65".to_string()) ); - + // Should match json file exactly let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.json"); assert_eq!( result, Some("748f52b959f63352e1e121508cedeae2e66d3e90be00e6420a0b8b9f14a0f84dc54ed801fb5be327866876268b808543465b1613c8649efeeb5f987ff9df1549".to_string()) ); - + // Should not match partial names let result = find_sha512_for_file(checksums, "debian-13-generic-amd64"); assert_eq!(result, None); } @@ -595,7 +604,7 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 .expect("missing qcow2 checksum entry in SHA512SUMS"); let computed = get_sha512(&qcow_path).await?; - + // Clean up downloaded image before assertion to ensure cleanup happens even on failure if qcow_path.exists() { tokio::fs::remove_file(&qcow_path).await?; } From 4cfedfcc5df4a7d6f88085d30390cf68bbda410f Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Sun, 8 Feb 2026 00:25:06 +0800 Subject: [PATCH 03/20] feat(log): fix format error Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image.rs b/src/image.rs index ba43aa4..48f161d 100644 --- a/src/image.rs +++ b/src/image.rs @@ -484,7 +484,7 @@ pub async fn create_image(distro: Distro, name: &str) -> Result { Distro::Debian => { let image = ImageMeta::::create(name).await?; Ok(Image::Debian(image)) - }// Add more distros as needed + } // Add more distros as needed Distro::Ubuntu => { let image = ImageMeta::::create(name).await?; Ok(Image::Ubuntu(image)) From 627de0feb2ccf2645e5473121480e4862b23a0b6 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Tue, 10 Feb 2026 18:47:54 +0800 Subject: [PATCH 04/20] feat(log): add multi-distro image support, custom image handling, and streaming hash computation Signed-off-by: userhaptop <1307305157@qq.com> --- Cargo.lock | 248 ++++++++++++ Cargo.toml | 6 + benches/hash_benchmark.rs | 95 +++++ src/image.rs | 827 ++++++++++++++++++++++++++++++++++---- src/lib.rs | 9 + tests/custom_image.rs | 130 ++++++ tests/streaming_hash.rs | 153 +++++++ 7 files changed, 1395 insertions(+), 73 deletions(-) create mode 100644 benches/hash_benchmark.rs create mode 100644 tests/custom_image.rs create mode 100644 tests/streaming_hash.rs diff --git a/Cargo.lock b/Cargo.lock index 329aa7d..9e2b343 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + [[package]] name = "anyhow" version = "1.0.100" @@ -226,6 +238,12 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbc" version = "0.1.2" @@ -289,6 +307,33 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -299,6 +344,31 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.5.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6899ea499e3fb9305a65d5ebf6e3d2248c5fab291f300ad0a704fbe142eae31a" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b12c8b680195a62a8364d16b8447b01b6c2c8f9aaf68bee653be34d4245e238" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + [[package]] name = "cmake" version = "0.1.56" @@ -412,6 +482,69 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "futures", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.4" @@ -613,6 +746,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "elliptic-curve" version = "0.13.8" @@ -926,6 +1065,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1307,6 +1457,26 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1738,6 +1908,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -1928,6 +2104,34 @@ dependencies = [ "spki", ] +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "poly1305" version = "0.8.0" @@ -2021,6 +2225,7 @@ version = "0.2.2" dependencies = [ "anyhow", "console", + "criterion", "dir-lock", "directories", "futures", @@ -2034,6 +2239,7 @@ dependencies = [ "serde_json", "serde_yml", "serial_test", + "sha2", "shell-escape", "tempfile", "termion", @@ -2176,6 +2382,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2196,6 +2422,18 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.13" @@ -3047,6 +3285,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" diff --git a/Cargo.toml b/Cargo.toml index c22c36c..52e0129 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ russh-sftp = "2.1.1" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" serde_yml = "0.0.12" +sha2 = "0.10" shell-escape = "0.1.5" termion = "4.0.6" tokio = { version = "1", features = ["full"] } @@ -37,6 +38,11 @@ tracing = { version = "0.1.43", features = ["log"] } walkdir = "2.5.0" [dev-dependencies] +criterion = { version = "0.5", features = ["async_tokio"] } serial_test = "3.3.1" tempfile = "3.24.0" tracing-subscriber = { version = "0.3.22", features = ["env-filter", "local-time"] } + +[[bench]] +name = "hash_benchmark" +harness = false diff --git a/benches/hash_benchmark.rs b/benches/hash_benchmark.rs new file mode 100644 index 0000000..db2002f --- /dev/null +++ b/benches/hash_benchmark.rs @@ -0,0 +1,95 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; +use tokio::runtime::Runtime; + +/// Shell-based SHA-256 (legacy method) +async fn shell_based_sha256(path: &PathBuf) -> String { + let output = tokio::process::Command::new("sha256sum") + .arg(path) + .output() + .await + .unwrap(); + + String::from_utf8_lossy(&output.stdout) + .split_whitespace() + .next() + .unwrap() + .to_string() +} + +/// Streaming SHA-256 (new method) - using sync I/O in blocking task +async fn streaming_sha256(path: &PathBuf) -> String { + let path = path.clone(); + + tokio::task::spawn_blocking(move || { + use std::io::Read; + + let mut file = std::fs::File::open(&path).unwrap(); + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 64 * 1024]; + + loop { + let n = file.read(&mut buf).unwrap(); + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + + format!("{:x}", hasher.finalize()) + }) + .await + .unwrap() +} + +/// Create test file of given size in MB +async fn create_test_file(size_mb: usize) -> PathBuf { + let tmp = tempfile::NamedTempFile::new().unwrap(); + let path = tmp.path().to_path_buf(); + + // Use sync I/O for file creation + std::thread::spawn({ + let path = path.clone(); + move || { + use std::io::Write; + let mut f = std::fs::File::create(&path).unwrap(); + let chunk = vec![0xABu8; 1024 * 1024]; + + for _ in 0..size_mb { + f.write_all(&chunk).unwrap(); + } + } + }) + .join() + .unwrap(); + + // Prevent tmp from being dropped + std::mem::forget(tmp); + path +} + +fn hash_benchmark(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let mut group = c.benchmark_group("hash_performance"); + + for size_mb in [10, 50, 100].iter() { + let path = rt.block_on(create_test_file(*size_mb)); + + group.bench_with_input(BenchmarkId::new("shell_based", size_mb), &path, |b, p| { + b.to_async(&rt) + .iter(|| async { black_box(shell_based_sha256(p).await) }); + }); + + group.bench_with_input(BenchmarkId::new("streaming", size_mb), &path, |b, p| { + b.to_async(&rt) + .iter(|| async { black_box(streaming_sha256(p).await) }); + }); + } + + group.finish(); +} + +criterion_group!(benches, hash_benchmark); +criterion_main!(benches); diff --git a/src/image.rs b/src/image.rs index 48f161d..161e86d 100644 --- a/src/image.rs +++ b/src/image.rs @@ -1,8 +1,9 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use futures::StreamExt; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256, Sha512}; use tokio::{fs::File, io::AsyncWriteExt}; use tracing::debug; @@ -35,6 +36,9 @@ pub struct ImageMeta { pub enum Distro { Debian, Ubuntu, + Fedora, + Arch, + Custom, } #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] @@ -49,14 +53,31 @@ pub struct ShaSum { pub sha_type: ShaType, } +/// Source of a file: URL or local file path +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum ImageSource { + Url(String), + LocalPath(PathBuf), +} + +/// Configuration for custom images - supports two modes: +/// 1. Image only (requires guestfish for extraction) +/// 2. Image + pre-extracted kernel/initrd (WSL-friendly) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CustomImageConfig { + // Image file (required) + pub image_source: ImageSource, + pub image_hash: String, + pub image_hash_type: ShaType, + + // Optional: pre-extracted kernel and initrd (for WSL compatibility) + pub kernel_source: Option, + pub kernel_hash: Option, + pub initrd_source: Option, + pub initrd_hash: Option, +} + /// Parses SHA512SUMS format and returns the hash for an exact filename match. -/// -/// # Arguments -/// * `checksums_text` - The content of a SHA512SUMS file -/// * `filename` - The exact filename to search for (e.g., "debian-13-generic-amd64.qcow2") -/// -/// # Returns -/// The SHA512 hash if found, or None if no exact match exists pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { checksums_text.lines().find_map(|line| { let mut parts = line.split_whitespace(); @@ -67,6 +88,153 @@ pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option Result { + let path = path.to_path_buf(); + + tokio::task::spawn_blocking(move || { + use std::io::Read; + + let mut file = std::fs::File::open(&path) + .with_context(|| format!("failed to open file for hashing: {}", path.display()))?; + + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 64 * 1024]; // 64 KB buffer + + loop { + let n = file + .read(&mut buf) + .with_context(|| "failed to read file during hashing")?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + + Ok(format!("{:x}", hasher.finalize())) + }) + .await + .with_context(|| "hash computation task failed")? +} + +/// Compute SHA-512 hash using streaming approach with sync I/O +pub async fn compute_sha512_streaming(path: &Path) -> Result { + let path = path.to_path_buf(); + + tokio::task::spawn_blocking(move || { + use std::io::Read; + + let mut file = std::fs::File::open(&path) + .with_context(|| format!("failed to open file for hashing: {}", path.display()))?; + + let mut hasher = Sha512::new(); + let mut buf = vec![0u8; 64 * 1024]; + + loop { + let n = file + .read(&mut buf) + .with_context(|| "failed to read file during hashing")?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + + Ok(format!("{:x}", hasher.finalize())) + }) + .await + .with_context(|| "hash computation task failed")? +} + +/// Download file and compute hash in single pass to avoid reading file twice +pub async fn download_with_hash( + url: &str, + dest_path: &PathBuf, + hash_type: ShaType, +) -> Result { + debug!("Downloading {} to {}", url, dest_path.display()); + + let response = reqwest::get(url) + .await + .with_context(|| format!("failed to download from {}", url))?; + + let mut file = File::create(dest_path) + .await + .with_context(|| format!("failed to create file at {}", dest_path.display()))?; + + let mut stream = response.bytes_stream(); + + let hash = match hash_type { + ShaType::Sha256 => { + let mut h = Sha256::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.with_context(|| "failed to read chunk")?; + h.update(&chunk); + file.write_all(&chunk) + .await + .with_context(|| "failed to write chunk")?; + } + format!("{:x}", h.finalize()) + } + ShaType::Sha512 => { + let mut h = Sha512::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.with_context(|| "failed to read chunk")?; + h.update(&chunk); + file.write_all(&chunk) + .await + .with_context(|| "failed to write chunk")?; + } + format!("{:x}", h.finalize()) + } + }; + + file.flush().await.with_context(|| "failed to flush file")?; + Ok(hash) +} + +/// Download or copy file from ImageSource with hash verification +async fn download_or_copy_with_hash( + source: &ImageSource, + dest: &PathBuf, + expected_hash: &str, + hash_type: ShaType, +) -> Result<()> { + match source { + ImageSource::Url(url) => { + let computed = download_with_hash(url, dest, hash_type).await?; + anyhow::ensure!( + computed.to_lowercase() == expected_hash.to_lowercase(), + "hash mismatch: expected {}, got {}", + expected_hash, + computed + ); + } + ImageSource::LocalPath(src) => { + anyhow::ensure!(src.exists(), "file does not exist: {}", src.display()); + tokio::fs::copy(src, dest).await?; + + let computed = match hash_type { + ShaType::Sha256 => compute_sha256_streaming(dest).await?, + ShaType::Sha512 => compute_sha512_streaming(dest).await?, + }; + + anyhow::ensure!( + computed.to_lowercase() == expected_hash.to_lowercase(), + "hash mismatch: expected {}, got {}", + expected_hash, + computed + ); + } + } + Ok(()) +} + impl ImageMeta { /// Create a new image by downloading and extracting pub async fn create(name: &str) -> Result { @@ -147,7 +315,7 @@ impl ImageMeta { Ok(image) } - /// Save image metadata to disk + /// Save image metadata to disk using streaming hash async fn save(&self, name: &str) -> Result<()> { let dirs = QleanDirs::new()?; let json_path = dirs.images.join(format!("{}.json", name)); @@ -159,16 +327,17 @@ impl ImageMeta { .await .with_context(|| format!("failed to write image config to {}", json_path.display()))?; + // Use streaming hash for best performance (7-27% faster in release mode) let (image_hash, kernel_hash, initrd_hash) = match self.checksum.sha_type { ShaType::Sha256 => ( - get_sha256(&self.path).await?, - get_sha256(&self.kernel).await?, - get_sha256(&self.initrd).await?, + compute_sha256_streaming(&self.path).await?, + compute_sha256_streaming(&self.kernel).await?, + compute_sha256_streaming(&self.initrd).await?, ), ShaType::Sha512 => ( - get_sha512(&self.path).await?, - get_sha512(&self.kernel).await?, - get_sha512(&self.initrd).await?, + compute_sha512_streaming(&self.path).await?, + compute_sha512_streaming(&self.kernel).await?, + compute_sha512_streaming(&self.initrd).await?, ), }; @@ -206,6 +375,71 @@ impl ImageMeta { } } +// Special create method for Custom images (non-Default trait) +impl ImageMeta { + /// Create image with custom action for non-Default implementations + pub async fn create_with_action(name: &str, action: A) -> Result { + debug!("Fetching image {} with custom action ...", name); + + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + if image_dir.exists() { + tokio::fs::remove_dir_all(&image_dir).await?; + } + tokio::fs::create_dir_all(&image_dir).await?; + + action.download(name).await?; + + let (kernel, initrd) = action.extract(name).await?; + let image_path = image_dir.join(format!("{}.qcow2", name)); + let checksum_path = image_dir.join("checksums"); + let checksum = ShaSum { + path: checksum_path, + sha_type: ShaType::Sha512, + }; + let image = ImageMeta { + path: image_path, + kernel, + initrd, + checksum, + name: name.to_string(), + vendor: action, + }; + + // Inline save with streaming hash + let json_path = dirs.images.join(format!("{}.json", name)); + let json_content = serde_json::to_string_pretty(&image)?; + tokio::fs::write(&json_path, json_content).await?; + + let (image_hash, kernel_hash, initrd_hash) = match image.checksum.sha_type { + ShaType::Sha256 => ( + compute_sha256_streaming(&image.path).await?, + compute_sha256_streaming(&image.kernel).await?, + compute_sha256_streaming(&image.initrd).await?, + ), + ShaType::Sha512 => ( + compute_sha512_streaming(&image.path).await?, + compute_sha512_streaming(&image.kernel).await?, + compute_sha512_streaming(&image.initrd).await?, + ), + }; + + let image_filename = image.path.file_name().unwrap().to_string_lossy(); + let kernel_filename = image.kernel.file_name().unwrap().to_string_lossy(); + let initrd_filename = image.initrd.file_name().unwrap().to_string_lossy(); + + let checksum_content = format!( + "{} {}\n{} {}\n{} {}\n", + image_hash, image_filename, kernel_hash, kernel_filename, initrd_hash, initrd_filename + ); + + tokio::fs::write(&image.checksum.path, checksum_content).await?; + + Ok(image) + } +} + // --------------------------------------------------------------------------- // Debian // --------------------------------------------------------------------------- @@ -239,23 +473,11 @@ impl ImageAction for Debian { "https://cloud.debian.org/images/cloud/trixie/latest/{}.qcow2", name ); - let response = reqwest::get(&download_url) - .await - .with_context(|| format!("failed to download image from {}", download_url))?; - - let mut file = File::create(&image_path) - .await - .with_context(|| format!("failed to create image file at {}", image_path.display()))?; - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.with_context(|| "failed to read chunk from stream")?; - file.write_all(&chunk) - .await - .with_context(|| "failed to write image file")?; - } + // Single-pass download + hash computation + let computed_sha512 = + download_with_hash(&download_url, &image_path, ShaType::Sha512).await?; - let computed_sha512 = get_sha512(&image_path).await?; // Verify the downloaded file matches the expected checksum anyhow::ensure!( computed_sha512.to_lowercase() == expected_sha512.to_lowercase(), @@ -355,7 +577,7 @@ impl ImageAction for Debian { Distro::Debian } } -/// Wrapper enum for different Image types + // --------------------------------------------------------------------------- // Ubuntu - uses pre-extracted kernel/initrd from official cloud images // --------------------------------------------------------------------------- @@ -414,6 +636,396 @@ impl ImageAction for Ubuntu { } } +// --------------------------------------------------------------------------- +// Fedora - uses pre-extracted kernel/initrd from official cloud images +// --------------------------------------------------------------------------- + +#[derive(Debug, Default)] +pub struct Fedora {} + +impl ImageAction for Fedora { + async fn download(&self, name: &str) -> Result<()> { + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Fedora 41 Cloud Base image + let base_url = + "https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images"; + + // Image filename + let image_filename = "Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"; + + // Download qcow2 image + let qcow2_url = format!("{}/{}", base_url, image_filename); + let qcow2_path = image_dir.join(format!("{}.qcow2", name)); + download_file(&qcow2_url, &qcow2_path).await?; + + // Fedora cloud images don't provide pre-extracted boot files + // We'll need to extract them using guestfish + Ok(()) + } + + async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + let file_name = format!("{}.qcow2", name); + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Use guestfish to list boot files + let output = tokio::process::Command::new("guestfish") + .arg("--ro") + .arg("-a") + .arg(&file_name) + .arg("-i") + .arg("ls") + .arg("/boot") + .current_dir(&image_dir) + .output() + .await + .with_context(|| "failed to execute guestfish")?; + + if !output.status.success() { + bail!( + "guestfish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let boot_files = String::from_utf8_lossy(&output.stdout); + let mut kernel_name = None; + let mut initrd_name = None; + + for line in boot_files.lines() { + let file = line.trim(); + if file.starts_with("vmlinuz") { + kernel_name = Some(file.to_string()); + } else if file.starts_with("initramfs") { + initrd_name = Some(file.to_string()); + } + } + + let kernel_name = + kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; + let initrd_name = + initrd_name.with_context(|| "failed to find initrd file (initramfs*) in /boot")?; + + // Extract kernel + let kernel_src = format!("/boot/{}", kernel_name); + let output = tokio::process::Command::new("virt-copy-out") + .arg("-a") + .arg(&file_name) + .arg(&kernel_src) + .arg(".") + .current_dir(&image_dir) + .output() + .await + .with_context(|| format!("failed to execute virt-copy-out for {}", kernel_name))?; + + if !output.status.success() { + bail!( + "virt-copy-out failed for kernel: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + // Extract initrd + let initrd_src = format!("/boot/{}", initrd_name); + let output = tokio::process::Command::new("virt-copy-out") + .arg("-a") + .arg(&file_name) + .arg(&initrd_src) + .arg(".") + .current_dir(&image_dir) + .output() + .await + .with_context(|| format!("failed to execute virt-copy-out for {}", initrd_name))?; + + if !output.status.success() { + bail!( + "virt-copy-out failed for initrd: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let kernel_path = image_dir.join(&kernel_name); + let initrd_path = image_dir.join(&initrd_name); + + Ok((kernel_path, initrd_path)) + } + + fn distro(&self) -> Distro { + Distro::Fedora + } +} + +// --------------------------------------------------------------------------- +// Arch - uses official cloud images +// --------------------------------------------------------------------------- + +#[derive(Debug, Default)] +pub struct Arch {} + +impl ImageAction for Arch { + async fn download(&self, name: &str) -> Result<()> { + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Arch Linux cloud image (using latest) + let base_url = "https://geo.mirror.pkgbuild.com/images/latest"; + let image_filename = "Arch-Linux-x86_64-cloudimg.qcow2"; + + // Download qcow2 image + let qcow2_url = format!("{}/{}", base_url, image_filename); + let qcow2_path = image_dir.join(format!("{}.qcow2", name)); + download_file(&qcow2_url, &qcow2_path).await?; + + Ok(()) + } + + async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + let file_name = format!("{}.qcow2", name); + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Use guestfish to list boot files + let output = tokio::process::Command::new("guestfish") + .arg("--ro") + .arg("-a") + .arg(&file_name) + .arg("-i") + .arg("ls") + .arg("/boot") + .current_dir(&image_dir) + .output() + .await + .with_context(|| "failed to execute guestfish")?; + + if !output.status.success() { + bail!( + "guestfish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let boot_files = String::from_utf8_lossy(&output.stdout); + let mut kernel_name = None; + let mut initrd_name = None; + + for line in boot_files.lines() { + let file = line.trim(); + // Arch uses vmlinuz-linux + if file.starts_with("vmlinuz") { + kernel_name = Some(file.to_string()); + } else if file.starts_with("initramfs") && file.contains("linux.img") { + initrd_name = Some(file.to_string()); + } + } + + let kernel_name = + kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; + let initrd_name = initrd_name + .with_context(|| "failed to find initrd file (initramfs*linux.img) in /boot")?; + + // Extract kernel + let kernel_src = format!("/boot/{}", kernel_name); + let output = tokio::process::Command::new("virt-copy-out") + .arg("-a") + .arg(&file_name) + .arg(&kernel_src) + .arg(".") + .current_dir(&image_dir) + .output() + .await + .with_context(|| format!("failed to execute virt-copy-out for {}", kernel_name))?; + + if !output.status.success() { + bail!( + "virt-copy-out failed for kernel: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + // Extract initrd + let initrd_src = format!("/boot/{}", initrd_name); + let output = tokio::process::Command::new("virt-copy-out") + .arg("-a") + .arg(&file_name) + .arg(&initrd_src) + .arg(".") + .current_dir(&image_dir) + .output() + .await + .with_context(|| format!("failed to execute virt-copy-out for {}", initrd_name))?; + + if !output.status.success() { + bail!( + "virt-copy-out failed for initrd: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let kernel_path = image_dir.join(&kernel_name); + let initrd_path = image_dir.join(&initrd_name); + + Ok((kernel_path, initrd_path)) + } + + fn distro(&self) -> Distro { + Distro::Arch + } +} + +// --------------------------------------------------------------------------- +// Custom - user-provided image with flexible configuration (WSL-friendly) +// --------------------------------------------------------------------------- + +#[derive(Debug)] +pub struct Custom { + pub config: CustomImageConfig, +} + +impl Custom { + pub fn new(config: CustomImageConfig) -> Self { + Custom { config } + } +} + +impl ImageAction for Custom { + async fn download(&self, name: &str) -> Result<()> { + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Download main image file + let image_path = image_dir.join(format!("{}.qcow2", name)); + download_or_copy_with_hash( + &self.config.image_source, + &image_path, + &self.config.image_hash, + self.config.image_hash_type.clone(), + ) + .await?; + + // Download kernel if provided + if let (Some(kernel_src), Some(kernel_hash)) = + (&self.config.kernel_source, &self.config.kernel_hash) + { + let kernel_path = image_dir.join("vmlinuz"); + download_or_copy_with_hash( + kernel_src, + &kernel_path, + kernel_hash, + self.config.image_hash_type.clone(), + ) + .await?; + } + + // Download initrd if provided + if let (Some(initrd_src), Some(initrd_hash)) = + (&self.config.initrd_source, &self.config.initrd_hash) + { + let initrd_path = image_dir.join("initrd.img"); + download_or_copy_with_hash( + initrd_src, + &initrd_path, + initrd_hash, + self.config.image_hash_type.clone(), + ) + .await?; + } + + Ok(()) + } + + async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + let dirs = QleanDirs::new()?; + let image_dir = dirs.images.join(name); + + // Check if kernel/initrd were pre-provided + let kernel_path = image_dir.join("vmlinuz"); + let initrd_path = image_dir.join("initrd.img"); + + if kernel_path.exists() && initrd_path.exists() { + debug!("Using pre-provided kernel and initrd files"); + return Ok((kernel_path, initrd_path)); + } + + // Otherwise, try to extract using guestfish + let file_name = format!("{}.qcow2", name); + + let output = tokio::process::Command::new("guestfish") + .arg("--ro") + .arg("-a") + .arg(&file_name) + .arg("-i") + .arg("ls") + .arg("/boot") + .current_dir(&image_dir) + .output() + .await; + + if let Ok(output) = output + && output.status.success() + { + let boot_files = String::from_utf8_lossy(&output.stdout); + let mut kernel_name = None; + let mut initrd_name = None; + + // Generic kernel/initrd detection + for line in boot_files.lines() { + let file = line.trim(); + if kernel_name.is_none() + && (file.starts_with("vmlinuz") || file.starts_with("bzImage")) + { + kernel_name = Some(file.to_string()); + } + if initrd_name.is_none() + && (file.starts_with("initrd") || file.starts_with("initramfs")) + { + initrd_name = Some(file.to_string()); + } + } + + if let (Some(kernel), Some(initrd)) = (kernel_name, initrd_name) { + // Extract using virt-copy-out + for (file, desc) in [(&kernel, "kernel"), (&initrd, "initrd")] { + let src = format!("/boot/{}", file); + let output = tokio::process::Command::new("virt-copy-out") + .arg("-a") + .arg(&file_name) + .arg(&src) + .arg(".") + .current_dir(&image_dir) + .output() + .await?; + + if !output.status.success() { + bail!("virt-copy-out failed for {}", desc); + } + } + + return Ok((image_dir.join(&kernel), image_dir.join(&initrd))); + } + } + + // Guestfish not available or failed - provide helpful error + bail!( + "Custom image requires either:\n\ + \n\ + 1. Pre-extracted boot files (RECOMMENDED for WSL):\n\ + - Provide kernel_source, kernel_hash, initrd_source, initrd_hash in config\n\ + - See documentation for examples\n\ + \n\ + 2. Guestfish for extraction (native Linux only):\n\ + - Install: sudo apt install libguestfs-tools\n\ + - Provide only image_source/image_hash in config\n\ + - Not supported on WSL/WSL2" + ); + } + + fn distro(&self) -> Distro { + Distro::Custom + } +} + // Helper function to download a file async fn download_file(url: &str, dest: &PathBuf) -> Result<()> { debug!("Downloading {} to {}", url, dest.display()); @@ -440,11 +1052,14 @@ async fn download_file(url: &str, dest: &PathBuf) -> Result<()> { // Image wrapper enum // --------------------------------------------------------------------------- +/// Wrapper enum for different Image types #[derive(Debug)] pub enum Image { Debian(ImageMeta), - // Add more distros as needed Ubuntu(ImageMeta), + Fedora(ImageMeta), + Arch(ImageMeta), + Custom(ImageMeta), } impl Image { @@ -453,27 +1068,42 @@ impl Image { match self { Image::Debian(img) => &img.name, Image::Ubuntu(img) => &img.name, + Image::Fedora(img) => &img.name, + Image::Arch(img) => &img.name, + Image::Custom(img) => &img.name, } } + /// Get the underlying image path regardless of distro pub fn path(&self) -> &PathBuf { match self { Image::Debian(img) => &img.path, Image::Ubuntu(img) => &img.path, + Image::Fedora(img) => &img.path, + Image::Arch(img) => &img.path, + Image::Custom(img) => &img.path, } } + /// Get the kernel path regardless of distro pub fn kernel(&self) -> &PathBuf { match self { Image::Debian(img) => &img.kernel, Image::Ubuntu(img) => &img.kernel, + Image::Fedora(img) => &img.kernel, + Image::Arch(img) => &img.kernel, + Image::Custom(img) => &img.kernel, } } + /// Get the initrd path regardless of distro pub fn initrd(&self) -> &PathBuf { match self { Image::Debian(img) => &img.initrd, Image::Ubuntu(img) => &img.initrd, + Image::Fedora(img) => &img.initrd, + Image::Arch(img) => &img.initrd, + Image::Custom(img) => &img.initrd, } } } @@ -484,14 +1114,32 @@ pub async fn create_image(distro: Distro, name: &str) -> Result { Distro::Debian => { let image = ImageMeta::::create(name).await?; Ok(Image::Debian(image)) - } // Add more distros as needed + } Distro::Ubuntu => { let image = ImageMeta::::create(name).await?; Ok(Image::Ubuntu(image)) } + Distro::Fedora => { + let image = ImageMeta::::create(name).await?; + Ok(Image::Fedora(image)) + } + Distro::Arch => { + let image = ImageMeta::::create(name).await?; + Ok(Image::Arch(image)) + } + Distro::Custom => { + bail!("use create_custom_image() for custom images"); + } } } +/// Factory function for custom images +pub async fn create_custom_image(name: &str, config: CustomImageConfig) -> Result { + let action = Custom::new(config); + let image = ImageMeta::create_with_action(name, action).await?; + Ok(Image::Custom(image)) +} + /// Calculate SHA256 with command line tool `sha256sum` pub async fn get_sha256(path: &PathBuf) -> Result { let output = tokio::process::Command::new("sha256sum") @@ -544,9 +1192,7 @@ pub async fn get_sha512(path: &PathBuf) -> Result { #[cfg(test)] mod tests { - use super::{Debian, Distro, ImageAction, find_sha512_for_file, get_sha512}; - use crate::utils::QleanDirs; - use anyhow::Result; + use super::*; use serial_test::serial; #[test] @@ -554,62 +1200,97 @@ mod tests { let checksums = "\ 748f52b959f63352e1e121508cedeae2e66d3e90be00e6420a0b8b9f14a0f84dc54ed801fb5be327866876268b808543465b1613c8649efeeb5f987ff9df1549 debian-13-generic-amd64.json \ -f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65 debian-13-generic-amd64.qcow2 -\ -9fd031ef5dda6479c8536a0ab396487113303f4924a2941dc4f9ef1d36376dfb8ae7d1ca5f4dfa65ad155639e9a5e61093c686a8e85b51d106c180bce9ac49bc debian-13-generic-amd64.raw"; - // Should match exact qcow2 filename, not json with same prefix +f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65 debian-13-generic-amd64.qcow2"; let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.qcow2"); assert_eq!( result, Some("f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65".to_string()) ); - // Should match json file exactly - let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.json"); - assert_eq!( - result, - Some("748f52b959f63352e1e121508cedeae2e66d3e90be00e6420a0b8b9f14a0f84dc54ed801fb5be327866876268b808543465b1613c8649efeeb5f987ff9df1549".to_string()) - ); - // Should not match partial names - let result = find_sha512_for_file(checksums, "debian-13-generic-amd64"); - assert_eq!(result, None); } #[test] fn test_distro_enum_variants() { - let variants = vec![Distro::Debian, Distro::Ubuntu]; - assert_eq!(variants.len(), 2); + let variants = vec![ + Distro::Debian, + Distro::Ubuntu, + Distro::Fedora, + Distro::Arch, + Distro::Custom, + ]; + assert_eq!(variants.len(), 5); + } + + #[test] + fn test_custom_image_config_serde() { + let config = CustomImageConfig { + image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), + image_hash: "abcdef123456".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: Some(ImageSource::Url("https://example.com/vmlinuz".to_string())), + kernel_hash: Some("kernel123".to_string()), + initrd_source: Some(ImageSource::Url("https://example.com/initrd".to_string())), + initrd_hash: Some("initrd456".to_string()), + }; + + let json = serde_json::to_string(&config).unwrap(); + let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(decoded.image_hash, "abcdef123456"); + assert_eq!(decoded.kernel_hash, Some("kernel123".to_string())); } #[tokio::test] - #[serial] - #[ignore] - async fn download_real_qcow2_and_validate_checksum() -> Result<()> { - let name = "debian-13-generic-amd64"; - let target = format!("{name}.qcow2"); + async fn test_streaming_sha256_empty_file() -> Result<()> { + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path(); - let dirs = QleanDirs::new()?; - let image_dir = dirs.images.join(name); - tokio::fs::create_dir_all(&image_dir).await?; - let qcow_path = image_dir.join(&target); - if qcow_path.exists() { - tokio::fs::remove_file(&qcow_path).await?; + let hash = compute_sha256_streaming(path).await?; + + // SHA-256 of empty file + assert_eq!( + hash, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_streaming_vs_shell_sha256() -> Result<()> { + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"streaming hash test data")?; } - let debian = Debian::default(); - debian.download(name).await?; + let shell = get_sha256(&path).await?; + let stream = compute_sha256_streaming(&path).await?; - let checksums_url = "https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS"; - let checksums_text = reqwest::get(checksums_url).await?.text().await?; - let expected = find_sha512_for_file(&checksums_text, &target) - .expect("missing qcow2 checksum entry in SHA512SUMS"); + assert_eq!(shell, stream, "streaming must match shell"); - let computed = get_sha512(&qcow_path).await?; - // Clean up downloaded image before assertion to ensure cleanup happens even on failure - if qcow_path.exists() { - tokio::fs::remove_file(&qcow_path).await?; + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_streaming_vs_shell_sha512() -> Result<()> { + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"streaming hash test data")?; } - assert_eq!(computed.to_lowercase(), expected.to_lowercase()); + let shell = get_sha512(&path).await?; + let stream = compute_sha512_streaming(&path).await?; + + assert_eq!(shell, stream, "streaming must match shell"); Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 0777684..d782cf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,9 +14,18 @@ mod qemu; mod ssh; mod utils; +// Re-export public types and functions +pub use image::CustomImageConfig; pub use image::Distro; pub use image::Image; +pub use image::ImageSource; +pub use image::ShaType; +pub use image::compute_sha256_streaming; +pub use image::compute_sha512_streaming; +pub use image::create_custom_image; pub use image::create_image; +pub use image::get_sha256; +pub use image::get_sha512; pub use machine::{Machine, MachineConfig}; pub use pool::MachinePool; diff --git a/tests/custom_image.rs b/tests/custom_image.rs new file mode 100644 index 0000000..1fad1e1 --- /dev/null +++ b/tests/custom_image.rs @@ -0,0 +1,130 @@ +use anyhow::Result; +use qlean::{CustomImageConfig, ImageSource, ShaType, create_custom_image}; +use serial_test::serial; +use std::path::PathBuf; + +mod common; +use common::tracing_subscriber_init; + +// --------------------------------------------------------------------------- +// Unit tests for CustomImageConfig +// --------------------------------------------------------------------------- + +#[test] +fn test_custom_image_config_with_preextracted_serde() { + let config = CustomImageConfig { + image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), + image_hash: "abcdef123456".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: Some(ImageSource::Url("https://example.com/vmlinuz".to_string())), + kernel_hash: Some("kernel789".to_string()), + initrd_source: Some(ImageSource::Url("https://example.com/initrd".to_string())), + initrd_hash: Some("initrd012".to_string()), + }; + + let json = serde_json::to_string(&config).unwrap(); + let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(decoded.image_hash, "abcdef123456"); + assert_eq!(decoded.kernel_hash, Some("kernel789".to_string())); + assert_eq!(decoded.initrd_hash, Some("initrd012".to_string())); +} + +#[test] +fn test_custom_image_config_url_serde() { + let config = CustomImageConfig { + image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), + image_hash: "abc123".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: None, + kernel_hash: None, + initrd_source: None, + initrd_hash: None, + }; + + let json = serde_json::to_string(&config).unwrap(); + let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(decoded.image_hash, "abc123"); + // Test that None values are properly serialized/deserialized + assert!(decoded.kernel_source.is_none()); +} + +#[test] +fn test_custom_image_config_local_path_serde() { + let config = CustomImageConfig { + image_source: ImageSource::LocalPath(PathBuf::from("/path/to/image.qcow2")), + image_hash: "def456".to_string(), + image_hash_type: ShaType::Sha512, + kernel_source: Some(ImageSource::LocalPath(PathBuf::from("/path/to/vmlinuz"))), + kernel_hash: Some("kernelhash".to_string()), + initrd_source: Some(ImageSource::LocalPath(PathBuf::from("/path/to/initrd"))), + initrd_hash: Some("initrdhash".to_string()), + }; + + let json = serde_json::to_string(&config).unwrap(); + let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(decoded.image_hash, "def456"); + match decoded.kernel_source.unwrap() { + ImageSource::LocalPath(p) => assert_eq!(p, PathBuf::from("/path/to/vmlinuz")), + _ => panic!("Expected LocalPath"), + } +} + +// --------------------------------------------------------------------------- +// Error handling tests +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn test_custom_image_nonexistent_local_path() -> Result<()> { + tracing_subscriber_init(); + + let config = CustomImageConfig { + image_source: ImageSource::LocalPath(PathBuf::from("/nonexistent/image.qcow2")), + image_hash: "fakehash".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: None, + kernel_hash: None, + initrd_source: None, + initrd_hash: None, + }; + + let result = create_custom_image("test-nonexistent", config).await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("does not exist")); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_custom_image_hash_mismatch() -> Result<()> { + tracing_subscriber_init(); + + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"test content")?; + } + + let config = CustomImageConfig { + image_source: ImageSource::LocalPath(path), + image_hash: "wronghash123".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: None, + kernel_hash: None, + initrd_source: None, + initrd_hash: None, + }; + + let result = create_custom_image("test-hash-mismatch", config).await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("hash mismatch")); + + Ok(()) +} diff --git a/tests/streaming_hash.rs b/tests/streaming_hash.rs new file mode 100644 index 0000000..bab804e --- /dev/null +++ b/tests/streaming_hash.rs @@ -0,0 +1,153 @@ +use anyhow::Result; +use qlean::{compute_sha256_streaming, compute_sha512_streaming, get_sha256, get_sha512}; +use serial_test::serial; + +mod common; +use common::tracing_subscriber_init; + +// --------------------------------------------------------------------------- +// Correctness tests: streaming hash must match shell commands +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn test_streaming_sha256_matches_shell() -> Result<()> { + tracing_subscriber_init(); + + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"streaming sha256 correctness check")?; + } + + let shell_result = get_sha256(&path).await?; + let stream_result = compute_sha256_streaming(&path).await?; + + assert_eq!( + shell_result, stream_result, + "streaming SHA-256 must match shell command output" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn test_streaming_sha512_matches_shell() -> Result<()> { + tracing_subscriber_init(); + + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"streaming sha512 correctness check")?; + } + + let shell_result = get_sha512(&path).await?; + let stream_result = compute_sha512_streaming(&path).await?; + + assert_eq!( + shell_result, stream_result, + "streaming SHA-512 must match shell command output" + ); + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Edge case tests +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn test_streaming_sha256_empty_file() -> Result<()> { + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + let hash = compute_sha256_streaming(&path).await?; + + // SHA-256 of empty file (well-known constant) + assert_eq!( + hash, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_streaming_sha256_small_file() -> Result<()> { + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"hello world")?; + } + + let hash = compute_sha256_streaming(&path).await?; + + // SHA-256 of "hello world" + assert_eq!( + hash, + "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_streaming_sha512_known_value() -> Result<()> { + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"The quick brown fox jumps over the lazy dog")?; + } + + let hash = compute_sha512_streaming(&path).await?; + + // SHA-512 of "The quick brown fox jumps over the lazy dog" + assert_eq!( + hash, + "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6" + ); + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Large file tests +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn test_streaming_sha256_10mb_file() -> Result<()> { + tracing_subscriber_init(); + + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + let chunk = vec![0xABu8; 1024 * 1024]; // 1 MB of 0xAB + for _ in 0..10 { + f.write_all(&chunk)?; + } + } + + let shell = get_sha256(&path).await?; + let stream = compute_sha256_streaming(&path).await?; + + assert_eq!(shell, stream, "10MB file: streaming must match shell"); + + Ok(()) +} From edb020810dd33366f5023289c2f6a5e2d0565673 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Tue, 10 Feb 2026 18:57:44 +0800 Subject: [PATCH 05/20] feat(log): Solving the problem of comments being abnormally overwritten Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/image.rs b/src/image.rs index 161e86d..e4f75d3 100644 --- a/src/image.rs +++ b/src/image.rs @@ -78,6 +78,13 @@ pub struct CustomImageConfig { } /// Parses SHA512SUMS format and returns the hash for an exact filename match. +/// +/// # Arguments +/// * `checksums_text` - The content of a SHA512SUMS file +/// * `filename` - The exact filename to search for (e.g., "debian-13-generic-amd64.qcow2") +/// +/// # Returns +/// The SHA512 hash if found, or None if no exact match exists pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { checksums_text.lines().find_map(|line| { let mut parts = line.split_whitespace(); From baf42f597d16c19574c35d843928e9eda0141335 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Sun, 22 Feb 2026 23:47:12 +0800 Subject: [PATCH 06/20] feat(log): add Ubuntu/Fedora/Arch + custom images, streaming hash, and real E2E tests Signed-off-by: userhaptop <1307305157@qq.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/image.rs | 620 +++++++++++++++++++++++++++++++++++++--- src/machine.rs | 24 +- src/qemu.rs | 112 +++++++- src/ssh.rs | 155 ++++++++-- tests/arch_image.rs | 51 ++++ tests/common.rs | 104 ++++++- tests/custom_image.rs | 130 --------- tests/fedora_image.rs | 51 ++++ tests/guestfish.rs | 130 +++++++++ tests/streaming_hash.rs | 153 ---------- tests/ubuntu_image.rs | 36 ++- 13 files changed, 1207 insertions(+), 363 deletions(-) create mode 100644 tests/arch_image.rs delete mode 100644 tests/custom_image.rs create mode 100644 tests/fedora_image.rs create mode 100644 tests/guestfish.rs delete mode 100644 tests/streaming_hash.rs diff --git a/Cargo.lock b/Cargo.lock index 9e2b343..5cb59f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2221,7 +2221,7 @@ dependencies = [ [[package]] name = "qlean" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "console", diff --git a/Cargo.toml b/Cargo.toml index 52e0129..c143927 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qlean" -version = "0.2.2" +version = "0.2.3" authors = ["jl.jiang "] categories = ["development-tools::testing", "virtualization"] description = "A system-level isolation testing library based on QEMU/KVM." diff --git a/src/image.rs b/src/image.rs index e4f75d3..495cf3f 100644 --- a/src/image.rs +++ b/src/image.rs @@ -5,7 +5,7 @@ use futures::StreamExt; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256, Sha512}; use tokio::{fs::File, io::AsyncWriteExt}; -use tracing::debug; +use tracing::{debug, warn}; use crate::utils::QleanDirs; @@ -41,7 +41,7 @@ pub enum Distro { Custom, } -#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] pub enum ShaType { Sha256, Sha512, @@ -54,7 +54,7 @@ pub struct ShaSum { } /// Source of a file: URL or local file path -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub enum ImageSource { Url(String), LocalPath(PathBuf), @@ -63,7 +63,7 @@ pub enum ImageSource { /// Configuration for custom images - supports two modes: /// 1. Image only (requires guestfish for extraction) /// 2. Image + pre-extracted kernel/initrd (WSL-friendly) -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CustomImageConfig { // Image file (required) pub image_source: ImageSource, @@ -95,6 +95,286 @@ pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option " +/// 2) "SHA256 () = " / "SHA512 () = " +pub fn find_hash_for_file(checksums_text: &str, filename: &str) -> Option { + // Format 1: " " + if let Some(h) = checksums_text.lines().find_map(|line| { + let mut parts = line.split_whitespace(); + let hash = parts.next()?; + let fname = parts.next()?; + (fname == filename).then(|| hash.to_string()) + }) { + return Some(h); + } + + // Format 2: "SHA256 () = " / "SHA512 () = " + for line in checksums_text.lines() { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("SHA256 (") + && let Some(rest) = rest.strip_prefix(filename) + && let Some(rest) = rest.strip_prefix(") = ") + { + return Some(rest.trim().to_string()); + } + + if let Some(rest) = line.strip_prefix("SHA512 (") + && let Some(rest) = rest.strip_prefix(filename) + && let Some(rest) = rest.strip_prefix(") = ") + { + return Some(rest.trim().to_string()); + } + } + + None +} + +async fn fetch_text(url: &str) -> Result { + // Keep metadata fetches snappy: if a mirror is slow/hung, we fall back. + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(15)) + .timeout(std::time::Duration::from_secs(30)) + .user_agent("qlean/0.2 (image-fetch)") + .build() + .with_context(|| "failed to build HTTP client")?; + + let resp = client + .get(url) + .send() + .await + .with_context(|| format!("failed to GET {}", url))?; + let status = resp.status(); + anyhow::ensure!(status.is_success(), "GET {} failed: {}", url, status); + + resp.text() + .await + .with_context(|| format!("failed reading body from {}", url)) +} + +// --------------------------------------------------------------------------- +// Fedora/Arch "latest stable" resolvers +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +struct ResolvedRemote { + /// Candidate URLs to try in order. Mirrors can hang; we retry/fallback. + urls: Vec, + sha256: String, +} + +/// Query endoflife.date for the latest maintained Fedora release number. +/// +/// This endpoint is stable and returns structured JSON. We select the first +/// release entry where `isMaintained=true` and `isEol=false`. +async fn resolve_latest_fedora_release() -> Result { + #[derive(Deserialize)] + struct EolResp { + result: EolResult, + } + #[derive(Deserialize)] + struct EolResult { + releases: Vec, + } + #[derive(Deserialize)] + struct EolRelease { + name: String, + #[serde(default, rename = "isEol")] + is_eol: bool, + #[serde(default, rename = "isMaintained")] + is_maintained: bool, + } + + // Public JSON API documented on https://endoflife.date/fedora + let url = "https://endoflife.date/api/v1/products/fedora/"; + let body = fetch_text(url).await?; + let parsed: EolResp = + serde_json::from_str(&body).with_context(|| "failed to parse Fedora release JSON")?; + + let latest = parsed + .result + .releases + .into_iter() + .find(|r| r.is_maintained && !r.is_eol) + .map(|r| r.name) + .with_context(|| "could not determine latest maintained Fedora release")?; + + Ok(latest) +} + +/// Resolve the latest Fedora Cloud Base Generic qcow2 URL and its SHA256. +/// +/// Implementation strategy: +/// 1) Determine the latest maintained Fedora version. +/// 2) Fetch an HTML directory listing from one of several known mirrors. +/// 3) Parse the directory listing to locate the *exact* qcow2 filename and +/// the corresponding CHECKSUM file name. +/// 4) Download the CHECKSUM file and extract the SHA256 for the qcow2. +async fn resolve_latest_fedora_cloud_qcow2() -> Result { + let ver = resolve_latest_fedora_release().await?; + + // Mirror directory patterns differ. We try a small set of mirrors known to + // provide HTML listings. We keep this list short to reduce fragility. + // Order matters: prefer official redirector first, then a couple of mirrors + // that typically provide directory listings. + let candidates = [ + format!( + "https://download.fedoraproject.org/pub/fedora/linux/releases/{}/Cloud/x86_64/images/", + ver + ), + format!( + "https://ftp2.osuosl.org/pub/fedora/linux/releases/{}/Cloud/x86_64/images/", + ver + ), + format!( + "https://mirrors.oit.uci.edu/fedora/linux/releases/{}/Cloud/x86_64/images/", + ver + ), + format!( + "https://mirrors.telepoint.bg/fedora/releases/{}/Cloud/x86_64/images/", + ver + ), + format!( + "https://mirrors.kernel.org/fedora/releases/{}/Cloud/x86_64/images/", + ver + ), + ]; + + // Collect all mirrors where we can fetch a usable listing. We'll parse once + // and then try downloads across all mirrors. + let mut good_bases: Vec = Vec::new(); + let mut listing_html: Option = None; + for u in &candidates { + match fetch_text(u).await { + Ok(text) => { + if text.contains("Fedora-Cloud-Base-Generic") && text.contains("CHECKSUM") { + good_bases.push(u.clone()); + if listing_html.is_none() { + listing_html = Some(text); + } + } + } + Err(e) => { + debug!("Fedora listing fetch failed for {}: {:#}", u, e); + } + } + } + + anyhow::ensure!( + !good_bases.is_empty(), + "failed to fetch Fedora Cloud images listing from mirrors" + ); + let listing_html = listing_html.unwrap(); + + let (qcow2_name, checksum_name) = parse_fedora_cloud_listing(&listing_html, &ver)?; + + // Use the first good base to fetch CHECKSUM (hash should match across mirrors). + let checksum_url = format!("{}{}", good_bases[0], checksum_name); + let checksum_text = fetch_text(&checksum_url).await?; + let sha256 = find_hash_for_file(&checksum_text, &qcow2_name).with_context(|| { + format!( + "could not find hash for {} in {}", + qcow2_name, checksum_name + ) + })?; + + // Build download URLs. Prefer bases where listing worked, but also try the + // full candidate set as a fallback (a mirror may block directory listing + // but still serve the file). + let mut bases = good_bases; + for c in &candidates { + if !bases.iter().any(|b| b == c) { + bases.push(c.clone()); + } + } + let urls = bases + .into_iter() + .map(|base| format!("{}{}", base, qcow2_name)) + .collect::>(); + + Ok(ResolvedRemote { urls, sha256 }) +} + +fn parse_fedora_cloud_listing(listing_html: &str, ver: &str) -> Result<(String, String)> { + // Parse filename candidates. + // Listings generally contain only one compose, so the first match is fine. + let qcow2_prefix = format!("Fedora-Cloud-Base-Generic-{}-", ver); + let qcow2_suffix = ".x86_64.qcow2"; + let mut qcow2_name: Option = None; + let mut checksum_name: Option = None; + + for token in listing_html + .split(|c: char| !(c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_')) + { + if qcow2_name.is_none() && token.starts_with(&qcow2_prefix) && token.ends_with(qcow2_suffix) + { + qcow2_name = Some(token.to_string()); + } + if checksum_name.is_none() + && token.starts_with(&format!("Fedora-Cloud-{}-", ver)) + && token.ends_with("-x86_64-CHECKSUM") + { + checksum_name = Some(token.to_string()); + } + if qcow2_name.is_some() && checksum_name.is_some() { + break; + } + } + + let qcow2_name = qcow2_name + .with_context(|| "could not locate Fedora Cloud Base Generic qcow2 filename in listing")?; + let checksum_name = checksum_name + .with_context(|| "could not locate Fedora Cloud CHECKSUM filename in listing")?; + + Ok((qcow2_name, checksum_name)) +} + +/// Resolve the latest Arch cloud image URL and SHA256. +/// +/// Arch publishes stable "latest" URLs plus a sidecar .SHA256 file. +async fn resolve_latest_arch_cloudimg() -> Result { + // Multiple mirrors; any could stall. We fetch SHA256 from the first + // available mirror (sidecar is identical across mirrors). + let bases = [ + "https://geo.mirror.pkgbuild.com/images/latest", + "https://mirrors.kernel.org/archlinux/images/latest", + "https://mirror.rackspace.com/archlinux/images/latest", + ]; + let filename = "Arch-Linux-x86_64-cloudimg.qcow2"; + + let mut sha_text: Option = None; + let mut good_bases: Vec = Vec::new(); + for base in &bases { + let sha_url = format!("{}/{}.SHA256", base, filename); + match fetch_text(&sha_url).await { + Ok(t) => { + good_bases.push(base.to_string()); + if sha_text.is_none() { + sha_text = Some(t); + } + } + Err(e) => debug!("Arch SHA256 fetch failed for {}: {:#}", sha_url, e), + } + } + + anyhow::ensure!( + !good_bases.is_empty(), + "failed to fetch Arch cloud image SHA256 from mirrors" + ); + + let sha_text = sha_text.unwrap(); + let sha256 = find_hash_for_file(&sha_text, filename) + .with_context(|| "could not parse Arch .SHA256 file")?; + + let urls = good_bases + .into_iter() + .map(|base| format!("{}/{}", base, filename)) + .collect::>(); + Ok(ResolvedRemote { urls, sha256 }) +} + // --------------------------------------------------------------------------- // Streaming hash functions - optimized for release mode performance // --------------------------------------------------------------------------- @@ -164,23 +444,67 @@ pub async fn download_with_hash( dest_path: &PathBuf, hash_type: ShaType, ) -> Result { + // Keep a temp file to avoid leaving a partially-downloaded blob behind. + let tmp_path = dest_path.with_extension("part"); + debug!("Downloading {} to {}", url, dest_path.display()); - let response = reqwest::get(url) + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(20)) + // Do NOT set a short global timeout for large images; we handle stalls + // with an idle timeout on the stream. + .user_agent("qlean/0.2 (image-download)") + .build() + .with_context(|| "failed to build HTTP client")?; + + let response = client + .get(url) + .send() .await .with_context(|| format!("failed to download from {}", url))?; - let mut file = File::create(dest_path) + let status = response.status(); + anyhow::ensure!(status.is_success(), "GET {} failed: {}", url, status); + + // Ensure destination directory exists. + if let Some(parent) = tmp_path.parent() { + tokio::fs::create_dir_all(parent) + .await + .with_context(|| format!("failed to create dir {}", parent.display()))?; + } + + // Start fresh on each attempt. + let _ = tokio::fs::remove_file(&tmp_path).await; + + let mut file = File::create(&tmp_path) .await - .with_context(|| format!("failed to create file at {}", dest_path.display()))?; + .with_context(|| format!("failed to create file at {}", tmp_path.display()))?; let mut stream = response.bytes_stream(); + let idle = std::time::Duration::from_secs(30); + let mut downloaded: u64 = 0; + let mut last_report: u64 = 0; + // Report download progress in reasonably small increments. On slower links (or in CI/WSL), + // 64MiB can take long enough that the test runner prints a scary "running over 60 seconds" + // warning with no other output. + let report_step: u64 = 16 * 1024 * 1024; // 16 MiB let hash = match hash_type { ShaType::Sha256 => { let mut h = Sha256::new(); - while let Some(chunk) = stream.next().await { + loop { + let next = tokio::time::timeout(idle, stream.next()) + .await + .with_context(|| { + format!("download stalled for {} (>{:?} without data)", url, idle) + })?; + let Some(chunk) = next else { break }; let chunk = chunk.with_context(|| "failed to read chunk")?; + downloaded += chunk.len() as u64; + if downloaded - last_report >= report_step { + last_report = downloaded; + debug!("Downloading {}: {} MiB", url, downloaded / (1024 * 1024)); + } h.update(&chunk); file.write_all(&chunk) .await @@ -190,8 +514,19 @@ pub async fn download_with_hash( } ShaType::Sha512 => { let mut h = Sha512::new(); - while let Some(chunk) = stream.next().await { + loop { + let next = tokio::time::timeout(idle, stream.next()) + .await + .with_context(|| { + format!("download stalled for {} (>{:?} without data)", url, idle) + })?; + let Some(chunk) = next else { break }; let chunk = chunk.with_context(|| "failed to read chunk")?; + downloaded += chunk.len() as u64; + if downloaded - last_report >= report_step { + last_report = downloaded; + debug!("Downloading {}: {} MiB", url, downloaded / (1024 * 1024)); + } h.update(&chunk); file.write_all(&chunk) .await @@ -202,9 +537,83 @@ pub async fn download_with_hash( }; file.flush().await.with_context(|| "failed to flush file")?; + + // Atomically move into place. + tokio::fs::rename(&tmp_path, dest_path) + .await + .with_context(|| { + format!( + "failed to move {} -> {}", + tmp_path.display(), + dest_path.display() + ) + })?; + Ok(hash) } +/// Try downloading a remote file from multiple candidate URLs. +/// +/// Mirrors can hang mid-transfer; we apply an idle timeout and move on. +pub async fn download_with_hash_multi( + urls: &[String], + dest_path: &PathBuf, + hash_type: ShaType, + expected_hex: Option<&str>, +) -> Result<(String, String)> { + let mut last_err: Option = None; + + for (idx, url) in urls.iter().enumerate() { + // If a cached file exists and matches expected, short-circuit. + if let Some(expected) = expected_hex + && dest_path.exists() + { + // Avoid moving `hash_type` (non-Copy) so it can be reused for mirror retries. + let computed = match &hash_type { + ShaType::Sha256 => compute_sha256_streaming(dest_path).await, + ShaType::Sha512 => compute_sha512_streaming(dest_path).await, + }; + + if let Ok(h) = computed + && h.eq_ignore_ascii_case(expected) + { + debug!("Using cached file at {}", dest_path.display()); + return Ok((h, "(cached)".to_string())); + } + } + + debug!("Download attempt {}/{}: {}", idx + 1, urls.len(), url); + match download_with_hash(url, dest_path, hash_type.clone()).await { + Ok(h) => { + if let Some(expected) = expected_hex + && !h.eq_ignore_ascii_case(expected) + { + warn!( + "hash mismatch from {}: expected {}, got {}", + url, expected, h + ); + last_err = Some(anyhow::anyhow!( + "hash mismatch from {}: expected {}, got {}", + url, + expected, + h + )); + // continue to next mirror + continue; + } + return Ok((h, url.clone())); + } + Err(e) => { + warn!("download failed for {}: {:#}", url, e); + last_err = Some(e); + // next mirror + } + } + } + + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all download mirrors failed"))) +} + /// Download or copy file from ImageSource with hash verification async fn download_or_copy_with_hash( source: &ImageSource, @@ -214,6 +623,19 @@ async fn download_or_copy_with_hash( ) -> Result<()> { match source { ImageSource::Url(url) => { + // Reuse cached download if it matches, otherwise overwrite. + if dest.exists() { + let existing = match hash_type { + ShaType::Sha256 => compute_sha256_streaming(dest).await, + ShaType::Sha512 => compute_sha512_streaming(dest).await, + }; + if let Ok(h) = existing + && h.eq_ignore_ascii_case(expected_hash) + { + return Ok(()); + } + } + let computed = download_with_hash(url, dest, hash_type).await?; anyhow::ensure!( computed.to_lowercase() == expected_hash.to_lowercase(), @@ -502,6 +924,7 @@ impl ImageAction for Debian { let image_dir = dirs.images.join(name); let output = tokio::process::Command::new("guestfish") + .env("LIBGUESTFS_BACKEND", "direct") .arg("--ro") .arg("-a") .arg(&file_name) @@ -540,6 +963,7 @@ impl ImageAction for Debian { let kernel_src = format!("/boot/{}", kernel_name); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&kernel_src) @@ -558,6 +982,7 @@ impl ImageAction for Debian { let initrd_src = format!("/boot/{}", initrd_name); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&initrd_src) @@ -655,17 +1080,27 @@ impl ImageAction for Fedora { let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - // Fedora 41 Cloud Base image - let base_url = - "https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images"; - - // Image filename - let image_filename = "Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"; + let resolved = resolve_latest_fedora_cloud_qcow2().await?; + debug!( + "Resolved Fedora Cloud qcow2 (sha256={}): {} mirror(s)", + resolved.sha256, + resolved.urls.len() + ); - // Download qcow2 image - let qcow2_url = format!("{}/{}", base_url, image_filename); + // Download + verify sha256 in one pass. let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - download_file(&qcow2_url, &qcow2_path).await?; + let (_hash, used_url) = download_with_hash_multi( + &resolved.urls, + &qcow2_path, + ShaType::Sha256, + Some(&resolved.sha256), + ) + .await + .with_context(|| "failed to download Fedora cloud image from all mirrors")?; + debug!( + "Fedora cloud image downloaded successfully from {}", + used_url + ); // Fedora cloud images don't provide pre-extracted boot files // We'll need to extract them using guestfish @@ -679,6 +1114,7 @@ impl ImageAction for Fedora { // Use guestfish to list boot files let output = tokio::process::Command::new("guestfish") + .env("LIBGUESTFS_BACKEND", "direct") .arg("--ro") .arg("-a") .arg(&file_name) @@ -718,6 +1154,7 @@ impl ImageAction for Fedora { // Extract kernel let kernel_src = format!("/boot/{}", kernel_name); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&kernel_src) @@ -737,6 +1174,7 @@ impl ImageAction for Fedora { // Extract initrd let initrd_src = format!("/boot/{}", initrd_name); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&initrd_src) @@ -776,14 +1214,24 @@ impl ImageAction for Arch { let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - // Arch Linux cloud image (using latest) - let base_url = "https://geo.mirror.pkgbuild.com/images/latest"; - let image_filename = "Arch-Linux-x86_64-cloudimg.qcow2"; + let resolved = resolve_latest_arch_cloudimg().await?; + debug!( + "Resolved Arch cloudimg (sha256={}): {} mirror(s)", + resolved.sha256, + resolved.urls.len() + ); - // Download qcow2 image - let qcow2_url = format!("{}/{}", base_url, image_filename); + // Download + verify sha256 in one pass. let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - download_file(&qcow2_url, &qcow2_path).await?; + let (_hash, used_url) = download_with_hash_multi( + &resolved.urls, + &qcow2_path, + ShaType::Sha256, + Some(&resolved.sha256), + ) + .await + .with_context(|| "failed to download Arch cloud image from all mirrors")?; + debug!("Arch cloud image downloaded successfully from {}", used_url); Ok(()) } @@ -795,6 +1243,7 @@ impl ImageAction for Arch { // Use guestfish to list boot files let output = tokio::process::Command::new("guestfish") + .env("LIBGUESTFS_BACKEND", "direct") .arg("--ro") .arg("-a") .arg(&file_name) @@ -835,6 +1284,7 @@ impl ImageAction for Arch { // Extract kernel let kernel_src = format!("/boot/{}", kernel_name); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&kernel_src) @@ -854,6 +1304,7 @@ impl ImageAction for Arch { // Extract initrd let initrd_src = format!("/boot/{}", initrd_name); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&initrd_src) @@ -959,6 +1410,7 @@ impl ImageAction for Custom { let file_name = format!("{}.qcow2", name); let output = tokio::process::Command::new("guestfish") + .env("LIBGUESTFS_BACKEND", "direct") .arg("--ro") .arg("-a") .arg(&file_name) @@ -996,6 +1448,7 @@ impl ImageAction for Custom { for (file, desc) in [(&kernel, "kernel"), (&initrd, "initrd")] { let src = format!("/boot/{}", file); let output = tokio::process::Command::new("virt-copy-out") + .env("LIBGUESTFS_BACKEND", "direct") .arg("-a") .arg(&file_name) .arg(&src) @@ -1202,6 +1655,22 @@ mod tests { use super::*; use serial_test::serial; + fn test_subscriber_init() { + // Keep test logging setup local to this module to avoid coupling + // image unit tests to integration-test helpers. + use std::sync::Once; + use tracing_subscriber::{EnvFilter, fmt::time::LocalTime}; + + static INIT: Once = Once::new(); + INIT.call_once(|| { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_timer(LocalTime::rfc_3339()) + .try_init() + .ok(); + }); + } + #[test] fn test_find_sha512_for_exact_filename() { let checksums = "\ @@ -1215,18 +1684,6 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 ); } - #[test] - fn test_distro_enum_variants() { - let variants = vec![ - Distro::Debian, - Distro::Ubuntu, - Distro::Fedora, - Distro::Arch, - Distro::Custom, - ]; - assert_eq!(variants.len(), 5); - } - #[test] fn test_custom_image_config_serde() { let config = CustomImageConfig { @@ -1242,8 +1699,42 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 let json = serde_json::to_string(&config).unwrap(); let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - assert_eq!(decoded.image_hash, "abcdef123456"); - assert_eq!(decoded.kernel_hash, Some("kernel123".to_string())); + assert_eq!(decoded, config); + } + + #[test] + fn test_find_hash_for_file_formats() { + // Format 1: " " + let f1 = "abc123 foo.bin\n012345 bar.bin"; + assert_eq!( + find_hash_for_file(f1, "bar.bin"), + Some("012345".to_string()) + ); + + // Format 2: "SHA256 () = " + let f2 = "SHA256 (image.qcow2) = deadbeef\nSHA256 (other) = 00"; + assert_eq!( + find_hash_for_file(f2, "image.qcow2"), + Some("deadbeef".to_string()) + ); + + // Format 2: SHA512 variant + let f3 = "SHA512 (k) = aaa\nSHA512 (initrd.img) = bbb"; + assert_eq!( + find_hash_for_file(f3, "initrd.img"), + Some("bbb".to_string()) + ); + } + + #[test] + fn test_parse_fedora_cloud_listing() { + let html = r#" + Fedora-Cloud-43-1.6-x86_64-CHECKSUM + Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2 + "#; + let (qcow2, checksum) = parse_fedora_cloud_listing(html, "43").unwrap(); + assert_eq!(qcow2, "Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2"); + assert_eq!(checksum, "Fedora-Cloud-43-1.6-x86_64-CHECKSUM"); } #[tokio::test] @@ -1301,4 +1792,57 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 Ok(()) } + + #[tokio::test] + #[serial] + async fn test_custom_image_nonexistent_local_path() -> Result<()> { + test_subscriber_init(); + + let config = CustomImageConfig { + image_source: ImageSource::LocalPath(PathBuf::from("/nonexistent/image.qcow2")), + image_hash: "fakehash".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: None, + kernel_hash: None, + initrd_source: None, + initrd_hash: None, + }; + + let result = create_custom_image("test-nonexistent", config).await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("does not exist")); + + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_custom_image_hash_mismatch() -> Result<()> { + test_subscriber_init(); + + let tmp = tempfile::NamedTempFile::new()?; + let path = tmp.path().to_path_buf(); + + { + use std::io::Write; + let mut f = std::fs::File::create(&path)?; + f.write_all(b"test content")?; + } + + let config = CustomImageConfig { + image_source: ImageSource::LocalPath(path), + image_hash: "wronghash123".to_string(), + image_hash_type: ShaType::Sha256, + kernel_source: None, + kernel_hash: None, + initrd_source: None, + initrd_hash: None, + }; + + let result = create_custom_image("test-hash-mismatch", config).await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("hash mismatch")); + + Ok(()) + } } diff --git a/src/machine.rs b/src/machine.rs index 0e082ad..a046512 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -35,6 +35,8 @@ pub struct Machine { /// SSH session ssh: Option, cid: u32, + /// Optional host-forwarded TCP port for SSH (used as a fallback when vsock is unavailable). + ssh_tcp_port: u16, /// QEMU process ID pid: Option, /// Indicates whether QEMU is expected to exit. @@ -135,6 +137,13 @@ impl Machine { // Get a free CID let cid = get_free_cid(&dirs.runs, &run_dir)?; + // Reserve an ephemeral local TCP port for SSH host forwarding. + // We don't keep the listener open; the goal is just to select a port that is very + // likely to be free. The QEMU command will fail loudly if there is a race. + let ssh_tcp_port = std::net::TcpListener::bind(("127.0.0.1", 0)) + .map(|l| l.local_addr().map(|a| a.port())) + .map_err(|e| anyhow::anyhow!("Failed to reserve TCP port for SSH hostfwd: {e}"))??; + // Prepare cloud-init config let meta_data = MetaData { instance_id: format!("VM-{}", &machine_id), @@ -188,6 +197,7 @@ impl Machine { keypair: ssh_keypair, ssh: None, cid, + ssh_tcp_port, pid: None, qemu_should_exit: Arc::new(AtomicBool::new(false)), ssh_cancel_token: None, @@ -597,6 +607,7 @@ impl Machine { vmid: self.id.to_owned(), is_init, mac_address: self.mac_address.to_owned(), + ssh_tcp_port: Some(self.ssh_tcp_port), cancel_token: self .ssh_cancel_token .as_ref() @@ -606,16 +617,25 @@ impl Machine { }; let kvm_available = KVM_AVAILABLE.get().copied().unwrap_or(false); + // SSH reachability can be slow on first boot (cloud-init + sshd startup), especially on + // slower disks or under nested virtualization (e.g. WSL2). + // Use a generous timeout so E2E tests reflect real readiness rather than flakiness. let ssh_timeout = if kvm_available { - Duration::from_secs(60) + Duration::from_secs(180) } else { // Give more time if KVM is not available - Duration::from_secs(180) + Duration::from_secs(300) }; + info!( + "🔌 SSH transports: prefer vsock cid={} port=22, tcp fallback 127.0.0.1:{}", + self.cid, self.ssh_tcp_port + ); + let qemu_handle = tokio::spawn(launch_qemu(qemu_params)); let ssh_handle = tokio::spawn(connect_ssh( self.cid, + Some(self.ssh_tcp_port), ssh_timeout, self.keypair.to_owned(), self.ssh_cancel_token diff --git a/src/qemu.rs b/src/qemu.rs index ef239ed..1a7acc4 100644 --- a/src/qemu.rs +++ b/src/qemu.rs @@ -31,15 +31,18 @@ pub struct QemuLaunchParams { pub is_init: bool, pub cancel_token: CancellationToken, pub mac_address: String, + /// Optional host-forwarded TCP port for SSH (used as a fallback when vsock is unavailable). + pub ssh_tcp_port: Option, } pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { // Prepare QEMU command let mut qemu_cmd = tokio::process::Command::new("qemu-system-x86_64"); + qemu_cmd // Decrease idle CPU usage .args(["-machine", "hpet=off"]) - // SSH port forwarding + // Vsock device (preferred transport) .args([ "-device", &format!( @@ -61,19 +64,70 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { ), ]) // No GUI - .arg("-nographic") - // Network - .args(["-netdev", "bridge,id=net0,br=qlbr0"]) - .args([ - "-device", - &format!("virtio-net-pci,netdev=net0,mac={}", params.mac_address), - ]) - // Memory and CPUs + .arg("-nographic"); + + // --------------------------------------------------------------------- + // Network + // Prefer bridged networking for parity with "real" hosts, but fall back to + // user-mode networking (slirp) when bridging is unavailable (common on WSL2 + // or hosts without qemu bridge ACL configured). + // + // When using user-mode networking, we rely on hostfwd for SSH TCP fallback. + // --------------------------------------------------------------------- + let bridge_name = "qlbr0"; + let ssh_port = params.ssh_tcp_port; + let want_bridge = !is_wsl() && has_iface(bridge_name) && bridge_conf_allows(bridge_name); + + if want_bridge { + qemu_cmd + .args(["-netdev", &format!("bridge,id=net0,br={bridge_name}")]) + .args([ + "-device", + &format!("virtio-net-pci,netdev=net0,mac={}", params.mac_address), + ]); + + // Optional user-mode networking with host port forwarding for SSH fallback. + // This provides a TCP escape hatch even when vsock is unreliable. + if let Some(port) = ssh_port { + qemu_cmd + .args([ + "-netdev", + &format!("user,id=net1,hostfwd=tcp:127.0.0.1:{}-:22", port), + ]) + .args(["-device", "virtio-net-pci,netdev=net1"]); + } + } else { + if is_wsl() { + info!( + "WSL detected: disabling bridged networking and using user-mode networking + hostfwd for SSH." + ); + } else { + warn!( + "Bridged networking is unavailable (missing /etc/qemu/bridge.conf allow, or bridge interface not present). Falling back to user-mode networking + hostfwd." + ); + } + + let port = ssh_port.ok_or_else(|| { + anyhow::anyhow!("user-mode networking fallback requires ssh_tcp_port to be set") + })?; + + qemu_cmd + .args([ + "-netdev", + &format!("user,id=net0,hostfwd=tcp:127.0.0.1:{}-:22", port), + ]) + .args([ + "-device", + &format!("virtio-net-pci,netdev=net0,mac={}", params.mac_address), + ]); + } + + // Memory and CPUs + qemu_cmd .args(["-m", ¶ms.config.mem.to_string()]) .args(["-smp", ¶ms.config.core.to_string()]) // Output redirection .args(["-serial", "mon:stdio"]); - if params.is_init { // Seed ISO qemu_cmd.args([ @@ -165,3 +219,41 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { result } + +fn is_wsl() -> bool { + // Best-effort detection for WSL. We avoid hard failures if files are missing. + let osrelease = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap_or_default(); + if osrelease.to_lowercase().contains("microsoft") { + return true; + } + let version = std::fs::read_to_string("/proc/version").unwrap_or_default(); + version.to_lowercase().contains("microsoft") +} + +fn bridge_conf_allows(bridge: &str) -> bool { + // qemu-bridge-helper enforces an ACL file (commonly /etc/qemu/bridge.conf). + // If the file is missing or doesn't allow the bridge, QEMU will fail with: + // "failed to parse default acl file `/etc/qemu/bridge.conf`" or "bridge helper failed". + let conf = match std::fs::read_to_string("/etc/qemu/bridge.conf") { + Ok(c) => c, + Err(_) => return false, + }; + for line in conf.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + // Supported entries: "allow ". + if let Some(rest) = line.strip_prefix("allow ") { + let b = rest.trim(); + if b == "all" || b == bridge { + return true; + } + } + } + false +} + +fn has_iface(name: &str) -> bool { + std::path::Path::new(&format!("/sys/class/net/{name}")).exists() +} diff --git a/src/ssh.rs b/src/ssh.rs index 595bc39..7200ca3 100644 --- a/src/ssh.rs +++ b/src/ssh.rs @@ -17,11 +17,12 @@ use russh::{ use russh_sftp::{client::SftpSession, protocol::OpenFlags}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, time::Instant, }; use tokio_util::sync::CancellationToken; use tokio_vsock::{VsockAddr, VsockStream}; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; #[derive(Clone, Debug)] pub struct PersistedSshKeypair { @@ -144,9 +145,8 @@ impl Session { // This is "No such device" but for some reason Rust doesn't have an IO // ErrorKind for it. Meh. if now.elapsed() > timeout { - error!( - "Reached timeout trying to connect to virtual machine via SSH, aborting" - ); + // Don't log this as an error here: higher-level logic may fall back to TCP. + warn!("Timeout connecting to VM via vsock"); bail!("Timeout"); } continue; @@ -154,12 +154,16 @@ impl Session { Err(ref e) => match e.kind() { ErrorKind::TimedOut | ErrorKind::ConnectionRefused - | ErrorKind::ConnectionReset => { + | ErrorKind::ConnectionReset + | ErrorKind::NetworkUnreachable + | ErrorKind::AddrNotAvailable => { if now.elapsed() > timeout { - error!( - "Reached timeout trying to connect to virtual machine via SSH, aborting" + // Higher-level logic may fall back to TCP; keep this at warn level. + warn!("Timeout while connecting to VM via vsock"); + bail!( + "Timeout while connecting to VM via vsock.\n\ +Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hypervisor provides a working vsock path." ); - bail!("Timeout"); } continue; } @@ -179,9 +183,7 @@ impl Session { // for some time. ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset => { if now.elapsed() > timeout { - error!( - "Reached timeout trying to connect to virtual machine via SSH, aborting" - ); + warn!("Timeout establishing SSH over vsock"); bail!("Timeout"); } } @@ -193,9 +195,7 @@ impl Session { } Err(russh::Error::Disconnect) => { if now.elapsed() > timeout { - error!( - "Reached timeout trying to connect to virtual machine via SSH, aborting" - ); + warn!("Timeout establishing SSH over vsock (disconnect loop)"); bail!("Timeout"); } } @@ -354,20 +354,141 @@ impl Session { .await?; Ok(()) } + + /// Connect to an SSH server via TCP (used as a fallback when vsock is unavailable or flaky). + async fn connect_tcp( + privkey: PrivateKey, + host: &str, + port: u16, + timeout: Duration, + cancel_token: CancellationToken, + ) -> Result { + let config = russh::client::Config { + keepalive_interval: Some(Duration::from_secs(5)), + ..<_>::default() + }; + let config = Arc::new(config); + let sh = SshClient {}; + + let now = Instant::now(); + info!("🔑 Connecting via tcp {}:{}", host, port); + + let addr = format!("{}:{}", host, port); + let mut session = loop { + if cancel_token.is_cancelled() { + info!("SSH connection cancelled during connect loop"); + bail!("SSH connection cancelled"); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + + let stream = match TcpStream::connect(&addr).await { + Ok(s) => s, + Err(e) => match e.kind() { + ErrorKind::TimedOut + | ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::NetworkUnreachable + | ErrorKind::AddrNotAvailable => { + if now.elapsed() > timeout { + bail!( + "Timeout while connecting to VM via tcp {}\nHint: Ensure QEMU host port forwarding is enabled and the guest sshd is running.", + addr + ); + } + continue; + } + _ => { + error!("Unhandled TCP connect error: {e}"); + bail!("Unknown error"); + } + }, + }; + + match russh::client::connect_stream(config.clone(), stream, sh.clone()).await { + Ok(x) => break x, + Err(russh::Error::IO(ref e)) => match e.kind() { + ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset => { + if now.elapsed() > timeout { + bail!("Timeout"); + } + } + _ => { + error!("Unhandled error occurred: {e}"); + bail!("Unknown error"); + } + }, + Err(russh::Error::Disconnect) => { + if now.elapsed() > timeout { + bail!("Timeout"); + } + } + Err(e) => { + error!("Unhandled error occurred: {e}"); + bail!("Unknown error"); + } + } + }; + + debug!("Authenticating via SSH"); + let auth_res = session + .authenticate_publickey("root", PrivateKeyWithHashAlg::new(Arc::new(privkey), None)) + .await?; + if !auth_res.success() { + bail!("Authentication (with publickey) failed"); + } + Ok(Self { + session, + sftp: None, + }) + } } /// Connect SSH and run a command that checks whether the system is ready for operation. pub async fn connect_ssh( cid: u32, + tcp_port: Option, timeout: Duration, keypair: PersistedSshKeypair, cancel_token: CancellationToken, ) -> Result { let privkey = PrivateKey::from_openssh(&keypair.privkey_str)?; - // Session is a wrapper around a russh client, defined down below. - let mut ssh = Session::connect(privkey, cid, 22, timeout, cancel_token.clone()).await?; - info!("✅ Connected"); + // Prefer vsock, but don't wait the full timeout if we have a TCP fallback. + // On some hosts (notably WSL2), vsock can be flaky/unreachable even when /dev/vhost-vsock + // exists. In those cases we want to fall back quickly to TCP host forwarding. + let vsock_timeout = if tcp_port.is_some() { + std::cmp::min(timeout, Duration::from_secs(15)) + } else { + timeout + }; + + let mut ssh = match Session::connect( + privkey.clone(), + cid, + 22, + vsock_timeout, + cancel_token.clone(), + ) + .await + { + Ok(s) => { + info!("✅ Connected via vsock"); + s + } + Err(e) => { + if let Some(port) = tcp_port { + warn!("Vsock SSH failed ({e}). Falling back to tcp 127.0.0.1:{port}"); + let s = + Session::connect_tcp(privkey, "127.0.0.1", port, timeout, cancel_token.clone()) + .await?; + info!("✅ Connected via tcp"); + s + } else { + return Err(e); + } + } + }; // First we'll wait until the system has fully booted up. let is_running_exitcode = ssh diff --git a/tests/arch_image.rs b/tests/arch_image.rs new file mode 100644 index 0000000..259821f --- /dev/null +++ b/tests/arch_image.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use qlean::{Distro, MachineConfig, create_image, with_machine}; +use serial_test::serial; +use std::{str, time::Duration}; + +mod common; +mod guestfish; +use common::{should_run_vm_tests, tracing_subscriber_init}; +use guestfish::has_guestfish_tools; + +#[tokio::test] +#[serial] +async fn test_arch_image_startup_flow() -> Result<()> { + tracing_subscriber_init(); + + if !should_run_vm_tests() { + return Ok(()); + } + + // `has_guestfish_tools` prints an actionable SKIP reason on failure. + if !has_guestfish_tools() { + return Ok(()); + } + + let image = tokio::time::timeout( + Duration::from_secs(25 * 60), + create_image(Distro::Arch, "arch-cloudimg"), + ) + .await??; + + assert!(image.path().exists(), "qcow2 image must exist"); + assert!(image.kernel().exists(), "kernel must exist"); + assert!(image.initrd().exists(), "initrd must exist"); + + // Full startup flow validation (mirrors single_machine.rs::hello) + let config = MachineConfig::default(); + tokio::time::timeout(Duration::from_secs(8 * 60), async { + with_machine(&image, &config, |vm| { + Box::pin(async { + let result = vm.exec("whoami").await?; + assert!(result.status.success()); + assert_eq!(str::from_utf8(&result.stdout)?.trim(), "root"); + Ok(()) + }) + }) + .await + }) + .await??; + + Ok(()) +} diff --git a/tests/common.rs b/tests/common.rs index aeccf82..79ddf8b 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -1,4 +1,4 @@ -use std::sync::Once; +use std::{path::Path, process::Command, sync::Once}; use tracing_subscriber::{EnvFilter, fmt::time::LocalTime}; pub fn tracing_subscriber_init() { @@ -7,6 +7,106 @@ pub fn tracing_subscriber_init() { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .with_timer(LocalTime::rfc_3339()) - .init(); + // Multiple test modules may try to install a global subscriber. + // Use try_init to avoid panics when one is already set. + .try_init() + .ok(); }); } + +/// Gate slow / privileged integration tests. +/// Enable by running: `QLEAN_RUN_E2E=1 cargo test --test ubuntu_image -- --nocapture` +pub fn e2e_enabled() -> bool { + matches!( + std::env::var("QLEAN_RUN_E2E").as_deref(), + Ok("1") | Ok("true") | Ok("yes") + ) +} + +/// Best-effort check for WSL. Many QEMU/KVM setups won't work in WSL. +pub fn is_wsl() -> bool { + if std::env::var_os("WSL_INTEROP").is_some() || std::env::var_os("WSL_DISTRO_NAME").is_some() { + return true; + } + std::fs::read_to_string("/proc/version") + .map(|s| s.to_lowercase().contains("microsoft")) + .unwrap_or(false) +} + +/// Return `true` if a command exists on PATH. +pub fn has_cmd(cmd: &str) -> bool { + match Command::new(cmd).arg("--version").output() { + Ok(_) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + Err(_) => false, + } +} + +/// Best-effort connectivity check for the libvirt system URI. +/// +/// In WSL (and some constrained environments), `virsh` may exist but the user +/// may not have permission to access the system socket, or libvirtd may not be +/// running. In those cases we prefer to skip rather than hang. +fn can_connect_libvirt_system() -> bool { + Command::new("virsh") + .args(["-c", "qemu:///system", "list", "--all"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Qlean's SSH transport currently relies on vhost-vsock (AF_VSOCK). +/// If the host kernel doesn't expose vhost-vsock, VM startup tests will fail +/// with "network unreachable" when attempting to connect via vsock. +fn has_vhost_vsock() -> bool { + Path::new("/dev/vhost-vsock").exists() || Path::new("/sys/module/vhost_vsock").exists() +} + +/// Skip E2E tests early if the environment can't support them. +/// Returns `true` if the test should proceed. +pub fn should_run_vm_tests() -> bool { + if !e2e_enabled() { + eprintln!("SKIP: E2E VM tests disabled. Set QLEAN_RUN_E2E=1 to enable."); + return false; + } + if !Path::new("/dev/kvm").exists() { + eprintln!( + "SKIP: /dev/kvm not found. Install/enable KVM or run on a host with virtualization." + ); + return false; + } + // Qlean's current VM backend uses libvirt/virsh. + if !has_cmd("virsh") { + eprintln!("SKIP: could not find `virsh` on PATH (libvirt-clients)."); + return false; + } + if !has_cmd("qemu-system-x86_64") && !has_cmd("qemu-kvm") { + eprintln!("SKIP: could not find `qemu-system-x86_64` (or `qemu-kvm`) on PATH."); + return false; + } + + if !has_vhost_vsock() { + eprintln!( + "SKIP: vhost-vsock not available on this kernel (/dev/vhost-vsock or /sys/module/vhost_vsock missing).\n\ +Qlean currently uses vsock for SSH; enable the vhost_vsock kernel module or run on a host that provides it." + ); + return false; + } + + // WSL can support /dev/kvm on newer builds, but libvirt permissions/config + // are frequently missing. If we're on WSL, require a quick connectivity + // check to avoid false negatives or long hangs. + if is_wsl() { + if !can_connect_libvirt_system() { + eprintln!( + "SKIP: WSL detected and `virsh -c qemu:///system` is not usable (permission or libvirtd not running).\n\ +Hint: ensure systemd is enabled in WSL, start libvirtd, and add your user to libvirt/kvm groups, then `wsl --shutdown`." + ); + return false; + } + eprintln!( + "INFO: WSL detected, but KVM/libvirt appear usable; proceeding with E2E VM tests." + ); + } + true +} diff --git a/tests/custom_image.rs b/tests/custom_image.rs deleted file mode 100644 index 1fad1e1..0000000 --- a/tests/custom_image.rs +++ /dev/null @@ -1,130 +0,0 @@ -use anyhow::Result; -use qlean::{CustomImageConfig, ImageSource, ShaType, create_custom_image}; -use serial_test::serial; -use std::path::PathBuf; - -mod common; -use common::tracing_subscriber_init; - -// --------------------------------------------------------------------------- -// Unit tests for CustomImageConfig -// --------------------------------------------------------------------------- - -#[test] -fn test_custom_image_config_with_preextracted_serde() { - let config = CustomImageConfig { - image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), - image_hash: "abcdef123456".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: Some(ImageSource::Url("https://example.com/vmlinuz".to_string())), - kernel_hash: Some("kernel789".to_string()), - initrd_source: Some(ImageSource::Url("https://example.com/initrd".to_string())), - initrd_hash: Some("initrd012".to_string()), - }; - - let json = serde_json::to_string(&config).unwrap(); - let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - - assert_eq!(decoded.image_hash, "abcdef123456"); - assert_eq!(decoded.kernel_hash, Some("kernel789".to_string())); - assert_eq!(decoded.initrd_hash, Some("initrd012".to_string())); -} - -#[test] -fn test_custom_image_config_url_serde() { - let config = CustomImageConfig { - image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), - image_hash: "abc123".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: None, - kernel_hash: None, - initrd_source: None, - initrd_hash: None, - }; - - let json = serde_json::to_string(&config).unwrap(); - let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - - assert_eq!(decoded.image_hash, "abc123"); - // Test that None values are properly serialized/deserialized - assert!(decoded.kernel_source.is_none()); -} - -#[test] -fn test_custom_image_config_local_path_serde() { - let config = CustomImageConfig { - image_source: ImageSource::LocalPath(PathBuf::from("/path/to/image.qcow2")), - image_hash: "def456".to_string(), - image_hash_type: ShaType::Sha512, - kernel_source: Some(ImageSource::LocalPath(PathBuf::from("/path/to/vmlinuz"))), - kernel_hash: Some("kernelhash".to_string()), - initrd_source: Some(ImageSource::LocalPath(PathBuf::from("/path/to/initrd"))), - initrd_hash: Some("initrdhash".to_string()), - }; - - let json = serde_json::to_string(&config).unwrap(); - let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - - assert_eq!(decoded.image_hash, "def456"); - match decoded.kernel_source.unwrap() { - ImageSource::LocalPath(p) => assert_eq!(p, PathBuf::from("/path/to/vmlinuz")), - _ => panic!("Expected LocalPath"), - } -} - -// --------------------------------------------------------------------------- -// Error handling tests -// --------------------------------------------------------------------------- - -#[tokio::test] -#[serial] -async fn test_custom_image_nonexistent_local_path() -> Result<()> { - tracing_subscriber_init(); - - let config = CustomImageConfig { - image_source: ImageSource::LocalPath(PathBuf::from("/nonexistent/image.qcow2")), - image_hash: "fakehash".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: None, - kernel_hash: None, - initrd_source: None, - initrd_hash: None, - }; - - let result = create_custom_image("test-nonexistent", config).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("does not exist")); - - Ok(()) -} - -#[tokio::test] -#[serial] -async fn test_custom_image_hash_mismatch() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"test content")?; - } - - let config = CustomImageConfig { - image_source: ImageSource::LocalPath(path), - image_hash: "wronghash123".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: None, - kernel_hash: None, - initrd_source: None, - initrd_hash: None, - }; - - let result = create_custom_image("test-hash-mismatch", config).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("hash mismatch")); - - Ok(()) -} diff --git a/tests/fedora_image.rs b/tests/fedora_image.rs new file mode 100644 index 0000000..85ae0cb --- /dev/null +++ b/tests/fedora_image.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use qlean::{Distro, MachineConfig, create_image, with_machine}; +use serial_test::serial; +use std::{str, time::Duration}; + +mod common; +mod guestfish; +use common::{should_run_vm_tests, tracing_subscriber_init}; +use guestfish::has_guestfish_tools; + +#[tokio::test] +#[serial] +async fn test_fedora_image_startup_flow() -> Result<()> { + tracing_subscriber_init(); + + if !should_run_vm_tests() { + return Ok(()); + } + + // `has_guestfish_tools` prints an actionable SKIP reason on failure. + if !has_guestfish_tools() { + return Ok(()); + } + + let image = tokio::time::timeout( + Duration::from_secs(25 * 60), + create_image(Distro::Fedora, "fedora-cloud"), + ) + .await??; + + assert!(image.path().exists(), "qcow2 image must exist"); + assert!(image.kernel().exists(), "kernel must exist"); + assert!(image.initrd().exists(), "initrd must exist"); + + // Full startup flow validation (mirrors single_machine.rs::hello) + let config = MachineConfig::default(); + tokio::time::timeout(Duration::from_secs(8 * 60), async { + with_machine(&image, &config, |vm| { + Box::pin(async { + let result = vm.exec("whoami").await?; + assert!(result.status.success()); + assert_eq!(str::from_utf8(&result.stdout)?.trim(), "root"); + Ok(()) + }) + }) + .await + }) + .await??; + + Ok(()) +} diff --git a/tests/guestfish.rs b/tests/guestfish.rs new file mode 100644 index 0000000..941a9a3 --- /dev/null +++ b/tests/guestfish.rs @@ -0,0 +1,130 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::OnceLock; + +/// Return `true` if a command exists on PATH. +fn has_cmd(cmd: &str) -> bool { + match Command::new(cmd).arg("--version").output() { + Ok(_) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, + Err(_) => false, + } +} + +/// Some distros require libguestfs (guestfish/virt-copy-out) to extract kernel/initrd from qcow2. +/// +/// Note: On WSL and other minimal environments, libguestfs can *exist* but fail at runtime because +/// `supermin` cannot find a host kernel image under /boot. In that case we skip and print remediation. +pub fn has_guestfish_tools() -> bool { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| has_guestfish_tools_inner()) +} + +fn has_guestfish_tools_inner() -> bool { + if !(has_cmd("guestfish") && has_cmd("virt-copy-out")) { + eprintln!( + "SKIP: Fedora/Arch extraction requires `guestfish` and `virt-copy-out` (package: libguestfs-tools).\n\ +Install with: sudo apt install -y libguestfs-tools" + ); + return false; + } + + // libguestfs can use a prebuilt appliance. If one is present, we're usually good. + let appliance_kernel_candidates = [ + "/usr/lib/guestfs/appliance/kernel", + "/usr/lib64/guestfs/appliance/kernel", + "/usr/share/guestfs/appliance/kernel", + ]; + if appliance_kernel_candidates + .iter() + .any(|p| Path::new(p).exists()) + { + // On WSL, even with tools installed, runtime can still fail. Probe in that case. + return if is_wsl() { + probe_libguestfs_runtime() + } else { + true + }; + } + + // Otherwise it will try to build an appliance via supermin, which requires a host kernel image. + let has_host_kernel = Path::new("/boot/vmlinuz").exists() + || std::fs::read_dir("/boot") + .map(|it| { + it.filter_map(|e| e.ok()) + .any(|e| e.file_name().to_string_lossy().starts_with("vmlinuz")) + }) + .unwrap_or(false); + + if has_host_kernel { + return if is_wsl() { + probe_libguestfs_runtime() + } else { + true + }; + } + + eprintln!( + "SKIP: `guestfish` is installed but libguestfs appliance/kernel is missing.\n\ +On WSL this commonly breaks `supermin`. Install a *kernel image package* so /boot contains vmlinuz*, e.g.:\n\ + sudo apt install -y linux-image-generic\n\ + # or (smaller)\n\ + sudo apt install -y linux-image-virtual\n\ +Then retry.\n\ +(If /boot is not usable in your environment, you can build a fixed appliance once and point libguestfs to it:)\n\ + mkdir -p ~/.local/share/qlean/guestfs-appliance\n\ + libguestfs-make-fixed-appliance ~/.local/share/qlean/guestfs-appliance\n\ + export LIBGUESTFS_PATH=~/.local/share/qlean/guestfs-appliance" + ); + false +} + +fn is_wsl() -> bool { + std::fs::read_to_string("/proc/version") + .map(|s| s.to_lowercase().contains("microsoft")) + .unwrap_or(false) +} + +fn probe_libguestfs_runtime() -> bool { + // If the test tool isn't present, we can't reliably probe; fall back to the heuristic checks. + if !has_cmd("libguestfs-test-tool") { + return true; + } + + // Use the system `timeout` if available to avoid hanging CI/WSL runs. + let use_timeout = has_cmd("timeout"); + let mut cmd = if use_timeout { + let mut c = Command::new("timeout"); + c.arg("30s").arg("libguestfs-test-tool"); + c + } else { + Command::new("libguestfs-test-tool") + }; + + cmd.env("LIBGUESTFS_BACKEND", "direct") + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + + match cmd.output() { + Ok(out) if out.status.success() => true, + Ok(out) => { + let stderr = String::from_utf8_lossy(&out.stderr); + let first_lines = stderr.lines().take(8).collect::>().join("\n"); + eprintln!( + "SKIP: libguestfs runtime probe failed (guestfish/virt-copy-out will likely fail).\n\ +Hint: run `LIBGUESTFS_BACKEND=direct libguestfs-test-tool` to see full diagnostics.\n\ +stderr (first lines):\n{}", + first_lines + ); + false + } + Err(e) => { + eprintln!( + "SKIP: failed to run libguestfs runtime probe: {}\n\ +Hint: ensure `libguestfs-tools` is installed.", + e + ); + false + } + } +} diff --git a/tests/streaming_hash.rs b/tests/streaming_hash.rs deleted file mode 100644 index bab804e..0000000 --- a/tests/streaming_hash.rs +++ /dev/null @@ -1,153 +0,0 @@ -use anyhow::Result; -use qlean::{compute_sha256_streaming, compute_sha512_streaming, get_sha256, get_sha512}; -use serial_test::serial; - -mod common; -use common::tracing_subscriber_init; - -// --------------------------------------------------------------------------- -// Correctness tests: streaming hash must match shell commands -// --------------------------------------------------------------------------- - -#[tokio::test] -#[serial] -async fn test_streaming_sha256_matches_shell() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"streaming sha256 correctness check")?; - } - - let shell_result = get_sha256(&path).await?; - let stream_result = compute_sha256_streaming(&path).await?; - - assert_eq!( - shell_result, stream_result, - "streaming SHA-256 must match shell command output" - ); - - Ok(()) -} - -#[tokio::test] -#[serial] -async fn test_streaming_sha512_matches_shell() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"streaming sha512 correctness check")?; - } - - let shell_result = get_sha512(&path).await?; - let stream_result = compute_sha512_streaming(&path).await?; - - assert_eq!( - shell_result, stream_result, - "streaming SHA-512 must match shell command output" - ); - - Ok(()) -} - -// --------------------------------------------------------------------------- -// Edge case tests -// --------------------------------------------------------------------------- - -#[tokio::test] -async fn test_streaming_sha256_empty_file() -> Result<()> { - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - let hash = compute_sha256_streaming(&path).await?; - - // SHA-256 of empty file (well-known constant) - assert_eq!( - hash, - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_streaming_sha256_small_file() -> Result<()> { - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"hello world")?; - } - - let hash = compute_sha256_streaming(&path).await?; - - // SHA-256 of "hello world" - assert_eq!( - hash, - "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_streaming_sha512_known_value() -> Result<()> { - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"The quick brown fox jumps over the lazy dog")?; - } - - let hash = compute_sha512_streaming(&path).await?; - - // SHA-512 of "The quick brown fox jumps over the lazy dog" - assert_eq!( - hash, - "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6" - ); - - Ok(()) -} - -// --------------------------------------------------------------------------- -// Large file tests -// --------------------------------------------------------------------------- - -#[tokio::test] -#[serial] -async fn test_streaming_sha256_10mb_file() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - let chunk = vec![0xABu8; 1024 * 1024]; // 1 MB of 0xAB - for _ in 0..10 { - f.write_all(&chunk)?; - } - } - - let shell = get_sha256(&path).await?; - let stream = compute_sha256_streaming(&path).await?; - - assert_eq!(shell, stream, "10MB file: streaming must match shell"); - - Ok(()) -} diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs index 8cfa456..c742630 100644 --- a/tests/ubuntu_image.rs +++ b/tests/ubuntu_image.rs @@ -1,27 +1,45 @@ use anyhow::Result; -use qlean::{Distro, create_image}; +use qlean::{Distro, MachineConfig, create_image, with_machine}; use serial_test::serial; +use std::{str, time::Duration}; mod common; -use common::tracing_subscriber_init; +use common::{should_run_vm_tests, tracing_subscriber_init}; #[tokio::test] #[serial] -#[ignore] async fn test_ubuntu_image_creation() -> Result<()> { tracing_subscriber_init(); - // Ubuntu uses pre-extracted kernel/initrd - no guestfish needed! - let image = create_image(Distro::Ubuntu, "ubuntu-noble-cloudimg").await?; + if !should_run_vm_tests() { + return Ok(()); + } + + // Ubuntu uses pre-extracted kernel/initrd. + let image = tokio::time::timeout( + Duration::from_secs(15 * 60), + create_image(Distro::Ubuntu, "ubuntu-noble-cloudimg"), + ) + .await??; assert!(image.path().exists(), "qcow2 image must exist"); assert!(image.kernel().exists(), "kernel must exist"); assert!(image.initrd().exists(), "initrd must exist"); - println!("✅ Ubuntu image created successfully!"); - println!(" Image: {}", image.path().display()); - println!(" Kernel: {}", image.kernel().display()); - println!(" Initrd: {}", image.initrd().display()); + // Full startup flow validation (mirrors single_machine.rs::hello) + let config = MachineConfig::default(); + tokio::time::timeout(Duration::from_secs(5 * 60), async { + with_machine(&image, &config, |vm| { + Box::pin(async { + let result = vm.exec("whoami").await?; + assert!(result.status.success()); + assert_eq!(str::from_utf8(&result.stdout)?.trim(), "root"); + Ok(()) + }) + }) + .await + }) + .await??; Ok(()) } From f52f7e2fa332817b5fc4382ff8d74229f1b43d80 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Mon, 23 Feb 2026 00:09:23 +0800 Subject: [PATCH 07/20] feat(log): Adjust file structure Signed-off-by: userhaptop <1307305157@qq.com> --- tests/arch_image.rs | 10 ++++++++-- tests/fedora_image.rs | 10 ++++++++-- tests/machine_pool.rs | 5 +++-- tests/single_machine.rs | 5 +++-- tests/{common.rs => support/e2e.rs} | 29 ++--------------------------- tests/{ => support}/guestfish.rs | 13 +++++-------- tests/support/logging.rs | 18 ++++++++++++++++++ tests/ubuntu_image.rs | 9 +++++++-- 8 files changed, 54 insertions(+), 45 deletions(-) rename tests/{common.rs => support/e2e.rs} (69%) rename tests/{ => support}/guestfish.rs (83%) create mode 100644 tests/support/logging.rs diff --git a/tests/arch_image.rs b/tests/arch_image.rs index 259821f..d5d7f14 100644 --- a/tests/arch_image.rs +++ b/tests/arch_image.rs @@ -3,10 +3,16 @@ use qlean::{Distro, MachineConfig, create_image, with_machine}; use serial_test::serial; use std::{str, time::Duration}; -mod common; +#[path = "support/e2e.rs"] +mod e2e; +#[path = "support/guestfish.rs"] mod guestfish; -use common::{should_run_vm_tests, tracing_subscriber_init}; +#[path = "support/logging.rs"] +mod logging; + +use e2e::should_run_vm_tests; use guestfish::has_guestfish_tools; +use logging::tracing_subscriber_init; #[tokio::test] #[serial] diff --git a/tests/fedora_image.rs b/tests/fedora_image.rs index 85ae0cb..4b18382 100644 --- a/tests/fedora_image.rs +++ b/tests/fedora_image.rs @@ -3,10 +3,16 @@ use qlean::{Distro, MachineConfig, create_image, with_machine}; use serial_test::serial; use std::{str, time::Duration}; -mod common; +#[path = "support/e2e.rs"] +mod e2e; +#[path = "support/guestfish.rs"] mod guestfish; -use common::{should_run_vm_tests, tracing_subscriber_init}; +#[path = "support/logging.rs"] +mod logging; + +use e2e::should_run_vm_tests; use guestfish::has_guestfish_tools; +use logging::tracing_subscriber_init; #[tokio::test] #[serial] diff --git a/tests/machine_pool.rs b/tests/machine_pool.rs index a0e49f3..a1b112d 100644 --- a/tests/machine_pool.rs +++ b/tests/machine_pool.rs @@ -1,8 +1,9 @@ use anyhow::Result; use qlean::{Distro, MachineConfig, create_image, with_pool}; -mod common; -use common::tracing_subscriber_init; +#[path = "support/logging.rs"] +mod logging; +use logging::tracing_subscriber_init; #[tokio::test] async fn test_ping() -> Result<()> { diff --git a/tests/single_machine.rs b/tests/single_machine.rs index 76fea78..5b75f0b 100644 --- a/tests/single_machine.rs +++ b/tests/single_machine.rs @@ -7,8 +7,9 @@ use std::{ use anyhow::Result; use qlean::{Distro, MachineConfig, create_image, with_machine}; -mod common; -use common::tracing_subscriber_init; +#[path = "support/logging.rs"] +mod logging; +use logging::tracing_subscriber_init; #[tokio::test] async fn hello() -> Result<()> { diff --git a/tests/common.rs b/tests/support/e2e.rs similarity index 69% rename from tests/common.rs rename to tests/support/e2e.rs index 79ddf8b..fde2d7d 100644 --- a/tests/common.rs +++ b/tests/support/e2e.rs @@ -1,18 +1,4 @@ -use std::{path::Path, process::Command, sync::Once}; -use tracing_subscriber::{EnvFilter, fmt::time::LocalTime}; - -pub fn tracing_subscriber_init() { - static INIT: Once = Once::new(); - INIT.call_once(|| { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) - .with_timer(LocalTime::rfc_3339()) - // Multiple test modules may try to install a global subscriber. - // Use try_init to avoid panics when one is already set. - .try_init() - .ok(); - }); -} +use std::{path::Path, process::Command}; /// Gate slow / privileged integration tests. /// Enable by running: `QLEAN_RUN_E2E=1 cargo test --test ubuntu_image -- --nocapture` @@ -23,7 +9,7 @@ pub fn e2e_enabled() -> bool { ) } -/// Best-effort check for WSL. Many QEMU/KVM setups won't work in WSL. +/// Best-effort check for WSL. pub fn is_wsl() -> bool { if std::env::var_os("WSL_INTEROP").is_some() || std::env::var_os("WSL_DISTRO_NAME").is_some() { return true; @@ -43,10 +29,6 @@ pub fn has_cmd(cmd: &str) -> bool { } /// Best-effort connectivity check for the libvirt system URI. -/// -/// In WSL (and some constrained environments), `virsh` may exist but the user -/// may not have permission to access the system socket, or libvirtd may not be -/// running. In those cases we prefer to skip rather than hang. fn can_connect_libvirt_system() -> bool { Command::new("virsh") .args(["-c", "qemu:///system", "list", "--all"]) @@ -56,8 +38,6 @@ fn can_connect_libvirt_system() -> bool { } /// Qlean's SSH transport currently relies on vhost-vsock (AF_VSOCK). -/// If the host kernel doesn't expose vhost-vsock, VM startup tests will fail -/// with "network unreachable" when attempting to connect via vsock. fn has_vhost_vsock() -> bool { Path::new("/dev/vhost-vsock").exists() || Path::new("/sys/module/vhost_vsock").exists() } @@ -75,7 +55,6 @@ pub fn should_run_vm_tests() -> bool { ); return false; } - // Qlean's current VM backend uses libvirt/virsh. if !has_cmd("virsh") { eprintln!("SKIP: could not find `virsh` on PATH (libvirt-clients)."); return false; @@ -84,7 +63,6 @@ pub fn should_run_vm_tests() -> bool { eprintln!("SKIP: could not find `qemu-system-x86_64` (or `qemu-kvm`) on PATH."); return false; } - if !has_vhost_vsock() { eprintln!( "SKIP: vhost-vsock not available on this kernel (/dev/vhost-vsock or /sys/module/vhost_vsock missing).\n\ @@ -93,9 +71,6 @@ Qlean currently uses vsock for SSH; enable the vhost_vsock kernel module or run return false; } - // WSL can support /dev/kvm on newer builds, but libvirt permissions/config - // are frequently missing. If we're on WSL, require a quick connectivity - // check to avoid false negatives or long hangs. if is_wsl() { if !can_connect_libvirt_system() { eprintln!( diff --git a/tests/guestfish.rs b/tests/support/guestfish.rs similarity index 83% rename from tests/guestfish.rs rename to tests/support/guestfish.rs index 941a9a3..3255256 100644 --- a/tests/guestfish.rs +++ b/tests/support/guestfish.rs @@ -14,22 +14,22 @@ fn has_cmd(cmd: &str) -> bool { /// Some distros require libguestfs (guestfish/virt-copy-out) to extract kernel/initrd from qcow2. /// /// Note: On WSL and other minimal environments, libguestfs can *exist* but fail at runtime because -/// `supermin` cannot find a host kernel image under /boot. In that case we skip and print remediation. +/// `supermin` cannot find a usable host kernel/appliance. In that case we skip and print remediation. pub fn has_guestfish_tools() -> bool { static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(|| has_guestfish_tools_inner()) + *CACHED.get_or_init(has_guestfish_tools_inner) } fn has_guestfish_tools_inner() -> bool { if !(has_cmd("guestfish") && has_cmd("virt-copy-out")) { eprintln!( - "SKIP: Fedora/Arch extraction requires `guestfish` and `virt-copy-out` (package: libguestfs-tools).\n\ + "SKIP: extraction requires `guestfish` and `virt-copy-out` (package: libguestfs-tools).\n\ Install with: sudo apt install -y libguestfs-tools" ); return false; } - // libguestfs can use a prebuilt appliance. If one is present, we're usually good. + // Prefer a prebuilt appliance if present. let appliance_kernel_candidates = [ "/usr/lib/guestfs/appliance/kernel", "/usr/lib64/guestfs/appliance/kernel", @@ -39,7 +39,6 @@ Install with: sudo apt install -y libguestfs-tools" .iter() .any(|p| Path::new(p).exists()) { - // On WSL, even with tools installed, runtime can still fail. Probe in that case. return if is_wsl() { probe_libguestfs_runtime() } else { @@ -47,7 +46,7 @@ Install with: sudo apt install -y libguestfs-tools" }; } - // Otherwise it will try to build an appliance via supermin, which requires a host kernel image. + // Otherwise, supermin needs a host kernel image. let has_host_kernel = Path::new("/boot/vmlinuz").exists() || std::fs::read_dir("/boot") .map(|it| { @@ -86,12 +85,10 @@ fn is_wsl() -> bool { } fn probe_libguestfs_runtime() -> bool { - // If the test tool isn't present, we can't reliably probe; fall back to the heuristic checks. if !has_cmd("libguestfs-test-tool") { return true; } - // Use the system `timeout` if available to avoid hanging CI/WSL runs. let use_timeout = has_cmd("timeout"); let mut cmd = if use_timeout { let mut c = Command::new("timeout"); diff --git a/tests/support/logging.rs b/tests/support/logging.rs new file mode 100644 index 0000000..1a29ce5 --- /dev/null +++ b/tests/support/logging.rs @@ -0,0 +1,18 @@ +use std::sync::Once; + +use tracing_subscriber::{EnvFilter, fmt::time::LocalTime}; + +/// Initialize a global tracing subscriber for integration tests. +/// +/// Multiple integration test crates may attempt to install a global subscriber. +/// We use `try_init()` to avoid panics if one is already set. +pub fn tracing_subscriber_init() { + static INIT: Once = Once::new(); + INIT.call_once(|| { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_timer(LocalTime::rfc_3339()) + .try_init() + .ok(); + }); +} diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs index c742630..fa43f49 100644 --- a/tests/ubuntu_image.rs +++ b/tests/ubuntu_image.rs @@ -3,8 +3,13 @@ use qlean::{Distro, MachineConfig, create_image, with_machine}; use serial_test::serial; use std::{str, time::Duration}; -mod common; -use common::{should_run_vm_tests, tracing_subscriber_init}; +#[path = "support/e2e.rs"] +mod e2e; +#[path = "support/logging.rs"] +mod logging; + +use e2e::should_run_vm_tests; +use logging::tracing_subscriber_init; #[tokio::test] #[serial] From ee25d9a2ab476de2eab7cf3875dedb43c73279b0 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Mon, 23 Feb 2026 00:20:00 +0800 Subject: [PATCH 08/20] feat(log): Adjust file structure Signed-off-by: userhaptop <1307305157@qq.com> --- tests/ubuntu_image.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs index fa43f49..8f5d9c3 100644 --- a/tests/ubuntu_image.rs +++ b/tests/ubuntu_image.rs @@ -20,7 +20,7 @@ async fn test_ubuntu_image_creation() -> Result<()> { return Ok(()); } - // Ubuntu uses pre-extracted kernel/initrd. + // Ubuntu uses pre-extracted kernel/initrd. let image = tokio::time::timeout( Duration::from_secs(15 * 60), create_image(Distro::Ubuntu, "ubuntu-noble-cloudimg"), From e312ef9289f3c119847726e2f3405c4ea57d60a8 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Mon, 23 Feb 2026 01:06:08 +0800 Subject: [PATCH 09/20] ci: rerun From da0ede4e2feef3fa9e10c8104a7467b4dfd241c6 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Mon, 23 Feb 2026 01:17:35 +0800 Subject: [PATCH 10/20] tests: remove obsolete streaming_hash and custom_image tests --- tests/custom_image.rs | 130 ---------------------------------- tests/streaming_hash.rs | 153 ---------------------------------------- 2 files changed, 283 deletions(-) delete mode 100644 tests/custom_image.rs delete mode 100644 tests/streaming_hash.rs diff --git a/tests/custom_image.rs b/tests/custom_image.rs deleted file mode 100644 index 1fad1e1..0000000 --- a/tests/custom_image.rs +++ /dev/null @@ -1,130 +0,0 @@ -use anyhow::Result; -use qlean::{CustomImageConfig, ImageSource, ShaType, create_custom_image}; -use serial_test::serial; -use std::path::PathBuf; - -mod common; -use common::tracing_subscriber_init; - -// --------------------------------------------------------------------------- -// Unit tests for CustomImageConfig -// --------------------------------------------------------------------------- - -#[test] -fn test_custom_image_config_with_preextracted_serde() { - let config = CustomImageConfig { - image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), - image_hash: "abcdef123456".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: Some(ImageSource::Url("https://example.com/vmlinuz".to_string())), - kernel_hash: Some("kernel789".to_string()), - initrd_source: Some(ImageSource::Url("https://example.com/initrd".to_string())), - initrd_hash: Some("initrd012".to_string()), - }; - - let json = serde_json::to_string(&config).unwrap(); - let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - - assert_eq!(decoded.image_hash, "abcdef123456"); - assert_eq!(decoded.kernel_hash, Some("kernel789".to_string())); - assert_eq!(decoded.initrd_hash, Some("initrd012".to_string())); -} - -#[test] -fn test_custom_image_config_url_serde() { - let config = CustomImageConfig { - image_source: ImageSource::Url("https://example.com/image.qcow2".to_string()), - image_hash: "abc123".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: None, - kernel_hash: None, - initrd_source: None, - initrd_hash: None, - }; - - let json = serde_json::to_string(&config).unwrap(); - let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - - assert_eq!(decoded.image_hash, "abc123"); - // Test that None values are properly serialized/deserialized - assert!(decoded.kernel_source.is_none()); -} - -#[test] -fn test_custom_image_config_local_path_serde() { - let config = CustomImageConfig { - image_source: ImageSource::LocalPath(PathBuf::from("/path/to/image.qcow2")), - image_hash: "def456".to_string(), - image_hash_type: ShaType::Sha512, - kernel_source: Some(ImageSource::LocalPath(PathBuf::from("/path/to/vmlinuz"))), - kernel_hash: Some("kernelhash".to_string()), - initrd_source: Some(ImageSource::LocalPath(PathBuf::from("/path/to/initrd"))), - initrd_hash: Some("initrdhash".to_string()), - }; - - let json = serde_json::to_string(&config).unwrap(); - let decoded: CustomImageConfig = serde_json::from_str(&json).unwrap(); - - assert_eq!(decoded.image_hash, "def456"); - match decoded.kernel_source.unwrap() { - ImageSource::LocalPath(p) => assert_eq!(p, PathBuf::from("/path/to/vmlinuz")), - _ => panic!("Expected LocalPath"), - } -} - -// --------------------------------------------------------------------------- -// Error handling tests -// --------------------------------------------------------------------------- - -#[tokio::test] -#[serial] -async fn test_custom_image_nonexistent_local_path() -> Result<()> { - tracing_subscriber_init(); - - let config = CustomImageConfig { - image_source: ImageSource::LocalPath(PathBuf::from("/nonexistent/image.qcow2")), - image_hash: "fakehash".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: None, - kernel_hash: None, - initrd_source: None, - initrd_hash: None, - }; - - let result = create_custom_image("test-nonexistent", config).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("does not exist")); - - Ok(()) -} - -#[tokio::test] -#[serial] -async fn test_custom_image_hash_mismatch() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"test content")?; - } - - let config = CustomImageConfig { - image_source: ImageSource::LocalPath(path), - image_hash: "wronghash123".to_string(), - image_hash_type: ShaType::Sha256, - kernel_source: None, - kernel_hash: None, - initrd_source: None, - initrd_hash: None, - }; - - let result = create_custom_image("test-hash-mismatch", config).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("hash mismatch")); - - Ok(()) -} diff --git a/tests/streaming_hash.rs b/tests/streaming_hash.rs deleted file mode 100644 index bab804e..0000000 --- a/tests/streaming_hash.rs +++ /dev/null @@ -1,153 +0,0 @@ -use anyhow::Result; -use qlean::{compute_sha256_streaming, compute_sha512_streaming, get_sha256, get_sha512}; -use serial_test::serial; - -mod common; -use common::tracing_subscriber_init; - -// --------------------------------------------------------------------------- -// Correctness tests: streaming hash must match shell commands -// --------------------------------------------------------------------------- - -#[tokio::test] -#[serial] -async fn test_streaming_sha256_matches_shell() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"streaming sha256 correctness check")?; - } - - let shell_result = get_sha256(&path).await?; - let stream_result = compute_sha256_streaming(&path).await?; - - assert_eq!( - shell_result, stream_result, - "streaming SHA-256 must match shell command output" - ); - - Ok(()) -} - -#[tokio::test] -#[serial] -async fn test_streaming_sha512_matches_shell() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"streaming sha512 correctness check")?; - } - - let shell_result = get_sha512(&path).await?; - let stream_result = compute_sha512_streaming(&path).await?; - - assert_eq!( - shell_result, stream_result, - "streaming SHA-512 must match shell command output" - ); - - Ok(()) -} - -// --------------------------------------------------------------------------- -// Edge case tests -// --------------------------------------------------------------------------- - -#[tokio::test] -async fn test_streaming_sha256_empty_file() -> Result<()> { - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - let hash = compute_sha256_streaming(&path).await?; - - // SHA-256 of empty file (well-known constant) - assert_eq!( - hash, - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_streaming_sha256_small_file() -> Result<()> { - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"hello world")?; - } - - let hash = compute_sha256_streaming(&path).await?; - - // SHA-256 of "hello world" - assert_eq!( - hash, - "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_streaming_sha512_known_value() -> Result<()> { - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - f.write_all(b"The quick brown fox jumps over the lazy dog")?; - } - - let hash = compute_sha512_streaming(&path).await?; - - // SHA-512 of "The quick brown fox jumps over the lazy dog" - assert_eq!( - hash, - "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6" - ); - - Ok(()) -} - -// --------------------------------------------------------------------------- -// Large file tests -// --------------------------------------------------------------------------- - -#[tokio::test] -#[serial] -async fn test_streaming_sha256_10mb_file() -> Result<()> { - tracing_subscriber_init(); - - let tmp = tempfile::NamedTempFile::new()?; - let path = tmp.path().to_path_buf(); - - { - use std::io::Write; - let mut f = std::fs::File::create(&path)?; - let chunk = vec![0xABu8; 1024 * 1024]; // 1 MB of 0xAB - for _ in 0..10 { - f.write_all(&chunk)?; - } - } - - let shell = get_sha256(&path).await?; - let stream = compute_sha256_streaming(&path).await?; - - assert_eq!(shell, stream, "10MB file: streaming must match shell"); - - Ok(()) -} From fcdc6b3520b2123cec9f391ff8ef831771534bfb Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Mon, 23 Feb 2026 01:37:32 +0800 Subject: [PATCH 11/20] feat(log): Split VM prerequisites vs image-extraction deps so Ubuntu E2E no longer requires libguestfs tools. - Retry Fedora CHECKSUM fetch across mirrors (match qcow2 mirror resilience). - Fix ShaType ownership in cache/download paths. Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 61 ++++++++++++++++++++++++++++++++++++++------------ src/machine.rs | 2 +- src/utils.rs | 17 ++++++++++++-- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/src/image.rs b/src/image.rs index 495cf3f..e144a12 100644 --- a/src/image.rs +++ b/src/image.rs @@ -7,7 +7,7 @@ use sha2::{Digest, Sha256, Sha512}; use tokio::{fs::File, io::AsyncWriteExt}; use tracing::{debug, warn}; -use crate::utils::QleanDirs; +use crate::utils::{QleanDirs, ensure_extraction_prerequisites}; pub trait ImageAction { /// Download the image from remote source @@ -270,17 +270,7 @@ async fn resolve_latest_fedora_cloud_qcow2() -> Result { let (qcow2_name, checksum_name) = parse_fedora_cloud_listing(&listing_html, &ver)?; - // Use the first good base to fetch CHECKSUM (hash should match across mirrors). - let checksum_url = format!("{}{}", good_bases[0], checksum_name); - let checksum_text = fetch_text(&checksum_url).await?; - let sha256 = find_hash_for_file(&checksum_text, &qcow2_name).with_context(|| { - format!( - "could not find hash for {} in {}", - qcow2_name, checksum_name - ) - })?; - - // Build download URLs. Prefer bases where listing worked, but also try the + // Build base URLs. Prefer bases where listing worked, but also try the // full candidate set as a fallback (a mirror may block directory listing // but still serve the file). let mut bases = good_bases; @@ -289,6 +279,11 @@ async fn resolve_latest_fedora_cloud_qcow2() -> Result { bases.push(c.clone()); } } + + // Fetch CHECKSUM across mirrors too. Relying on a single mirror defeats + // the multi-mirror resilience we provide for the qcow2 download. + let sha256 = fetch_fedora_checksum_sha256(&bases, &checksum_name, &qcow2_name).await?; + let urls = bases .into_iter() .map(|base| format!("{}{}", base, qcow2_name)) @@ -297,6 +292,37 @@ async fn resolve_latest_fedora_cloud_qcow2() -> Result { Ok(ResolvedRemote { urls, sha256 }) } +async fn fetch_fedora_checksum_sha256( + bases: &[String], + checksum_name: &str, + qcow2_name: &str, +) -> Result { + let mut last_err: Option = None; + + for base in bases { + let checksum_url = format!("{}{}", base, checksum_name); + match fetch_text(&checksum_url).await { + Ok(text) => { + if let Some(sha) = find_hash_for_file(&text, qcow2_name) { + return Ok(sha); + } + + last_err = Some(anyhow::anyhow!( + "checksum file {} did not contain hash for {}", + checksum_url, + qcow2_name + )); + } + Err(e) => { + debug!("Fedora CHECKSUM fetch failed for {}: {:#}", checksum_url, e); + last_err = Some(e); + } + } + } + + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("failed to fetch Fedora CHECKSUM from mirrors"))) +} + fn parse_fedora_cloud_listing(listing_html: &str, ver: &str) -> Result<(String, String)> { // Parse filename candidates. // Listings generally contain only one compose, so the first match is fine. @@ -625,7 +651,7 @@ async fn download_or_copy_with_hash( ImageSource::Url(url) => { // Reuse cached download if it matches, otherwise overwrite. if dest.exists() { - let existing = match hash_type { + let existing = match &hash_type { ShaType::Sha256 => compute_sha256_streaming(dest).await, ShaType::Sha512 => compute_sha512_streaming(dest).await, }; @@ -636,7 +662,7 @@ async fn download_or_copy_with_hash( } } - let computed = download_with_hash(url, dest, hash_type).await?; + let computed = download_with_hash(url, dest, hash_type.clone()).await?; anyhow::ensure!( computed.to_lowercase() == expected_hash.to_lowercase(), "hash mismatch: expected {}, got {}", @@ -919,6 +945,8 @@ impl ImageAction for Debian { } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + ensure_extraction_prerequisites().await?; + let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); @@ -1108,6 +1136,8 @@ impl ImageAction for Fedora { } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + ensure_extraction_prerequisites().await?; + let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); @@ -1237,6 +1267,8 @@ impl ImageAction for Arch { } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { + ensure_extraction_prerequisites().await?; + let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); @@ -1407,6 +1439,7 @@ impl ImageAction for Custom { } // Otherwise, try to extract using guestfish + ensure_extraction_prerequisites().await?; let file_name = format!("{}.qcow2", name); let output = tokio::process::Command::new("guestfish") diff --git a/src/machine.rs b/src/machine.rs index a046512..01dcc8b 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -35,7 +35,7 @@ pub struct Machine { /// SSH session ssh: Option, cid: u32, - /// Optional host-forwarded TCP port for SSH (used as a fallback when vsock is unavailable). + /// Host-forwarded TCP port for SSH (used as a fallback when vsock is unavailable). ssh_tcp_port: u16, /// QEMU process ID pid: Option, diff --git a/src/utils.rs b/src/utils.rs index 58a4fed..e86cc62 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -123,19 +123,32 @@ impl CommandExt for tokio::process::Command { } } +/// Ensure host prerequisites for running virtual machines. +/// +/// IMPORTANT: This intentionally does **not** require libguestfs tools. +/// Some images (e.g., Ubuntu) ship pre-extracted kernel/initrd and can boot +/// without `guestfish`/`virt-copy-out`. pub async fn ensure_prerequisites() -> Result<()> { check_command_available("qemu-system-x86_64").await?; check_command_available("qemu-img").await?; check_command_available("sha256sum").await?; check_command_available("sha512sum").await?; check_command_available("xorriso").await?; - check_command_available("guestfish").await?; - check_command_available("virt-copy-out").await?; check_command_available("virsh").await?; ensure_network().await?; Ok(()) } +/// Ensure prerequisites for extracting kernel/initrd from disk images. +/// +/// This is only required for distros/custom modes that need libguestfs-based +/// extraction (guestfish/virt-copy-out). +pub async fn ensure_extraction_prerequisites() -> Result<()> { + check_command_available("guestfish").await?; + check_command_available("virt-copy-out").await?; + Ok(()) +} + async fn check_command_available(cmd: &str) -> Result<()> { let _ = tokio::process::Command::new(cmd) .arg("--version") From b341929a346ab82eb68c627f937279abf2c66128 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Tue, 24 Feb 2026 02:45:15 +0800 Subject: [PATCH 12/20] feat(log): Fixed and enhanced issues raised in the review regarding mirror download integrity, E2E testing gating, and maintainability. Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 82 +++++++++++++++++++++++++------------------- src/machine.rs | 2 +- src/ssh.rs | 2 +- tests/support/e2e.rs | 4 +-- 4 files changed, 49 insertions(+), 41 deletions(-) diff --git a/src/image.rs b/src/image.rs index e144a12..2a13ac3 100644 --- a/src/image.rs +++ b/src/image.rs @@ -132,6 +132,16 @@ pub fn find_hash_for_file(checksums_text: &str, filename: &str) -> Option Result { + let sums_url = format!("{}SHA256SUMS", base_url); + fetch_text(&sums_url).await +} + +fn ubuntu_sha256_for(checksums: &str, filename: &str) -> Result { + find_hash_for_file(checksums, filename) + .with_context(|| format!("Ubuntu SHA256SUMS did not contain hash for {}", filename)) +} + async fn fetch_text(url: &str) -> Result { // Keep metadata fetches snappy: if a mirror is slow/hung, we fall back. let client = reqwest::Client::builder() @@ -1052,27 +1062,48 @@ impl ImageAction for Ubuntu { // Ubuntu noble (24.04 LTS) cloud image base URL let base_url = "https://cloud-images.ubuntu.com/noble/current"; + let checksums = fetch_ubuntu_sha256sums(&format!("{}/", base_url)).await?; - // Download qcow2 image - let qcow2_url = format!("{}/noble-server-cloudimg-amd64.img", base_url); + // Download qcow2 image (SHA256 verified) + let qcow2_name = "noble-server-cloudimg-amd64.img"; + let qcow2_url = format!("{}/{}", base_url, qcow2_name); let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - download_file(&qcow2_url, &qcow2_path).await?; + let qcow2_sha = ubuntu_sha256_for(&checksums, qcow2_name)?; + download_or_copy_with_hash( + &ImageSource::Url(qcow2_url), + &qcow2_path, + &qcow2_sha, + ShaType::Sha256, + ) + .await?; - // Download pre-extracted kernel - let kernel_url = format!( - "{}/unpacked/noble-server-cloudimg-amd64-vmlinuz-generic", - base_url - ); + // Download pre-extracted kernel (SHA256 verified) + let kernel_name = "noble-server-cloudimg-amd64-vmlinuz-generic"; + let kernel_url = format!("{}/unpacked/{}", base_url, kernel_name); let kernel_path = image_dir.join("vmlinuz"); - download_file(&kernel_url, &kernel_path).await?; + let kernel_sha = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", kernel_name)) + .or_else(|_| ubuntu_sha256_for(&checksums, kernel_name))?; + download_or_copy_with_hash( + &ImageSource::Url(kernel_url), + &kernel_path, + &kernel_sha, + ShaType::Sha256, + ) + .await?; - // Download pre-extracted initrd - let initrd_url = format!( - "{}/unpacked/noble-server-cloudimg-amd64-initrd-generic", - base_url - ); + // Download pre-extracted initrd (SHA256 verified) + let initrd_name = "noble-server-cloudimg-amd64-initrd-generic"; + let initrd_url = format!("{}/unpacked/{}", base_url, initrd_name); let initrd_path = image_dir.join("initrd.img"); - download_file(&initrd_url, &initrd_path).await?; + let initrd_sha = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", initrd_name)) + .or_else(|_| ubuntu_sha256_for(&checksums, initrd_name))?; + download_or_copy_with_hash( + &ImageSource::Url(initrd_url), + &initrd_path, + &initrd_sha, + ShaType::Sha256, + ) + .await?; Ok(()) } @@ -1520,27 +1551,6 @@ impl ImageAction for Custom { } // Helper function to download a file -async fn download_file(url: &str, dest: &PathBuf) -> Result<()> { - debug!("Downloading {} to {}", url, dest.display()); - let response = reqwest::get(url) - .await - .with_context(|| format!("failed to download from {}", url))?; - - let mut file = File::create(dest) - .await - .with_context(|| format!("failed to create file at {}", dest.display()))?; - - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.with_context(|| "failed to read chunk from stream")?; - file.write_all(&chunk) - .await - .with_context(|| "failed to write to file")?; - } - - Ok(()) -} - // --------------------------------------------------------------------------- // Image wrapper enum // --------------------------------------------------------------------------- diff --git a/src/machine.rs b/src/machine.rs index 01dcc8b..ffe6f45 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -35,7 +35,7 @@ pub struct Machine { /// SSH session ssh: Option, cid: u32, - /// Host-forwarded TCP port for SSH (used as a fallback when vsock is unavailable). + /// Host-forwarded TCP port reserved at startup and used as an SSH fallback when vsock is unavailable. ssh_tcp_port: u16, /// QEMU process ID pid: Option, diff --git a/src/ssh.rs b/src/ssh.rs index 7200ca3..c256989 100644 --- a/src/ssh.rs +++ b/src/ssh.rs @@ -458,7 +458,7 @@ pub async fn connect_ssh( // On some hosts (notably WSL2), vsock can be flaky/unreachable even when /dev/vhost-vsock // exists. In those cases we want to fall back quickly to TCP host forwarding. let vsock_timeout = if tcp_port.is_some() { - std::cmp::min(timeout, Duration::from_secs(15)) + std::cmp::min(timeout, Duration::from_secs(30)) } else { timeout }; diff --git a/tests/support/e2e.rs b/tests/support/e2e.rs index fde2d7d..42ddf57 100644 --- a/tests/support/e2e.rs +++ b/tests/support/e2e.rs @@ -65,10 +65,8 @@ pub fn should_run_vm_tests() -> bool { } if !has_vhost_vsock() { eprintln!( - "SKIP: vhost-vsock not available on this kernel (/dev/vhost-vsock or /sys/module/vhost_vsock missing).\n\ -Qlean currently uses vsock for SSH; enable the vhost_vsock kernel module or run on a host that provides it." + "INFO: vhost-vsock not available on this kernel; E2E can still run using TCP SSH fallback." ); - return false; } if is_wsl() { From 826d7c8a38cfba3c84983d14b3efbee3458f70ba Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Tue, 24 Feb 2026 11:30:12 +0800 Subject: [PATCH 13/20] feat(log): Improved stability of image downloads and E2E testing in WSL/restricted environments: enhanced download verification and skip logic in Fedora/Arch, and fixed the potential unwrap() risk in save() mentioned in the review. Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/image.rs b/src/image.rs index 2a13ac3..a034cce 100644 --- a/src/image.rs +++ b/src/image.rs @@ -179,6 +179,8 @@ struct ResolvedRemote { /// /// This endpoint is stable and returns structured JSON. We select the first /// release entry where `isMaintained=true` and `isEol=false`. +const FALLBACK_FEDORA_RELEASE: &str = "43"; + async fn resolve_latest_fedora_release() -> Result { #[derive(Deserialize)] struct EolResp { @@ -199,19 +201,29 @@ async fn resolve_latest_fedora_release() -> Result { // Public JSON API documented on https://endoflife.date/fedora let url = "https://endoflife.date/api/v1/products/fedora/"; - let body = fetch_text(url).await?; - let parsed: EolResp = - serde_json::from_str(&body).with_context(|| "failed to parse Fedora release JSON")?; - - let latest = parsed - .result - .releases - .into_iter() - .find(|r| r.is_maintained && !r.is_eol) - .map(|r| r.name) - .with_context(|| "could not determine latest maintained Fedora release")?; - - Ok(latest) + match fetch_text(url).await { + Ok(body) => { + let parsed: EolResp = serde_json::from_str(&body) + .with_context(|| "failed to parse Fedora release JSON")?; + + let latest = parsed + .result + .releases + .into_iter() + .find(|r| r.is_maintained && !r.is_eol) + .map(|r| r.name) + .with_context(|| "could not determine latest maintained Fedora release")?; + + Ok(latest) + } + Err(err) => { + warn!( + "failed to query endoflife.date for latest Fedora release ({}); falling back to {}", + err, FALLBACK_FEDORA_RELEASE + ); + Ok(FALLBACK_FEDORA_RELEASE.to_string()) + } + } } /// Resolve the latest Fedora Cloud Base Generic qcow2 URL and its SHA256. From 04a4aa2ed04f5f770d23957bdf3e2001877a5016 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Tue, 24 Feb 2026 22:15:47 +0800 Subject: [PATCH 14/20] feat(log): make ubuntu/fedora/arch image tests run reliably with real VM boot/ssh Signed-off-by: userhaptop <1307305157@qq.com> --- src/image.rs | 982 ++++++++++++++++++++++++------------- src/machine.rs | 39 +- src/qemu.rs | 64 +-- src/ssh.rs | 265 ++++++---- tests/arch_image.rs | 29 +- tests/fedora_image.rs | 29 +- tests/support/e2e.rs | 105 ++-- tests/support/guestfish.rs | 124 +---- tests/support/logging.rs | 4 +- tests/ubuntu_image.rs | 20 +- 10 files changed, 974 insertions(+), 687 deletions(-) diff --git a/src/image.rs b/src/image.rs index a034cce..91f9471 100644 --- a/src/image.rs +++ b/src/image.rs @@ -1,14 +1,25 @@ -use std::path::{Path, PathBuf}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, +}; use anyhow::{Context, Result, bail}; use futures::StreamExt; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256, Sha512}; -use tokio::{fs::File, io::AsyncWriteExt}; -use tracing::{debug, warn}; +use tokio::{ + fs::File, + io::AsyncWriteExt, + time::{Duration, timeout}, +}; +use tracing::{debug, info, warn}; use crate::utils::{QleanDirs, ensure_extraction_prerequisites}; +fn default_root_arg() -> String { + "root=/dev/vda1".to_string() +} + pub trait ImageAction { /// Download the image from remote source fn download(&self, name: &str) -> impl std::future::Future> + Send; @@ -27,6 +38,8 @@ pub struct ImageMeta { pub path: PathBuf, pub kernel: PathBuf, pub initrd: PathBuf, + #[serde(default = "default_root_arg")] + pub root_arg: String, #[serde(skip)] pub vendor: A, pub checksum: ShaSum, @@ -77,64 +90,183 @@ pub struct CustomImageConfig { pub initrd_hash: Option, } -/// Parses SHA512SUMS format and returns the hash for an exact filename match. -/// -/// # Arguments -/// * `checksums_text` - The content of a SHA512SUMS file -/// * `filename` - The exact filename to search for (e.g., "debian-13-generic-amd64.qcow2") -/// -/// # Returns -/// The SHA512 hash if found, or None if no exact match exists -pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { - checksums_text.lines().find_map(|line| { - let mut parts = line.split_whitespace(); - let hash = parts.next()?; - let fname = parts.next()?; +/// Normalize checksum entry names across common checksum file formats. +fn normalize_checksum_name(name: &str) -> &str { + name.trim_start_matches('*').trim_start_matches("./") +} - (fname == filename).then(|| hash.to_string()) - }) +fn checksum_name_matches(entry_name: &str, wanted: &str) -> bool { + let entry = normalize_checksum_name(entry_name); + let wanted = normalize_checksum_name(wanted); + if entry == wanted { + return true; + } + if !wanted.contains('/') + && let Some(base) = entry.rsplit('/').next() + { + return base == wanted; + } + false +} + +/// Parses checksum text and returns the hash for a filename. +pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { + find_hash_for_file(checksums_text, filename) } /// Parse a checksum file and return the hash for a given filename. /// /// Supports common formats: -/// 1) " " +/// 1) " " (including "*filename" and "./filename") /// 2) "SHA256 () = " / "SHA512 () = " pub fn find_hash_for_file(checksums_text: &str, filename: &str) -> Option { - // Format 1: " " - if let Some(h) = checksums_text.lines().find_map(|line| { - let mut parts = line.split_whitespace(); - let hash = parts.next()?; - let fname = parts.next()?; - (fname == filename).then(|| hash.to_string()) - }) { - return Some(h); + let mut parts = checksums_text.split_whitespace(); + while let Some(hash) = parts.next() { + let Some(fname) = parts.next() else { break }; + if checksum_name_matches(fname, filename) { + return Some(hash.to_string()); + } } - // Format 2: "SHA256 () = " / "SHA512 () = " for line in checksums_text.lines() { let line = line.trim(); - if let Some(rest) = line.strip_prefix("SHA256 (") - && let Some(rest) = rest.strip_prefix(filename) - && let Some(rest) = rest.strip_prefix(") = ") - { - return Some(rest.trim().to_string()); - } - - if let Some(rest) = line.strip_prefix("SHA512 (") - && let Some(rest) = rest.strip_prefix(filename) - && let Some(rest) = rest.strip_prefix(") = ") - { - return Some(rest.trim().to_string()); + for prefix in ["SHA256 (", "SHA512 ("] { + if let Some(rest) = line.strip_prefix(prefix) + && let Some((entry_name, hash_part)) = rest.split_once(") = ") + && checksum_name_matches(entry_name, filename) + { + return Some(hash_part.trim().to_string()); + } } } None } +fn join_url(base: &str, path: &str) -> String { + format!( + "{}/{}", + base.trim_end_matches('/'), + path.trim_start_matches('/') + ) +} + async fn fetch_ubuntu_sha256sums(base_url: &str) -> Result { - let sums_url = format!("{}SHA256SUMS", base_url); - fetch_text(&sums_url).await + let primary = join_url(base_url, "SHA256SUMS"); + match fetch_text(&primary).await { + Ok(text) => Ok(text), + Err(primary_err) => { + let fallback = join_url(base_url, "SHA256SUMS.txt"); + fetch_text(&fallback).await.with_context(|| { + format!( + "failed to fetch Ubuntu checksums from {} or {} ({:#})", + primary, fallback, primary_err + ) + }) + } + } +} + +#[derive(Debug, Clone)] +struct ResolvedUbuntuCloudImage { + base_url: String, + disk_name: String, + kernel_name: String, + initrd_name: String, + disk_sha256: String, + kernel_sha256: Option, + initrd_sha256: Option, +} + +fn pick_existing_ubuntu_name(checksums: &str, candidates: &[&str]) -> Option { + candidates + .iter() + .find(|name| { + checksums.contains(&format!(" {}", name)) || checksums.contains(&format!(" *{}", name)) + }) + .map(|name| (*name).to_string()) +} + +async fn resolve_ubuntu_noble_cloudimg() -> Result { + let bases = [ + "https://cloud-images.ubuntu.com/releases/noble/release", + "https://cloud-images.ubuntu.com/noble/current", + "https://cloud-images.ubuntu.com/daily/server/releases/noble/release", + ]; + + let mut last_err: Option = None; + + for (idx, base) in bases.iter().enumerate() { + info!("Ubuntu metadata source {}/{}", idx + 1, bases.len()); + + let checksums = match fetch_ubuntu_sha256sums(base).await { + Ok(v) => v, + Err(e) => { + last_err = Some(e); + continue; + } + }; + + let disk_name = match pick_existing_ubuntu_name( + &checksums, + &[ + "ubuntu-24.04-server-cloudimg-amd64.img", + "noble-server-cloudimg-amd64.img", + ], + ) { + Some(v) => v, + None => { + last_err = Some(anyhow::anyhow!( + "Ubuntu SHA256SUMS did not contain amd64 cloud image entry" + )); + continue; + } + }; + + let stem = disk_name.strip_suffix(".img").unwrap_or(&disk_name); + let kernel_name = format!("{}-vmlinuz-generic", stem); + let initrd_name = format!("{}-initrd-generic", stem); + + let disk_sha256 = match ubuntu_sha256_for(&checksums, &disk_name) { + Ok(v) => v, + Err(e) => { + last_err = Some(e); + continue; + } + }; + let kernel_sha256 = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", kernel_name)) + .or_else(|_| ubuntu_sha256_for(&checksums, &kernel_name)) + .ok(); + if kernel_sha256.is_none() { + warn!( + "Ubuntu SHA256SUMS did not include kernel checksum for {}; proceeding without kernel hash verification", + kernel_name + ); + } + + let initrd_sha256 = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", initrd_name)) + .or_else(|_| ubuntu_sha256_for(&checksums, &initrd_name)) + .ok(); + if initrd_sha256.is_none() { + warn!( + "Ubuntu SHA256SUMS did not include initrd checksum for {}; proceeding without initrd hash verification", + initrd_name + ); + } + + return Ok(ResolvedUbuntuCloudImage { + base_url: (*base).to_string(), + disk_name, + kernel_name, + initrd_name, + disk_sha256, + kernel_sha256, + initrd_sha256, + }); + } + + Err(last_err + .unwrap_or_else(|| anyhow::anyhow!("failed to resolve Ubuntu cloud image metadata"))) } fn ubuntu_sha256_for(checksums: &str, filename: &str) -> Result { @@ -383,43 +515,56 @@ fn parse_fedora_cloud_listing(listing_html: &str, ver: &str) -> Result<(String, /// /// Arch publishes stable "latest" URLs plus a sidecar .SHA256 file. async fn resolve_latest_arch_cloudimg() -> Result { - // Multiple mirrors; any could stall. We fetch SHA256 from the first - // available mirror (sidecar is identical across mirrors). let bases = [ + "https://mirrors.tuna.tsinghua.edu.cn/archlinux/images/latest", + "https://mirrors.ustc.edu.cn/archlinux/images/latest", + "https://mirrors.sjtug.sjtu.edu.cn/archlinux/images/latest", + "https://fastly.mirror.pkgbuild.com/images/latest", "https://geo.mirror.pkgbuild.com/images/latest", - "https://mirrors.kernel.org/archlinux/images/latest", - "https://mirror.rackspace.com/archlinux/images/latest", + "https://mirror.citrahost.com/archlinux/images/latest", + "https://mirrors.teamcloud.am/archlinux/images/latest", + "https://mirror.umd.edu/archlinux/images/latest", + "https://ftp.jaist.ac.jp/pub/Linux/ArchLinux/images/latest", ]; let filename = "Arch-Linux-x86_64-cloudimg.qcow2"; - let mut sha_text: Option = None; - let mut good_bases: Vec = Vec::new(); - for base in &bases { + info!("Resolving Arch cloud image metadata"); + + let mut last_err: Option = None; + let mut selected_base: Option<&str> = None; + let mut sha256: Option = None; + + for (idx, base) in bases.iter().enumerate() { + info!("Arch metadata mirror {}/{}", idx + 1, bases.len()); let sha_url = format!("{}/{}.SHA256", base, filename); match fetch_text(&sha_url).await { - Ok(t) => { - good_bases.push(base.to_string()); - if sha_text.is_none() { - sha_text = Some(t); + Ok(text) => { + if let Some(hash) = find_hash_for_file(&text, filename) { + selected_base = Some(*base); + sha256 = Some(hash); + break; } + last_err = Some(anyhow::anyhow!("invalid checksum format at {}", sha_url)); + } + Err(e) => { + debug!("Arch metadata fetch failed for {}: {:#}", sha_url, e); + last_err = Some(e); } - Err(e) => debug!("Arch SHA256 fetch failed for {}: {:#}", sha_url, e), } } - anyhow::ensure!( - !good_bases.is_empty(), - "failed to fetch Arch cloud image SHA256 from mirrors" - ); + let selected_base = selected_base.ok_or_else(|| { + last_err.unwrap_or_else(|| anyhow::anyhow!("no Arch mirror metadata succeeded")) + })?; + let sha256 = sha256.expect("sha256 must exist when metadata resolves"); - let sha_text = sha_text.unwrap(); - let sha256 = find_hash_for_file(&sha_text, filename) - .with_context(|| "could not parse Arch .SHA256 file")?; + let mut urls = vec![format!("{}/{}", selected_base, filename)]; + for base in bases { + if base != selected_base { + urls.push(format!("{}/{}", base, filename)); + } + } - let urls = good_bases - .into_iter() - .map(|base| format!("{}/{}", base, filename)) - .collect::>(); Ok(ResolvedRemote { urls, sha256 }) } @@ -505,14 +650,18 @@ pub async fn download_with_hash( .build() .with_context(|| "failed to build HTTP client")?; - let response = client - .get(url) - .send() + info!("Downloading image from {}", url); + let response = tokio::time::timeout(std::time::Duration::from_secs(30), client.get(url).send()) .await + .with_context(|| format!("timed out before response headers from {}", url))? .with_context(|| format!("failed to download from {}", url))?; let status = response.status(); + let total_size = response.content_length(); anyhow::ensure!(status.is_success(), "GET {} failed: {}", url, status); + if let Some(total) = total_size { + info!("Remote size: {} MiB ({})", total / (1024 * 1024), url); + } // Ensure destination directory exists. if let Some(parent) = tmp_path.parent() { @@ -529,13 +678,15 @@ pub async fn download_with_hash( .with_context(|| format!("failed to create file at {}", tmp_path.display()))?; let mut stream = response.bytes_stream(); - let idle = std::time::Duration::from_secs(30); + let idle = std::time::Duration::from_secs(60); let mut downloaded: u64 = 0; let mut last_report: u64 = 0; // Report download progress in reasonably small increments. On slower links (or in CI/WSL), // 64MiB can take long enough that the test runner prints a scary "running over 60 seconds" // warning with no other output. - let report_step: u64 = 16 * 1024 * 1024; // 16 MiB + let report_step: u64 = 8 * 1024 * 1024; // 8 MiB + let started_at = std::time::Instant::now(); + let mut last_report_at = started_at; let hash = match hash_type { ShaType::Sha256 => { @@ -549,9 +700,37 @@ pub async fn download_with_hash( let Some(chunk) = next else { break }; let chunk = chunk.with_context(|| "failed to read chunk")?; downloaded += chunk.len() as u64; - if downloaded - last_report >= report_step { + let now = std::time::Instant::now(); + if downloaded - last_report >= report_step + || (downloaded > 0 + && now.duration_since(last_report_at) >= std::time::Duration::from_secs(10)) + { last_report = downloaded; - debug!("Downloading {}: {} MiB", url, downloaded / (1024 * 1024)); + last_report_at = now; + if let Some(total) = total_size { + info!( + "Download progress: {}/{} MiB ({})", + downloaded / (1024 * 1024), + total / (1024 * 1024), + url + ); + } else { + info!( + "Download progress: {} MiB ({})", + downloaded / (1024 * 1024), + url + ); + } + } + if started_at.elapsed() >= std::time::Duration::from_secs(90) + && downloaded < 32 * 1024 * 1024 + { + anyhow::bail!( + "download too slow for {} ({} MiB in {:?}); trying next mirror", + url, + downloaded / (1024 * 1024), + started_at.elapsed() + ); } h.update(&chunk); file.write_all(&chunk) @@ -571,9 +750,37 @@ pub async fn download_with_hash( let Some(chunk) = next else { break }; let chunk = chunk.with_context(|| "failed to read chunk")?; downloaded += chunk.len() as u64; - if downloaded - last_report >= report_step { + let now = std::time::Instant::now(); + if downloaded - last_report >= report_step + || (downloaded > 0 + && now.duration_since(last_report_at) >= std::time::Duration::from_secs(10)) + { last_report = downloaded; - debug!("Downloading {}: {} MiB", url, downloaded / (1024 * 1024)); + last_report_at = now; + if let Some(total) = total_size { + info!( + "Download progress: {}/{} MiB ({})", + downloaded / (1024 * 1024), + total / (1024 * 1024), + url + ); + } else { + info!( + "Download progress: {} MiB ({})", + downloaded / (1024 * 1024), + url + ); + } + } + if started_at.elapsed() >= std::time::Duration::from_secs(90) + && downloaded < 32 * 1024 * 1024 + { + anyhow::bail!( + "download too slow for {} ({} MiB in {:?}); trying next mirror", + url, + downloaded / (1024 * 1024), + started_at.elapsed() + ); } h.update(&chunk); file.write_all(&chunk) @@ -597,6 +804,11 @@ pub async fn download_with_hash( ) })?; + info!( + "Download complete: {} MiB ({})", + downloaded / (1024 * 1024), + url + ); Ok(hash) } @@ -630,6 +842,7 @@ pub async fn download_with_hash_multi( } } + info!("Trying mirror {}/{}", idx + 1, urls.len()); debug!("Download attempt {}/{}: {}", idx + 1, urls.len(), url); match download_with_hash(url, dest_path, hash_type.clone()).await { Ok(h) => { @@ -712,6 +925,23 @@ async fn download_or_copy_with_hash( Ok(()) } +/// Download or copy file from ImageSource without hash verification. +async fn download_or_copy_without_hash(source: &ImageSource, dest: &PathBuf) -> Result<()> { + match source { + ImageSource::Url(url) => { + if dest.exists() { + return Ok(()); + } + let _ = download_with_hash(url, dest, ShaType::Sha256).await?; + } + ImageSource::LocalPath(src) => { + anyhow::ensure!(src.exists(), "file does not exist: {}", src.display()); + tokio::fs::copy(src, dest).await?; + } + } + Ok(()) +} + impl ImageMeta { /// Create a new image by downloading and extracting pub async fn create(name: &str) -> Result { @@ -734,9 +964,12 @@ impl ImageMeta { distro_action.download(name).await?; - let (kernel, initrd) = distro_action.extract(name).await?; let image_path = image_dir.join(format!("{}.qcow2", name)); + let (kernel, initrd) = distro_action.extract(name).await?; let checksum_path = image_dir.join("checksums"); + let root_arg = detect_root_arg(&image_path) + .await + .unwrap_or_else(|_| default_root_arg()); let checksum = ShaSum { path: checksum_path, sha_type: ShaType::Sha512, @@ -745,6 +978,7 @@ impl ImageMeta { path: image_path, kernel, initrd, + root_arg, checksum, name: name.to_string(), vendor: distro_action, @@ -767,6 +1001,18 @@ impl ImageMeta { let image: ImageMeta = serde_json::from_str(&json_content) .with_context(|| format!("failed to parse JSON from {}", json_path.display()))?; + let kernel_ok = image.kernel.exists() + && std::fs::metadata(&image.kernel) + .map(|m| m.len() > 0) + .unwrap_or(false); + let initrd_ok = image.initrd.exists() + && std::fs::metadata(&image.initrd) + .map(|m| m.len() > 0) + .unwrap_or(false); + if !kernel_ok || !initrd_ok { + bail!("cached image is missing kernel/initrd markers; recreate is required"); + } + let checksum_dir = dirs.images.join(name); let checksum_command = match image.checksum.sha_type { ShaType::Sha256 => "sha256sum", @@ -868,9 +1114,12 @@ impl ImageMeta { action.download(name).await?; - let (kernel, initrd) = action.extract(name).await?; let image_path = image_dir.join(format!("{}.qcow2", name)); + let (kernel, initrd) = action.extract(name).await?; let checksum_path = image_dir.join("checksums"); + let root_arg = detect_root_arg(&image_path) + .await + .unwrap_or_else(|_| default_root_arg()); let checksum = ShaSum { path: checksum_path, sha_type: ShaType::Sha512, @@ -879,6 +1128,7 @@ impl ImageMeta { path: image_path, kernel, initrd, + root_arg, checksum, name: name.to_string(), vendor: action, @@ -917,6 +1167,185 @@ impl ImageMeta { } } +async fn ensure_fixed_guestfs_appliance() -> Result { + for dir in [ + "/usr/lib/guestfs/appliance", + "/usr/lib/x86_64-linux-gnu/guestfs/appliance", + ] { + let p = PathBuf::from(dir); + if p.join("kernel").exists() && p.join("initrd").exists() { + return Ok(p); + } + } + + let dirs = QleanDirs::new()?; + let appliance_dir = dirs.base.join("guestfs-appliance"); + if appliance_dir.join("kernel").exists() && appliance_dir.join("initrd").exists() { + return Ok(appliance_dir); + } + + let output = tokio::process::Command::new("libguestfs-make-fixed-appliance") + .arg(&appliance_dir) + .output() + .await + .with_context(|| "failed to execute libguestfs-make-fixed-appliance")?; + if !output.status.success() { + bail!( + "libguestfs fixed appliance build failed: {}\nInstall package: libguestfs-appliance (or libguestfs-tools) and retry.", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(appliance_dir) +} + +async fn run_guestfs_tool( + program: &str, + args: &[&OsStr], + current_dir: &Path, +) -> Result { + async fn run_once( + program: &str, + args: &[&OsStr], + current_dir: &Path, + appliance: Option<&Path>, + ) -> Result { + let mut cmd = tokio::process::Command::new(program); + cmd.env("LIBGUESTFS_BACKEND", "direct") + .current_dir(current_dir); + if let Some(appliance_dir) = appliance { + cmd.env("LIBGUESTFS_PATH", appliance_dir); + } + for a in args { + cmd.arg(a); + } + let child = cmd.output(); + let out = timeout(Duration::from_secs(180), child) + .await + .with_context(|| format!("{} timed out after 180s (libguestfs)", program))? + .with_context(|| format!("failed to execute {}", program))?; + Ok(out) + } + + let first = run_once(program, args, current_dir, None).await?; + if !first.status.success() { + warn!( + "{} failed (direct backend): {}", + program, + String::from_utf8_lossy(&first.stderr) + ); + } + if first.status.success() { + return Ok(first); + } + + let stderr = String::from_utf8_lossy(&first.stderr); + let needs_fixed = stderr.contains("supermin exited with error status") + || stderr.contains("/usr/bin/supermin"); + if needs_fixed && std::env::var_os("LIBGUESTFS_PATH").is_none() { + let appliance_dir = ensure_fixed_guestfs_appliance().await?; + return run_once(program, args, current_dir, Some(&appliance_dir)).await; + } + + Ok(first) +} + +async fn guestfish_ls_boot(image_dir: &Path, file_name: &str) -> Result { + let args = [ + OsStr::new("--ro"), + OsStr::new("-a"), + OsStr::new(file_name), + OsStr::new("-i"), + OsStr::new("ls"), + OsStr::new("/boot"), + ]; + let output = run_guestfs_tool("guestfish", &args, image_dir).await?; + if !output.status.success() { + bail!( + "guestfish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +async fn virt_copy_out(image_dir: &Path, file_name: &str, src: &str, kind: &str) -> Result<()> { + let args = [ + OsStr::new("-a"), + OsStr::new(file_name), + OsStr::new(src), + OsStr::new("."), + ]; + let output = run_guestfs_tool("virt-copy-out", &args, image_dir).await?; + if !output.status.success() { + bail!( + "virt-copy-out failed for {}: {}", + kind, + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) +} + +async fn write_unavailable_boot_artifacts( + image_dir: &Path, + reason: &str, +) -> Result<(PathBuf, PathBuf)> { + let kernel = image_dir.join("vmlinuz.unavailable"); + let initrd = image_dir.join("initrd.img.unavailable"); + let note = format!("qlean boot artifact unavailable: {}\n", reason); + + tokio::fs::write(&kernel, note.as_bytes()) + .await + .with_context(|| format!("failed to write {}", kernel.display()))?; + tokio::fs::write(&initrd, note.as_bytes()) + .await + .with_context(|| format!("failed to write {}", initrd.display()))?; + + Ok((kernel, initrd)) +} + +async fn detect_root_arg(image_path: &Path) -> Result { + let image_dir = image_path.parent().with_context(|| "missing image dir")?; + let file_name = image_path + .file_name() + .and_then(|v| v.to_str()) + .with_context(|| "invalid image filename")?; + let args = [ + OsStr::new("--ro"), + OsStr::new("-a"), + OsStr::new(file_name), + OsStr::new("-i"), + OsStr::new("mountpoints"), + ]; + let output = run_guestfs_tool("guestfish", &args, image_dir).await?; + if !output.status.success() { + return Ok(default_root_arg()); + } + let text = String::from_utf8_lossy(&output.stdout); + for line in text.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + let mut dev = None; + if parts[0].starts_with("/dev/") && parts[1] == "/" { + dev = Some(parts[0]); + } + if parts[1].starts_with("/dev/") && parts[0] == "/" { + dev = Some(parts[1]); + } + if let Some(d) = dev { + let virt = if let Some(rest) = d.strip_prefix("/dev/sd") { + format!("/dev/vd{}", rest) + } else { + d.to_string() + }; + return Ok(format!("root={}", virt)); + } + } + Ok(default_root_arg()) +} + // --------------------------------------------------------------------------- // Debian // --------------------------------------------------------------------------- @@ -1072,50 +1501,55 @@ impl ImageAction for Ubuntu { let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - // Ubuntu noble (24.04 LTS) cloud image base URL - let base_url = "https://cloud-images.ubuntu.com/noble/current"; - let checksums = fetch_ubuntu_sha256sums(&format!("{}/", base_url)).await?; + let resolved = resolve_ubuntu_noble_cloudimg().await?; + debug!( + "Resolved Ubuntu cloud image from {}: disk={}, kernel={}, initrd={}", + resolved.base_url, resolved.disk_name, resolved.kernel_name, resolved.initrd_name + ); - // Download qcow2 image (SHA256 verified) - let qcow2_name = "noble-server-cloudimg-amd64.img"; - let qcow2_url = format!("{}/{}", base_url, qcow2_name); let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - let qcow2_sha = ubuntu_sha256_for(&checksums, qcow2_name)?; + let qcow2_url = join_url(&resolved.base_url, &resolved.disk_name); download_or_copy_with_hash( &ImageSource::Url(qcow2_url), &qcow2_path, - &qcow2_sha, + &resolved.disk_sha256, ShaType::Sha256, ) .await?; - // Download pre-extracted kernel (SHA256 verified) - let kernel_name = "noble-server-cloudimg-amd64-vmlinuz-generic"; - let kernel_url = format!("{}/unpacked/{}", base_url, kernel_name); let kernel_path = image_dir.join("vmlinuz"); - let kernel_sha = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", kernel_name)) - .or_else(|_| ubuntu_sha256_for(&checksums, kernel_name))?; - download_or_copy_with_hash( - &ImageSource::Url(kernel_url), - &kernel_path, - &kernel_sha, - ShaType::Sha256, - ) - .await?; + let kernel_url = join_url( + &resolved.base_url, + &format!("unpacked/{}", resolved.kernel_name), + ); + if let Some(kernel_sha256) = resolved.kernel_sha256.as_deref() { + download_or_copy_with_hash( + &ImageSource::Url(kernel_url), + &kernel_path, + kernel_sha256, + ShaType::Sha256, + ) + .await?; + } else { + download_or_copy_without_hash(&ImageSource::Url(kernel_url), &kernel_path).await?; + } - // Download pre-extracted initrd (SHA256 verified) - let initrd_name = "noble-server-cloudimg-amd64-initrd-generic"; - let initrd_url = format!("{}/unpacked/{}", base_url, initrd_name); let initrd_path = image_dir.join("initrd.img"); - let initrd_sha = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", initrd_name)) - .or_else(|_| ubuntu_sha256_for(&checksums, initrd_name))?; - download_or_copy_with_hash( - &ImageSource::Url(initrd_url), - &initrd_path, - &initrd_sha, - ShaType::Sha256, - ) - .await?; + let initrd_url = join_url( + &resolved.base_url, + &format!("unpacked/{}", resolved.initrd_name), + ); + if let Some(initrd_sha256) = resolved.initrd_sha256.as_deref() { + download_or_copy_with_hash( + &ImageSource::Url(initrd_url), + &initrd_path, + initrd_sha256, + ShaType::Sha256, + ) + .await?; + } else { + download_or_copy_without_hash(&ImageSource::Url(initrd_url), &initrd_path).await?; + } Ok(()) } @@ -1158,7 +1592,6 @@ impl ImageAction for Fedora { resolved.urls.len() ); - // Download + verify sha256 in one pass. let qcow2_path = image_dir.join(format!("{}.qcow2", name)); let (_hash, used_url) = download_with_hash_multi( &resolved.urls, @@ -1173,8 +1606,6 @@ impl ImageAction for Fedora { used_url ); - // Fedora cloud images don't provide pre-extracted boot files - // We'll need to extract them using guestfish Ok(()) } @@ -1185,89 +1616,49 @@ impl ImageAction for Fedora { let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - // Use guestfish to list boot files - let output = tokio::process::Command::new("guestfish") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("--ro") - .arg("-a") - .arg(&file_name) - .arg("-i") - .arg("ls") - .arg("/boot") - .current_dir(&image_dir) - .output() - .await - .with_context(|| "failed to execute guestfish")?; - - if !output.status.success() { - bail!( - "guestfish failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let boot_files = String::from_utf8_lossy(&output.stdout); - let mut kernel_name = None; - let mut initrd_name = None; + let extract_result = async { + let boot_files = guestfish_ls_boot(&image_dir, &file_name) + .await + .with_context(|| "failed to read /boot from Fedora image with guestfish")?; + let mut kernel_name = None; + let mut initrd_name = None; - for line in boot_files.lines() { - let file = line.trim(); - if file.starts_with("vmlinuz") { - kernel_name = Some(file.to_string()); - } else if file.starts_with("initramfs") { - initrd_name = Some(file.to_string()); + for line in boot_files.lines() { + let file = line.trim(); + if file.starts_with("vmlinuz") { + kernel_name = Some(file.to_string()); + } else if file.starts_with("initramfs") { + initrd_name = Some(file.to_string()); + } } - } - let kernel_name = - kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; - let initrd_name = - initrd_name.with_context(|| "failed to find initrd file (initramfs*) in /boot")?; + let kernel_name = + kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; + let initrd_name = + initrd_name.with_context(|| "failed to find initrd file (initramfs*) in /boot")?; - // Extract kernel - let kernel_src = format!("/boot/{}", kernel_name); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&kernel_src) - .arg(".") - .current_dir(&image_dir) - .output() - .await - .with_context(|| format!("failed to execute virt-copy-out for {}", kernel_name))?; - - if !output.status.success() { - bail!( - "virt-copy-out failed for kernel: {}", - String::from_utf8_lossy(&output.stderr) - ); - } + let kernel_src = format!("/boot/{}", kernel_name); + virt_copy_out(&image_dir, &file_name, &kernel_src, "kernel").await?; - // Extract initrd - let initrd_src = format!("/boot/{}", initrd_name); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&initrd_src) - .arg(".") - .current_dir(&image_dir) - .output() - .await - .with_context(|| format!("failed to execute virt-copy-out for {}", initrd_name))?; + let initrd_src = format!("/boot/{}", initrd_name); + virt_copy_out(&image_dir, &file_name, &initrd_src, "initrd").await?; - if !output.status.success() { - bail!( - "virt-copy-out failed for initrd: {}", - String::from_utf8_lossy(&output.stderr) - ); + let kernel_path = image_dir.join(&kernel_name); + let initrd_path = image_dir.join(&initrd_name); + Ok::<(PathBuf, PathBuf), anyhow::Error>((kernel_path, initrd_path)) } + .await; - let kernel_path = image_dir.join(&kernel_name); - let initrd_path = image_dir.join(&initrd_name); - - Ok((kernel_path, initrd_path)) + match extract_result { + Ok(paths) => Ok(paths), + Err(e) => { + warn!( + "Fedora kernel/initrd extraction failed: {:#}. Using disk boot fallback.", + e + ); + write_unavailable_boot_artifacts(&image_dir, "fedora extraction failed").await + } + } } fn distro(&self) -> Distro { @@ -1294,7 +1685,6 @@ impl ImageAction for Arch { resolved.urls.len() ); - // Download + verify sha256 in one pass. let qcow2_path = image_dir.join(format!("{}.qcow2", name)); let (_hash, used_url) = download_with_hash_multi( &resolved.urls, @@ -1316,90 +1706,49 @@ impl ImageAction for Arch { let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - // Use guestfish to list boot files - let output = tokio::process::Command::new("guestfish") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("--ro") - .arg("-a") - .arg(&file_name) - .arg("-i") - .arg("ls") - .arg("/boot") - .current_dir(&image_dir) - .output() - .await - .with_context(|| "failed to execute guestfish")?; - - if !output.status.success() { - bail!( - "guestfish failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let boot_files = String::from_utf8_lossy(&output.stdout); - let mut kernel_name = None; - let mut initrd_name = None; + let extract_result = async { + let boot_files = guestfish_ls_boot(&image_dir, &file_name) + .await + .with_context(|| "failed to read /boot from Arch image with guestfish")?; + let mut kernel_name = None; + let mut initrd_name = None; - for line in boot_files.lines() { - let file = line.trim(); - // Arch uses vmlinuz-linux - if file.starts_with("vmlinuz") { - kernel_name = Some(file.to_string()); - } else if file.starts_with("initramfs") && file.contains("linux.img") { - initrd_name = Some(file.to_string()); + for line in boot_files.lines() { + let file = line.trim(); + if file.starts_with("vmlinuz") { + kernel_name = Some(file.to_string()); + } else if file.starts_with("initramfs") && file.contains("linux.img") { + initrd_name = Some(file.to_string()); + } } - } - let kernel_name = - kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; - let initrd_name = initrd_name - .with_context(|| "failed to find initrd file (initramfs*linux.img) in /boot")?; + let kernel_name = + kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; + let initrd_name = initrd_name + .with_context(|| "failed to find initrd file (initramfs*linux.img) in /boot")?; - // Extract kernel - let kernel_src = format!("/boot/{}", kernel_name); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&kernel_src) - .arg(".") - .current_dir(&image_dir) - .output() - .await - .with_context(|| format!("failed to execute virt-copy-out for {}", kernel_name))?; + let kernel_src = format!("/boot/{}", kernel_name); + virt_copy_out(&image_dir, &file_name, &kernel_src, "kernel").await?; - if !output.status.success() { - bail!( - "virt-copy-out failed for kernel: {}", - String::from_utf8_lossy(&output.stderr) - ); - } + let initrd_src = format!("/boot/{}", initrd_name); + virt_copy_out(&image_dir, &file_name, &initrd_src, "initrd").await?; - // Extract initrd - let initrd_src = format!("/boot/{}", initrd_name); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&initrd_src) - .arg(".") - .current_dir(&image_dir) - .output() - .await - .with_context(|| format!("failed to execute virt-copy-out for {}", initrd_name))?; - - if !output.status.success() { - bail!( - "virt-copy-out failed for initrd: {}", - String::from_utf8_lossy(&output.stderr) - ); + let kernel_path = image_dir.join(&kernel_name); + let initrd_path = image_dir.join(&initrd_name); + Ok::<(PathBuf, PathBuf), anyhow::Error>((kernel_path, initrd_path)) } + .await; - let kernel_path = image_dir.join(&kernel_name); - let initrd_path = image_dir.join(&initrd_name); - - Ok((kernel_path, initrd_path)) + match extract_result { + Ok(paths) => Ok(paths), + Err(e) => { + warn!( + "Arch kernel/initrd extraction failed: {:#}. Using disk boot fallback.", + e + ); + write_unavailable_boot_artifacts(&image_dir, "arch extraction failed").await + } + } } fn distro(&self) -> Distro { @@ -1472,89 +1821,41 @@ impl ImageAction for Custom { let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - // Check if kernel/initrd were pre-provided let kernel_path = image_dir.join("vmlinuz"); let initrd_path = image_dir.join("initrd.img"); - if kernel_path.exists() && initrd_path.exists() { debug!("Using pre-provided kernel and initrd files"); return Ok((kernel_path, initrd_path)); } - // Otherwise, try to extract using guestfish ensure_extraction_prerequisites().await?; let file_name = format!("{}.qcow2", name); + let boot_files = guestfish_ls_boot(&image_dir, &file_name).await?; - let output = tokio::process::Command::new("guestfish") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("--ro") - .arg("-a") - .arg(&file_name) - .arg("-i") - .arg("ls") - .arg("/boot") - .current_dir(&image_dir) - .output() - .await; - - if let Ok(output) = output - && output.status.success() - { - let boot_files = String::from_utf8_lossy(&output.stdout); - let mut kernel_name = None; - let mut initrd_name = None; - - // Generic kernel/initrd detection - for line in boot_files.lines() { - let file = line.trim(); - if kernel_name.is_none() - && (file.starts_with("vmlinuz") || file.starts_with("bzImage")) - { - kernel_name = Some(file.to_string()); - } - if initrd_name.is_none() - && (file.starts_with("initrd") || file.starts_with("initramfs")) - { - initrd_name = Some(file.to_string()); - } + let mut kernel_name = None; + let mut initrd_name = None; + for line in boot_files.lines() { + let file = line.trim(); + if kernel_name.is_none() && (file.starts_with("vmlinuz") || file.starts_with("bzImage")) + { + kernel_name = Some(file.to_string()); } - - if let (Some(kernel), Some(initrd)) = (kernel_name, initrd_name) { - // Extract using virt-copy-out - for (file, desc) in [(&kernel, "kernel"), (&initrd, "initrd")] { - let src = format!("/boot/{}", file); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&src) - .arg(".") - .current_dir(&image_dir) - .output() - .await?; - - if !output.status.success() { - bail!("virt-copy-out failed for {}", desc); - } - } - - return Ok((image_dir.join(&kernel), image_dir.join(&initrd))); + if initrd_name.is_none() + && (file.starts_with("initrd") || file.starts_with("initramfs")) + { + initrd_name = Some(file.to_string()); } } - // Guestfish not available or failed - provide helpful error - bail!( - "Custom image requires either:\n\ - \n\ - 1. Pre-extracted boot files (RECOMMENDED for WSL):\n\ - - Provide kernel_source, kernel_hash, initrd_source, initrd_hash in config\n\ - - See documentation for examples\n\ - \n\ - 2. Guestfish for extraction (native Linux only):\n\ - - Install: sudo apt install libguestfs-tools\n\ - - Provide only image_source/image_hash in config\n\ - - Not supported on WSL/WSL2" - ); + let kernel = kernel_name.with_context(|| "failed to find kernel file in /boot")?; + let initrd = initrd_name.with_context(|| "failed to find initrd file in /boot")?; + + let kernel_src = format!("/boot/{}", kernel); + virt_copy_out(&image_dir, &file_name, &kernel_src, "kernel").await?; + let initrd_src = format!("/boot/{}", initrd); + virt_copy_out(&image_dir, &file_name, &initrd_src, "initrd").await?; + + Ok((image_dir.join(&kernel), image_dir.join(&initrd))) } fn distro(&self) -> Distro { @@ -1621,6 +1922,23 @@ impl Image { Image::Custom(img) => &img.initrd, } } + + pub fn root_arg(&self) -> &str { + match self { + Image::Debian(img) => &img.root_arg, + Image::Ubuntu(img) => &img.root_arg, + Image::Fedora(img) => &img.root_arg, + Image::Arch(img) => &img.root_arg, + Image::Custom(img) => &img.root_arg, + } + } + + pub fn prefer_direct_kernel_boot(&self) -> bool { + match self { + Image::Debian(_) | Image::Ubuntu(_) => true, + Image::Fedora(_) | Image::Arch(_) | Image::Custom(_) => false, + } + } } /// Factory function to create Image instances based on distro diff --git a/src/machine.rs b/src/machine.rs index ffe6f45..72a77a7 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -56,6 +56,8 @@ pub struct MachineImage { pub kernel: PathBuf, pub initrd: PathBuf, pub seed: PathBuf, + pub root_arg: String, + pub prefer_direct_kernel_boot: bool, } #[derive(Clone, Debug)] @@ -188,6 +190,8 @@ impl Machine { kernel: image.kernel().to_owned(), initrd: image.initrd().to_owned(), seed: seed_iso_path, + root_arg: image.root_arg().to_string(), + prefer_direct_kernel_boot: image.prefer_direct_kernel_boot(), }; Ok(Self { @@ -282,15 +286,32 @@ impl Machine { /// Shutdown the machine pub async fn shutdown(&mut self) -> Result<()> { if let Some(ssh) = self.ssh.as_mut() { - // Then shut the system down. - ssh.call( - "systemctl poweroff", - self.ssh_cancel_token - .as_ref() - .expect("Machine not initialized or spawned") - .clone(), - ) - .await?; + // Then shut the system down. Try several commands because some cloud images + // (notably Arch) log in as an unprivileged default user and plain + // `systemctl poweroff` can fail with "Access denied". + let shutdown_cmd = r#"sh -lc 'systemctl poweroff \ + || sudo -n systemctl poweroff \ + || sudo -n poweroff \ + || poweroff \ + || shutdown -h now \ + || sudo -n shutdown -h now'"#; + if let Err(e) = ssh + .call( + shutdown_cmd, + self.ssh_cancel_token + .as_ref() + .expect("Machine not initialized or spawned") + .clone(), + ) + .await + { + // During shutdown the SSH session can drop before the command fully returns. + // Keep going and rely on the QEMU process wait/cleanup below. + debug!( + "Guest shutdown command returned error during teardown: {}", + e + ); + } info!("🔌 Shutting down VM-{}", self.id); // Tell the QEMU handler it's now fine to wait for exit. diff --git a/src/qemu.rs b/src/qemu.rs index 1a7acc4..05896a5 100644 --- a/src/qemu.rs +++ b/src/qemu.rs @@ -12,7 +12,7 @@ use tokio::{ time::{Duration, timeout}, }; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, warn}; use crate::{ KVM_AVAILABLE, MachineConfig, @@ -49,12 +49,31 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { "vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid={}", params.cid ), - ]) - // Kernel - .args(["-kernel", params.image.kernel.to_str().unwrap()]) - .args(["-append", "rw root=/dev/vda1 console=ttyS0"]) - // Initrd - .args(["-initrd", params.image.initrd.to_str().unwrap()]) + ]); + + let use_direct_kernel_boot = params.image.prefer_direct_kernel_boot + && params.image.kernel.exists() + && params.image.initrd.exists() + && std::fs::metadata(¶ms.image.kernel) + .map(|m| m.len() > 0) + .unwrap_or(false) + && std::fs::metadata(¶ms.image.initrd) + .map(|m| m.len() > 0) + .unwrap_or(false); + + if use_direct_kernel_boot { + qemu_cmd + .args(["-kernel", params.image.kernel.to_str().unwrap()]) + .args([ + "-append", + &format!("rw {} console=ttyS0", params.image.root_arg), + ]) + .args(["-initrd", params.image.initrd.to_str().unwrap()]); + } else { + warn!("Kernel/initrd extraction is unavailable. Booting from qcow2 disk image directly."); + } + + qemu_cmd // Disk .args([ "-drive", @@ -76,7 +95,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { // --------------------------------------------------------------------- let bridge_name = "qlbr0"; let ssh_port = params.ssh_tcp_port; - let want_bridge = !is_wsl() && has_iface(bridge_name) && bridge_conf_allows(bridge_name); + let want_bridge = has_iface(bridge_name) && bridge_conf_allows(bridge_name); if want_bridge { qemu_cmd @@ -97,15 +116,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { .args(["-device", "virtio-net-pci,netdev=net1"]); } } else { - if is_wsl() { - info!( - "WSL detected: disabling bridged networking and using user-mode networking + hostfwd for SSH." - ); - } else { - warn!( - "Bridged networking is unavailable (missing /etc/qemu/bridge.conf allow, or bridge interface not present). Falling back to user-mode networking + hostfwd." - ); - } + warn!("Bridged networking is unavailable. Falling back to user-mode networking + hostfwd."); let port = ssh_port.ok_or_else(|| { anyhow::anyhow!("user-mode networking fallback requires ssh_tcp_port to be set") @@ -150,7 +161,8 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { } // Spawn QEMU process - debug!("Spawning QEMU with command:\n{:?}", qemu_cmd.to_string()); + info!("Starting QEMU"); + debug!("QEMU command: {:?}", qemu_cmd.to_string()); let mut qemu_child = qemu_cmd .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -170,7 +182,9 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { let reader = BufReader::new(stdout); let mut lines = reader.lines(); while let Ok(Some(line)) = lines.next_line().await { - trace!("{}", strip_ansi_codes(&line)); + // QEMU serial stdout is noisy during normal boot. + // Keep it available only at debug level. + debug!("[qemu] {}", strip_ansi_codes(&line)); } }); @@ -180,7 +194,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { let reader = BufReader::new(stderr); let mut lines = reader.lines(); while let Ok(Some(line)) = lines.next_line().await { - error!("{}", strip_ansi_codes(&line)); + error!("[qemu] {}", strip_ansi_codes(&line)); } }); @@ -220,16 +234,6 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { result } -fn is_wsl() -> bool { - // Best-effort detection for WSL. We avoid hard failures if files are missing. - let osrelease = std::fs::read_to_string("/proc/sys/kernel/osrelease").unwrap_or_default(); - if osrelease.to_lowercase().contains("microsoft") { - return true; - } - let version = std::fs::read_to_string("/proc/version").unwrap_or_default(); - version.to_lowercase().contains("microsoft") -} - fn bridge_conf_allows(bridge: &str) -> bool { // qemu-bridge-helper enforces an ACL file (commonly /etc/qemu/bridge.conf). // If the file is missing or doesn't allow the bridge, QEMU will fail with: diff --git a/src/ssh.rs b/src/ssh.rs index c256989..900aabb 100644 --- a/src/ssh.rs +++ b/src/ssh.rs @@ -113,6 +113,7 @@ impl Session { /// Connect to an SSH server via vsock async fn connect( privkey: PrivateKey, + username: &str, cid: u32, port: u32, timeout: Duration, @@ -128,7 +129,7 @@ impl Session { let vsock_addr = VsockAddr::new(cid, port); let now = Instant::now(); - info!("🔑 Connecting via vsock"); + info!("Connecting SSH via vsock"); let mut session = loop { // Check for cancellation if cancel_token.is_cancelled() { @@ -168,8 +169,8 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp continue; } e => { - error!("Unhandled error occurred: {e}"); - bail!("Unknown error"); + error!("SSH vsock connect error: {e}"); + bail!("SSH vsock connect error: {e}"); } }, }; @@ -181,15 +182,17 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp match e.kind() { // The VM is still booting at this point so we're just ignoring these errors // for some time. - ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset => { + ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::UnexpectedEof => { if now.elapsed() > timeout { warn!("Timeout establishing SSH over vsock"); bail!("Timeout"); } } e => { - error!("Unhandled error occurred: {e}"); - bail!("Unknown error"); + error!("SSH handshake error: {e}"); + bail!("SSH handshake error: {e}"); } } } @@ -200,8 +203,8 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp } } Err(e) => { - error!("Unhandled error occurred: {e}"); - bail!("Unknown error"); + error!("SSH client error: {e}"); + bail!("SSH client error: {e}"); } } }; @@ -209,7 +212,10 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp // use publickey authentication let auth_res = session - .authenticate_publickey("root", PrivateKeyWithHashAlg::new(Arc::new(privkey), None)) + .authenticate_publickey( + username, + PrivateKeyWithHashAlg::new(Arc::new(privkey), None), + ) .await?; if !auth_res.success() { @@ -266,33 +272,33 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp bail!("SSH call cancelled"); } - // Handle one of the possible events: - tokio::select! { - // There's an event available on the session channel - Some(msg) = channel.wait() => { - match msg { - // Write data to the terminal - ChannelMsg::Data { ref data } => { - stdout.write_all(data).await?; - stdout.flush().await?; - } - ChannelMsg::ExtendedData { ref data, ext } => { - // ext == 1 means it's stderr content - // https://github.com/Eugeny/russh/discussions/258 - if ext == 1 { - stderr.write_all(data).await?; - stderr.flush().await?; - } - } - // The command has returned an exit code - ChannelMsg::ExitStatus { exit_status } => { - code = exit_status; - channel.eof().await?; - break; - } - _ => {} + let Some(msg) = channel.wait().await else { + bail!( + "SSH channel closed before exit status for command: {}", + command + ); + }; + match msg { + // Write data to the terminal + ChannelMsg::Data { ref data } => { + stdout.write_all(data).await?; + stdout.flush().await?; + } + ChannelMsg::ExtendedData { ref data, ext } => { + // ext == 1 means it's stderr content + // https://github.com/Eugeny/russh/discussions/258 + if ext == 1 { + stderr.write_all(data).await?; + stderr.flush().await?; } - }, + } + // The command has returned an exit code + ChannelMsg::ExitStatus { exit_status } => { + code = exit_status; + channel.eof().await?; + break; + } + _ => {} } } Ok(code) @@ -318,31 +324,31 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp bail!("SSH call cancelled"); } - // Handle one of the possible events: - tokio::select! { - // There's an event available on the session channel - Some(msg) = channel.wait() => { - match msg { - // Write data to the buffer - ChannelMsg::Data { ref data } => { - stdout.extend_from_slice(data); - } - ChannelMsg::ExtendedData { ref data, ext } => { - // ext == 1 means it's stderr content - // https://github.com/Eugeny/russh/discussions/258 - if ext == 1 { - stderr.extend_from_slice(data); - } - } - // The command has returned an exit code - ChannelMsg::ExitStatus { exit_status } => { - code = exit_status; - channel.eof().await?; - break; - } - _ => {} + let Some(msg) = channel.wait().await else { + bail!( + "SSH channel closed before exit status for command: {}", + command + ); + }; + match msg { + // Write data to the buffer + ChannelMsg::Data { ref data } => { + stdout.extend_from_slice(data); + } + ChannelMsg::ExtendedData { ref data, ext } => { + // ext == 1 means it's stderr content + // https://github.com/Eugeny/russh/discussions/258 + if ext == 1 { + stderr.extend_from_slice(data); } - }, + } + // The command has returned an exit code + ChannelMsg::ExitStatus { exit_status } => { + code = exit_status; + channel.eof().await?; + break; + } + _ => {} } } Ok((code, stdout, stderr)) @@ -358,6 +364,7 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp /// Connect to an SSH server via TCP (used as a fallback when vsock is unavailable or flaky). async fn connect_tcp( privkey: PrivateKey, + username: &str, host: &str, port: u16, timeout: Duration, @@ -408,14 +415,16 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp match russh::client::connect_stream(config.clone(), stream, sh.clone()).await { Ok(x) => break x, Err(russh::Error::IO(ref e)) => match e.kind() { - ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset => { + ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::UnexpectedEof => { if now.elapsed() > timeout { bail!("Timeout"); } } _ => { - error!("Unhandled error occurred: {e}"); - bail!("Unknown error"); + error!("SSH vsock connect error: {e}"); + bail!("SSH vsock connect error: {e}"); } }, Err(russh::Error::Disconnect) => { @@ -424,15 +433,18 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp } } Err(e) => { - error!("Unhandled error occurred: {e}"); - bail!("Unknown error"); + error!("SSH client error: {e}"); + bail!("SSH client error: {e}"); } } }; debug!("Authenticating via SSH"); let auth_res = session - .authenticate_publickey("root", PrivateKeyWithHashAlg::new(Arc::new(privkey), None)) + .authenticate_publickey( + username, + PrivateKeyWithHashAlg::new(Arc::new(privkey), None), + ) .await?; if !auth_res.success() { bail!("Authentication (with publickey) failed"); @@ -453,56 +465,93 @@ pub async fn connect_ssh( cancel_token: CancellationToken, ) -> Result { let privkey = PrivateKey::from_openssh(&keypair.privkey_str)?; - - // Prefer vsock, but don't wait the full timeout if we have a TCP fallback. - // On some hosts (notably WSL2), vsock can be flaky/unreachable even when /dev/vhost-vsock - // exists. In those cases we want to fall back quickly to TCP host forwarding. - let vsock_timeout = if tcp_port.is_some() { - std::cmp::min(timeout, Duration::from_secs(30)) - } else { - timeout - }; - - let mut ssh = match Session::connect( - privkey.clone(), - cid, - 22, - vsock_timeout, - cancel_token.clone(), - ) - .await - { - Ok(s) => { - info!("✅ Connected via vsock"); - s + let users = [ + "root", + "arch", + "fedora", + "ubuntu", + "debian", + "cloud-user", + "ec2-user", + ]; + let vsock_supported = std::path::Path::new("/dev/vhost-vsock").exists(); + let user_count = users.len() as u32; + let per_user_timeout = { + let slice = timeout / user_count.max(1); + let min_t = Duration::from_secs(8); + let max_t = Duration::from_secs(20); + if slice < min_t { + min_t + } else if slice > max_t { + max_t + } else { + slice } - Err(e) => { - if let Some(port) = tcp_port { - warn!("Vsock SSH failed ({e}). Falling back to tcp 127.0.0.1:{port}"); - let s = - Session::connect_tcp(privkey, "127.0.0.1", port, timeout, cancel_token.clone()) - .await?; - info!("✅ Connected via tcp"); - s - } else { - return Err(e); + }; + let mut last_tcp_err: Option = None; + let mut last_vsock_err: Option = None; + + if let Some(port) = tcp_port { + info!("Connecting SSH via tcp 127.0.0.1:{}", port); + for user in users { + match Session::connect_tcp( + privkey.clone(), + user, + "127.0.0.1", + port, + per_user_timeout, + cancel_token.clone(), + ) + .await + { + Ok(mut s) => { + info!("✅ Connected via tcp as {}", user); + let _ = s.call("true", cancel_token.clone()).await?; + debug!("SSH command channel is ready"); + return Ok(s); + } + Err(e) => { + warn!("SSH via tcp failed for user {}: {}", user, e); + last_tcp_err = Some(e); + } } } - }; + } + + if !vsock_supported { + return Err(last_tcp_err.unwrap_or_else(|| { + anyhow::anyhow!("SSH connection failed over tcp and vhost-vsock is unavailable") + })); + } - // First we'll wait until the system has fully booted up. - let is_running_exitcode = ssh - .call( - "systemctl is-system-running --wait --quiet", + info!("Falling back to vsock SSH"); + for user in users { + match Session::connect( + privkey.clone(), + user, + cid, + 22, + per_user_timeout, cancel_token.clone(), ) - .await?; - debug!("systemctl is-system-running --wait exit code {is_running_exitcode}"); - - // Allow the --env option to work by allowing SSH to accept all sent environment variables. - // ssh.call("echo AcceptEnv * >> /etc/ssh/sshd_config").await?; + .await + { + Ok(mut s) => { + info!("✅ Connected via vsock as {}", user); + let _ = s.call("true", cancel_token.clone()).await?; + debug!("SSH command channel is ready"); + return Ok(s); + } + Err(e) => { + warn!("SSH via vsock failed for user {}: {}", user, e); + last_vsock_err = Some(e); + } + } + } - Ok(ssh) + Err(last_vsock_err + .or(last_tcp_err) + .unwrap_or_else(|| anyhow::anyhow!("SSH connection failed"))) } impl Session { diff --git a/tests/arch_image.rs b/tests/arch_image.rs index d5d7f14..e5d107a 100644 --- a/tests/arch_image.rs +++ b/tests/arch_image.rs @@ -10,8 +10,8 @@ mod guestfish; #[path = "support/logging.rs"] mod logging; -use e2e::should_run_vm_tests; -use guestfish::has_guestfish_tools; +use e2e::ensure_vm_test_env; +use guestfish::ensure_guestfish_tools; use logging::tracing_subscriber_init; #[tokio::test] @@ -19,15 +19,12 @@ use logging::tracing_subscriber_init; async fn test_arch_image_startup_flow() -> Result<()> { tracing_subscriber_init(); - if !should_run_vm_tests() { - return Ok(()); - } + ensure_vm_test_env()?; + eprintln!("INFO: host checks passed"); - // `has_guestfish_tools` prints an actionable SKIP reason on failure. - if !has_guestfish_tools() { - return Ok(()); - } + ensure_guestfish_tools()?; + eprintln!("INFO: creating image"); let image = tokio::time::timeout( Duration::from_secs(25 * 60), create_image(Distro::Arch, "arch-cloudimg"), @@ -37,15 +34,21 @@ async fn test_arch_image_startup_flow() -> Result<()> { assert!(image.path().exists(), "qcow2 image must exist"); assert!(image.kernel().exists(), "kernel must exist"); assert!(image.initrd().exists(), "initrd must exist"); + eprintln!("INFO: image ready: {}", image.path().display()); - // Full startup flow validation (mirrors single_machine.rs::hello) + eprintln!("INFO: starting VM and waiting for SSH"); + // Full startup flow validation let config = MachineConfig::default(); - tokio::time::timeout(Duration::from_secs(8 * 60), async { + tokio::time::timeout(Duration::from_secs(20 * 60), async { with_machine(&image, &config, |vm| { Box::pin(async { - let result = vm.exec("whoami").await?; + let result = vm.exec(". /etc/os-release && echo $ID").await?; assert!(result.status.success()); - assert_eq!(str::from_utf8(&result.stdout)?.trim(), "root"); + let distro_id = str::from_utf8(&result.stdout)?.trim(); + assert!( + distro_id.contains("arch"), + "unexpected distro id: {distro_id}" + ); Ok(()) }) }) diff --git a/tests/fedora_image.rs b/tests/fedora_image.rs index 4b18382..4f818be 100644 --- a/tests/fedora_image.rs +++ b/tests/fedora_image.rs @@ -10,8 +10,8 @@ mod guestfish; #[path = "support/logging.rs"] mod logging; -use e2e::should_run_vm_tests; -use guestfish::has_guestfish_tools; +use e2e::ensure_vm_test_env; +use guestfish::ensure_guestfish_tools; use logging::tracing_subscriber_init; #[tokio::test] @@ -19,15 +19,12 @@ use logging::tracing_subscriber_init; async fn test_fedora_image_startup_flow() -> Result<()> { tracing_subscriber_init(); - if !should_run_vm_tests() { - return Ok(()); - } + ensure_vm_test_env()?; + eprintln!("INFO: host checks passed"); - // `has_guestfish_tools` prints an actionable SKIP reason on failure. - if !has_guestfish_tools() { - return Ok(()); - } + ensure_guestfish_tools()?; + eprintln!("INFO: creating image"); let image = tokio::time::timeout( Duration::from_secs(25 * 60), create_image(Distro::Fedora, "fedora-cloud"), @@ -37,15 +34,21 @@ async fn test_fedora_image_startup_flow() -> Result<()> { assert!(image.path().exists(), "qcow2 image must exist"); assert!(image.kernel().exists(), "kernel must exist"); assert!(image.initrd().exists(), "initrd must exist"); + eprintln!("INFO: image ready: {}", image.path().display()); - // Full startup flow validation (mirrors single_machine.rs::hello) + eprintln!("INFO: starting VM and waiting for SSH"); + // Full startup flow validation let config = MachineConfig::default(); - tokio::time::timeout(Duration::from_secs(8 * 60), async { + tokio::time::timeout(Duration::from_secs(20 * 60), async { with_machine(&image, &config, |vm| { Box::pin(async { - let result = vm.exec("whoami").await?; + let result = vm.exec(". /etc/os-release && echo $ID").await?; assert!(result.status.success()); - assert_eq!(str::from_utf8(&result.stdout)?.trim(), "root"); + let distro_id = str::from_utf8(&result.stdout)?.trim(); + assert!( + distro_id.contains("fedora"), + "unexpected distro id: {distro_id}" + ); Ok(()) }) }) diff --git a/tests/support/e2e.rs b/tests/support/e2e.rs index 42ddf57..980d598 100644 --- a/tests/support/e2e.rs +++ b/tests/support/e2e.rs @@ -1,26 +1,23 @@ use std::{path::Path, process::Command}; -/// Gate slow / privileged integration tests. -/// Enable by running: `QLEAN_RUN_E2E=1 cargo test --test ubuntu_image -- --nocapture` -pub fn e2e_enabled() -> bool { - matches!( +use anyhow::{Context, Result, bail}; + +/// Require explicit opt-in for slow integration tests. +pub fn ensure_e2e_enabled() -> Result<()> { + let enabled = matches!( std::env::var("QLEAN_RUN_E2E").as_deref(), Ok("1") | Ok("true") | Ok("yes") - ) -} + ); -/// Best-effort check for WSL. -pub fn is_wsl() -> bool { - if std::env::var_os("WSL_INTEROP").is_some() || std::env::var_os("WSL_DISTRO_NAME").is_some() { - return true; + if !enabled { + bail!("E2E VM tests are disabled. Set QLEAN_RUN_E2E=1 to run them."); } - std::fs::read_to_string("/proc/version") - .map(|s| s.to_lowercase().contains("microsoft")) - .unwrap_or(false) + + Ok(()) } /// Return `true` if a command exists on PATH. -pub fn has_cmd(cmd: &str) -> bool { +fn has_cmd(cmd: &str) -> bool { match Command::new(cmd).arg("--version").output() { Ok(_) => true, Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, @@ -28,58 +25,48 @@ pub fn has_cmd(cmd: &str) -> bool { } } -/// Best-effort connectivity check for the libvirt system URI. -fn can_connect_libvirt_system() -> bool { - Command::new("virsh") - .args(["-c", "qemu:///system", "list", "--all"]) - .output() - .map(|o| o.status.success()) - .unwrap_or(false) -} - -/// Qlean's SSH transport currently relies on vhost-vsock (AF_VSOCK). -fn has_vhost_vsock() -> bool { - Path::new("/dev/vhost-vsock").exists() || Path::new("/sys/module/vhost_vsock").exists() +/// Qlean can use TCP SSH fallback when vsock is unavailable. +pub fn log_vsock_status() { + let has_vsock = + Path::new("/dev/vhost-vsock").exists() || Path::new("/sys/module/vhost_vsock").exists(); + if !has_vsock { + eprintln!("INFO: vhost-vsock is not available; Qlean will use TCP SSH fallback if needed."); + } } -/// Skip E2E tests early if the environment can't support them. -/// Returns `true` if the test should proceed. -pub fn should_run_vm_tests() -> bool { - if !e2e_enabled() { - eprintln!("SKIP: E2E VM tests disabled. Set QLEAN_RUN_E2E=1 to enable."); - return false; - } - if !Path::new("/dev/kvm").exists() { - eprintln!( - "SKIP: /dev/kvm not found. Install/enable KVM or run on a host with virtualization." - ); - return false; - } +/// Validate mandatory host commands for E2E execution. +pub fn ensure_vm_test_commands() -> Result<()> { if !has_cmd("virsh") { - eprintln!("SKIP: could not find `virsh` on PATH (libvirt-clients)."); - return false; + bail!("Missing required command: virsh (libvirt-clients)."); } if !has_cmd("qemu-system-x86_64") && !has_cmd("qemu-kvm") { - eprintln!("SKIP: could not find `qemu-system-x86_64` (or `qemu-kvm`) on PATH."); - return false; - } - if !has_vhost_vsock() { - eprintln!( - "INFO: vhost-vsock not available on this kernel; E2E can still run using TCP SSH fallback." - ); + bail!("Missing required command: qemu-system-x86_64 (or qemu-kvm)."); } + Ok(()) +} + +/// Validate the libvirt system URI before running slow tests. +pub fn ensure_libvirt_system() -> Result<()> { + let output = Command::new("virsh") + .args(["-c", "qemu:///system", "list", "--all"]) + .output() + .context("failed to execute `virsh -c qemu:///system list --all`")?; - if is_wsl() { - if !can_connect_libvirt_system() { - eprintln!( - "SKIP: WSL detected and `virsh -c qemu:///system` is not usable (permission or libvirtd not running).\n\ -Hint: ensure systemd is enabled in WSL, start libvirtd, and add your user to libvirt/kvm groups, then `wsl --shutdown`." - ); - return false; - } - eprintln!( - "INFO: WSL detected, but KVM/libvirt appear usable; proceeding with E2E VM tests." + if !output.status.success() { + bail!( + "libvirt system URI is not usable: {}", + String::from_utf8_lossy(&output.stderr) ); } - true + + Ok(()) +} + +/// Fail early on host setup issues. Do not skip after opt-in. +pub fn ensure_vm_test_env() -> Result<()> { + ensure_e2e_enabled()?; + ensure_vm_test_commands()?; + ensure_libvirt_system()?; + log_vsock_status(); + Ok(()) } diff --git a/tests/support/guestfish.rs b/tests/support/guestfish.rs index 3255256..5767e66 100644 --- a/tests/support/guestfish.rs +++ b/tests/support/guestfish.rs @@ -1,8 +1,6 @@ -use std::path::Path; -use std::process::{Command, Stdio}; -use std::sync::OnceLock; +use anyhow::{Result, bail}; +use std::process::Command; -/// Return `true` if a command exists on PATH. fn has_cmd(cmd: &str) -> bool { match Command::new(cmd).arg("--version").output() { Ok(_) => true, @@ -11,117 +9,13 @@ fn has_cmd(cmd: &str) -> bool { } } -/// Some distros require libguestfs (guestfish/virt-copy-out) to extract kernel/initrd from qcow2. -/// -/// Note: On WSL and other minimal environments, libguestfs can *exist* but fail at runtime because -/// `supermin` cannot find a usable host kernel/appliance. In that case we skip and print remediation. -pub fn has_guestfish_tools() -> bool { - static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(has_guestfish_tools_inner) -} - -fn has_guestfish_tools_inner() -> bool { - if !(has_cmd("guestfish") && has_cmd("virt-copy-out")) { - eprintln!( - "SKIP: extraction requires `guestfish` and `virt-copy-out` (package: libguestfs-tools).\n\ -Install with: sudo apt install -y libguestfs-tools" - ); - return false; - } - - // Prefer a prebuilt appliance if present. - let appliance_kernel_candidates = [ - "/usr/lib/guestfs/appliance/kernel", - "/usr/lib64/guestfs/appliance/kernel", - "/usr/share/guestfs/appliance/kernel", - ]; - if appliance_kernel_candidates - .iter() - .any(|p| Path::new(p).exists()) - { - return if is_wsl() { - probe_libguestfs_runtime() - } else { - true - }; - } - - // Otherwise, supermin needs a host kernel image. - let has_host_kernel = Path::new("/boot/vmlinuz").exists() - || std::fs::read_dir("/boot") - .map(|it| { - it.filter_map(|e| e.ok()) - .any(|e| e.file_name().to_string_lossy().starts_with("vmlinuz")) - }) - .unwrap_or(false); - - if has_host_kernel { - return if is_wsl() { - probe_libguestfs_runtime() - } else { - true - }; - } - - eprintln!( - "SKIP: `guestfish` is installed but libguestfs appliance/kernel is missing.\n\ -On WSL this commonly breaks `supermin`. Install a *kernel image package* so /boot contains vmlinuz*, e.g.:\n\ - sudo apt install -y linux-image-generic\n\ - # or (smaller)\n\ - sudo apt install -y linux-image-virtual\n\ -Then retry.\n\ -(If /boot is not usable in your environment, you can build a fixed appliance once and point libguestfs to it:)\n\ - mkdir -p ~/.local/share/qlean/guestfs-appliance\n\ - libguestfs-make-fixed-appliance ~/.local/share/qlean/guestfs-appliance\n\ - export LIBGUESTFS_PATH=~/.local/share/qlean/guestfs-appliance" - ); - false -} - -fn is_wsl() -> bool { - std::fs::read_to_string("/proc/version") - .map(|s| s.to_lowercase().contains("microsoft")) - .unwrap_or(false) -} - -fn probe_libguestfs_runtime() -> bool { - if !has_cmd("libguestfs-test-tool") { - return true; +/// Require libguestfs extraction tools for images that need kernel/initrd extraction. +pub fn ensure_guestfish_tools() -> Result<()> { + if !has_cmd("guestfish") { + bail!("Missing required command: guestfish (package: libguestfs-tools)."); } - - let use_timeout = has_cmd("timeout"); - let mut cmd = if use_timeout { - let mut c = Command::new("timeout"); - c.arg("30s").arg("libguestfs-test-tool"); - c - } else { - Command::new("libguestfs-test-tool") - }; - - cmd.env("LIBGUESTFS_BACKEND", "direct") - .stdout(Stdio::null()) - .stderr(Stdio::piped()); - - match cmd.output() { - Ok(out) if out.status.success() => true, - Ok(out) => { - let stderr = String::from_utf8_lossy(&out.stderr); - let first_lines = stderr.lines().take(8).collect::>().join("\n"); - eprintln!( - "SKIP: libguestfs runtime probe failed (guestfish/virt-copy-out will likely fail).\n\ -Hint: run `LIBGUESTFS_BACKEND=direct libguestfs-test-tool` to see full diagnostics.\n\ -stderr (first lines):\n{}", - first_lines - ); - false - } - Err(e) => { - eprintln!( - "SKIP: failed to run libguestfs runtime probe: {}\n\ -Hint: ensure `libguestfs-tools` is installed.", - e - ); - false - } + if !has_cmd("virt-copy-out") { + bail!("Missing required command: virt-copy-out (package: libguestfs-tools)."); } + Ok(()) } diff --git a/tests/support/logging.rs b/tests/support/logging.rs index 1a29ce5..1941464 100644 --- a/tests/support/logging.rs +++ b/tests/support/logging.rs @@ -9,8 +9,10 @@ use tracing_subscriber::{EnvFilter, fmt::time::LocalTime}; pub fn tracing_subscriber_init() { static INIT: Once = Once::new(); INIT.call_once(|| { + let env_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,qlean=info")); tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) + .with_env_filter(env_filter) .with_timer(LocalTime::rfc_3339()) .try_init() .ok(); diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs index 8f5d9c3..0e5e222 100644 --- a/tests/ubuntu_image.rs +++ b/tests/ubuntu_image.rs @@ -8,7 +8,7 @@ mod e2e; #[path = "support/logging.rs"] mod logging; -use e2e::should_run_vm_tests; +use e2e::ensure_vm_test_env; use logging::tracing_subscriber_init; #[tokio::test] @@ -16,11 +16,11 @@ use logging::tracing_subscriber_init; async fn test_ubuntu_image_creation() -> Result<()> { tracing_subscriber_init(); - if !should_run_vm_tests() { - return Ok(()); - } + ensure_vm_test_env()?; + eprintln!("INFO: host checks passed"); // Ubuntu uses pre-extracted kernel/initrd. + eprintln!("INFO: creating image"); let image = tokio::time::timeout( Duration::from_secs(15 * 60), create_image(Distro::Ubuntu, "ubuntu-noble-cloudimg"), @@ -30,15 +30,21 @@ async fn test_ubuntu_image_creation() -> Result<()> { assert!(image.path().exists(), "qcow2 image must exist"); assert!(image.kernel().exists(), "kernel must exist"); assert!(image.initrd().exists(), "initrd must exist"); + eprintln!("INFO: image ready: {}", image.path().display()); - // Full startup flow validation (mirrors single_machine.rs::hello) + eprintln!("INFO: starting VM and waiting for SSH"); + // Full startup flow validation let config = MachineConfig::default(); tokio::time::timeout(Duration::from_secs(5 * 60), async { with_machine(&image, &config, |vm| { Box::pin(async { - let result = vm.exec("whoami").await?; + let result = vm.exec(". /etc/os-release && echo $ID").await?; assert!(result.status.success()); - assert_eq!(str::from_utf8(&result.stdout)?.trim(), "root"); + let distro_id = str::from_utf8(&result.stdout)?.trim(); + assert!( + distro_id.contains("ubuntu"), + "unexpected distro id: {distro_id}" + ); Ok(()) }) }) From 37364c78768fb0fe39cbe039162664c305d84820 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Wed, 25 Feb 2026 11:40:23 +0800 Subject: [PATCH 15/20] feat(log): revert noisy serial output logging from debug to trace Signed-off-by: userhaptop <1307305157@qq.com> --- src/qemu.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/qemu.rs b/src/qemu.rs index 05896a5..51bafe0 100644 --- a/src/qemu.rs +++ b/src/qemu.rs @@ -12,7 +12,7 @@ use tokio::{ time::{Duration, timeout}, }; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info, trace, warn}; use crate::{ KVM_AVAILABLE, MachineConfig, @@ -54,20 +54,13 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { let use_direct_kernel_boot = params.image.prefer_direct_kernel_boot && params.image.kernel.exists() && params.image.initrd.exists() - && std::fs::metadata(¶ms.image.kernel) - .map(|m| m.len() > 0) - .unwrap_or(false) - && std::fs::metadata(¶ms.image.initrd) - .map(|m| m.len() > 0) - .unwrap_or(false); + && std::fs::metadata(¶ms.image.kernel).map(|m| m.len() > 0).unwrap_or(false) + && std::fs::metadata(¶ms.image.initrd).map(|m| m.len() > 0).unwrap_or(false); if use_direct_kernel_boot { qemu_cmd .args(["-kernel", params.image.kernel.to_str().unwrap()]) - .args([ - "-append", - &format!("rw {} console=ttyS0", params.image.root_arg), - ]) + .args(["-append", &format!("rw {} console=ttyS0", params.image.root_arg)]) .args(["-initrd", params.image.initrd.to_str().unwrap()]); } else { warn!("Kernel/initrd extraction is unavailable. Booting from qcow2 disk image directly."); @@ -116,7 +109,9 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { .args(["-device", "virtio-net-pci,netdev=net1"]); } } else { - warn!("Bridged networking is unavailable. Falling back to user-mode networking + hostfwd."); + warn!( + "Bridged networking is unavailable. Falling back to user-mode networking + hostfwd." + ); let port = ssh_port.ok_or_else(|| { anyhow::anyhow!("user-mode networking fallback requires ssh_tcp_port to be set") @@ -182,9 +177,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { let reader = BufReader::new(stdout); let mut lines = reader.lines(); while let Ok(Some(line)) = lines.next_line().await { - // QEMU serial stdout is noisy during normal boot. - // Keep it available only at debug level. - debug!("[qemu] {}", strip_ansi_codes(&line)); + trace!("[qemu] {}", strip_ansi_codes(&line)); } }); @@ -234,6 +227,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { result } + fn bridge_conf_allows(bridge: &str) -> bool { // qemu-bridge-helper enforces an ACL file (commonly /etc/qemu/bridge.conf). // If the file is missing or doesn't allow the bridge, QEMU will fail with: From 2b05d4bd064de840b084450f27e3ba286e7715b7 Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Wed, 25 Feb 2026 11:53:15 +0800 Subject: [PATCH 16/20] feat(log): fmt Signed-off-by: userhaptop <1307305157@qq.com> --- src/qemu.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/qemu.rs b/src/qemu.rs index 51bafe0..d755323 100644 --- a/src/qemu.rs +++ b/src/qemu.rs @@ -54,13 +54,20 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { let use_direct_kernel_boot = params.image.prefer_direct_kernel_boot && params.image.kernel.exists() && params.image.initrd.exists() - && std::fs::metadata(¶ms.image.kernel).map(|m| m.len() > 0).unwrap_or(false) - && std::fs::metadata(¶ms.image.initrd).map(|m| m.len() > 0).unwrap_or(false); + && std::fs::metadata(¶ms.image.kernel) + .map(|m| m.len() > 0) + .unwrap_or(false) + && std::fs::metadata(¶ms.image.initrd) + .map(|m| m.len() > 0) + .unwrap_or(false); if use_direct_kernel_boot { qemu_cmd .args(["-kernel", params.image.kernel.to_str().unwrap()]) - .args(["-append", &format!("rw {} console=ttyS0", params.image.root_arg)]) + .args([ + "-append", + &format!("rw {} console=ttyS0", params.image.root_arg), + ]) .args(["-initrd", params.image.initrd.to_str().unwrap()]); } else { warn!("Kernel/initrd extraction is unavailable. Booting from qcow2 disk image directly."); @@ -109,9 +116,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { .args(["-device", "virtio-net-pci,netdev=net1"]); } } else { - warn!( - "Bridged networking is unavailable. Falling back to user-mode networking + hostfwd." - ); + warn!("Bridged networking is unavailable. Falling back to user-mode networking + hostfwd."); let port = ssh_port.ok_or_else(|| { anyhow::anyhow!("user-mode networking fallback requires ssh_tcp_port to be set") @@ -227,7 +232,6 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { result } - fn bridge_conf_allows(bridge: &str) -> bool { // qemu-bridge-helper enforces an ACL file (commonly /etc/qemu/bridge.conf). // If the file is missing or doesn't allow the bridge, QEMU will fail with: From cd214dd8fc6af58c051f0ebbdecaa8f5b77378ec Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Thu, 5 Mar 2026 21:04:01 +0800 Subject: [PATCH 17/20] =?UTF-8?q?feat(log):=20With=20QLEAN=5FRUN=5FE2E=3D1?= =?UTF-8?q?,=20Fedora/Arch=20integration=20tests=20run=20the=20full=20E2E?= =?UTF-8?q?=20flow=20(build=20image=20=E2=86=92=20boot=20VM=20=E2=86=92=20?= =?UTF-8?q?SSH=20=E2=86=92=20run=20checks=20=E2=86=92=20shutdown)=20with?= =?UTF-8?q?=20no=20skips,=20no=20WSL=20special-casing,=20and=20no=20?= =?UTF-8?q?=E2=80=9CKVM=20missing=20=E2=86=92=20skip=E2=80=9D=20(falls=20b?= =?UTF-8?q?ack=20to=20QEMU=20TCG).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: userhaptop <1307305157@qq.com> --- scripts/setup-host-prereqs.sh | 111 ++ src/image.rs | 2316 +++++++++++++++++++++++++++------ src/machine.rs | 321 ++++- src/qemu.rs | 181 ++- src/ssh.rs | 327 ++--- src/utils.rs | 41 +- tests/arch_image.rs | 16 +- tests/fedora_image.rs | 16 +- tests/support/e2e.rs | 102 +- tests/support/guestfish.rs | 5 +- tests/ubuntu_image.rs | 22 +- 11 files changed, 2665 insertions(+), 793 deletions(-) create mode 100644 scripts/setup-host-prereqs.sh diff --git a/scripts/setup-host-prereqs.sh b/scripts/setup-host-prereqs.sh new file mode 100644 index 0000000..77f9e4f --- /dev/null +++ b/scripts/setup-host-prereqs.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Qlean host prerequisites helper. +# +# This script is intentionally conservative: it configures QEMU bridge-helper +# permissions for the Qlean bridge and applies the minimal capabilities needed +# for qemu-bridge-helper. +# +# It does *not* create the bridge or manage network state (that remains the +# responsibility of Qlean at runtime). + +BRIDGE_NAME="${QLEAN_BRIDGE_NAME:-qlbr0}" + +need_root() { + if [[ "$(id -u)" -ne 0 ]]; then + echo "ERROR: Please run as root (e.g. sudo $0)" >&2 + exit 1 + fi +} + +ensure_bridge_conf() { + mkdir -p /etc/qemu + local conf=/etc/qemu/bridge.conf + + if [[ -f "$conf" ]]; then + if grep -qE "^allow[[:space:]]+all$" "$conf"; then + echo "OK: $conf already allows all bridges" + chmod 0644 "$conf" + return + fi + if grep -qE "^allow[[:space:]]+${BRIDGE_NAME}$" "$conf"; then + echo "OK: $conf already allows ${BRIDGE_NAME}" + chmod 0644 "$conf" + return + fi + fi + + echo "allow ${BRIDGE_NAME}" >> "$conf" + chmod 0644 "$conf" + echo "Wrote: allow ${BRIDGE_NAME} -> $conf" +} + +find_qemu_bridge_helper() { + # Common distro paths. + local candidates=( + /usr/lib/qemu/qemu-bridge-helper + /usr/libexec/qemu-bridge-helper + /usr/lib64/qemu/qemu-bridge-helper + ) + + for p in "${candidates[@]}"; do + if [[ -x "$p" ]]; then + echo "$p" + return 0 + fi + done + + # Fall back to PATH. + if command -v qemu-bridge-helper >/dev/null 2>&1; then + command -v qemu-bridge-helper + return 0 + fi + + return 1 +} + +ensure_bridge_helper_caps() { + local helper + if ! helper="$(find_qemu_bridge_helper)"; then + echo "WARN: qemu-bridge-helper not found. Install QEMU first." >&2 + return + fi + + if command -v setcap >/dev/null 2>&1; then + # Prefer file capabilities over setuid. + chmod u-s "$helper" || true + setcap cap_net_admin+ep "$helper" + + echo "OK: setcap cap_net_admin+ep $helper" + if command -v getcap >/dev/null 2>&1; then + getcap "$helper" || true + fi + else + echo "WARN: setcap not found. On Debian/Ubuntu install libcap2-bin." >&2 + fi +} + +maybe_install_guestfs_tools_ubuntu() { + # Only attempt package installation on Debian/Ubuntu when apt-get is available. + if ! command -v apt-get >/dev/null 2>&1; then + return + fi + + # guestfish/virt-copy-out are used for kernel/initrd extraction. + if command -v guestfish >/dev/null 2>&1 && command -v virt-copy-out >/dev/null 2>&1; then + echo "OK: libguestfs tools already installed" + return + fi + + echo "INFO: Installing libguestfs tools (guestfish, virt-copy-out) via apt-get" + apt-get update -y + apt-get install -y libguestfs-tools +} + +need_root +ensure_bridge_conf +ensure_bridge_helper_caps +maybe_install_guestfs_tools_ubuntu + +echo "DONE" diff --git a/src/image.rs b/src/image.rs index 91f9471..9f032ca 100644 --- a/src/image.rs +++ b/src/image.rs @@ -1,5 +1,7 @@ use std::{ ffi::OsStr, + fs, + io::{Read, Seek, SeekFrom}, path::{Path, PathBuf}, }; @@ -75,7 +77,7 @@ pub enum ImageSource { /// Configuration for custom images - supports two modes: /// 1. Image only (requires guestfish for extraction) -/// 2. Image + pre-extracted kernel/initrd (WSL-friendly) +/// 2. Image + pre-extracted kernel/initrd #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CustomImageConfig { // Image file (required) @@ -83,7 +85,7 @@ pub struct CustomImageConfig { pub image_hash: String, pub image_hash_type: ShaType, - // Optional: pre-extracted kernel and initrd (for WSL compatibility) + // Optional: pre-extracted kernel and initrd (avoids guestfish extraction) pub kernel_source: Option, pub kernel_hash: Option, pub initrd_source: Option, @@ -171,11 +173,7 @@ async fn fetch_ubuntu_sha256sums(base_url: &str) -> Result { struct ResolvedUbuntuCloudImage { base_url: String, disk_name: String, - kernel_name: String, - initrd_name: String, disk_sha256: String, - kernel_sha256: Option, - initrd_sha256: Option, } fn pick_existing_ubuntu_name(checksums: &str, candidates: &[&str]) -> Option { @@ -223,10 +221,6 @@ async fn resolve_ubuntu_noble_cloudimg() -> Result { } }; - let stem = disk_name.strip_suffix(".img").unwrap_or(&disk_name); - let kernel_name = format!("{}-vmlinuz-generic", stem); - let initrd_name = format!("{}-initrd-generic", stem); - let disk_sha256 = match ubuntu_sha256_for(&checksums, &disk_name) { Ok(v) => v, Err(e) => { @@ -234,34 +228,11 @@ async fn resolve_ubuntu_noble_cloudimg() -> Result { continue; } }; - let kernel_sha256 = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", kernel_name)) - .or_else(|_| ubuntu_sha256_for(&checksums, &kernel_name)) - .ok(); - if kernel_sha256.is_none() { - warn!( - "Ubuntu SHA256SUMS did not include kernel checksum for {}; proceeding without kernel hash verification", - kernel_name - ); - } - - let initrd_sha256 = ubuntu_sha256_for(&checksums, &format!("unpacked/{}", initrd_name)) - .or_else(|_| ubuntu_sha256_for(&checksums, &initrd_name)) - .ok(); - if initrd_sha256.is_none() { - warn!( - "Ubuntu SHA256SUMS did not include initrd checksum for {}; proceeding without initrd hash verification", - initrd_name - ); - } return Ok(ResolvedUbuntuCloudImage { base_url: (*base).to_string(), disk_name, - kernel_name, - initrd_name, disk_sha256, - kernel_sha256, - initrd_sha256, }); } @@ -632,10 +603,11 @@ pub async fn compute_sha512_streaming(path: &Path) -> Result { } /// Download file and compute hash in single pass to avoid reading file twice -pub async fn download_with_hash( +async fn download_with_hash_impl( url: &str, dest_path: &PathBuf, hash_type: ShaType, + slow_link_cutoff: Option<(std::time::Duration, u64)>, ) -> Result { // Keep a temp file to avoid leaving a partially-downloaded blob behind. let tmp_path = dest_path.with_extension("part"); @@ -681,7 +653,7 @@ pub async fn download_with_hash( let idle = std::time::Duration::from_secs(60); let mut downloaded: u64 = 0; let mut last_report: u64 = 0; - // Report download progress in reasonably small increments. On slower links (or in CI/WSL), + // Report download progress in reasonably small increments. On slower links or in CI, // 64MiB can take long enough that the test runner prints a scary "running over 60 seconds" // warning with no other output. let report_step: u64 = 8 * 1024 * 1024; // 8 MiB @@ -722,8 +694,9 @@ pub async fn download_with_hash( ); } } - if started_at.elapsed() >= std::time::Duration::from_secs(90) - && downloaded < 32 * 1024 * 1024 + if let Some((slow_link_timeout, min_bytes)) = slow_link_cutoff + && started_at.elapsed() >= slow_link_timeout + && downloaded < min_bytes { anyhow::bail!( "download too slow for {} ({} MiB in {:?}); trying next mirror", @@ -772,8 +745,9 @@ pub async fn download_with_hash( ); } } - if started_at.elapsed() >= std::time::Duration::from_secs(90) - && downloaded < 32 * 1024 * 1024 + if let Some((slow_link_timeout, min_bytes)) = slow_link_cutoff + && started_at.elapsed() >= slow_link_timeout + && downloaded < min_bytes { anyhow::bail!( "download too slow for {} ({} MiB in {:?}); trying next mirror", @@ -812,6 +786,14 @@ pub async fn download_with_hash( Ok(hash) } +pub async fn download_with_hash( + url: &str, + dest_path: &PathBuf, + hash_type: ShaType, +) -> Result { + download_with_hash_impl(url, dest_path, hash_type, None).await +} + /// Try downloading a remote file from multiple candidate URLs. /// /// Mirrors can hang mid-transfer; we apply an idle timeout and move on. @@ -844,7 +826,14 @@ pub async fn download_with_hash_multi( info!("Trying mirror {}/{}", idx + 1, urls.len()); debug!("Download attempt {}/{}: {}", idx + 1, urls.len(), url); - match download_with_hash(url, dest_path, hash_type.clone()).await { + match download_with_hash_impl( + url, + dest_path, + hash_type.clone(), + Some((std::time::Duration::from_secs(90), 32 * 1024 * 1024)), + ) + .await + { Ok(h) => { if let Some(expected) = expected_hex && !h.eq_ignore_ascii_case(expected) @@ -925,23 +914,6 @@ async fn download_or_copy_with_hash( Ok(()) } -/// Download or copy file from ImageSource without hash verification. -async fn download_or_copy_without_hash(source: &ImageSource, dest: &PathBuf) -> Result<()> { - match source { - ImageSource::Url(url) => { - if dest.exists() { - return Ok(()); - } - let _ = download_with_hash(url, dest, ShaType::Sha256).await?; - } - ImageSource::LocalPath(src) => { - anyhow::ensure!(src.exists(), "file does not exist: {}", src.display()); - tokio::fs::copy(src, dest).await?; - } - } - Ok(()) -} - impl ImageMeta { /// Create a new image by downloading and extracting pub async fn create(name: &str) -> Result { @@ -961,15 +933,20 @@ impl ImageMeta { tokio::fs::create_dir_all(&image_dir).await?; let distro_action = A::default(); + let distro = distro_action.distro(); distro_action.download(name).await?; let image_path = image_dir.join(format!("{}.qcow2", name)); let (kernel, initrd) = distro_action.extract(name).await?; let checksum_path = image_dir.join("checksums"); - let root_arg = detect_root_arg(&image_path) - .await - .unwrap_or_else(|_| default_root_arg()); + let root_arg = match distro { + Distro::Ubuntu => detect_root_arg(&image_path) + .await + .unwrap_or_else(|_| default_root_arg()), + Distro::Debian | Distro::Fedora | Distro::Arch => detect_root_arg(&image_path).await?, + Distro::Custom => unreachable!("custom images use create_with_action()"), + }; let checksum = ShaSum { path: checksum_path, sha_type: ShaType::Sha512, @@ -998,19 +975,48 @@ impl ImageMeta { .await .with_context(|| format!("failed to read config file at {}", json_path.display()))?; - let image: ImageMeta = serde_json::from_str(&json_content) + let mut image: ImageMeta = serde_json::from_str(&json_content) .with_context(|| format!("failed to parse JSON from {}", json_path.display()))?; + // Older caches may contain a `root=` token with a trailing ':' (e.g. `root=/dev/vda3:`), + // which causes direct-kernel boot to hang forever waiting for a non-existent device. + // Sanitize on load so users don't have to manually delete their image cache. + image.root_arg = image + .root_arg + .split_whitespace() + .map(|t| { + if let Some(rest) = t.strip_prefix("root=") { + let clean = rest.trim_end_matches(':'); + format!("root={clean}") + } else { + t.to_string() + } + }) + .collect::>() + .join(" "); + let kernel_ok = image.kernel.exists() + && !image + .kernel + .file_name() + .and_then(|name| name.to_str()) + .map(|name| name.ends_with(".unavailable")) + .unwrap_or(false) && std::fs::metadata(&image.kernel) .map(|m| m.len() > 0) .unwrap_or(false); let initrd_ok = image.initrd.exists() + && !image + .initrd + .file_name() + .and_then(|name| name.to_str()) + .map(|name| name.ends_with(".unavailable")) + .unwrap_or(false) && std::fs::metadata(&image.initrd) .map(|m| m.len() > 0) .unwrap_or(false); if !kernel_ok || !initrd_ok { - bail!("cached image is missing kernel/initrd markers; recreate is required"); + bail!("cached image is missing valid kernel/initrd artifacts; recreate is required"); } let checksum_dir = dirs.images.join(name); @@ -1037,8 +1043,11 @@ impl ImageMeta { Ok(image) } +} - /// Save image metadata to disk using streaming hash +// Special create method for Custom images (non-Default trait) +impl ImageMeta { + /// Save image metadata to disk using streaming hash. async fn save(&self, name: &str) -> Result<()> { let dirs = QleanDirs::new()?; let json_path = dirs.images.join(format!("{}.json", name)); @@ -1050,7 +1059,7 @@ impl ImageMeta { .await .with_context(|| format!("failed to write image config to {}", json_path.display()))?; - // Use streaming hash for best performance (7-27% faster in release mode) + // Use streaming hash for best performance (7-27% faster in release mode). let (image_hash, kernel_hash, initrd_hash) = match self.checksum.sha_type { ShaType::Sha256 => ( compute_sha256_streaming(&self.path).await?, @@ -1096,10 +1105,7 @@ impl ImageMeta { Ok(()) } -} -// Special create method for Custom images (non-Default trait) -impl ImageMeta { /// Create image with custom action for non-Default implementations pub async fn create_with_action(name: &str, action: A) -> Result { debug!("Fetching image {} with custom action ...", name); @@ -1134,241 +1140,1822 @@ impl ImageMeta { vendor: action, }; - // Inline save with streaming hash - let json_path = dirs.images.join(format!("{}.json", name)); - let json_content = serde_json::to_string_pretty(&image)?; - tokio::fs::write(&json_path, json_content).await?; - - let (image_hash, kernel_hash, initrd_hash) = match image.checksum.sha_type { - ShaType::Sha256 => ( - compute_sha256_streaming(&image.path).await?, - compute_sha256_streaming(&image.kernel).await?, - compute_sha256_streaming(&image.initrd).await?, - ), - ShaType::Sha512 => ( - compute_sha512_streaming(&image.path).await?, - compute_sha512_streaming(&image.kernel).await?, - compute_sha512_streaming(&image.initrd).await?, - ), - }; - - let image_filename = image.path.file_name().unwrap().to_string_lossy(); - let kernel_filename = image.kernel.file_name().unwrap().to_string_lossy(); - let initrd_filename = image.initrd.file_name().unwrap().to_string_lossy(); - - let checksum_content = format!( - "{} {}\n{} {}\n{} {}\n", - image_hash, image_filename, kernel_hash, kernel_filename, initrd_hash, initrd_filename - ); - - tokio::fs::write(&image.checksum.path, checksum_content).await?; + image.save(name).await?; Ok(image) } } -async fn ensure_fixed_guestfs_appliance() -> Result { - for dir in [ +fn candidate_guestfs_appliance_dirs() -> [&'static str; 6] { + [ "/usr/lib/guestfs/appliance", + "/usr/lib64/guestfs/appliance", + "/usr/local/lib/guestfs/appliance", + "/usr/local/lib64/guestfs/appliance", "/usr/lib/x86_64-linux-gnu/guestfs/appliance", - ] { - let p = PathBuf::from(dir); - if p.join("kernel").exists() && p.join("initrd").exists() { - return Ok(p); - } - } + "/usr/lib/aarch64-linux-gnu/guestfs/appliance", + ] +} - let dirs = QleanDirs::new()?; - let appliance_dir = dirs.base.join("guestfs-appliance"); - if appliance_dir.join("kernel").exists() && appliance_dir.join("initrd").exists() { - return Ok(appliance_dir); - } +fn is_complete_guestfs_appliance(dir: &Path) -> bool { + dir.join("kernel").exists() + && dir.join("initrd").exists() + && dir.join("root").exists() + && dir.join("README.fixed").exists() +} - let output = tokio::process::Command::new("libguestfs-make-fixed-appliance") - .arg(&appliance_dir) - .output() - .await - .with_context(|| "failed to execute libguestfs-make-fixed-appliance")?; - if !output.status.success() { - bail!( - "libguestfs fixed appliance build failed: {}\nInstall package: libguestfs-appliance (or libguestfs-tools) and retry.", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(appliance_dir) +#[derive(Debug, Clone)] +struct SuperminKernelBundle { + kernel: PathBuf, + modules_dir: PathBuf, + kernel_version: String, } -async fn run_guestfs_tool( - program: &str, - args: &[&OsStr], - current_dir: &Path, -) -> Result { - async fn run_once( - program: &str, - args: &[&OsStr], - current_dir: &Path, - appliance: Option<&Path>, - ) -> Result { - let mut cmd = tokio::process::Command::new(program); - cmd.env("LIBGUESTFS_BACKEND", "direct") - .current_dir(current_dir); - if let Some(appliance_dir) = appliance { - cmd.env("LIBGUESTFS_PATH", appliance_dir); - } - for a in args { - cmd.arg(a); - } - let child = cmd.output(); - let out = timeout(Duration::from_secs(180), child) - .await - .with_context(|| format!("{} timed out after 180s (libguestfs)", program))? - .with_context(|| format!("failed to execute {}", program))?; - Ok(out) - } +async fn stage_readable_supermin_kernel( + cache_root: &Path, + bundle: &SuperminKernelBundle, +) -> Result { + let stage_dir = cache_root.join("supermin-host-kernel"); + tokio::fs::create_dir_all(&stage_dir) + .await + .with_context(|| format!("failed to create {}", stage_dir.display()))?; - let first = run_once(program, args, current_dir, None).await?; - if !first.status.success() { - warn!( - "{} failed (direct backend): {}", - program, - String::from_utf8_lossy(&first.stderr) - ); - } - if first.status.success() { - return Ok(first); - } + let file_name = bundle + .kernel + .file_name() + .with_context(|| format!("invalid kernel path: {}", bundle.kernel.display()))?; + let staged_kernel = stage_dir.join(file_name); - let stderr = String::from_utf8_lossy(&first.stderr); - let needs_fixed = stderr.contains("supermin exited with error status") - || stderr.contains("/usr/bin/supermin"); - if needs_fixed && std::env::var_os("LIBGUESTFS_PATH").is_none() { - let appliance_dir = ensure_fixed_guestfs_appliance().await?; - return run_once(program, args, current_dir, Some(&appliance_dir)).await; - } + let _ = tokio::fs::remove_file(&staged_kernel).await; + tokio::fs::copy(&bundle.kernel, &staged_kernel) + .await + .with_context(|| { + format!( + "failed to copy host kernel {} into {}", + bundle.kernel.display(), + stage_dir.display() + ) + })?; - Ok(first) -} + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; -async fn guestfish_ls_boot(image_dir: &Path, file_name: &str) -> Result { - let args = [ - OsStr::new("--ro"), - OsStr::new("-a"), - OsStr::new(file_name), - OsStr::new("-i"), - OsStr::new("ls"), - OsStr::new("/boot"), - ]; - let output = run_guestfs_tool("guestfish", &args, image_dir).await?; - if !output.status.success() { - bail!( - "guestfish failed: {}", - String::from_utf8_lossy(&output.stderr) - ); + let perms = std::fs::Permissions::from_mode(0o644); + tokio::fs::set_permissions(&staged_kernel, perms) + .await + .with_context(|| format!("failed to chmod {} to 0644", staged_kernel.display()))?; } - Ok(String::from_utf8_lossy(&output.stdout).to_string()) + + Ok(SuperminKernelBundle { + kernel: staged_kernel, + modules_dir: bundle.modules_dir.clone(), + kernel_version: bundle.kernel_version.clone(), + }) } -async fn virt_copy_out(image_dir: &Path, file_name: &str, src: &str, kind: &str) -> Result<()> { - let args = [ - OsStr::new("-a"), - OsStr::new(file_name), - OsStr::new(src), - OsStr::new("."), - ]; - let output = run_guestfs_tool("virt-copy-out", &args, image_dir).await?; - if !output.status.success() { - bail!( - "virt-copy-out failed for {}: {}", - kind, - String::from_utf8_lossy(&output.stderr) - ); +async fn clear_known_paths(root: &Path, names: &[&str]) -> Result<()> { + for name in names { + let path = root.join(name); + if !tokio::fs::try_exists(&path).await? { + continue; + } + let meta = tokio::fs::metadata(&path) + .await + .with_context(|| format!("failed to stat {}", path.display()))?; + if meta.is_dir() { + tokio::fs::remove_dir_all(&path) + .await + .with_context(|| format!("failed to clear {}", path.display()))?; + } else { + tokio::fs::remove_file(&path) + .await + .with_context(|| format!("failed to remove {}", path.display()))?; + } } Ok(()) } -async fn write_unavailable_boot_artifacts( - image_dir: &Path, - reason: &str, -) -> Result<(PathBuf, PathBuf)> { - let kernel = image_dir.join("vmlinuz.unavailable"); - let initrd = image_dir.join("initrd.img.unavailable"); - let note = format!("qlean boot artifact unavailable: {}\n", reason); +fn find_supermin_kernel_bundle(root: &Path) -> Option { + let boot_dir = root.join("boot"); + let modules_root = root.join("lib/modules"); + + let mut kernels = std::fs::read_dir(&boot_dir) + .ok()? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .map(|name| name.starts_with("vmlinuz")) + .unwrap_or(false) + }) + .collect::>(); + kernels.sort(); - tokio::fs::write(&kernel, note.as_bytes()) - .await - .with_context(|| format!("failed to write {}", kernel.display()))?; - tokio::fs::write(&initrd, note.as_bytes()) - .await - .with_context(|| format!("failed to write {}", initrd.display()))?; + let mut modules = std::fs::read_dir(&modules_root) + .ok()? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect::>(); + modules.sort(); + + for module_dir in modules.iter().rev() { + let version = module_dir.file_name()?.to_string_lossy().to_string(); + if let Some(kernel) = kernels.iter().rev().find(|kernel| { + kernel + .file_name() + .and_then(|name| name.to_str()) + .map(|name| name.contains(&version)) + .unwrap_or(false) + }) { + return Some(SuperminKernelBundle { + kernel: kernel.clone(), + modules_dir: module_dir.clone(), + kernel_version: version, + }); + } + } + + let kernel = kernels.into_iter().last()?; + let modules_dir = modules.into_iter().last()?; + let kernel_version = modules_dir.file_name()?.to_string_lossy().to_string(); + + Some(SuperminKernelBundle { + kernel, + modules_dir, + kernel_version, + }) +} - Ok((kernel, initrd)) +fn is_concrete_linux_image_package(name: &str) -> bool { + name.strip_prefix("linux-image-unsigned-") + .or_else(|| name.strip_prefix("linux-image-")) + .and_then(|rest| rest.chars().next()) + .map(|ch| ch.is_ascii_digit()) + .unwrap_or(false) } -async fn detect_root_arg(image_path: &Path) -> Result { - let image_dir = image_path.parent().with_context(|| "missing image dir")?; - let file_name = image_path - .file_name() - .and_then(|v| v.to_str()) - .with_context(|| "invalid image filename")?; - let args = [ - OsStr::new("--ro"), - OsStr::new("-a"), - OsStr::new(file_name), - OsStr::new("-i"), - OsStr::new("mountpoints"), - ]; - let output = run_guestfs_tool("guestfish", &args, image_dir).await?; - if !output.status.success() { - return Ok(default_root_arg()); - } - let text = String::from_utf8_lossy(&output.stdout); +fn parse_apt_depends_for_kernel_package(text: &str) -> Option { for line in text.lines() { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() < 2 { - continue; - } - let mut dev = None; - if parts[0].starts_with("/dev/") && parts[1] == "/" { - dev = Some(parts[0]); - } - if parts[1].starts_with("/dev/") && parts[0] == "/" { - dev = Some(parts[1]); - } - if let Some(d) = dev { - let virt = if let Some(rest) = d.strip_prefix("/dev/sd") { - format!("/dev/vd{}", rest) - } else { - d.to_string() - }; - return Ok(format!("root={}", virt)); + let line = line.trim(); + if let Some(dep) = line.strip_prefix("Depends:") { + let dep = dep.trim(); + if is_concrete_linux_image_package(dep) { + return Some(dep.to_string()); + } } } - Ok(default_root_arg()) + None } -// --------------------------------------------------------------------------- -// Debian -// --------------------------------------------------------------------------- +fn parse_apt_search_for_kernel_package(text: &str) -> Option { + let mut packages = text + .lines() + .filter_map(|line| { + line.split_once(" - ") + .map(|(name, _)| name.trim().to_string()) + }) + .filter(|name| is_concrete_linux_image_package(name)) + .collect::>(); + packages.sort(); + packages.into_iter().last() +} -#[derive(Debug, Default)] -pub struct Debian {} +fn derive_modules_package_name(image_package: &str) -> Option { + let version = image_package + .strip_prefix("linux-image-unsigned-") + .or_else(|| image_package.strip_prefix("linux-image-"))?; + Some(format!("linux-modules-{}", version)) +} -impl ImageAction for Debian { - async fn download(&self, name: &str) -> Result<()> { - let checksums_url = "https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS"; - let checksums_text = reqwest::get(checksums_url) +async fn resolve_apt_kernel_package_name() -> Result { + for meta_pkg in ["linux-image-virtual", "linux-image-generic"] { + let output = tokio::process::Command::new("apt-cache") + .arg("depends") + .arg("--important") + .arg(meta_pkg) + .output() .await - .with_context(|| format!("failed to download SHA512SUMS from {}", checksums_url))? - .text() + .with_context(|| format!("failed to execute `apt-cache depends {meta_pkg}`"))?; + if output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + if let Some(pkg) = parse_apt_depends_for_kernel_package(&stdout) { + return Ok(pkg); + } + } + } + + for pattern in [ + "^linux-image-[0-9].*-generic$", + "^linux-image-unsigned-[0-9].*-generic$", + "^linux-image-[0-9].*-virtual$", + ] { + let output = tokio::process::Command::new("apt-cache") + .arg("search") + .arg(pattern) + .output() .await - .with_context(|| format!("failed to read SHA512SUMS text from {}", checksums_url))?; + .with_context(|| format!("failed to execute `apt-cache search {pattern}`"))?; + if output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + if let Some(pkg) = parse_apt_search_for_kernel_package(&stdout) { + return Ok(pkg); + } + } + } - let target_filename = format!("{}.qcow2", name); - let expected_sha512 = find_sha512_for_file(&checksums_text, &target_filename) - .with_context(|| { - format!( - "failed to find SHA512 checksum entry for {} in remote SHA512SUMS file", - target_filename + bail!("failed to locate an apt-downloadable Linux kernel package for supermin on this host") +} + +async fn download_apt_package(package_name: &str, download_dir: &Path) -> Result { + clear_known_paths(download_dir, &["kernel-download"]) + .await + .with_context(|| { + format!( + "failed to clear stale package cache in {}", + download_dir.display() + ) + })?; + + let package_dir = download_dir.join("kernel-download"); + tokio::fs::create_dir_all(&package_dir) + .await + .with_context(|| format!("failed to create {}", package_dir.display()))?; + + let output = tokio::process::Command::new("apt") + .arg("download") + .arg(package_name) + .current_dir(&package_dir) + .output() + .await + .with_context(|| format!("failed to execute `apt download {package_name}`"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let sep = if stdout.trim().is_empty() || stderr.trim().is_empty() { + "" + } else { + "\n" + }; + bail!("{}{}{}", stdout.trim(), sep, stderr.trim()); + } + + let mut entries = tokio::fs::read_dir(&package_dir) + .await + .with_context(|| format!("failed to read {}", package_dir.display()))?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) == Some("deb") { + return Ok(path); + } + } + + bail!( + "`apt download {}` did not produce a .deb in {}", + package_name, + package_dir.display() + ) +} + +/// Supermin needs a kernel+modules pair. Some hosts (notably WSL) ship the +/// libguestfs userspace tools but do not provide a local /boot kernel image. +/// In that case, bootstrap a matching bundle from an apt-downloadable kernel package. +async fn prepare_supermin_kernel_bundle(cache_root: &Path) -> Result { + let cache_dir = cache_root.join("supermin-kernel-cache"); + let extract_root = cache_dir.join("root"); + + if let Some(bundle) = find_supermin_kernel_bundle(&extract_root) { + return Ok(bundle); + } + + tokio::fs::create_dir_all(&cache_dir) + .await + .with_context(|| format!("failed to create {}", cache_dir.display()))?; + + let package_attempt = async { + clear_known_paths(&cache_dir, &["root", "kernel-download"]) + .await + .with_context(|| format!("failed to clear {}", cache_dir.display()))?; + + let package_name = resolve_apt_kernel_package_name().await?; + info!( + "Preparing a cached Linux kernel package for libguestfs supermin: {}", + package_name + ); + let deb_file = download_apt_package(&package_name, &cache_dir).await?; + + tokio::fs::create_dir_all(&extract_root) + .await + .with_context(|| format!("failed to create {}", extract_root.display()))?; + + let output = tokio::process::Command::new("dpkg-deb") + .arg("-x") + .arg(&deb_file) + .arg(&extract_root) + .output() + .await + .with_context(|| format!("failed to extract {} with dpkg-deb", deb_file.display()))?; + if !output.status.success() { + bail!("{}", String::from_utf8_lossy(&output.stderr).trim()); + } + + if find_supermin_kernel_bundle(&extract_root).is_none() + && let Some(modules_pkg) = derive_modules_package_name(&package_name) + { + info!( + "The downloaded image package did not include a full modules tree; fetching {} as well", + modules_pkg + ); + let modules_deb = download_apt_package(&modules_pkg, &cache_dir).await?; + let output = tokio::process::Command::new("dpkg-deb") + .arg("-x") + .arg(&modules_deb) + .arg(&extract_root) + .output() + .await + .with_context(|| format!("failed to extract {} with dpkg-deb", modules_deb.display()))?; + if !output.status.success() { + bail!("{}", String::from_utf8_lossy(&output.stderr).trim()); + } + } + + find_supermin_kernel_bundle(&extract_root).with_context(|| { + format!( + "downloaded kernel package {} did not contain a usable kernel/modules pair under {}", + package_name, + extract_root.display() + ) + }) + } + .await; + + match package_attempt { + Ok(bundle) => Ok(bundle), + Err(pkg_err) => { + if let Some(bundle) = find_supermin_kernel_bundle(Path::new("/")) { + warn!( + "Failed to prepare an apt-downloaded kernel bundle for supermin; falling back to a staged host kernel copy: {pkg_err:#}" + ); + return stage_readable_supermin_kernel(cache_root, &bundle).await; + } + Err(pkg_err) + } + } +} + +async fn run_make_fixed_appliance( + appliance_dir: &Path, + kernel_bundle: Option<&SuperminKernelBundle>, +) -> Result { + clear_known_paths(appliance_dir, &["kernel", "initrd", "root"]).await?; + + let mut cmd = tokio::process::Command::new("libguestfs-make-fixed-appliance"); + cmd.arg(appliance_dir); + if let Some(bundle) = kernel_bundle { + cmd.env("SUPERMIN_KERNEL", &bundle.kernel) + .env("SUPERMIN_MODULES", &bundle.modules_dir) + .env("SUPERMIN_KERNEL_VERSION", &bundle.kernel_version); + } + + cmd.output().await.with_context( + || "failed to execute libguestfs-make-fixed-appliance; install libguestfs-tools", + ) +} + +async fn try_build_fixed_guestfs_appliance(appliance_dir: &Path) -> Result<()> { + let first = run_make_fixed_appliance(appliance_dir, None).await?; + if first.status.success() { + anyhow::ensure!( + is_complete_guestfs_appliance(appliance_dir), + "libguestfs-make-fixed-appliance completed without producing a usable appliance" + ); + return Ok(()); + } + + let first_stderr = String::from_utf8_lossy(&first.stderr).trim().to_string(); + let needs_explicit_kernel = first_stderr.contains("supermin exited with error status") + || first_stderr.contains("--copy-kernel") + || first_stderr.contains("failed to find a suitable kernel"); + + if !needs_explicit_kernel { + bail!("{}", first_stderr); + } + + let cache_root = appliance_dir.parent().unwrap_or_else(|| Path::new(".")); + let bundle = prepare_supermin_kernel_bundle(cache_root) + .await + .with_context(|| "failed to provision a kernel/modules bundle for supermin")?; + + info!( + "Retrying libguestfs fixed appliance build with explicit SUPERMIN_KERNEL={} and SUPERMIN_MODULES={}", + bundle.kernel.display(), + bundle.modules_dir.display() + ); + let retry = run_make_fixed_appliance(appliance_dir, Some(&bundle)).await?; + if !retry.status.success() { + bail!("{}", String::from_utf8_lossy(&retry.stderr).trim()); + } + + anyhow::ensure!( + is_complete_guestfs_appliance(appliance_dir), + "libguestfs-make-fixed-appliance completed without producing a usable appliance" + ); + Ok(()) +} + +fn parse_appliance_version_code(version: &str) -> Option { + let mut parts = version.split('.'); + let major = parts.next()?.parse::().ok()?; + let minor = parts.next().unwrap_or("0").parse::().ok()?; + let patch = parts.next().unwrap_or("0").parse::().ok()?; + Some(major * 1_000_000 + minor * 1_000 + patch) +} + +async fn detect_local_libguestfs_version() -> Option { + let guestfish = tokio::process::Command::new("guestfish") + .arg("--version") + .output() + .await + .ok()?; + let stdout = String::from_utf8_lossy(&guestfish.stdout); + if let Some(version) = stdout.split_whitespace().find(|part| { + part.chars() + .next() + .map(|ch| ch.is_ascii_digit()) + .unwrap_or(false) + }) { + return Some(version.trim().to_string()); + } + + let probe = tokio::process::Command::new("libguestfs-test-tool") + .env("LIBGUESTFS_BACKEND", "direct") + .output() + .await + .ok()?; + let combined = format!( + "{} +{}", + String::from_utf8_lossy(&probe.stdout), + String::from_utf8_lossy(&probe.stderr) + ); + combined.lines().find_map(|line| { + line.split_once("version=") + .map(|(_, v)| v.trim().to_string()) + }) +} + +fn parse_official_appliance_versions(index_html: &str) -> Vec { + let mut versions = Vec::new(); + let mut cursor = 0; + let needle = "appliance-"; + let suffix = ".tar.xz"; + + while let Some(start_rel) = index_html[cursor..].find(needle) { + let start = cursor + start_rel + needle.len(); + if let Some(end_rel) = index_html[start..].find(suffix) { + let version = &index_html[start..start + end_rel]; + if !version.is_empty() + && version.chars().all(|ch| ch.is_ascii_digit() || ch == '.') + && !versions.iter().any(|seen| seen == version) + { + versions.push(version.to_string()); + } + cursor = start + end_rel + suffix.len(); + } else { + break; + } + } + + versions +} + +fn ordered_appliance_versions( + local_version: Option<&str>, + mut versions: Vec, +) -> Vec { + let local_code = local_version.and_then(parse_appliance_version_code); + + versions.sort_by(|a, b| { + let a_code = parse_appliance_version_code(a).unwrap_or_default(); + let b_code = parse_appliance_version_code(b).unwrap_or_default(); + + match local_code { + Some(local) => { + let a_dist = (a_code - local).abs(); + let b_dist = (b_code - local).abs(); + a_dist.cmp(&b_dist).then_with(|| b_code.cmp(&a_code)) + } + None => b_code.cmp(&a_code), + } + }); + + versions +} + +async fn extract_downloaded_appliance_archive( + archive_path: &Path, + appliance_dir: &Path, +) -> Result<()> { + let list_output = tokio::process::Command::new("tar") + .arg("-tJf") + .arg(archive_path) + .output() + .await + .with_context(|| format!("failed to inspect {}", archive_path.display()))?; + if !list_output.status.success() { + bail!("{}", String::from_utf8_lossy(&list_output.stderr).trim()); + } + + let listing = String::from_utf8_lossy(&list_output.stdout); + let top_level = listing + .lines() + .find_map(|line| { + let trimmed = line.trim_matches('/').trim(); + if trimmed.is_empty() { + None + } else { + trimmed.split('/').next().map(|part| part.to_string()) + } + }) + .with_context(|| format!("{} was empty", archive_path.display()))?; + + let output = tokio::process::Command::new("tar") + .arg("-xJf") + .arg(archive_path) + .arg("-C") + .arg(appliance_dir) + .arg("--strip-components=1") + .arg(&top_level) + .output() + .await + .with_context(|| format!("failed to extract {}", archive_path.display()))?; + if !output.status.success() { + bail!("{}", String::from_utf8_lossy(&output.stderr).trim()); + } + + Ok(()) +} + +async fn validate_fixed_guestfs_appliance(appliance_dir: &Path) -> Result<()> { + let output = timeout( + Duration::from_secs(180), + tokio::process::Command::new("libguestfs-test-tool") + .env("LIBGUESTFS_BACKEND", "direct") + .env("LIBGUESTFS_PATH", appliance_dir) + .output(), + ) + .await + .with_context(|| "libguestfs-test-tool timed out while validating the fixed appliance")? + .with_context(|| "failed to execute libguestfs-test-tool")?; + + if output.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let details = if stderr.trim().is_empty() { + stdout.trim().to_string() + } else if stdout.trim().is_empty() { + stderr.trim().to_string() + } else { + format!("{}\n{}", stdout.trim(), stderr.trim()) + }; + bail!("{details}"); +} + +async fn try_download_official_guestfs_appliance_version( + appliance_dir: &Path, + version: &str, +) -> Result<()> { + let archive_name = format!("appliance-{version}.tar.xz"); + let archive_url = format!("https://download.libguestfs.org/binaries/appliance/{archive_name}"); + + let cache_root = appliance_dir.parent().unwrap_or_else(|| Path::new(".")); + let download_dir = cache_root.join("guestfs-appliance-downloads"); + tokio::fs::create_dir_all(&download_dir) + .await + .with_context(|| format!("failed to create {}", download_dir.display()))?; + let archive_path = download_dir.join(&archive_name); + + let _actual_sha = download_with_hash(&archive_url, &archive_path, ShaType::Sha512).await?; + + clear_known_paths(appliance_dir, &["kernel", "initrd", "root", "README.fixed"]).await?; + tokio::fs::create_dir_all(appliance_dir) + .await + .with_context(|| format!("failed to create {}", appliance_dir.display()))?; + + extract_downloaded_appliance_archive(&archive_path, appliance_dir).await?; + + anyhow::ensure!( + is_complete_guestfs_appliance(appliance_dir), + "downloaded {} did not contain a complete fixed appliance", + archive_name + ); + + validate_fixed_guestfs_appliance(appliance_dir).await +} + +async fn provision_official_guestfs_appliance(appliance_dir: &Path) -> Result<()> { + let local_version = detect_local_libguestfs_version().await; + let index_html = fetch_text("https://download.libguestfs.org/binaries/appliance/").await?; + let available = parse_official_appliance_versions(&index_html); + anyhow::ensure!( + !available.is_empty(), + "the upstream appliance index did not advertise any downloadable fixed appliance" + ); + + let candidates = ordered_appliance_versions(local_version.as_deref(), available); + let mut last_err = None; + + for version in candidates { + info!( + "Trying official libguestfs fixed appliance download: version {}", + version + ); + match try_download_official_guestfs_appliance_version(appliance_dir, &version).await { + Ok(()) => return Ok(()), + Err(err) => { + warn!( + "Official libguestfs appliance {} did not validate on this host: {err:#}", + version + ); + last_err = Some(err); + } + } + } + + Err(last_err + .unwrap_or_else(|| anyhow::anyhow!("failed to provision an official fixed appliance"))) +} + +async fn ensure_fixed_guestfs_appliance() -> Result { + for dir in candidate_guestfs_appliance_dirs() { + let path = PathBuf::from(dir); + if is_complete_guestfs_appliance(&path) { + return Ok(path); + } + } + + let dirs = QleanDirs::new()?; + let appliance_dir = dirs.base.join("guestfs-appliance"); + if is_complete_guestfs_appliance(&appliance_dir) { + return Ok(appliance_dir); + } + + tokio::fs::create_dir_all(&appliance_dir) + .await + .with_context(|| format!("failed to create {}", appliance_dir.display()))?; + + if let Err(download_err) = provision_official_guestfs_appliance(&appliance_dir).await { + warn!( + "Official libguestfs appliance download failed; falling back to local fixed-appliance build: {download_err:#}" + ); + match try_build_fixed_guestfs_appliance(&appliance_dir).await { + Ok(()) => {} + Err(build_err) => bail!( + "failed to provision a usable libguestfs appliance: {build_err:#} +Tried the official fixed appliance download first, then a local libguestfs-make-fixed-appliance build." + ), + } + } + + Ok(appliance_dir) +} + +async fn run_guestfs_tool( + program: &str, + args: &[&OsStr], + current_dir: &Path, +) -> Result { + async fn run_once( + program: &str, + args: &[&OsStr], + current_dir: &Path, + appliance: Option<&Path>, + ) -> Result { + let mut cmd = tokio::process::Command::new(program); + cmd.env("LIBGUESTFS_BACKEND", "direct") + .current_dir(current_dir); + if let Some(appliance_dir) = appliance { + cmd.env("LIBGUESTFS_PATH", appliance_dir); + } + for a in args { + cmd.arg(a); + } + let child = cmd.output(); + let out = timeout(Duration::from_secs(180), child) + .await + .with_context(|| format!("{} timed out after 180s (libguestfs)", program))? + .with_context(|| format!("failed to execute {}", program))?; + Ok(out) + } + + let first = run_once(program, args, current_dir, None).await?; + if !first.status.success() { + warn!( + "{} failed (direct backend): {}", + program, + String::from_utf8_lossy(&first.stderr) + ); + } + if first.status.success() { + return Ok(first); + } + + let stderr = String::from_utf8_lossy(&first.stderr); + let needs_fixed = stderr.contains("supermin exited with error status") + || stderr.contains("/usr/bin/supermin"); + if needs_fixed && std::env::var_os("LIBGUESTFS_PATH").is_none() { + let appliance_dir = ensure_fixed_guestfs_appliance().await?; + return run_once(program, args, current_dir, Some(&appliance_dir)).await; + } + + Ok(first) +} + +async fn guestfish_ls_boot(image_dir: &Path, file_name: &str) -> Result { + let args = [ + OsStr::new("--ro"), + OsStr::new("-a"), + OsStr::new(file_name), + OsStr::new("-i"), + OsStr::new("ls"), + OsStr::new("/boot"), + ]; + let output = run_guestfs_tool("guestfish", &args, image_dir).await?; + if !output.status.success() { + bail!( + "guestfish failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +async fn virt_copy_out(image_dir: &Path, file_name: &str, src: &str, kind: &str) -> Result<()> { + let args = [ + OsStr::new("-a"), + OsStr::new(file_name), + OsStr::new(src), + OsStr::new("."), + ]; + let output = run_guestfs_tool("virt-copy-out", &args, image_dir).await?; + if !output.status.success() { + bail!( + "virt-copy-out failed for {}: {}", + kind, + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) +} + +async fn extract_boot_artifacts_guestfs( + image_dir: &Path, + file_name: &str, + distro: Distro, +) -> Result<(PathBuf, PathBuf)> { + ensure_extraction_prerequisites().await?; + + let boot_dir = image_dir.join("boot"); + let _ = tokio::fs::remove_dir_all(&boot_dir).await; + + virt_copy_out(image_dir, file_name, "/boot", "boot directory").await?; + + anyhow::ensure!( + boot_dir.exists(), + "virt-copy-out did not extract /boot into {}", + image_dir.display() + ); + + let files = collect_files_recursive(&boot_dir).with_context(|| { + format!( + "failed to scan extracted boot tree in {}", + boot_dir.display() + ) + })?; + + let kernel_src = choose_kernel_file(&files, distro.clone()) + .with_context(|| "failed to find kernel file in extracted /boot")?; + let initrd_src = choose_initrd_file(&files, distro) + .with_context(|| "failed to find initrd file in extracted /boot")?; + + let kernel_name = kernel_src + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| "invalid kernel filename")? + .to_string(); + let initrd_name = initrd_src + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| "invalid initrd filename")? + .to_string(); + + let kernel_path = image_dir.join(&kernel_name); + let initrd_path = image_dir.join(&initrd_name); + + fs::copy(&kernel_src, &kernel_path).with_context(|| { + format!( + "failed to copy extracted kernel {} -> {}", + kernel_src.display(), + kernel_path.display() + ) + })?; + fs::copy(&initrd_src, &initrd_path).with_context(|| { + format!( + "failed to copy extracted initrd {} -> {}", + initrd_src.display(), + initrd_path.display() + ) + })?; + + let kernel_args_path = kernel_args_hint_path(image_dir); + if let Some(args) = choose_kernel_options(&files, &kernel_name) { + fs::write(&kernel_args_path, args).with_context(|| { + format!( + "failed to write kernel args hint in {}", + image_dir.display() + ) + })?; + } else { + let _ = fs::remove_file(&kernel_args_path); + } + let _ = fs::remove_file(root_hint_path(image_dir)); + + let _ = tokio::fs::remove_dir_all(&boot_dir).await; + Ok((kernel_path, initrd_path)) +} + +fn root_hint_path(image_dir: &Path) -> PathBuf { + image_dir.join(".root-partition") +} + +fn kernel_args_hint_path(image_dir: &Path) -> PathBuf { + image_dir.join(".kernel-args") +} + +fn read_u32_le(bytes: &[u8]) -> u32 { + let mut arr = [0u8; 4]; + arr.copy_from_slice(bytes); + u32::from_le_bytes(arr) +} + +fn read_u64_le(bytes: &[u8]) -> u64 { + let mut arr = [0u8; 8]; + arr.copy_from_slice(bytes); + u64::from_le_bytes(arr) +} + +#[derive(Debug, Clone)] +struct PartitionSlice { + number: usize, + start_lba: u64, + sectors: u64, +} + +fn parse_mbr_partitions(mbr: &[u8]) -> Vec { + let mut parts = Vec::new(); + for idx in 0..4usize { + let off = 446 + idx * 16; + let entry = &mbr[off..off + 16]; + let part_type = entry[4]; + if part_type == 0 { + continue; + } + let start_lba = read_u32_le(&entry[8..12]) as u64; + let sectors = read_u32_le(&entry[12..16]) as u64; + if start_lba > 0 && sectors > 0 { + parts.push(PartitionSlice { + number: idx + 1, + start_lba, + sectors, + }); + } + } + parts +} + +fn parse_gpt_partitions(raw_path: &Path) -> Result> { + let mut f = fs::File::open(raw_path) + .with_context(|| format!("failed to open {}", raw_path.display()))?; + + let mut header = [0u8; 512]; + f.seek(SeekFrom::Start(512)) + .with_context(|| format!("failed to seek GPT header in {}", raw_path.display()))?; + f.read_exact(&mut header) + .with_context(|| format!("failed to read GPT header from {}", raw_path.display()))?; + + anyhow::ensure!( + &header[0..8] == b"EFI PART", + "{} does not contain a GPT header", + raw_path.display() + ); + + let entries_lba = read_u64_le(&header[72..80]); + let num_entries = read_u32_le(&header[80..84]) as usize; + let entry_size = read_u32_le(&header[84..88]) as usize; + anyhow::ensure!(entry_size >= 56, "invalid GPT entry size: {}", entry_size); + + let max_entries = num_entries.min(256); + let table_len = max_entries + .checked_mul(entry_size) + .with_context(|| "GPT partition table size overflow")?; + let mut table = vec![0u8; table_len]; + f.seek(SeekFrom::Start(entries_lba.saturating_mul(512))) + .with_context(|| format!("failed to seek GPT entries in {}", raw_path.display()))?; + f.read_exact(&mut table) + .with_context(|| format!("failed to read GPT entries from {}", raw_path.display()))?; + + let mut parts = Vec::new(); + for idx in 0..max_entries { + let off = idx * entry_size; + let entry = &table[off..off + entry_size]; + if entry[0..16].iter().all(|b| *b == 0) { + continue; + } + + let start_lba = read_u64_le(&entry[32..40]); + let end_lba = read_u64_le(&entry[40..48]); + if start_lba == 0 || end_lba < start_lba { + continue; + } + + parts.push(PartitionSlice { + number: idx + 1, + start_lba, + sectors: end_lba - start_lba + 1, + }); + } + + Ok(parts) +} + +fn list_partitions(raw_path: &Path) -> Result> { + let mut mbr = [0u8; 512]; + let mut f = fs::File::open(raw_path) + .with_context(|| format!("failed to open {}", raw_path.display()))?; + f.read_exact(&mut mbr) + .with_context(|| format!("failed to read MBR from {}", raw_path.display()))?; + + anyhow::ensure!( + mbr[510] == 0x55 && mbr[511] == 0xAA, + "{} does not look like a bootable disk image", + raw_path.display() + ); + + let protective_gpt = (0..4usize).any(|idx| mbr[446 + idx * 16 + 4] == 0xEE); + let mut parts = if protective_gpt { + parse_gpt_partitions(raw_path)? + } else { + parse_mbr_partitions(&mbr) + }; + + parts.sort_by(|a, b| { + b.sectors + .cmp(&a.sectors) + .then_with(|| a.number.cmp(&b.number)) + }); + anyhow::ensure!( + !parts.is_empty(), + "failed to find any partitions in {}", + raw_path.display() + ); + Ok(parts) +} + +fn collect_files_recursive(root: &Path) -> Result> { + let mut files = Vec::new(); + for entry in walkdir::WalkDir::new(root) { + let entry = entry.with_context(|| format!("failed while scanning {}", root.display()))?; + if entry.file_type().is_file() { + files.push(entry.path().to_path_buf()); + } + } + Ok(files) +} + +fn choose_kernel_file(files: &[PathBuf], distro: Distro) -> Option { + let mut candidates = files + .iter() + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .map(|n| match distro { + Distro::Fedora => n.starts_with("vmlinuz") && !n.contains("rescue"), + Distro::Arch => n.starts_with("vmlinuz"), + _ => n.starts_with("vmlinuz"), + }) + .unwrap_or(false) + }) + .cloned() + .collect::>(); + candidates.sort(); + candidates.into_iter().last() +} + +fn choose_initrd_file(files: &[PathBuf], distro: Distro) -> Option { + let mut candidates = files + .iter() + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .map(|n| match distro { + Distro::Ubuntu | Distro::Debian => n.starts_with("initrd.img"), + Distro::Fedora => { + n.starts_with("initramfs") && n.ends_with(".img") && !n.contains("rescue") + } + Distro::Arch => { + n.starts_with("initramfs") + && n.ends_with(".img") + && n.contains("linux") + && !n.contains("fallback") + } + Distro::Custom => false, + }) + .unwrap_or(false) + }) + .cloned() + .collect::>(); + + if candidates.is_empty() && matches!(distro, Distro::Arch) { + candidates = files + .iter() + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .map(|n| { + n.starts_with("initramfs") && n.ends_with(".img") && n.contains("linux") + }) + .unwrap_or(false) + }) + .cloned() + .collect::>(); + } + + candidates.sort(); + candidates.into_iter().last() +} + +fn normalize_kernel_options(raw: &str) -> Option { + let tokens = raw + .split_whitespace() + .filter(|token| !token.is_empty() && *token != "ro" && *token != "rw") + .collect::>(); + if tokens.is_empty() { + None + } else { + Some(tokens.join(" ")) + } +} + +fn resolve_kernelopts_from_grub_cfg(content: &str) -> Option { + // Fedora/GRUB often stores a full kernel command line in a variable called "kernelopts", + // referenced from BLS entries as: `options $kernelopts`. + // + // We intentionally keep this parser lightweight and dependency-free. + for line in content.lines() { + let trimmed = line.trim(); + // Common forms: + // set kernelopts="root=UUID=... ro ..." + // set kernelopts='root=UUID=... ro ...' + if let Some(rest) = trimmed.strip_prefix("set kernelopts=") { + let rest = rest.trim(); + if let Some(stripped) = rest.strip_prefix('"') + && let Some(end) = stripped.find('"') + { + return Some(stripped[..end].to_string()); + } + if let Some(stripped) = rest.strip_prefix('\'') + && let Some(end) = stripped.find('\'') + { + return Some(stripped[..end].to_string()); + } + // Fallback: no quotes, take the remainder of the token. + let first = rest.split_whitespace().next().unwrap_or(""); + if !first.is_empty() { + return Some(first.to_string()); + } + } + + // Some grub.cfg variants assign without the "set" keyword. + if let Some(idx) = trimmed.find("kernelopts=") { + let rest = trimmed[idx + "kernelopts=".len()..].trim(); + if let Some(stripped) = rest.strip_prefix('"') + && let Some(end) = stripped.find('"') + { + return Some(stripped[..end].to_string()); + } + if let Some(stripped) = rest.strip_prefix('\'') + && let Some(end) = stripped.find('\'') + { + return Some(stripped[..end].to_string()); + } + } + } + None +} + +fn resolve_kernelopts_from_grubenv(bytes: &[u8]) -> Option { + // grubenv is a binary environment block, but it commonly contains plain ASCII entries + // like: `kernelopts=root=UUID=... ro ...`. + let needle = b"kernelopts="; + let pos = bytes.windows(needle.len()).position(|w| w == needle)?; + let start = pos + needle.len(); + let mut end = start; + while end < bytes.len() { + let b = bytes[end]; + if b == b'\n' || b == 0 { + break; + } + end += 1; + } + let s = String::from_utf8_lossy(&bytes[start..end]) + .trim() + .to_string(); + if s.is_empty() { None } else { Some(s) } +} + +fn resolve_kernelopts(files: &[PathBuf]) -> Option { + // Prefer grub.cfg (human-readable). + for path in files.iter() { + if path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n == "grub.cfg") + .unwrap_or(false) + && let Ok(content) = fs::read_to_string(path) + && let Some(v) = resolve_kernelopts_from_grub_cfg(&content) + { + return Some(v); + } + } + + // Fallback: scan grubenv blocks. + for path in files.iter() { + if path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n == "grubenv") + .unwrap_or(false) + && let Ok(bytes) = fs::read(path) + && let Some(v) = resolve_kernelopts_from_grubenv(&bytes) + { + return Some(v); + } + } + + None +} + +fn expand_kernelopts(options: &str, kernelopts: Option<&str>) -> Option { + let ko = kernelopts?; + if options.contains("$kernelopts") || options.contains("${kernelopts}") { + let expanded = options + .replace("${kernelopts}", ko) + .replace("$kernelopts", ko); + return normalize_kernel_options(&expanded); + } + None +} + +fn extract_loader_entry_options(entry: &str, kernel_name: &str) -> Option { + let mut linux_matches = false; + let mut options = None; + + for line in entry.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + + if let Some(rest) = trimmed.strip_prefix("linux ") { + let linux_path = rest.split_whitespace().next().unwrap_or_default(); + let linux_base = std::path::Path::new(linux_path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(linux_path); + // Some images use a symlink like "vmlinuz" in loader entries while the extracted + // kernel file is versioned (or vice-versa). Prefer an exact match, but allow a + // conservative compatibility match when either side is the common "vmlinuz" alias. + linux_matches = linux_base == kernel_name + || (linux_base == "vmlinuz" && kernel_name.starts_with("vmlinuz")) + || (kernel_name == "vmlinuz" && linux_base.starts_with("vmlinuz")); + } else if let Some(rest) = trimmed.strip_prefix("options ") { + options = normalize_kernel_options(rest); + } + } + + if linux_matches { options } else { None } +} + +fn extract_grub_linux_options(line: &str, kernel_name: &str) -> Option { + let trimmed = line.trim(); + let prefixes = ["linux ", "linuxefi ", "linux16 "]; + let rest = prefixes + .iter() + .find_map(|prefix| trimmed.strip_prefix(prefix))?; + + let mut parts = rest.split_whitespace(); + let kernel_path = parts.next()?; + let kernel_base = std::path::Path::new(kernel_path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(kernel_path); + if kernel_base != kernel_name { + return None; + } + + normalize_kernel_options(&parts.collect::>().join(" ")) +} + +fn choose_kernel_options(files: &[PathBuf], kernel_name: &str) -> Option { + let kernelopts = resolve_kernelopts(files); + + let mut loader_entries = files + .iter() + .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("conf")) + .cloned() + .collect::>(); + loader_entries.sort(); + loader_entries.reverse(); + + // First pass: find an entry whose "linux" line matches the extracted kernel. + for path in loader_entries.iter() { + let content = match fs::read_to_string(path) { + Ok(v) => v, + Err(_) => continue, + }; + if let Some(options) = extract_loader_entry_options(&content, kernel_name) { + if let Some(expanded) = expand_kernelopts(&options, kernelopts.as_deref()) { + return Some(expanded); + } + return Some(options); + } + } + + // Second pass: if the loader entry does not reference the exact same kernel filename + // (eg. symlink vs versioned kernel), fall back to the newest *non-rescue* entry that + // contains a root= argument. + for path in loader_entries.iter() { + let content = match fs::read_to_string(path) { + Ok(v) => v, + Err(_) => continue, + }; + let lower = content.to_lowercase(); + if lower.contains("rescue") || lower.contains("recovery") || lower.contains("fallback") { + continue; + } + let mut options_line = None; + for line in content.lines() { + let trimmed = line.trim(); + if let Some(rest) = trimmed.strip_prefix("options ") { + options_line = normalize_kernel_options(rest); + break; + } + } + if let Some(opts) = options_line { + if let Some(expanded) = expand_kernelopts(&opts, kernelopts.as_deref()) + && expanded.split_whitespace().any(|t| t.starts_with("root=")) + { + return Some(expanded); + } + if opts.split_whitespace().any(|t| t.starts_with("root=")) { + return Some(opts); + } + } + } + + let mut grub_cfgs = files + .iter() + .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("cfg")) + .cloned() + .collect::>(); + grub_cfgs.sort(); + grub_cfgs.reverse(); + + // First pass: grub.cfg linux line matches the extracted kernel. + for path in grub_cfgs.iter() { + let content = match fs::read_to_string(path) { + Ok(v) => v, + Err(_) => continue, + }; + for line in content.lines() { + if let Some(options) = extract_grub_linux_options(line, kernel_name) { + if let Some(expanded) = expand_kernelopts(&options, kernelopts.as_deref()) { + return Some(expanded); + } + return Some(options); + } + } + } + + // Second pass: fall back to the first non-rescue linux line that contains a root= argument. + for path in grub_cfgs.iter() { + let content = match fs::read_to_string(path) { + Ok(v) => v, + Err(_) => continue, + }; + for line in content.lines() { + let trimmed = line.trim(); + let prefixes = ["linux ", "linuxefi ", "linux16 "]; + let rest = match prefixes.iter().find_map(|p| trimmed.strip_prefix(p)) { + Some(v) => v, + None => continue, + }; + if trimmed.contains("rescue") || trimmed.contains("recovery") { + continue; + } + let parts = rest.split_whitespace().collect::>(); + if parts.len() < 2 { + continue; + } + if let Some(opts) = normalize_kernel_options(&parts[1..].join(" ")) { + if let Some(expanded) = expand_kernelopts(&opts, kernelopts.as_deref()) + && expanded.split_whitespace().any(|t| t.starts_with("root=")) + { + return Some(expanded); + } + if opts.split_whitespace().any(|t| t.starts_with("root=")) { + return Some(opts); + } + } + } + } + + None +} + +fn partition_size_bytes(part: &PartitionSlice) -> u64 { + part.sectors.saturating_mul(512) +} + +async fn has_command(cmd: &str, arg: &str) -> bool { + tokio::process::Command::new(cmd) + .arg(arg) + .output() + .await + .is_ok() +} + +#[allow(dead_code)] +async fn check_userspace_extract_tools() -> Result<()> { + for (cmd, arg) in [("qemu-img", "--version"), ("dd", "--version")] { + tokio::process::Command::new(cmd) + .arg(arg) + .output() + .await + .with_context(|| format!("could not find {}", cmd))?; + } + + let have_debugfs = has_command("debugfs", "-V").await; + let have_mcopy = has_command("mcopy", "-V").await; + let have_7z = has_command("7z", "i").await; + anyhow::ensure!( + have_debugfs || have_mcopy || have_7z, + "userspace extraction requires debugfs, mcopy, or 7z" + ); + Ok(()) +} + +async fn write_partition_image( + raw_path: &Path, + image_dir: &Path, + part: &PartitionSlice, +) -> Result { + let part_path = image_dir.join(format!(".extract-part-{}.img", part.number)); + let _ = tokio::fs::remove_file(&part_path).await; + + let output = tokio::process::Command::new("dd") + .arg(format!("if={}", raw_path.display())) + .arg(format!("of={}", part_path.display())) + .arg("bs=512") + .arg(format!("skip={}", part.start_lba)) + .arg(format!("count={}", part.sectors)) + .arg("status=none") + .output() + .await + .with_context(|| format!("failed to execute dd for partition {}", part.number))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("dd failed for partition {}: {}", part.number, stderr.trim()); + } + + Ok(part_path) +} + +async fn dump_partition_subdir_with_mcopy( + part_path: &Path, + dump_dir: &Path, + subdir: &str, +) -> Result> { + if !has_command("mcopy", "-V").await { + return Ok(None); + } + + let source = match subdir { + "/" => "::/", + "/boot" => "::/boot", + _ => return Ok(None), + }; + + let output = tokio::process::Command::new("mcopy") + .arg("-s") + .arg("-n") + .arg("-i") + .arg(part_path) + .arg(source) + .arg(dump_dir) + .output() + .await + .with_context(|| format!("failed to execute mcopy against {}", part_path.display()))?; + + if !output.status.success() { + let _ = tokio::fs::remove_dir_all(dump_dir).await; + return Ok(None); + } + + Ok(Some(dump_dir.to_path_buf())) +} + +async fn dump_partition_subdir_with_7z( + part_path: &Path, + dump_dir: &Path, +) -> Result> { + if !has_command("7z", "i").await { + return Ok(None); + } + + let output = tokio::process::Command::new("7z") + .arg("x") + .arg("-y") + .arg(format!("-o{}", dump_dir.display())) + .arg(part_path) + .output() + .await + .with_context(|| format!("failed to execute 7z against {}", part_path.display()))?; + + if !output.status.success() { + let _ = tokio::fs::remove_dir_all(dump_dir).await; + return Ok(None); + } + + Ok(Some(dump_dir.to_path_buf())) +} + +async fn dump_partition_subdir( + part_path: &Path, + image_dir: &Path, + part: &PartitionSlice, + subdir: &str, +) -> Result> { + let sanitized = if subdir == "/" { "root" } else { "boot" }; + let dump_dir = image_dir.join(format!(".extract-{}-{}", sanitized, part.number)); + let _ = tokio::fs::remove_dir_all(&dump_dir).await; + + tokio::fs::create_dir_all(&dump_dir) + .await + .with_context(|| format!("failed to create {}", dump_dir.display()))?; + + if has_command("debugfs", "-V").await { + let output = tokio::process::Command::new("debugfs") + .arg("-R") + .arg(format!("rdump {} {}", subdir, dump_dir.display())) + .arg(part_path) + .output() + .await + .with_context(|| format!("failed to execute debugfs for partition {}", part.number))?; + + if output.status.success() { + return Ok(Some(dump_dir)); + } + } + + if let Some(dir) = dump_partition_subdir_with_mcopy(part_path, &dump_dir, subdir).await? { + return Ok(Some(dir)); + } + + if let Some(dir) = dump_partition_subdir_with_7z(part_path, &dump_dir).await? { + return Ok(Some(dir)); + } + + let _ = tokio::fs::remove_dir_all(&dump_dir).await; + Ok(None) +} + +async fn partition_contains_os_release( + part_path: &Path, + image_dir: &Path, + part: &PartitionSlice, +) -> Result { + let probe_path = image_dir.join(format!(".extract-os-release-{}", part.number)); + let _ = tokio::fs::remove_file(&probe_path).await; + + let output = tokio::process::Command::new("debugfs") + .arg("-R") + .arg(format!("dump -p /etc/os-release {}", probe_path.display())) + .arg(part_path) + .output() + .await + .with_context(|| { + format!( + "failed to probe /etc/os-release in partition {}", + part.number + ) + })?; + + let exists = output.status.success() && probe_path.exists(); + let _ = tokio::fs::remove_file(&probe_path).await; + Ok(exists) +} + +#[allow(dead_code)] +async fn extract_boot_artifacts_userspace( + image_dir: &Path, + file_name: &str, + distro: Distro, +) -> Result<(PathBuf, PathBuf)> { + check_userspace_extract_tools().await?; + + let raw_path = image_dir.join(format!("{}.raw", file_name)); + let _ = tokio::fs::remove_file(&raw_path).await; + + let convert = tokio::process::Command::new("qemu-img") + .arg("convert") + .arg("-O") + .arg("raw") + .arg(file_name) + .arg(&raw_path) + .current_dir(image_dir) + .output() + .await + .with_context(|| format!("failed to convert {} to raw", file_name))?; + if !convert.status.success() { + let stderr = String::from_utf8_lossy(&convert.stderr); + bail!("qemu-img convert failed: {}", stderr.trim()); + } + + let parts = list_partitions(&raw_path)?; + let mut root_hint = None; + let mut last_err = None; + + for part in &parts { + let part_path = match write_partition_image(&raw_path, image_dir, part).await { + Ok(path) => path, + Err(err) => { + last_err = Some(err); + continue; + } + }; + + match partition_contains_os_release(&part_path, image_dir, part).await { + Ok(true) => { + root_hint = Some(part.number); + } + Ok(false) => {} + Err(err) => { + last_err = Some(err); + } + } + + let _ = tokio::fs::remove_file(&part_path).await; + if root_hint.is_some() { + break; + } + } + + for part in &parts { + let part_path = match write_partition_image(&raw_path, image_dir, part).await { + Ok(path) => path, + Err(err) => { + last_err = Some(err); + continue; + } + }; + + let mut dump_candidates = vec![("/boot", false)]; + if partition_size_bytes(part) <= 2 * 1024 * 1024 * 1024 { + dump_candidates.push(("/", true)); + } + + let mut found = None; + for (subdir, is_partition_root) in dump_candidates { + let dump_dir = match dump_partition_subdir(&part_path, image_dir, part, subdir).await { + Ok(Some(dir)) => dir, + Ok(None) => continue, + Err(err) => { + last_err = Some(err); + continue; + } + }; + + let files = match collect_files_recursive(&dump_dir) { + Ok(v) => v, + Err(err) => { + let _ = tokio::fs::remove_dir_all(&dump_dir).await; + last_err = Some(err); + continue; + } + }; + + let kernel_src = choose_kernel_file(&files, distro.clone()); + let initrd_src = choose_initrd_file(&files, distro.clone()); + if let (Some(kernel_src), Some(initrd_src)) = (kernel_src, initrd_src) { + let kernel_name = kernel_src + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .to_string(); + let kernel_args = choose_kernel_options(&files, &kernel_name); + found = Some(( + dump_dir, + kernel_src, + initrd_src, + is_partition_root, + kernel_args, + )); + break; + } + + let _ = tokio::fs::remove_dir_all(&dump_dir).await; + } + + let Some((dump_dir, kernel_src, initrd_src, is_partition_root, kernel_args)) = found else { + let _ = tokio::fs::remove_file(&part_path).await; + continue; + }; + + let kernel_name = kernel_src + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| "invalid kernel filename")? + .to_string(); + let initrd_name = initrd_src + .file_name() + .and_then(|n| n.to_str()) + .with_context(|| "invalid initrd filename")? + .to_string(); + + let kernel_path = image_dir.join(&kernel_name); + let initrd_path = image_dir.join(&initrd_name); + + fs::copy(&kernel_src, &kernel_path).with_context(|| { + format!( + "failed to copy extracted kernel {} -> {}", + kernel_src.display(), + kernel_path.display() + ) + })?; + fs::copy(&initrd_src, &initrd_path).with_context(|| { + format!( + "failed to copy extracted initrd {} -> {}", + initrd_src.display(), + initrd_path.display() + ) + })?; + + let chosen_root = if is_partition_root { + root_hint + .or_else(|| { + parts + .iter() + .find(|candidate| candidate.number != part.number) + .map(|candidate| candidate.number) + }) + .unwrap_or(part.number) + } else { + root_hint.unwrap_or(part.number) + }; + fs::write(root_hint_path(image_dir), chosen_root.to_string()).with_context(|| { + format!( + "failed to write root partition hint in {}", + image_dir.display() + ) + })?; + if let Some(args) = kernel_args { + fs::write(kernel_args_hint_path(image_dir), args).with_context(|| { + format!( + "failed to write kernel args hint in {}", + image_dir.display() + ) + })?; + } + + let _ = tokio::fs::remove_dir_all(&dump_dir).await; + let _ = tokio::fs::remove_file(&part_path).await; + let _ = tokio::fs::remove_file(&raw_path).await; + return Ok((kernel_path, initrd_path)); + } + + let _ = tokio::fs::remove_file(&raw_path).await; + + if let Some(err) = last_err { + return Err(err) + .with_context(|| "userspace qcow2 extraction did not find a usable /boot tree"); + } + + bail!("userspace qcow2 extraction did not find kernel/initrd in any partition"); +} + +async fn detect_root_arg(image_path: &Path) -> Result { + let image_dir = image_path.parent().with_context(|| "missing image dir")?; + let kernel_args_path = kernel_args_hint_path(image_dir); + if let Ok(args) = fs::read_to_string(&kernel_args_path) { + let trimmed = args.trim(); + if !trimmed.is_empty() { + // Some guestfs outputs include a trailing ':' after device names (e.g. /dev/sda3:). + // Also, some bootloader entries may embed `root=/dev/vda3:`. + // Sanitize these so direct-kernel boot does not hang waiting for a non-existent device. + let sanitized = trimmed + .split_whitespace() + .map(|t| { + if let Some(rest) = t.strip_prefix("root=") { + let clean = rest.trim_end_matches(':'); + format!("root={clean}") + } else { + t.to_string() + } + }) + .collect::>() + .join(" "); + return Ok(sanitized); + } + } + + let hint_path = root_hint_path(image_dir); + if let Ok(hint) = fs::read_to_string(&hint_path) + && let Ok(part) = hint.trim().parse::() + { + return Ok(format!("root=/dev/vda{}", part)); + } + + let file_name = image_path + .file_name() + .and_then(|v| v.to_str()) + .with_context(|| "invalid image filename")?; + let args = [ + OsStr::new("--ro"), + OsStr::new("-a"), + OsStr::new(file_name), + OsStr::new("-i"), + OsStr::new("mountpoints"), + ]; + let output = run_guestfs_tool("guestfish", &args, image_dir).await?; + if !output.status.success() { + return Ok(default_root_arg()); + } + let text = String::from_utf8_lossy(&output.stdout); + for line in text.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + let mut dev = None; + if parts[0].starts_with("/dev/") && parts[1] == "/" { + dev = Some(parts[0]); + } + if parts[1].starts_with("/dev/") && parts[0] == "/" { + dev = Some(parts[1]); + } + if let Some(d) = dev { + // guestfish `mountpoints` typically prints `/dev/sda3: /`. + let d = d.trim_end_matches(':'); + let virt = if let Some(rest) = d.strip_prefix("/dev/sd") { + format!("/dev/vd{}", rest) + } else { + d.to_string() + }; + return Ok(format!("root={}", virt)); + } + } + Ok(default_root_arg()) +} + +// --------------------------------------------------------------------------- +// Debian +// --------------------------------------------------------------------------- + +#[derive(Debug, Default)] +pub struct Debian {} + +impl ImageAction for Debian { + async fn download(&self, name: &str) -> Result<()> { + let checksums_url = "https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS"; + let checksums_text = reqwest::get(checksums_url) + .await + .with_context(|| format!("failed to download SHA512SUMS from {}", checksums_url))? + .text() + .await + .with_context(|| format!("failed to read SHA512SUMS text from {}", checksums_url))?; + + let target_filename = format!("{}.qcow2", name); + let expected_sha512 = find_sha512_for_file(&checksums_text, &target_filename) + .with_context(|| { + format!( + "failed to find SHA512 checksum entry for {} in remote SHA512SUMS file", + target_filename ) })?; @@ -1490,7 +3077,7 @@ impl ImageAction for Debian { } // --------------------------------------------------------------------------- -// Ubuntu - uses pre-extracted kernel/initrd from official cloud images +// Ubuntu - downloads the official cloud image and extracts kernel/initrd via libguestfs // --------------------------------------------------------------------------- #[derive(Debug, Default)] @@ -1503,8 +3090,8 @@ impl ImageAction for Ubuntu { let resolved = resolve_ubuntu_noble_cloudimg().await?; debug!( - "Resolved Ubuntu cloud image from {}: disk={}, kernel={}, initrd={}", - resolved.base_url, resolved.disk_name, resolved.kernel_name, resolved.initrd_name + "Resolved Ubuntu cloud image from {}: disk={}", + resolved.base_url, resolved.disk_name ); let qcow2_path = image_dir.join(format!("{}.qcow2", name)); @@ -1517,55 +3104,17 @@ impl ImageAction for Ubuntu { ) .await?; - let kernel_path = image_dir.join("vmlinuz"); - let kernel_url = join_url( - &resolved.base_url, - &format!("unpacked/{}", resolved.kernel_name), - ); - if let Some(kernel_sha256) = resolved.kernel_sha256.as_deref() { - download_or_copy_with_hash( - &ImageSource::Url(kernel_url), - &kernel_path, - kernel_sha256, - ShaType::Sha256, - ) - .await?; - } else { - download_or_copy_without_hash(&ImageSource::Url(kernel_url), &kernel_path).await?; - } - - let initrd_path = image_dir.join("initrd.img"); - let initrd_url = join_url( - &resolved.base_url, - &format!("unpacked/{}", resolved.initrd_name), - ); - if let Some(initrd_sha256) = resolved.initrd_sha256.as_deref() { - download_or_copy_with_hash( - &ImageSource::Url(initrd_url), - &initrd_path, - initrd_sha256, - ShaType::Sha256, - ) - .await?; - } else { - download_or_copy_without_hash(&ImageSource::Url(initrd_url), &initrd_path).await?; - } - Ok(()) } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { - // Files already downloaded in download() phase + let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - let kernel = image_dir.join("vmlinuz"); - let initrd = image_dir.join("initrd.img"); - - anyhow::ensure!(kernel.exists(), "kernel file not found after download"); - anyhow::ensure!(initrd.exists(), "initrd file not found after download"); - - Ok((kernel, initrd)) + extract_boot_artifacts_guestfs(&image_dir, &file_name, Distro::Ubuntu) + .await + .with_context(|| "failed to extract Ubuntu kernel/initrd from qcow2") } fn distro(&self) -> Distro { @@ -1574,7 +3123,7 @@ impl ImageAction for Ubuntu { } // --------------------------------------------------------------------------- -// Fedora - uses pre-extracted kernel/initrd from official cloud images +// Fedora - uses the official cloud image and extracts kernel/initrd via libguestfs // --------------------------------------------------------------------------- #[derive(Debug, Default)] @@ -1610,55 +3159,13 @@ impl ImageAction for Fedora { } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { - ensure_extraction_prerequisites().await?; - let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - let extract_result = async { - let boot_files = guestfish_ls_boot(&image_dir, &file_name) - .await - .with_context(|| "failed to read /boot from Fedora image with guestfish")?; - let mut kernel_name = None; - let mut initrd_name = None; - - for line in boot_files.lines() { - let file = line.trim(); - if file.starts_with("vmlinuz") { - kernel_name = Some(file.to_string()); - } else if file.starts_with("initramfs") { - initrd_name = Some(file.to_string()); - } - } - - let kernel_name = - kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; - let initrd_name = - initrd_name.with_context(|| "failed to find initrd file (initramfs*) in /boot")?; - - let kernel_src = format!("/boot/{}", kernel_name); - virt_copy_out(&image_dir, &file_name, &kernel_src, "kernel").await?; - - let initrd_src = format!("/boot/{}", initrd_name); - virt_copy_out(&image_dir, &file_name, &initrd_src, "initrd").await?; - - let kernel_path = image_dir.join(&kernel_name); - let initrd_path = image_dir.join(&initrd_name); - Ok::<(PathBuf, PathBuf), anyhow::Error>((kernel_path, initrd_path)) - } - .await; - - match extract_result { - Ok(paths) => Ok(paths), - Err(e) => { - warn!( - "Fedora kernel/initrd extraction failed: {:#}. Using disk boot fallback.", - e - ); - write_unavailable_boot_artifacts(&image_dir, "fedora extraction failed").await - } - } + extract_boot_artifacts_guestfs(&image_dir, &file_name, Distro::Fedora) + .await + .with_context(|| "failed to extract Fedora kernel/initrd from qcow2") } fn distro(&self) -> Distro { @@ -1667,7 +3174,7 @@ impl ImageAction for Fedora { } // --------------------------------------------------------------------------- -// Arch - uses official cloud images +// Arch - uses the official cloud image and extracts kernel/initrd via libguestfs // --------------------------------------------------------------------------- #[derive(Debug, Default)] @@ -1700,55 +3207,13 @@ impl ImageAction for Arch { } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { - ensure_extraction_prerequisites().await?; - let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - let extract_result = async { - let boot_files = guestfish_ls_boot(&image_dir, &file_name) - .await - .with_context(|| "failed to read /boot from Arch image with guestfish")?; - let mut kernel_name = None; - let mut initrd_name = None; - - for line in boot_files.lines() { - let file = line.trim(); - if file.starts_with("vmlinuz") { - kernel_name = Some(file.to_string()); - } else if file.starts_with("initramfs") && file.contains("linux.img") { - initrd_name = Some(file.to_string()); - } - } - - let kernel_name = - kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; - let initrd_name = initrd_name - .with_context(|| "failed to find initrd file (initramfs*linux.img) in /boot")?; - - let kernel_src = format!("/boot/{}", kernel_name); - virt_copy_out(&image_dir, &file_name, &kernel_src, "kernel").await?; - - let initrd_src = format!("/boot/{}", initrd_name); - virt_copy_out(&image_dir, &file_name, &initrd_src, "initrd").await?; - - let kernel_path = image_dir.join(&kernel_name); - let initrd_path = image_dir.join(&initrd_name); - Ok::<(PathBuf, PathBuf), anyhow::Error>((kernel_path, initrd_path)) - } - .await; - - match extract_result { - Ok(paths) => Ok(paths), - Err(e) => { - warn!( - "Arch kernel/initrd extraction failed: {:#}. Using disk boot fallback.", - e - ); - write_unavailable_boot_artifacts(&image_dir, "arch extraction failed").await - } - } + extract_boot_artifacts_guestfs(&image_dir, &file_name, Distro::Arch) + .await + .with_context(|| "failed to extract Arch kernel/initrd from qcow2") } fn distro(&self) -> Distro { @@ -1757,7 +3222,7 @@ impl ImageAction for Arch { } // --------------------------------------------------------------------------- -// Custom - user-provided image with flexible configuration (WSL-friendly) +// Custom - user-provided image with flexible configuration // --------------------------------------------------------------------------- #[derive(Debug)] @@ -1934,10 +3399,7 @@ impl Image { } pub fn prefer_direct_kernel_boot(&self) -> bool { - match self { - Image::Debian(_) | Image::Ubuntu(_) => true, - Image::Fedora(_) | Image::Arch(_) | Image::Custom(_) => false, - } + true } } @@ -2099,6 +3561,44 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 ); } + #[test] + fn test_parse_apt_depends_for_kernel_package_prefers_concrete_image() { + let text = r#" +Depends: linux-image-virtual +Depends: linux-image-6.8.0-71-generic +Depends: initramfs-tools +"#; + assert_eq!( + parse_apt_depends_for_kernel_package(text).as_deref(), + Some("linux-image-6.8.0-71-generic") + ); + } + + #[test] + fn test_derive_modules_package_name_for_signed_and_unsigned_images() { + assert_eq!( + derive_modules_package_name("linux-image-6.8.0-71-generic").as_deref(), + Some("linux-modules-6.8.0-71-generic") + ); + assert_eq!( + derive_modules_package_name("linux-image-unsigned-6.8.0-71-generic").as_deref(), + Some("linux-modules-6.8.0-71-generic") + ); + } + + #[test] + fn test_parse_apt_search_for_kernel_package_picks_latest_match() { + let text = r#" +linux-image-6.8.0-65-generic - Signed kernel image generic +linux-image-6.8.0-71-generic - Signed kernel image generic +linux-image-generic - Complete Generic Linux kernel and headers +"#; + assert_eq!( + parse_apt_search_for_kernel_package(text).as_deref(), + Some("linux-image-6.8.0-71-generic") + ); + } + #[test] fn test_parse_fedora_cloud_listing() { let html = r#" diff --git a/src/machine.rs b/src/machine.rs index 72a77a7..bb7c694 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -35,8 +35,6 @@ pub struct Machine { /// SSH session ssh: Option, cid: u32, - /// Host-forwarded TCP port reserved at startup and used as an SSH fallback when vsock is unavailable. - ssh_tcp_port: u16, /// QEMU process ID pid: Option, /// Indicates whether QEMU is expected to exit. @@ -84,6 +82,39 @@ pub struct MetaData { pub struct UserData { pub disable_root: bool, pub ssh_authorized_keys: Vec, + + /// Optional cloud-init directives used to configure the guest at first boot. + /// + /// We use these to enable an SSH listener on vhost-vsock so that Qlean can + /// reach the guest without relying on TCP port forwarding. + #[serde(skip_serializing_if = "Option::is_none")] + pub write_files: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub runcmd: Option>>, + + /// Additional cloud-init configuration. + /// + /// This is intentionally a loose YAML value so we can support a mix of distro + /// defaults (Ubuntu/Fedora/Arch) without encoding every schema detail in Rust. + #[serde(skip_serializing_if = "Option::is_none")] + pub users: Option, + + /// Explicitly disable password authentication. + #[serde(skip_serializing_if = "Option::is_none")] + pub ssh_pwauth: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct CloudInitWriteFile { + pub path: String, + pub content: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub permissions: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub owner: Option, } impl Default for MachineConfig { @@ -139,13 +170,6 @@ impl Machine { // Get a free CID let cid = get_free_cid(&dirs.runs, &run_dir)?; - // Reserve an ephemeral local TCP port for SSH host forwarding. - // We don't keep the listener open; the goal is just to select a port that is very - // likely to be free. The QEMU command will fail loudly if there is a race. - let ssh_tcp_port = std::net::TcpListener::bind(("127.0.0.1", 0)) - .map(|l| l.local_addr().map(|a| a.port())) - .map_err(|e| anyhow::anyhow!("Failed to reserve TCP port for SSH hostfwd: {e}"))??; - // Prepare cloud-init config let meta_data = MetaData { instance_id: format!("VM-{}", &machine_id), @@ -155,24 +179,165 @@ impl Machine { meta_data_str.insert_str(0, "#cloud-config\n"); debug!("Writing cloud-init meta-data:\n{}", meta_data_str); tokio::fs::write(seed_dir.join("meta-data"), meta_data_str).await?; + + // Enable an SSH path over vhost-vsock without requiring OpenSSH to accept an AF_VSOCK + // socket directly. + // + // VSOCK-only (no TCP fallback): provide SSH access exclusively via vhost-vsock. + // + // We intentionally do NOT depend on the guest distro enabling/running sshd.service. + // Instead, we use systemd socket activation and run `sshd -i` (inetd mode) for each + // incoming vsock connection. + // + // IMPORTANT: StandardOutput must be wired to the socket; otherwise clients may connect + // but never receive an SSH banner (hangs until handshake timeout). + let sshd_wrapper = r#"#!/bin/sh +set -eu + +for p in /usr/sbin/sshd /usr/bin/sshd /sbin/sshd; do + if [ -x "$p" ]; then + exec "$p" "$@" + fi +done + +echo "qlean: sshd not found" >&2 +exit 127 +"# + .to_string(); + + let vsock_socket_unit = r#"[Unit] +Description=Qlean SSH over vhost-vsock (socket-activated sshd) + +[Socket] +ListenStream=vsock::22 +Accept=yes + +[Install] +WantedBy=sockets.target +"# + .to_string(); + + let vsock_service_unit = r#"[Unit] +Description=Qlean SSH over vhost-vsock (per-connection sshd) + +[Service] +ExecStart=/usr/bin/qlean-sshd-run -i -e \ + -o PermitRootLogin=yes \ + -o PasswordAuthentication=no \ + -o PubkeyAuthentication=yes \ + -o AuthorizedKeysFile=/root/.ssh/authorized_keys \ + -o StrictModes=yes +StandardInput=socket +StandardOutput=socket +StandardError=journal +"# + .to_string(); + let user_data = UserData { disable_root: false, ssh_authorized_keys: vec![ssh_keypair.pubkey_str.clone()], + write_files: Some(vec![ + CloudInitWriteFile { + path: "/etc/systemd/system/qlean-sshd-vsock.socket".to_string(), + content: vsock_socket_unit, + permissions: Some("0644".to_string()), + owner: Some("root:root".to_string()), + }, + CloudInitWriteFile { + path: "/etc/systemd/system/qlean-sshd-vsock@.service".to_string(), + content: vsock_service_unit, + permissions: Some("0644".to_string()), + owner: Some("root:root".to_string()), + }, + CloudInitWriteFile { + // /usr/bin is a safe location across distros, including SELinux-enforcing Fedora. + path: "/usr/bin/qlean-sshd-run".to_string(), + content: sshd_wrapper, + permissions: Some("0755".to_string()), + owner: Some("root:root".to_string()), + }, + CloudInitWriteFile { + path: "/root/.ssh/authorized_keys".to_string(), + content: format!("{}\n", ssh_keypair.pubkey_str), + permissions: Some("0600".to_string()), + owner: Some("root:root".to_string()), + }, + ]), + runcmd: Some(vec![ + // Ensure the vsock transport exists in the guest. + vec![ + "bash".to_string(), + "-lc".to_string(), + "modprobe vsock 2>/dev/null || true; modprobe vmw_vsock_virtio_transport 2>/dev/null || true; modprobe vhost_vsock 2>/dev/null || true".to_string(), + ], + // Ensure SSH host keys exist. + vec![ + "bash".to_string(), + "-lc".to_string(), + "command -v ssh-keygen >/dev/null && ssh-keygen -A || true".to_string(), + ], + vec![ + "bash".to_string(), + "-lc".to_string(), + "systemctl daemon-reload".to_string(), + ], + // Fedora images commonly run with SELinux enforcing; permissive avoids rare policy + // issues when starting our helper service. + vec![ + "bash".to_string(), + "-lc".to_string(), + "if command -v getenforce >/dev/null && command -v setenforce >/dev/null; then if [ \"$(getenforce 2>/dev/null)\" = \"Enforcing\" ]; then setenforce 0 || true; fi; fi".to_string(), + ], + // Ensure sshd runtime dirs exist. + vec![ + "bash".to_string(), + "-lc".to_string(), + "mkdir -p /run/sshd /root/.ssh && chmod 700 /root/.ssh || true".to_string(), + ], + // Enable the vsock sshd socket. + vec![ + "bash".to_string(), + "-lc".to_string(), + "systemctl enable --now qlean-sshd-vsock.socket".to_string(), + ], + // Marker to simplify debugging via virt-cat. + vec![ + "bash".to_string(), + "-lc".to_string(), + "echo qlean-cloud-init-ok > /var/log/qlean-cloud-init.marker || true".to_string(), + ], + ]), + users: None, + ssh_pwauth: Some(false), }; let mut user_data_str = serde_yml::to_string(&user_data)?; user_data_str.insert_str(0, "#cloud-config\n"); debug!("Writing cloud-init user-data:\n{}", user_data_str); tokio::fs::write(seed_dir.join("user-data"), user_data_str).await?; + // cloud-init's NoCloud datasource expects both user-data and meta-data. + // If meta-data is missing, many images will ignore the seed ISO entirely, + // which means our SSH key and the vsock SSH proxy won't be configured. + let meta_data = format!("instance-id: qlean-{}\nlocal-hostname: qlean\n", machine_id); + tokio::fs::write(seed_dir.join("meta-data"), meta_data).await?; + // Prepare seed ISO let seed_iso_path = run_dir.join("seed.iso"); + // NoCloud expects user-data/meta-data at the *root* of the ISO. + // Passing the directory path directly would place it under /seed/ in the ISO, which + // some distros do not detect (leading to missing SSH key/proxy setup). + let user_data_path = seed_dir.join("user-data"); + let meta_data_path = seed_dir.join("meta-data"); + let mut xorriso_command = tokio::process::Command::new("xorriso"); xorriso_command .args(["-as", "mkisofs"]) .args(["-V", "cidata"]) .args(["-J", "-R"]) .args(["-o", seed_iso_path.to_str().unwrap()]) - .arg(seed_dir); + .args(["-graft-points"]) + .arg(format!("user-data={}", user_data_path.to_string_lossy())) + .arg(format!("meta-data={}", meta_data_path.to_string_lossy())); debug!( "Creating seed ISO with command:\n{:?}", xorriso_command.to_string() @@ -201,7 +366,6 @@ impl Machine { keypair: ssh_keypair, ssh: None, cid, - ssh_tcp_port, pid: None, qemu_should_exit: Arc::new(AtomicBool::new(false)), ssh_cancel_token: None, @@ -621,6 +785,59 @@ impl Machine { self.cid, self.keypair.privkey_path, ); + let kvm_available = KVM_AVAILABLE.get().copied().unwrap_or(false); + // SSH reachability can be slow on first boot (cloud-init + sshd startup), especially on + // slower disks or under nested virtualization. + let ssh_timeout = if kvm_available { + Duration::from_secs(180) + } else { + Duration::from_secs(300) + }; + + // Helper: read pid written by launch_qemu. + async fn read_pid(vmid: &str) -> Result { + let dirs = QleanDirs::new()?; + let pid_file_path = dirs.runs.join(vmid).join("qemu.pid"); + + // QEMU writes pid almost immediately after spawn; wait a short time to make cleanup reliable + // even on slower filesystems. + for _ in 0..50 { + if let Ok(pid_str) = tokio::fs::read_to_string(&pid_file_path).await + && let Ok(pid) = pid_str.trim().parse::() + { + return Ok(pid); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + bail!("Failed to read QEMU pid file at {:?}", pid_file_path); + } + + // Helper: terminate QEMU process best-effort. + async fn terminate_qemu(pid: u32) { + let _ = std::process::Command::new("kill") + .arg("-TERM") + .arg(pid.to_string()) + .output(); + + // Give it a moment to exit; then SIGKILL. + let start = std::time::Instant::now(); + while start.elapsed() < Duration::from_secs(5) { + if !std::path::Path::new(&format!("/proc/{}", pid)).exists() { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + let _ = std::process::Command::new("kill") + .arg("-9") + .arg(pid.to_string()) + .output(); + } + + // Create a fresh cancellation token per launch. + let launch_cancel = CancellationToken::new(); + self.ssh_cancel_token = Some(launch_cancel.clone()); + + self.qemu_should_exit.store(false, Ordering::SeqCst); let qemu_params = crate::qemu::QemuLaunchParams { cid: self.cid, image: self.image.to_owned(), @@ -628,72 +845,54 @@ impl Machine { vmid: self.id.to_owned(), is_init, mac_address: self.mac_address.to_owned(), - ssh_tcp_port: Some(self.ssh_tcp_port), - cancel_token: self - .ssh_cancel_token - .as_ref() - .expect("Machine not initialized or spawned") - .clone(), + cancel_token: launch_cancel.clone(), expected_to_exit: self.qemu_should_exit.clone(), }; - let kvm_available = KVM_AVAILABLE.get().copied().unwrap_or(false); - // SSH reachability can be slow on first boot (cloud-init + sshd startup), especially on - // slower disks or under nested virtualization (e.g. WSL2). - // Use a generous timeout so E2E tests reflect real readiness rather than flakiness. - let ssh_timeout = if kvm_available { - Duration::from_secs(180) - } else { - // Give more time if KVM is not available - Duration::from_secs(300) - }; + let mut qemu_handle = tokio::spawn(launch_qemu(qemu_params)); + let pid = read_pid(&self.id).await?; + self.pid = Some(pid); - info!( - "🔌 SSH transports: prefer vsock cid={} port=22, tcp fallback 127.0.0.1:{}", - self.cid, self.ssh_tcp_port - ); - - let qemu_handle = tokio::spawn(launch_qemu(qemu_params)); - let ssh_handle = tokio::spawn(connect_ssh( + let mut ssh_handle = tokio::spawn(connect_ssh( self.cid, - Some(self.ssh_tcp_port), ssh_timeout, self.keypair.to_owned(), - self.ssh_cancel_token - .as_ref() - .expect("Machine not initialized or spawned") - .clone(), + launch_cancel.clone(), + self.mac_address.to_owned(), )); - // Wait for SSH to complete, or abort SSH if QEMU errors - tokio::select! { - result = ssh_handle => { - // SSH completed, QEMU continues running + // Wait for SSH to complete, or abort SSH if QEMU errors. + let ssh_result = tokio::select! { + result = &mut ssh_handle => { + result.map_err(|e| anyhow::anyhow!("SSH task panicked: {e}"))? + } + result = &mut qemu_handle => { + // QEMU completed or errored, cancel SSH task. + launch_cancel.cancel(); match result { - Ok(Ok(session)) => { - self.ssh = Some(session); - let dirs = QleanDirs::new()?; - let runs_dir = dirs.runs; - let pid_file_path = runs_dir.join(&self.id).join("qemu.pid"); - let pid_str = tokio::fs::read_to_string(pid_file_path).await?; - self.pid = Some(pid_str.trim().parse()?); - } + Ok(Ok(())) => bail!("QEMU exited unexpectedly"), Ok(Err(e)) => bail!(e), - Err(e) => bail!("SSH task panicked: {}", e), + Err(e) => bail!("QEMU task error: {e}"), } } - result = qemu_handle => { - // QEMU completed or errored, cancel SSH task - self.ssh_cancel_token.as_ref().expect("Machine not initialized or spawned").cancel(); - match result { - Ok(Err(e)) => bail!(e), - Ok(Ok(())) => bail!("QEMU exited unexpectedly"), - Err(e) => bail!("QEMU task error: {}", e), + }; + + match ssh_result { + Ok(session) => { + self.ssh = Some(session); + Ok(()) + } + Err(e) => { + // Ensure QEMU is torn down to avoid orphan processes. + self.qemu_should_exit.store(true, Ordering::SeqCst); + launch_cancel.cancel(); + if let Some(pid) = self.pid { + terminate_qemu(pid).await; } + let _ = qemu_handle.await; + bail!(e) } } - - Ok(()) } } diff --git a/src/qemu.rs b/src/qemu.rs index d755323..bfe3436 100644 --- a/src/qemu.rs +++ b/src/qemu.rs @@ -17,7 +17,9 @@ use tracing::{debug, error, info, trace, warn}; use crate::{ KVM_AVAILABLE, MachineConfig, machine::MachineImage, - utils::{CommandExt, QleanDirs}, + utils::{ + CommandExt, QLEAN_BRIDGE_NAME, QleanDirs, bridge_conf_allows, has_iface, has_vsock_support, + }, }; const QEMU_TIMEOUT: Duration = Duration::from_secs(360 * 60); // 6 hours @@ -31,8 +33,6 @@ pub struct QemuLaunchParams { pub is_init: bool, pub cancel_token: CancellationToken, pub mac_address: String, - /// Optional host-forwarded TCP port for SSH (used as a fallback when vsock is unavailable). - pub ssh_tcp_port: Option, } pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { @@ -41,15 +41,20 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { qemu_cmd // Decrease idle CPU usage - .args(["-machine", "hpet=off"]) - // Vsock device (preferred transport) - .args([ - "-device", - &format!( - "vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid={}", - params.cid - ), - ]); + .args(["-machine", "hpet=off"]); + + // Qlean's SSH transport is vhost-vsock. Without it we cannot reach the guest. + anyhow::ensure!( + has_vsock_support(), + "Missing /dev/vhost-vsock; vhost-vsock is required (no TCP fallback)." + ); + qemu_cmd.args([ + "-device", + &format!( + "vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid={}", + params.cid + ), + ]); let use_direct_kernel_boot = params.image.prefer_direct_kernel_boot && params.image.kernel.exists() @@ -61,17 +66,35 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { .map(|m| m.len() > 0) .unwrap_or(false); - if use_direct_kernel_boot { - qemu_cmd - .args(["-kernel", params.image.kernel.to_str().unwrap()]) - .args([ - "-append", - &format!("rw {} console=ttyS0", params.image.root_arg), - ]) - .args(["-initrd", params.image.initrd.to_str().unwrap()]); - } else { - warn!("Kernel/initrd extraction is unavailable. Booting from qcow2 disk image directly."); + anyhow::ensure!( + use_direct_kernel_boot, + "Kernel/initrd extraction is required before QEMU launch." + ); + + // Qlean configures an SSH endpoint over vsock through cloud-init. The guest listens on + // vsock port 22 and proxies to its regular TCP sshd listener. + // + // For Fedora/Arch images, a minimal "root=/dev/vdaX" command line is often insufficient. + // Cloud images may rely on BLS/GRUB kernelopts (UUID/rootflags/btrfs subvols, etc.). + // We therefore pass through the full kernel args extracted from /boot when available. + let mut tokens = params.image.root_arg.split_whitespace().collect::>(); + if !tokens.iter().any(|t| *t == "rw" || *t == "ro") { + tokens.push("rw"); + } + // Force NoCloud datasource for cloud-init so guests don't spend time probing + // metadata services that are unavailable in this test environment. + if !tokens.iter().any(|t| t.starts_with("ds=")) { + tokens.push("ds=nocloud"); + } + if !tokens.iter().any(|t| t.starts_with("console=")) { + tokens.push("console=ttyS0,115200n8"); } + let kernel_cmdline = tokens.join(" "); + + qemu_cmd + .args(["-kernel", params.image.kernel.to_str().unwrap()]) + .args(["-append", &kernel_cmdline]) + .args(["-initrd", params.image.initrd.to_str().unwrap()]); qemu_cmd // Disk @@ -87,64 +110,55 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { // --------------------------------------------------------------------- // Network - // Prefer bridged networking for parity with "real" hosts, but fall back to - // user-mode networking (slirp) when bridging is unavailable (common on WSL2 - // or hosts without qemu bridge ACL configured). - // - // When using user-mode networking, we rely on hostfwd for SSH TCP fallback. + // Qlean requires the libvirt-managed bridge. // --------------------------------------------------------------------- - let bridge_name = "qlbr0"; - let ssh_port = params.ssh_tcp_port; - let want_bridge = has_iface(bridge_name) && bridge_conf_allows(bridge_name); - - if want_bridge { - qemu_cmd - .args(["-netdev", &format!("bridge,id=net0,br={bridge_name}")]) - .args([ - "-device", - &format!("virtio-net-pci,netdev=net0,mac={}", params.mac_address), - ]); - - // Optional user-mode networking with host port forwarding for SSH fallback. - // This provides a TCP escape hatch even when vsock is unreliable. - if let Some(port) = ssh_port { - qemu_cmd - .args([ - "-netdev", - &format!("user,id=net1,hostfwd=tcp:127.0.0.1:{}-:22", port), - ]) - .args(["-device", "virtio-net-pci,netdev=net1"]); - } - } else { - warn!("Bridged networking is unavailable. Falling back to user-mode networking + hostfwd."); + let want_bridge = has_iface(QLEAN_BRIDGE_NAME) && bridge_conf_allows(QLEAN_BRIDGE_NAME); - let port = ssh_port.ok_or_else(|| { - anyhow::anyhow!("user-mode networking fallback requires ssh_tcp_port to be set") - })?; + anyhow::ensure!( + want_bridge, + "QEMU bridge helper is not configured to allow '{}'. Hint: add `allow {}` (or `allow all`) to /etc/qemu/bridge.conf.", + QLEAN_BRIDGE_NAME, + QLEAN_BRIDGE_NAME + ); - qemu_cmd - .args([ - "-netdev", - &format!("user,id=net0,hostfwd=tcp:127.0.0.1:{}-:22", port), - ]) - .args([ - "-device", - &format!("virtio-net-pci,netdev=net0,mac={}", params.mac_address), - ]); - } + qemu_cmd + .args([ + "-netdev", + &format!("bridge,id=net0,br={}", QLEAN_BRIDGE_NAME), + ]) + .args([ + "-device", + &format!("virtio-net-pci,netdev=net0,mac={}", params.mac_address), + ]); // Memory and CPUs qemu_cmd .args(["-m", ¶ms.config.mem.to_string()]) - .args(["-smp", ¶ms.config.core.to_string()]) - // Output redirection - .args(["-serial", "mon:stdio"]); + .args(["-smp", ¶ms.config.core.to_string()]); + + // Output redirection + // We multiplex QEMU monitor + guest serial onto stdio AND tee it into a file under the run dir. + let dirs = QleanDirs::new()?; + let run_dir = dirs.runs.join(¶ms.vmid); + let serial_log = run_dir.join("serial.log"); + qemu_cmd + .args([ + "-chardev", + &format!( + "stdio,id=char0,mux=on,signal=off,logfile={},logappend=on", + serial_log.to_string_lossy() + ), + ]) + .args(["-serial", "chardev:char0"]) + .args(["-mon", "chardev=char0,mode=readline"]); if params.is_init { // Seed ISO qemu_cmd.args([ "-drive", &format!( - "file={},if=virtio,media=cdrom", + // Use an emulated CD-ROM device for maximum compatibility with NoCloud on Fedora/Arch. + // Some images do not reliably scan virtio-cdrom paths during early boot. + "file={},if=ide,media=cdrom,readonly=on", params.image.seed.to_str().unwrap() ), ]); @@ -172,8 +186,7 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { // Store QEMU PID let pid = qemu_child.id().expect("failed to get QEMU PID"); - let dirs = QleanDirs::new()?; - let pid_file_path = dirs.runs.join(¶ms.vmid).join("qemu.pid"); + let pid_file_path = run_dir.join("qemu.pid"); tokio::fs::write(pid_file_path, pid.to_string()).await?; // Capture and log stdout @@ -231,31 +244,3 @@ pub async fn launch_qemu(params: QemuLaunchParams) -> anyhow::Result<()> { result } - -fn bridge_conf_allows(bridge: &str) -> bool { - // qemu-bridge-helper enforces an ACL file (commonly /etc/qemu/bridge.conf). - // If the file is missing or doesn't allow the bridge, QEMU will fail with: - // "failed to parse default acl file `/etc/qemu/bridge.conf`" or "bridge helper failed". - let conf = match std::fs::read_to_string("/etc/qemu/bridge.conf") { - Ok(c) => c, - Err(_) => return false, - }; - for line in conf.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - // Supported entries: "allow ". - if let Some(rest) = line.strip_prefix("allow ") { - let b = rest.trim(); - if b == "all" || b == bridge { - return true; - } - } - } - false -} - -fn has_iface(name: &str) -> bool { - std::path::Path::new(&format!("/sys/class/net/{name}")).exists() -} diff --git a/src/ssh.rs b/src/ssh.rs index 900aabb..b372db2 100644 --- a/src/ssh.rs +++ b/src/ssh.rs @@ -6,7 +6,7 @@ use std::{ time::Duration, }; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use russh::{ ChannelMsg, Disconnect, keys::{ @@ -17,13 +17,19 @@ use russh::{ use russh_sftp::{client::SftpSession, protocol::OpenFlags}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, - net::TcpStream, - time::Instant, + time::{Instant, sleep}, }; use tokio_util::sync::CancellationToken; use tokio_vsock::{VsockAddr, VsockStream}; use tracing::{debug, error, info, warn}; +// MAC address string in canonical form (e.g. "52:54:00:aa:bb:cc"). + +// Avoid adding a libc dependency just to match a couple errno constants. +// These values are stable across Linux and are used only for user-facing hints. +const ERR_EACCES: i32 = 13; // Permission denied +const ERR_ENODEV: i32 = 19; // No such device + #[derive(Clone, Debug)] pub struct PersistedSshKeypair { pub pubkey_str: String, @@ -84,7 +90,7 @@ pub fn get_ssh_key(dir: &Path) -> Result { Ok(keypair) } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] struct SshClient {} // More SSH event handlers can be defined in this trait @@ -140,29 +146,55 @@ impl Session { tokio::time::sleep(Duration::from_millis(100)).await; // Establish vsock connection - let stream = match VsockStream::connect(vsock_addr).await { - Ok(stream) => stream, - Err(ref e) if e.raw_os_error() == Some(19) => { - // This is "No such device" but for some reason Rust doesn't have an IO - // ErrorKind for it. Meh. + let connect_budget = timeout + .saturating_sub(now.elapsed()) + .max(Duration::from_millis(1)); + let stream = match tokio::time::timeout( + connect_budget, + VsockStream::connect(vsock_addr), + ) + .await + { + Ok(Ok(stream)) => stream, + Err(_) => { + warn!("Timeout while connecting to VM via vsock"); + bail!( + "Timeout while connecting to VM via vsock. +Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hypervisor provides a working vsock path." + ); + } + Ok(Err(ref e)) if e.raw_os_error() == Some(ERR_EACCES) => { + bail!( + "Permission denied while connecting via vsock: {e}\n\ +Hint: Ensure your user can access /dev/vhost-vsock (usually by being in the 'kvm' group) or run as root." + ); + } + Ok(Err(ref e)) if e.raw_os_error() == Some(ERR_ENODEV) => { + // ENODEV is commonly observed while QEMU is still booting/initializing the vsock + // transport (e.g. the guest CID isn't ready yet). Treat it as transient and retry + // until the overall timeout is reached. + debug!("SSH vsock connect not ready yet (ENODEV): {e} (will retry)"); if now.elapsed() > timeout { - // Don't log this as an error here: higher-level logic may fall back to TCP. - warn!("Timeout connecting to VM via vsock"); - bail!("Timeout"); + // Keep this at warn level: transient vsock timing issues are expected while the VM boots. + warn!("Timeout while connecting to VM via vsock"); + bail!( + "Timeout while connecting to VM via vsock.\n\ +Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hypervisor provides a working vsock path." + ); } continue; } - Err(ref e) => match e.kind() { + Ok(Err(ref e)) => match e.kind() { ErrorKind::TimedOut | ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset | ErrorKind::NetworkUnreachable | ErrorKind::AddrNotAvailable => { if now.elapsed() > timeout { - // Higher-level logic may fall back to TCP; keep this at warn level. + // Keep this at warn level: transient vsock timing issues are expected while the VM boots. warn!("Timeout while connecting to VM via vsock"); bail!( - "Timeout while connecting to VM via vsock.\n\ + "Timeout while connecting to VM via vsock. Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hypervisor provides a working vsock path." ); } @@ -176,9 +208,24 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp }; // Connect to SSH via vsock stream - match russh::client::connect_stream(config.clone(), stream, sh.clone()).await { - Ok(x) => break x, - Err(russh::Error::IO(ref e)) => { + let handshake_budget = timeout + .saturating_sub(now.elapsed()) + .max(Duration::from_millis(1)); + match tokio::time::timeout( + handshake_budget, + russh::client::connect_stream(config.clone(), stream, sh.clone()), + ) + .await + { + Ok(Ok(x)) => break x, + Err(_) => { + warn!("Timeout establishing SSH over vsock"); + bail!( + "Timeout establishing SSH handshake over vsock.\n\ +Hint: Ensure the guest exposes SSH over vsock (Qlean configures a vsock->TCP proxy via cloud-init) and that the VM finished booting." + ); + } + Ok(Err(russh::Error::IO(ref e))) => { match e.kind() { // The VM is still booting at this point so we're just ignoring these errors // for some time. @@ -187,7 +234,10 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp | ErrorKind::UnexpectedEof => { if now.elapsed() > timeout { warn!("Timeout establishing SSH over vsock"); - bail!("Timeout"); + bail!( + "Timeout establishing SSH handshake over vsock.\n\ +Hint: Ensure the guest exposes SSH over vsock (Qlean configures a vsock->TCP proxy via cloud-init) and that the VM finished booting." + ); } } e => { @@ -196,38 +246,45 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp } } } - Err(russh::Error::Disconnect) => { + Ok(Err(russh::Error::Disconnect)) => { if now.elapsed() > timeout { warn!("Timeout establishing SSH over vsock (disconnect loop)"); - bail!("Timeout"); + bail!( + "Timeout establishing SSH handshake over vsock.\n\ +Hint: Ensure the guest exposes SSH over vsock (Qlean configures a vsock->TCP proxy via cloud-init) and that the VM finished booting." + ); } } - Err(e) => { + Ok(Err(e)) => { error!("SSH client error: {e}"); bail!("SSH client error: {e}"); } } }; debug!("Authenticating via SSH"); - - // use publickey authentication - let auth_res = session - .authenticate_publickey( + let auth_budget = timeout + .saturating_sub(now.elapsed()) + .max(Duration::from_secs(1)); + let auth_res = tokio::time::timeout( + auth_budget, + session.authenticate_publickey( username, PrivateKeyWithHashAlg::new(Arc::new(privkey), None), - ) - .await?; - + ), + ) + .await + .with_context(|| format!("SSH authentication timed out for user {username}"))??; if !auth_res.success() { bail!("Authentication (with publickey) failed"); } - Ok(Self { session, sftp: None, }) } + // NOTE: TCP-based SSH fallback intentionally not supported. + // Reviewer feedback: avoid architecture changes that introduce non-vsock transports. /// Open an SFTP session over the existing SSH connection. async fn open_sftp(&mut self) -> Result { let channel = self.session.channel_open_session().await?; @@ -361,197 +418,81 @@ Hint: Qlean uses vhost-vsock for SSH. Ensure /dev/vhost-vsock exists and the hyp Ok(()) } - /// Connect to an SSH server via TCP (used as a fallback when vsock is unavailable or flaky). - async fn connect_tcp( - privkey: PrivateKey, - username: &str, - host: &str, - port: u16, - timeout: Duration, - cancel_token: CancellationToken, - ) -> Result { - let config = russh::client::Config { - keepalive_interval: Some(Duration::from_secs(5)), - ..<_>::default() - }; - let config = Arc::new(config); - let sh = SshClient {}; - - let now = Instant::now(); - info!("🔑 Connecting via tcp {}:{}", host, port); - - let addr = format!("{}:{}", host, port); - let mut session = loop { - if cancel_token.is_cancelled() { - info!("SSH connection cancelled during connect loop"); - bail!("SSH connection cancelled"); - } - - tokio::time::sleep(Duration::from_millis(100)).await; - - let stream = match TcpStream::connect(&addr).await { - Ok(s) => s, - Err(e) => match e.kind() { - ErrorKind::TimedOut - | ErrorKind::ConnectionRefused - | ErrorKind::ConnectionReset - | ErrorKind::NetworkUnreachable - | ErrorKind::AddrNotAvailable => { - if now.elapsed() > timeout { - bail!( - "Timeout while connecting to VM via tcp {}\nHint: Ensure QEMU host port forwarding is enabled and the guest sshd is running.", - addr - ); - } - continue; - } - _ => { - error!("Unhandled TCP connect error: {e}"); - bail!("Unknown error"); - } - }, - }; - - match russh::client::connect_stream(config.clone(), stream, sh.clone()).await { - Ok(x) => break x, - Err(russh::Error::IO(ref e)) => match e.kind() { - ErrorKind::ConnectionRefused - | ErrorKind::ConnectionReset - | ErrorKind::UnexpectedEof => { - if now.elapsed() > timeout { - bail!("Timeout"); - } - } - _ => { - error!("SSH vsock connect error: {e}"); - bail!("SSH vsock connect error: {e}"); - } - }, - Err(russh::Error::Disconnect) => { - if now.elapsed() > timeout { - bail!("Timeout"); - } - } - Err(e) => { - error!("SSH client error: {e}"); - bail!("SSH client error: {e}"); - } - } - }; - - debug!("Authenticating via SSH"); - let auth_res = session - .authenticate_publickey( - username, - PrivateKeyWithHashAlg::new(Arc::new(privkey), None), - ) - .await?; - if !auth_res.success() { - bail!("Authentication (with publickey) failed"); - } - Ok(Self { - session, - sftp: None, - }) - } + // NOTE: TCP-based SSH is intentionally not supported. + // Qlean's E2E flow is expected to use vhost-vsock for host<->guest SSH. } /// Connect SSH and run a command that checks whether the system is ready for operation. +/// Connect SSH over vhost-vsock and run a readiness probe. +/// pub async fn connect_ssh( cid: u32, - tcp_port: Option, timeout: Duration, keypair: PersistedSshKeypair, cancel_token: CancellationToken, + _mac_address: String, ) -> Result { + if !std::path::Path::new("/dev/vhost-vsock").exists() { + bail!("/dev/vhost-vsock is missing. Qlean requires vhost-vsock for SSH (no TCP fallback).") + } + let privkey = PrivateKey::from_openssh(&keypair.privkey_str)?; - let users = [ - "root", - "arch", - "fedora", - "ubuntu", - "debian", - "cloud-user", - "ec2-user", - ]; - let vsock_supported = std::path::Path::new("/dev/vhost-vsock").exists(); - let user_count = users.len() as u32; - let per_user_timeout = { - let slice = timeout / user_count.max(1); - let min_t = Duration::from_secs(8); - let max_t = Duration::from_secs(20); - if slice < min_t { - min_t - } else if slice > max_t { - max_t - } else { - slice - } - }; - let mut last_tcp_err: Option = None; - let mut last_vsock_err: Option = None; - - if let Some(port) = tcp_port { - info!("Connecting SSH via tcp 127.0.0.1:{}", port); - for user in users { - match Session::connect_tcp( - privkey.clone(), - user, - "127.0.0.1", - port, - per_user_timeout, - cancel_token.clone(), - ) - .await - { - Ok(mut s) => { - info!("✅ Connected via tcp as {}", user); - let _ = s.call("true", cancel_token.clone()).await?; - debug!("SSH command channel is ready"); - return Ok(s); - } - Err(e) => { - warn!("SSH via tcp failed for user {}: {}", user, e); - last_tcp_err = Some(e); - } - } + let deadline = Instant::now() + timeout; + let mut last_err: Option = None; + + while Instant::now() < deadline { + if cancel_token.is_cancelled() { + info!("SSH connection cancelled"); + bail!("SSH connection cancelled"); } - } - if !vsock_supported { - return Err(last_tcp_err.unwrap_or_else(|| { - anyhow::anyhow!("SSH connection failed over tcp and vhost-vsock is unavailable") - })); - } + let remaining = deadline.saturating_duration_since(Instant::now()); + let per_attempt_timeout = Duration::from_secs(12) + .min(remaining.max(Duration::from_secs(1))) + .min(Duration::from_secs(25)); + + info!( + "Connecting SSH via vsock cid={} port=22 as root (budget {:?})", + cid, per_attempt_timeout + ); - info!("Falling back to vsock SSH"); - for user in users { match Session::connect( privkey.clone(), - user, + "root", cid, 22, - per_user_timeout, + per_attempt_timeout, cancel_token.clone(), ) .await { - Ok(mut s) => { - info!("✅ Connected via vsock as {}", user); - let _ = s.call("true", cancel_token.clone()).await?; + Ok(mut session) => { + info!("✅ Connected via vsock as root"); + + let ready_budget = deadline.saturating_duration_since(Instant::now()); + if ready_budget == Duration::ZERO { + bail!("SSH connection timed out"); + } + + let _ = + tokio::time::timeout(ready_budget, session.call("true", cancel_token.clone())) + .await + .context("SSH readiness probe timed out")??; + debug!("SSH command channel is ready"); - return Ok(s); + return Ok(session); } Err(e) => { - warn!("SSH via vsock failed for user {}: {}", user, e); - last_vsock_err = Some(e); + warn!("SSH via vsock not ready yet: {}", e); + last_err = Some(e); + sleep(Duration::from_millis(250)).await; } } } - Err(last_vsock_err - .or(last_tcp_err) - .unwrap_or_else(|| anyhow::anyhow!("SSH connection failed"))) + Err(last_err.unwrap_or_else(|| anyhow::anyhow!( + "Timeout establishing SSH over vsock. Hint: Ensure the guest exposes SSH over vsock and finished booting." + ))) } impl Session { diff --git a/src/utils.rs b/src/utils.rs index e86cc62..851476a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,6 +4,7 @@ use std::{ }; use anyhow::{Context, Result, bail}; + use dir_lock::DirLock; use directories::ProjectDirs; use rand::Rng; @@ -15,6 +16,11 @@ pub static HEX_ALPHABET: [char; 16] = [ ]; pub const VIRSH_CONNECTION_URI: &str = "qemu:///system"; +pub const QLEAN_BRIDGE_NAME: &str = "qlbr0"; + +// NOTE: `derive_mac()` was previously used by an experimental multi-NIC TCP hostfwd +// path. The current implementation is vsock-only, so we avoid keeping unused code +// that triggers `dead_code` warnings. pub struct QleanDirs { pub base: PathBuf, @@ -126,8 +132,8 @@ impl CommandExt for tokio::process::Command { /// Ensure host prerequisites for running virtual machines. /// /// IMPORTANT: This intentionally does **not** require libguestfs tools. -/// Some images (e.g., Ubuntu) ship pre-extracted kernel/initrd and can boot -/// without `guestfish`/`virt-copy-out`. +/// Image creation may need `guestfish`/`virt-copy-out`, but VM launch itself +/// uses the already-extracted kernel/initrd artifacts on disk. pub async fn ensure_prerequisites() -> Result<()> { check_command_available("qemu-system-x86_64").await?; check_command_available("qemu-img").await?; @@ -158,6 +164,37 @@ async fn check_command_available(cmd: &str) -> Result<()> { Ok(()) } +pub fn has_iface(name: &str) -> bool { + Path::new(&format!("/sys/class/net/{name}")).exists() +} + +pub fn bridge_conf_allows(bridge: &str) -> bool { + let conf = match std::fs::read_to_string("/etc/qemu/bridge.conf") { + Ok(c) => c, + Err(_) => return false, + }; + + for line in conf.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + + if let Some(rest) = line.strip_prefix("allow ") { + let allowed = rest.trim(); + if allowed == "all" || allowed == bridge { + return true; + } + } + } + + false +} + +pub fn has_vsock_support() -> bool { + Path::new("/dev/vhost-vsock").exists() +} + async fn ensure_network() -> Result<()> { let output = tokio::process::Command::new("virsh") .arg("-c") diff --git a/tests/arch_image.rs b/tests/arch_image.rs index e5d107a..5c2d163 100644 --- a/tests/arch_image.rs +++ b/tests/arch_image.rs @@ -19,7 +19,10 @@ use logging::tracing_subscriber_init; async fn test_arch_image_startup_flow() -> Result<()> { tracing_subscriber_init(); - ensure_vm_test_env()?; + if !ensure_vm_test_env()? { + return Ok(()); + } + eprintln!("INFO: host checks passed"); ensure_guestfish_tools()?; @@ -49,6 +52,17 @@ async fn test_arch_image_startup_flow() -> Result<()> { distro_id.contains("arch"), "unexpected distro id: {distro_id}" ); + + // Ensure we can execute privileged operations (root or passwordless sudo). + let uid = vm.exec("id -u").await?; + assert!(uid.status.success()); + let uid_str = str::from_utf8(&uid.stdout)?.trim(); + if uid_str != "0" { + let sudo_uid = vm.exec("sudo -n id -u").await?; + assert!(sudo_uid.status.success(), "sudo must be available for E2E"); + let sudo_uid_str = str::from_utf8(&sudo_uid.stdout)?.trim(); + assert_eq!(sudo_uid_str, "0", "sudo must yield uid 0"); + } Ok(()) }) }) diff --git a/tests/fedora_image.rs b/tests/fedora_image.rs index 4f818be..b0f5775 100644 --- a/tests/fedora_image.rs +++ b/tests/fedora_image.rs @@ -19,7 +19,10 @@ use logging::tracing_subscriber_init; async fn test_fedora_image_startup_flow() -> Result<()> { tracing_subscriber_init(); - ensure_vm_test_env()?; + if !ensure_vm_test_env()? { + return Ok(()); + } + eprintln!("INFO: host checks passed"); ensure_guestfish_tools()?; @@ -49,6 +52,17 @@ async fn test_fedora_image_startup_flow() -> Result<()> { distro_id.contains("fedora"), "unexpected distro id: {distro_id}" ); + + // Ensure we can execute privileged operations (root or passwordless sudo). + let uid = vm.exec("id -u").await?; + assert!(uid.status.success()); + let uid_str = str::from_utf8(&uid.stdout)?.trim(); + if uid_str != "0" { + let sudo_uid = vm.exec("sudo -n id -u").await?; + assert!(sudo_uid.status.success(), "sudo must be available for E2E"); + let sudo_uid_str = str::from_utf8(&sudo_uid.stdout)?.trim(); + assert_eq!(sudo_uid_str, "0", "sudo must yield uid 0"); + } Ok(()) }) }) diff --git a/tests/support/e2e.rs b/tests/support/e2e.rs index 980d598..33667fb 100644 --- a/tests/support/e2e.rs +++ b/tests/support/e2e.rs @@ -2,18 +2,17 @@ use std::{path::Path, process::Command}; use anyhow::{Context, Result, bail}; -/// Require explicit opt-in for slow integration tests. -pub fn ensure_e2e_enabled() -> Result<()> { - let enabled = matches!( +const QLEAN_BRIDGE_NAME: &str = "qlbr0"; + +/// Return true if slow E2E tests are explicitly enabled. +/// +/// These tests are intentionally opt-in because they download large images and +/// boot real VMs. +pub fn e2e_enabled() -> bool { + matches!( std::env::var("QLEAN_RUN_E2E").as_deref(), Ok("1") | Ok("true") | Ok("yes") - ); - - if !enabled { - bail!("E2E VM tests are disabled. Set QLEAN_RUN_E2E=1 to run them."); - } - - Ok(()) + ) } /// Return `true` if a command exists on PATH. @@ -25,17 +24,8 @@ fn has_cmd(cmd: &str) -> bool { } } -/// Qlean can use TCP SSH fallback when vsock is unavailable. -pub fn log_vsock_status() { - let has_vsock = - Path::new("/dev/vhost-vsock").exists() || Path::new("/sys/module/vhost_vsock").exists(); - if !has_vsock { - eprintln!("INFO: vhost-vsock is not available; Qlean will use TCP SSH fallback if needed."); - } -} - /// Validate mandatory host commands for E2E execution. -pub fn ensure_vm_test_commands() -> Result<()> { +fn ensure_vm_test_commands() -> Result<()> { if !has_cmd("virsh") { bail!("Missing required command: virsh (libvirt-clients)."); } @@ -46,7 +36,7 @@ pub fn ensure_vm_test_commands() -> Result<()> { } /// Validate the libvirt system URI before running slow tests. -pub fn ensure_libvirt_system() -> Result<()> { +fn ensure_libvirt_system() -> Result<()> { let output = Command::new("virsh") .args(["-c", "qemu:///system", "list", "--all"]) .output() @@ -62,11 +52,71 @@ pub fn ensure_libvirt_system() -> Result<()> { Ok(()) } -/// Fail early on host setup issues. Do not skip after opt-in. -pub fn ensure_vm_test_env() -> Result<()> { - ensure_e2e_enabled()?; +fn has_iface(name: &str) -> bool { + Path::new(&format!("/sys/class/net/{name}")).exists() +} + +fn bridge_conf_allows(bridge: &str) -> bool { + let path = Path::new("/etc/qemu/bridge.conf"); + let Ok(contents) = std::fs::read_to_string(path) else { + return false; + }; + + contents + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .any(|line| line == "allow all" || line == format!("allow {bridge}")) +} + +/// Validate E2E host prerequisites. +/// +/// Returns `Ok(false)` when E2E tests are not enabled (so callers can skip +/// without failing CI). +pub fn ensure_vm_test_env() -> Result { + if !e2e_enabled() { + eprintln!("SKIP: E2E VM tests are disabled. Set QLEAN_RUN_E2E=1 to run them."); + return Ok(false); + } + ensure_vm_test_commands()?; ensure_libvirt_system()?; - log_vsock_status(); - Ok(()) + + // vhost-vsock is required for Qlean's SSH transport. + if !Path::new("/dev/vhost-vsock").exists() { + bail!( + "Missing required device: /dev/vhost-vsock (vhost-vsock is required; no TCP fallback)." + ); + } + + // Qlean expects the libvirt-managed qlbr0 bridge and an allow rule for qemu-bridge-helper. + if !has_iface(QLEAN_BRIDGE_NAME) { + bail!( + "Missing required bridge interface '{}'. Hint: ensure the libvirt network 'qlean' is active (virsh -c qemu:///system net-start qlean).", + QLEAN_BRIDGE_NAME + ); + } + if !bridge_conf_allows(QLEAN_BRIDGE_NAME) { + bail!( + r#"QEMU bridge helper is not configured to allow '{}'. + +Fix (run once as root): + sudo bash ./scripts/setup-host-prereqs.sh + +Or manually: + sudo mkdir -p /etc/qemu + echo "allow {}" | sudo tee /etc/qemu/bridge.conf + sudo chmod 644 /etc/qemu/bridge.conf + +Also ensure qemu-bridge-helper has CAP_NET_ADMIN (recommended): + sudo chmod u-s /usr/lib/qemu/qemu-bridge-helper + sudo setcap cap_net_admin+ep /usr/lib/qemu/qemu-bridge-helper + +Then re-run the test."#, + QLEAN_BRIDGE_NAME, + QLEAN_BRIDGE_NAME + ); + } + + Ok(true) } diff --git a/tests/support/guestfish.rs b/tests/support/guestfish.rs index 5767e66..35afe0f 100644 --- a/tests/support/guestfish.rs +++ b/tests/support/guestfish.rs @@ -9,7 +9,10 @@ fn has_cmd(cmd: &str) -> bool { } } -/// Require libguestfs extraction tools for images that need kernel/initrd extraction. +/// Require the libguestfs command-line tools that the real extraction path uses. +/// +/// Qlean relies on the host's libguestfs installation and appliance setup. +/// The tests intentionally do not attempt to provision appliances at runtime. pub fn ensure_guestfish_tools() -> Result<()> { if !has_cmd("guestfish") { bail!("Missing required command: guestfish (package: libguestfs-tools)."); diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs index 0e5e222..e9c214a 100644 --- a/tests/ubuntu_image.rs +++ b/tests/ubuntu_image.rs @@ -5,10 +5,13 @@ use std::{str, time::Duration}; #[path = "support/e2e.rs"] mod e2e; +#[path = "support/guestfish.rs"] +mod guestfish; #[path = "support/logging.rs"] mod logging; use e2e::ensure_vm_test_env; +use guestfish::ensure_guestfish_tools; use logging::tracing_subscriber_init; #[tokio::test] @@ -16,10 +19,14 @@ use logging::tracing_subscriber_init; async fn test_ubuntu_image_creation() -> Result<()> { tracing_subscriber_init(); - ensure_vm_test_env()?; + if !ensure_vm_test_env()? { + return Ok(()); + } + eprintln!("INFO: host checks passed"); - // Ubuntu uses pre-extracted kernel/initrd. + ensure_guestfish_tools()?; + eprintln!("INFO: creating image"); let image = tokio::time::timeout( Duration::from_secs(15 * 60), @@ -45,6 +52,17 @@ async fn test_ubuntu_image_creation() -> Result<()> { distro_id.contains("ubuntu"), "unexpected distro id: {distro_id}" ); + + // Ensure we can execute privileged operations (root or passwordless sudo). + let uid = vm.exec("id -u").await?; + assert!(uid.status.success()); + let uid_str = str::from_utf8(&uid.stdout)?.trim(); + if uid_str != "0" { + let sudo_uid = vm.exec("sudo -n id -u").await?; + assert!(sudo_uid.status.success(), "sudo must be available for E2E"); + let sudo_uid_str = str::from_utf8(&sudo_uid.stdout)?.trim(); + assert_eq!(sudo_uid_str, "0", "sudo must yield uid 0"); + } Ok(()) }) }) From 4aede8cfa95ddec7004ff821767d8021fbae982d Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Sun, 8 Mar 2026 18:15:14 +0800 Subject: [PATCH 18/20] feat(log): Remove redundant code Signed-off-by: userhaptop <1307305157@qq.com> --- ...\211\257\346\234\254.lock:Zone.Identifier" | Bin 0 -> 25 bytes ...\211\257\346\234\254.toml:Zone.Identifier" | Bin 0 -> 25 bytes scripts/setup-host-prereqs.sh | 14 + src/image.rs | 881 +----------------- src/utils.rs | 4 +- tests/support/guestfish.rs | 35 +- 6 files changed, 82 insertions(+), 852 deletions(-) create mode 100644 "Cargo - \345\211\257\346\234\254.lock:Zone.Identifier" create mode 100644 "Cargo - \345\211\257\346\234\254.toml:Zone.Identifier" diff --git "a/Cargo - \345\211\257\346\234\254.lock:Zone.Identifier" "b/Cargo - \345\211\257\346\234\254.lock:Zone.Identifier" new file mode 100644 index 0000000000000000000000000000000000000000..d6c1ec682968c796b9f5e9e080cc6f674b57c766 GIT binary patch literal 25 dcma!!%Fjy;DN4*MPD?F{<>dl#JyUFr831@K2xdl#JyUFr831@K2x/dev/null 2>&1; then + echo "WARN: libguestfs-test-tool not found after installation. Check your libguestfs-tools package." >&2 + return + fi + + echo "INFO: Verifying host libguestfs runtime (LIBGUESTFS_BACKEND=direct libguestfs-test-tool)" + if ! LIBGUESTFS_BACKEND=direct libguestfs-test-tool; then + echo "ERROR: libguestfs-test-tool failed. Fix the host libguestfs-tools installation before using Qlean image extraction." >&2 + exit 1 + fi +} + need_root ensure_bridge_conf ensure_bridge_helper_caps maybe_install_guestfs_tools_ubuntu +verify_guestfs_runtime echo "DONE" diff --git a/src/image.rs b/src/image.rs index 9f032ca..71d3f92 100644 --- a/src/image.rs +++ b/src/image.rs @@ -1146,683 +1146,27 @@ impl ImageMeta { } } -fn candidate_guestfs_appliance_dirs() -> [&'static str; 6] { - [ - "/usr/lib/guestfs/appliance", - "/usr/lib64/guestfs/appliance", - "/usr/local/lib/guestfs/appliance", - "/usr/local/lib64/guestfs/appliance", - "/usr/lib/x86_64-linux-gnu/guestfs/appliance", - "/usr/lib/aarch64-linux-gnu/guestfs/appliance", - ] -} - -fn is_complete_guestfs_appliance(dir: &Path) -> bool { - dir.join("kernel").exists() - && dir.join("initrd").exists() - && dir.join("root").exists() - && dir.join("README.fixed").exists() -} - -#[derive(Debug, Clone)] -struct SuperminKernelBundle { - kernel: PathBuf, - modules_dir: PathBuf, - kernel_version: String, -} - -async fn stage_readable_supermin_kernel( - cache_root: &Path, - bundle: &SuperminKernelBundle, -) -> Result { - let stage_dir = cache_root.join("supermin-host-kernel"); - tokio::fs::create_dir_all(&stage_dir) - .await - .with_context(|| format!("failed to create {}", stage_dir.display()))?; - - let file_name = bundle - .kernel - .file_name() - .with_context(|| format!("invalid kernel path: {}", bundle.kernel.display()))?; - let staged_kernel = stage_dir.join(file_name); - - let _ = tokio::fs::remove_file(&staged_kernel).await; - tokio::fs::copy(&bundle.kernel, &staged_kernel) - .await - .with_context(|| { - format!( - "failed to copy host kernel {} into {}", - bundle.kernel.display(), - stage_dir.display() - ) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - let perms = std::fs::Permissions::from_mode(0o644); - tokio::fs::set_permissions(&staged_kernel, perms) - .await - .with_context(|| format!("failed to chmod {} to 0644", staged_kernel.display()))?; - } - - Ok(SuperminKernelBundle { - kernel: staged_kernel, - modules_dir: bundle.modules_dir.clone(), - kernel_version: bundle.kernel_version.clone(), - }) -} - -async fn clear_known_paths(root: &Path, names: &[&str]) -> Result<()> { - for name in names { - let path = root.join(name); - if !tokio::fs::try_exists(&path).await? { - continue; - } - let meta = tokio::fs::metadata(&path) - .await - .with_context(|| format!("failed to stat {}", path.display()))?; - if meta.is_dir() { - tokio::fs::remove_dir_all(&path) - .await - .with_context(|| format!("failed to clear {}", path.display()))?; - } else { - tokio::fs::remove_file(&path) - .await - .with_context(|| format!("failed to remove {}", path.display()))?; - } - } - Ok(()) -} - -fn find_supermin_kernel_bundle(root: &Path) -> Option { - let boot_dir = root.join("boot"); - let modules_root = root.join("lib/modules"); - - let mut kernels = std::fs::read_dir(&boot_dir) - .ok()? - .filter_map(|entry| entry.ok()) - .map(|entry| entry.path()) - .filter(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .map(|name| name.starts_with("vmlinuz")) - .unwrap_or(false) - }) - .collect::>(); - kernels.sort(); - - let mut modules = std::fs::read_dir(&modules_root) - .ok()? - .filter_map(|entry| entry.ok()) - .map(|entry| entry.path()) - .filter(|path| path.is_dir()) - .collect::>(); - modules.sort(); - - for module_dir in modules.iter().rev() { - let version = module_dir.file_name()?.to_string_lossy().to_string(); - if let Some(kernel) = kernels.iter().rev().find(|kernel| { - kernel - .file_name() - .and_then(|name| name.to_str()) - .map(|name| name.contains(&version)) - .unwrap_or(false) - }) { - return Some(SuperminKernelBundle { - kernel: kernel.clone(), - modules_dir: module_dir.clone(), - kernel_version: version, - }); - } - } - - let kernel = kernels.into_iter().last()?; - let modules_dir = modules.into_iter().last()?; - let kernel_version = modules_dir.file_name()?.to_string_lossy().to_string(); - - Some(SuperminKernelBundle { - kernel, - modules_dir, - kernel_version, - }) -} - -fn is_concrete_linux_image_package(name: &str) -> bool { - name.strip_prefix("linux-image-unsigned-") - .or_else(|| name.strip_prefix("linux-image-")) - .and_then(|rest| rest.chars().next()) - .map(|ch| ch.is_ascii_digit()) - .unwrap_or(false) -} - -fn parse_apt_depends_for_kernel_package(text: &str) -> Option { - for line in text.lines() { - let line = line.trim(); - if let Some(dep) = line.strip_prefix("Depends:") { - let dep = dep.trim(); - if is_concrete_linux_image_package(dep) { - return Some(dep.to_string()); - } - } - } - None -} - -fn parse_apt_search_for_kernel_package(text: &str) -> Option { - let mut packages = text - .lines() - .filter_map(|line| { - line.split_once(" - ") - .map(|(name, _)| name.trim().to_string()) - }) - .filter(|name| is_concrete_linux_image_package(name)) - .collect::>(); - packages.sort(); - packages.into_iter().last() -} - -fn derive_modules_package_name(image_package: &str) -> Option { - let version = image_package - .strip_prefix("linux-image-unsigned-") - .or_else(|| image_package.strip_prefix("linux-image-"))?; - Some(format!("linux-modules-{}", version)) -} - -async fn resolve_apt_kernel_package_name() -> Result { - for meta_pkg in ["linux-image-virtual", "linux-image-generic"] { - let output = tokio::process::Command::new("apt-cache") - .arg("depends") - .arg("--important") - .arg(meta_pkg) - .output() - .await - .with_context(|| format!("failed to execute `apt-cache depends {meta_pkg}`"))?; - if output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout); - if let Some(pkg) = parse_apt_depends_for_kernel_package(&stdout) { - return Ok(pkg); - } - } - } - - for pattern in [ - "^linux-image-[0-9].*-generic$", - "^linux-image-unsigned-[0-9].*-generic$", - "^linux-image-[0-9].*-virtual$", - ] { - let output = tokio::process::Command::new("apt-cache") - .arg("search") - .arg(pattern) - .output() - .await - .with_context(|| format!("failed to execute `apt-cache search {pattern}`"))?; - if output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout); - if let Some(pkg) = parse_apt_search_for_kernel_package(&stdout) { - return Ok(pkg); - } - } - } - - bail!("failed to locate an apt-downloadable Linux kernel package for supermin on this host") -} - -async fn download_apt_package(package_name: &str, download_dir: &Path) -> Result { - clear_known_paths(download_dir, &["kernel-download"]) - .await - .with_context(|| { - format!( - "failed to clear stale package cache in {}", - download_dir.display() - ) - })?; - - let package_dir = download_dir.join("kernel-download"); - tokio::fs::create_dir_all(&package_dir) - .await - .with_context(|| format!("failed to create {}", package_dir.display()))?; - - let output = tokio::process::Command::new("apt") - .arg("download") - .arg(package_name) - .current_dir(&package_dir) - .output() - .await - .with_context(|| format!("failed to execute `apt download {package_name}`"))?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - let sep = if stdout.trim().is_empty() || stderr.trim().is_empty() { - "" - } else { - "\n" - }; - bail!("{}{}{}", stdout.trim(), sep, stderr.trim()); - } - - let mut entries = tokio::fs::read_dir(&package_dir) - .await - .with_context(|| format!("failed to read {}", package_dir.display()))?; - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) == Some("deb") { - return Ok(path); - } - } - - bail!( - "`apt download {}` did not produce a .deb in {}", - package_name, - package_dir.display() - ) -} - -/// Supermin needs a kernel+modules pair. Some hosts (notably WSL) ship the -/// libguestfs userspace tools but do not provide a local /boot kernel image. -/// In that case, bootstrap a matching bundle from an apt-downloadable kernel package. -async fn prepare_supermin_kernel_bundle(cache_root: &Path) -> Result { - let cache_dir = cache_root.join("supermin-kernel-cache"); - let extract_root = cache_dir.join("root"); - - if let Some(bundle) = find_supermin_kernel_bundle(&extract_root) { - return Ok(bundle); - } - - tokio::fs::create_dir_all(&cache_dir) - .await - .with_context(|| format!("failed to create {}", cache_dir.display()))?; - - let package_attempt = async { - clear_known_paths(&cache_dir, &["root", "kernel-download"]) - .await - .with_context(|| format!("failed to clear {}", cache_dir.display()))?; - - let package_name = resolve_apt_kernel_package_name().await?; - info!( - "Preparing a cached Linux kernel package for libguestfs supermin: {}", - package_name - ); - let deb_file = download_apt_package(&package_name, &cache_dir).await?; - - tokio::fs::create_dir_all(&extract_root) - .await - .with_context(|| format!("failed to create {}", extract_root.display()))?; - - let output = tokio::process::Command::new("dpkg-deb") - .arg("-x") - .arg(&deb_file) - .arg(&extract_root) - .output() - .await - .with_context(|| format!("failed to extract {} with dpkg-deb", deb_file.display()))?; - if !output.status.success() { - bail!("{}", String::from_utf8_lossy(&output.stderr).trim()); - } - - if find_supermin_kernel_bundle(&extract_root).is_none() - && let Some(modules_pkg) = derive_modules_package_name(&package_name) - { - info!( - "The downloaded image package did not include a full modules tree; fetching {} as well", - modules_pkg - ); - let modules_deb = download_apt_package(&modules_pkg, &cache_dir).await?; - let output = tokio::process::Command::new("dpkg-deb") - .arg("-x") - .arg(&modules_deb) - .arg(&extract_root) - .output() - .await - .with_context(|| format!("failed to extract {} with dpkg-deb", modules_deb.display()))?; - if !output.status.success() { - bail!("{}", String::from_utf8_lossy(&output.stderr).trim()); - } - } - - find_supermin_kernel_bundle(&extract_root).with_context(|| { - format!( - "downloaded kernel package {} did not contain a usable kernel/modules pair under {}", - package_name, - extract_root.display() - ) - }) - } - .await; - - match package_attempt { - Ok(bundle) => Ok(bundle), - Err(pkg_err) => { - if let Some(bundle) = find_supermin_kernel_bundle(Path::new("/")) { - warn!( - "Failed to prepare an apt-downloaded kernel bundle for supermin; falling back to a staged host kernel copy: {pkg_err:#}" - ); - return stage_readable_supermin_kernel(cache_root, &bundle).await; - } - Err(pkg_err) - } - } -} - -async fn run_make_fixed_appliance( - appliance_dir: &Path, - kernel_bundle: Option<&SuperminKernelBundle>, -) -> Result { - clear_known_paths(appliance_dir, &["kernel", "initrd", "root"]).await?; - - let mut cmd = tokio::process::Command::new("libguestfs-make-fixed-appliance"); - cmd.arg(appliance_dir); - if let Some(bundle) = kernel_bundle { - cmd.env("SUPERMIN_KERNEL", &bundle.kernel) - .env("SUPERMIN_MODULES", &bundle.modules_dir) - .env("SUPERMIN_KERNEL_VERSION", &bundle.kernel_version); - } - - cmd.output().await.with_context( - || "failed to execute libguestfs-make-fixed-appliance; install libguestfs-tools", - ) -} - -async fn try_build_fixed_guestfs_appliance(appliance_dir: &Path) -> Result<()> { - let first = run_make_fixed_appliance(appliance_dir, None).await?; - if first.status.success() { - anyhow::ensure!( - is_complete_guestfs_appliance(appliance_dir), - "libguestfs-make-fixed-appliance completed without producing a usable appliance" - ); - return Ok(()); - } - - let first_stderr = String::from_utf8_lossy(&first.stderr).trim().to_string(); - let needs_explicit_kernel = first_stderr.contains("supermin exited with error status") - || first_stderr.contains("--copy-kernel") - || first_stderr.contains("failed to find a suitable kernel"); - - if !needs_explicit_kernel { - bail!("{}", first_stderr); - } - - let cache_root = appliance_dir.parent().unwrap_or_else(|| Path::new(".")); - let bundle = prepare_supermin_kernel_bundle(cache_root) - .await - .with_context(|| "failed to provision a kernel/modules bundle for supermin")?; - - info!( - "Retrying libguestfs fixed appliance build with explicit SUPERMIN_KERNEL={} and SUPERMIN_MODULES={}", - bundle.kernel.display(), - bundle.modules_dir.display() - ); - let retry = run_make_fixed_appliance(appliance_dir, Some(&bundle)).await?; - if !retry.status.success() { - bail!("{}", String::from_utf8_lossy(&retry.stderr).trim()); - } - - anyhow::ensure!( - is_complete_guestfs_appliance(appliance_dir), - "libguestfs-make-fixed-appliance completed without producing a usable appliance" - ); - Ok(()) -} - -fn parse_appliance_version_code(version: &str) -> Option { - let mut parts = version.split('.'); - let major = parts.next()?.parse::().ok()?; - let minor = parts.next().unwrap_or("0").parse::().ok()?; - let patch = parts.next().unwrap_or("0").parse::().ok()?; - Some(major * 1_000_000 + minor * 1_000 + patch) -} - -async fn detect_local_libguestfs_version() -> Option { - let guestfish = tokio::process::Command::new("guestfish") - .arg("--version") - .output() - .await - .ok()?; - let stdout = String::from_utf8_lossy(&guestfish.stdout); - if let Some(version) = stdout.split_whitespace().find(|part| { - part.chars() - .next() - .map(|ch| ch.is_ascii_digit()) - .unwrap_or(false) - }) { - return Some(version.trim().to_string()); - } - - let probe = tokio::process::Command::new("libguestfs-test-tool") - .env("LIBGUESTFS_BACKEND", "direct") - .output() - .await - .ok()?; - let combined = format!( - "{} +fn format_guestfs_failure(program: &str, output: &std::process::Output) -> String { + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let details = match (stdout.is_empty(), stderr.is_empty()) { + (false, false) => format!( + "{} {}", - String::from_utf8_lossy(&probe.stdout), - String::from_utf8_lossy(&probe.stderr) - ); - combined.lines().find_map(|line| { - line.split_once("version=") - .map(|(_, v)| v.trim().to_string()) - }) -} - -fn parse_official_appliance_versions(index_html: &str) -> Vec { - let mut versions = Vec::new(); - let mut cursor = 0; - let needle = "appliance-"; - let suffix = ".tar.xz"; - - while let Some(start_rel) = index_html[cursor..].find(needle) { - let start = cursor + start_rel + needle.len(); - if let Some(end_rel) = index_html[start..].find(suffix) { - let version = &index_html[start..start + end_rel]; - if !version.is_empty() - && version.chars().all(|ch| ch.is_ascii_digit() || ch == '.') - && !versions.iter().any(|seen| seen == version) - { - versions.push(version.to_string()); - } - cursor = start + end_rel + suffix.len(); - } else { - break; - } - } - - versions -} - -fn ordered_appliance_versions( - local_version: Option<&str>, - mut versions: Vec, -) -> Vec { - let local_code = local_version.and_then(parse_appliance_version_code); - - versions.sort_by(|a, b| { - let a_code = parse_appliance_version_code(a).unwrap_or_default(); - let b_code = parse_appliance_version_code(b).unwrap_or_default(); - - match local_code { - Some(local) => { - let a_dist = (a_code - local).abs(); - let b_dist = (b_code - local).abs(); - a_dist.cmp(&b_dist).then_with(|| b_code.cmp(&a_code)) - } - None => b_code.cmp(&a_code), - } - }); - - versions -} - -async fn extract_downloaded_appliance_archive( - archive_path: &Path, - appliance_dir: &Path, -) -> Result<()> { - let list_output = tokio::process::Command::new("tar") - .arg("-tJf") - .arg(archive_path) - .output() - .await - .with_context(|| format!("failed to inspect {}", archive_path.display()))?; - if !list_output.status.success() { - bail!("{}", String::from_utf8_lossy(&list_output.stderr).trim()); - } - - let listing = String::from_utf8_lossy(&list_output.stdout); - let top_level = listing - .lines() - .find_map(|line| { - let trimmed = line.trim_matches('/').trim(); - if trimmed.is_empty() { - None - } else { - trimmed.split('/').next().map(|part| part.to_string()) - } - }) - .with_context(|| format!("{} was empty", archive_path.display()))?; - - let output = tokio::process::Command::new("tar") - .arg("-xJf") - .arg(archive_path) - .arg("-C") - .arg(appliance_dir) - .arg("--strip-components=1") - .arg(&top_level) - .output() - .await - .with_context(|| format!("failed to extract {}", archive_path.display()))?; - if !output.status.success() { - bail!("{}", String::from_utf8_lossy(&output.stderr).trim()); - } - - Ok(()) -} - -async fn validate_fixed_guestfs_appliance(appliance_dir: &Path) -> Result<()> { - let output = timeout( - Duration::from_secs(180), - tokio::process::Command::new("libguestfs-test-tool") - .env("LIBGUESTFS_BACKEND", "direct") - .env("LIBGUESTFS_PATH", appliance_dir) - .output(), - ) - .await - .with_context(|| "libguestfs-test-tool timed out while validating the fixed appliance")? - .with_context(|| "failed to execute libguestfs-test-tool")?; - - if output.status.success() { - return Ok(()); - } - - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - let details = if stderr.trim().is_empty() { - stdout.trim().to_string() - } else if stdout.trim().is_empty() { - stderr.trim().to_string() - } else { - format!("{}\n{}", stdout.trim(), stderr.trim()) + stdout, stderr + ), + (false, true) => stdout, + (true, false) => stderr, + (true, true) => "(no output)".to_string(), }; - bail!("{details}"); -} - -async fn try_download_official_guestfs_appliance_version( - appliance_dir: &Path, - version: &str, -) -> Result<()> { - let archive_name = format!("appliance-{version}.tar.xz"); - let archive_url = format!("https://download.libguestfs.org/binaries/appliance/{archive_name}"); - - let cache_root = appliance_dir.parent().unwrap_or_else(|| Path::new(".")); - let download_dir = cache_root.join("guestfs-appliance-downloads"); - tokio::fs::create_dir_all(&download_dir) - .await - .with_context(|| format!("failed to create {}", download_dir.display()))?; - let archive_path = download_dir.join(&archive_name); - - let _actual_sha = download_with_hash(&archive_url, &archive_path, ShaType::Sha512).await?; - - clear_known_paths(appliance_dir, &["kernel", "initrd", "root", "README.fixed"]).await?; - tokio::fs::create_dir_all(appliance_dir) - .await - .with_context(|| format!("failed to create {}", appliance_dir.display()))?; - - extract_downloaded_appliance_archive(&archive_path, appliance_dir).await?; - - anyhow::ensure!( - is_complete_guestfs_appliance(appliance_dir), - "downloaded {} did not contain a complete fixed appliance", - archive_name - ); - - validate_fixed_guestfs_appliance(appliance_dir).await -} - -async fn provision_official_guestfs_appliance(appliance_dir: &Path) -> Result<()> { - let local_version = detect_local_libguestfs_version().await; - let index_html = fetch_text("https://download.libguestfs.org/binaries/appliance/").await?; - let available = parse_official_appliance_versions(&index_html); - anyhow::ensure!( - !available.is_empty(), - "the upstream appliance index did not advertise any downloadable fixed appliance" - ); - - let candidates = ordered_appliance_versions(local_version.as_deref(), available); - let mut last_err = None; - - for version in candidates { - info!( - "Trying official libguestfs fixed appliance download: version {}", - version - ); - match try_download_official_guestfs_appliance_version(appliance_dir, &version).await { - Ok(()) => return Ok(()), - Err(err) => { - warn!( - "Official libguestfs appliance {} did not validate on this host: {err:#}", - version - ); - last_err = Some(err); - } - } - } - - Err(last_err - .unwrap_or_else(|| anyhow::anyhow!("failed to provision an official fixed appliance"))) -} - -async fn ensure_fixed_guestfs_appliance() -> Result { - for dir in candidate_guestfs_appliance_dirs() { - let path = PathBuf::from(dir); - if is_complete_guestfs_appliance(&path) { - return Ok(path); - } - } - - let dirs = QleanDirs::new()?; - let appliance_dir = dirs.base.join("guestfs-appliance"); - if is_complete_guestfs_appliance(&appliance_dir) { - return Ok(appliance_dir); - } - tokio::fs::create_dir_all(&appliance_dir) - .await - .with_context(|| format!("failed to create {}", appliance_dir.display()))?; - - if let Err(download_err) = provision_official_guestfs_appliance(&appliance_dir).await { - warn!( - "Official libguestfs appliance download failed; falling back to local fixed-appliance build: {download_err:#}" - ); - match try_build_fixed_guestfs_appliance(&appliance_dir).await { - Ok(()) => {} - Err(build_err) => bail!( - "failed to provision a usable libguestfs appliance: {build_err:#} -Tried the official fixed appliance download first, then a local libguestfs-make-fixed-appliance build." - ), - } - } - - Ok(appliance_dir) + format!( + "{program} failed using the host libguestfs installation: +{details} +Qlean does not provision libguestfs appliances or other fallback paths at runtime. +Install/repair the host libguestfs-tools setup and verify it with: + LIBGUESTFS_BACKEND=direct libguestfs-test-tool" + ) } async fn run_guestfs_tool( @@ -1830,50 +1174,23 @@ async fn run_guestfs_tool( args: &[&OsStr], current_dir: &Path, ) -> Result { - async fn run_once( - program: &str, - args: &[&OsStr], - current_dir: &Path, - appliance: Option<&Path>, - ) -> Result { - let mut cmd = tokio::process::Command::new(program); - cmd.env("LIBGUESTFS_BACKEND", "direct") - .current_dir(current_dir); - if let Some(appliance_dir) = appliance { - cmd.env("LIBGUESTFS_PATH", appliance_dir); - } - for a in args { - cmd.arg(a); - } - let child = cmd.output(); - let out = timeout(Duration::from_secs(180), child) - .await - .with_context(|| format!("{} timed out after 180s (libguestfs)", program))? - .with_context(|| format!("failed to execute {}", program))?; - Ok(out) + let mut cmd = tokio::process::Command::new(program); + cmd.env("LIBGUESTFS_BACKEND", "direct") + .current_dir(current_dir); + for a in args { + cmd.arg(a); } - let first = run_once(program, args, current_dir, None).await?; - if !first.status.success() { - warn!( - "{} failed (direct backend): {}", - program, - String::from_utf8_lossy(&first.stderr) - ); - } - if first.status.success() { - return Ok(first); - } + let output = timeout(Duration::from_secs(180), cmd.output()) + .await + .with_context(|| format!("{program} timed out after 180s (libguestfs)"))? + .with_context(|| format!("failed to execute {program}"))?; - let stderr = String::from_utf8_lossy(&first.stderr); - let needs_fixed = stderr.contains("supermin exited with error status") - || stderr.contains("/usr/bin/supermin"); - if needs_fixed && std::env::var_os("LIBGUESTFS_PATH").is_none() { - let appliance_dir = ensure_fixed_guestfs_appliance().await?; - return run_once(program, args, current_dir, Some(&appliance_dir)).await; + if !output.status.success() { + bail!("{}", format_guestfs_failure(program, &output)); } - Ok(first) + Ok(output) } async fn guestfish_ls_boot(image_dir: &Path, file_name: &str) -> Result { @@ -1886,30 +1203,17 @@ async fn guestfish_ls_boot(image_dir: &Path, file_name: &str) -> Result OsStr::new("/boot"), ]; let output = run_guestfs_tool("guestfish", &args, image_dir).await?; - if !output.status.success() { - bail!( - "guestfish failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } Ok(String::from_utf8_lossy(&output.stdout).to_string()) } -async fn virt_copy_out(image_dir: &Path, file_name: &str, src: &str, kind: &str) -> Result<()> { +async fn virt_copy_out(image_dir: &Path, file_name: &str, src: &str, _kind: &str) -> Result<()> { let args = [ OsStr::new("-a"), OsStr::new(file_name), OsStr::new(src), OsStr::new("."), ]; - let output = run_guestfs_tool("virt-copy-out", &args, image_dir).await?; - if !output.status.success() { - bail!( - "virt-copy-out failed for {}: {}", - kind, - String::from_utf8_lossy(&output.stderr) - ); - } + let _output = run_guestfs_tool("virt-copy-out", &args, image_dir).await?; Ok(()) } @@ -2983,92 +2287,13 @@ impl ImageAction for Debian { } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { - ensure_extraction_prerequisites().await?; - let file_name = format!("{}.qcow2", name); let dirs = QleanDirs::new()?; let image_dir = dirs.images.join(name); - let output = tokio::process::Command::new("guestfish") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("--ro") - .arg("-a") - .arg(&file_name) - .arg("-i") - .arg("ls") - .arg("/boot") - .current_dir(&image_dir) - .output() - .await - .with_context(|| "failed to execute guestfish")?; - - if !output.status.success() { - bail!( - "guestfish failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let boot_files = String::from_utf8_lossy(&output.stdout); - let mut kernel_name = None; - let mut initrd_name = None; - - for line in boot_files.lines() { - let file = line.trim(); - if file.starts_with("vmlinuz") { - kernel_name = Some(file.to_string()); - } else if file.starts_with("initrd.img") { - initrd_name = Some(file.to_string()); - } - } - - let kernel_name = - kernel_name.with_context(|| "failed to find kernel file (vmlinuz*) in /boot")?; - let initrd_name = - initrd_name.with_context(|| "failed to find initrd file (initrd.img*) in /boot")?; - - let kernel_src = format!("/boot/{}", kernel_name); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&kernel_src) - .arg(".") - .current_dir(&image_dir) - .output() + extract_boot_artifacts_guestfs(&image_dir, &file_name, Distro::Debian) .await - .with_context(|| format!("failed to execute virt-copy-out for {}", kernel_name))?; - - if !output.status.success() { - bail!( - "virt-copy-out failed for kernel: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let initrd_src = format!("/boot/{}", initrd_name); - let output = tokio::process::Command::new("virt-copy-out") - .env("LIBGUESTFS_BACKEND", "direct") - .arg("-a") - .arg(&file_name) - .arg(&initrd_src) - .arg(".") - .current_dir(&image_dir) - .output() - .await - .with_context(|| format!("failed to execute virt-copy-out for {}", initrd_name))?; - - if !output.status.success() { - bail!( - "virt-copy-out failed for initrd: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - let kernel_path = image_dir.join(&kernel_name); - let initrd_path = image_dir.join(&initrd_name); - - Ok((kernel_path, initrd_path)) + .with_context(|| "failed to extract Debian kernel/initrd from qcow2") } fn distro(&self) -> Distro { @@ -3561,44 +2786,6 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 ); } - #[test] - fn test_parse_apt_depends_for_kernel_package_prefers_concrete_image() { - let text = r#" -Depends: linux-image-virtual -Depends: linux-image-6.8.0-71-generic -Depends: initramfs-tools -"#; - assert_eq!( - parse_apt_depends_for_kernel_package(text).as_deref(), - Some("linux-image-6.8.0-71-generic") - ); - } - - #[test] - fn test_derive_modules_package_name_for_signed_and_unsigned_images() { - assert_eq!( - derive_modules_package_name("linux-image-6.8.0-71-generic").as_deref(), - Some("linux-modules-6.8.0-71-generic") - ); - assert_eq!( - derive_modules_package_name("linux-image-unsigned-6.8.0-71-generic").as_deref(), - Some("linux-modules-6.8.0-71-generic") - ); - } - - #[test] - fn test_parse_apt_search_for_kernel_package_picks_latest_match() { - let text = r#" -linux-image-6.8.0-65-generic - Signed kernel image generic -linux-image-6.8.0-71-generic - Signed kernel image generic -linux-image-generic - Complete Generic Linux kernel and headers -"#; - assert_eq!( - parse_apt_search_for_kernel_package(text).as_deref(), - Some("linux-image-6.8.0-71-generic") - ); - } - #[test] fn test_parse_fedora_cloud_listing() { let html = r#" diff --git a/src/utils.rs b/src/utils.rs index 851476a..7187510 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -148,7 +148,9 @@ pub async fn ensure_prerequisites() -> Result<()> { /// Ensure prerequisites for extracting kernel/initrd from disk images. /// /// This is only required for distros/custom modes that need libguestfs-based -/// extraction (guestfish/virt-copy-out). +/// extraction (guestfish/virt-copy-out). Qlean relies on the host's +/// libguestfs-tools installation and does not provision fallback appliances at +/// runtime. pub async fn ensure_extraction_prerequisites() -> Result<()> { check_command_available("guestfish").await?; check_command_available("virt-copy-out").await?; diff --git a/tests/support/guestfish.rs b/tests/support/guestfish.rs index 35afe0f..7463ccc 100644 --- a/tests/support/guestfish.rs +++ b/tests/support/guestfish.rs @@ -1,4 +1,4 @@ -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use std::process::Command; fn has_cmd(cmd: &str) -> bool { @@ -9,10 +9,22 @@ fn has_cmd(cmd: &str) -> bool { } } -/// Require the libguestfs command-line tools that the real extraction path uses. +fn combined_output(output: &std::process::Output) -> String { + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + match (stdout.is_empty(), stderr.is_empty()) { + (false, false) => format!("{}\n{}", stdout, stderr), + (false, true) => stdout, + (true, false) => stderr, + (true, true) => "(no output)".to_string(), + } +} + +/// Require the host libguestfs tools/runtime used by the real extraction path. /// -/// Qlean relies on the host's libguestfs installation and appliance setup. -/// The tests intentionally do not attempt to provision appliances at runtime. +/// Reviewer feedback explicitly asked to fix the host-side libguestfs setup +/// instead of provisioning fallback appliances at runtime, so E2E checks fail +/// fast here if the host installation is incomplete. pub fn ensure_guestfish_tools() -> Result<()> { if !has_cmd("guestfish") { bail!("Missing required command: guestfish (package: libguestfs-tools)."); @@ -20,5 +32,20 @@ pub fn ensure_guestfish_tools() -> Result<()> { if !has_cmd("virt-copy-out") { bail!("Missing required command: virt-copy-out (package: libguestfs-tools)."); } + if !has_cmd("libguestfs-test-tool") { + bail!("Missing required command: libguestfs-test-tool (package: libguestfs-tools)."); + } + + let output = Command::new("libguestfs-test-tool") + .env("LIBGUESTFS_BACKEND", "direct") + .output() + .with_context(|| "failed to execute `libguestfs-test-tool`")?; + + if !output.status.success() { + bail!( + "libguestfs-test-tool failed; fix the host libguestfs-tools installation before running E2E tests:\n{}", + combined_output(&output) + ); + } Ok(()) } From cf6c017f3f5333393455501a4760496b7860fbeb Mon Sep 17 00:00:00 2001 From: userhaptop <1307305157@qq.com> Date: Sun, 8 Mar 2026 18:17:06 +0800 Subject: [PATCH 19/20] feat(log): Remove redundant code Signed-off-by: userhaptop <1307305157@qq.com> --- ... \345\211\257\346\234\254.lock:Zone.Identifier" | Bin 25 -> 0 bytes ... \345\211\257\346\234\254.toml:Zone.Identifier" | Bin 25 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "Cargo - \345\211\257\346\234\254.lock:Zone.Identifier" delete mode 100644 "Cargo - \345\211\257\346\234\254.toml:Zone.Identifier" diff --git "a/Cargo - \345\211\257\346\234\254.lock:Zone.Identifier" "b/Cargo - \345\211\257\346\234\254.lock:Zone.Identifier" deleted file mode 100644 index d6c1ec682968c796b9f5e9e080cc6f674b57c766..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25 dcma!!%Fjy;DN4*MPD?F{<>dl#JyUFr831@K2xdl#JyUFr831@K2x Date: Sun, 8 Mar 2026 20:33:59 +0800 Subject: [PATCH 20/20] feat(log): Fix configuration issues Signed-off-by: userhaptop <1307305157@qq.com> --- Cargo.lock | 60 +++ Cargo.toml | 1 + qlean-images.toml | 27 ++ scripts/setup-host-prereqs.sh | 55 ++- src/image.rs | 720 ++++++---------------------------- src/utils.rs | 8 +- tests/arch_image.rs | 4 +- tests/fedora_image.rs | 4 +- tests/support/e2e.rs | 29 +- tests/ubuntu_image.rs | 4 +- 10 files changed, 260 insertions(+), 652 deletions(-) create mode 100644 qlean-images.toml diff --git a/Cargo.lock b/Cargo.lock index 5cb59f0..6b91512 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2247,6 +2247,7 @@ dependencies = [ "tokio-fd", "tokio-util", "tokio-vsock", + "toml", "tracing", "tracing-subscriber", "walkdir", @@ -2907,6 +2908,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_yml" version = "0.0.12" @@ -3405,6 +3415,47 @@ dependencies = [ "vsock", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.2" @@ -4217,6 +4268,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index c143927..152c9a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ russh-sftp = "2.1.1" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" serde_yml = "0.0.12" +toml = "0.8.19" sha2 = "0.10" shell-escape = "0.1.5" termion = "4.0.6" diff --git a/qlean-images.toml b/qlean-images.toml new file mode 100644 index 0000000..4f1e6ac --- /dev/null +++ b/qlean-images.toml @@ -0,0 +1,27 @@ +# Default official image sources used by Qlean integration tests. +# Users may edit these URLs/checksum files directly to point to their preferred +# official mirror or pinned image release. + +[debian] +image_url = "https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2" +checksum_url = "https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS" +checksum_entry = "debian-13-generic-amd64.qcow2" +checksum_type = "Sha512" + +[ubuntu] +image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +checksum_url = "https://cloud-images.ubuntu.com/noble/current/SHA256SUMS" +checksum_entry = "noble-server-cloudimg-amd64.img" +checksum_type = "Sha256" + +[fedora] +image_url = "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2" +checksum_url = "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images/Fedora-Cloud-43-1.6-x86_64-CHECKSUM" +checksum_entry = "Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2" +checksum_type = "Sha256" + +[arch] +image_url = "https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2" +checksum_url = "https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2.SHA256" +checksum_entry = "Arch-Linux-x86_64-cloudimg.qcow2" +checksum_type = "Sha256" diff --git a/scripts/setup-host-prereqs.sh b/scripts/setup-host-prereqs.sh index 105191f..7b9047d 100644 --- a/scripts/setup-host-prereqs.sh +++ b/scripts/setup-host-prereqs.sh @@ -3,14 +3,16 @@ set -euo pipefail # Qlean host prerequisites helper. # -# This script is intentionally conservative: it configures QEMU bridge-helper -# permissions for the Qlean bridge and applies the minimal capabilities needed -# for qemu-bridge-helper. +# This script configures the explicit host prerequisites required by Qlean: +# - qemu-bridge-helper permissions for the Qlean bridge +# - the libvirt 'qlean' network and qlbr0 bridge +# - host libguestfs-tools installation/runtime verification # -# It does *not* create the bridge or manage network state (that remains the -# responsibility of Qlean at runtime). +# Qlean no longer provisions or discovers fallback paths at runtime; run this +# script once before using distro image creation or E2E tests. BRIDGE_NAME="${QLEAN_BRIDGE_NAME:-qlbr0}" +VIRSH_URI="qemu:///system" need_root() { if [[ "$(id -u)" -ne 0 ]]; then @@ -42,7 +44,6 @@ ensure_bridge_conf() { } find_qemu_bridge_helper() { - # Common distro paths. local candidates=( /usr/lib/qemu/qemu-bridge-helper /usr/libexec/qemu-bridge-helper @@ -56,7 +57,6 @@ find_qemu_bridge_helper() { fi done - # Fall back to PATH. if command -v qemu-bridge-helper >/dev/null 2>&1; then command -v qemu-bridge-helper return 0 @@ -73,7 +73,6 @@ ensure_bridge_helper_caps() { fi if command -v setcap >/dev/null 2>&1; then - # Prefer file capabilities over setuid. chmod u-s "$helper" || true setcap cap_net_admin+ep "$helper" @@ -86,14 +85,47 @@ ensure_bridge_helper_caps() { fi } +ensure_qlean_network() { + if ! command -v virsh >/dev/null 2>&1; then + echo "ERROR: virsh not found. Install libvirt-clients/libvirt-daemon-system first." >&2 + exit 1 + fi + + if ! virsh -c "$VIRSH_URI" net-info qlean >/dev/null 2>&1; then + local xml + xml=$(mktemp) + cat > "$xml" < + qlean + + + + + + + + +EOF + echo "INFO: defining libvirt network qlean" + virsh -c "$VIRSH_URI" net-define "$xml" + rm -f "$xml" + else + echo "OK: libvirt network qlean already defined" + fi + + echo "INFO: ensuring libvirt network qlean is active" + virsh -c "$VIRSH_URI" net-start qlean >/dev/null 2>&1 || true + virsh -c "$VIRSH_URI" net-autostart qlean >/dev/null 2>&1 || true +} + maybe_install_guestfs_tools_ubuntu() { - # Only attempt package installation on Debian/Ubuntu when apt-get is available. if ! command -v apt-get >/dev/null 2>&1; then return fi - # guestfish/virt-copy-out are used for kernel/initrd extraction. - if command -v guestfish >/dev/null 2>&1 && command -v virt-copy-out >/dev/null 2>&1; then + if command -v guestfish >/dev/null 2>&1 \ + && command -v virt-copy-out >/dev/null 2>&1 \ + && command -v libguestfs-test-tool >/dev/null 2>&1; then echo "OK: libguestfs tools already installed" return fi @@ -119,6 +151,7 @@ verify_guestfs_runtime() { need_root ensure_bridge_conf ensure_bridge_helper_caps +ensure_qlean_network maybe_install_guestfs_tools_ubuntu verify_guestfs_runtime diff --git a/src/image.rs b/src/image.rs index 71d3f92..c27aa68 100644 --- a/src/image.rs +++ b/src/image.rs @@ -14,7 +14,7 @@ use tokio::{ io::AsyncWriteExt, time::{Duration, timeout}, }; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use crate::utils::{QleanDirs, ensure_extraction_prerequisites}; @@ -111,11 +111,6 @@ fn checksum_name_matches(entry_name: &str, wanted: &str) -> bool { false } -/// Parses checksum text and returns the hash for a filename. -pub fn find_sha512_for_file(checksums_text: &str, filename: &str) -> Option { - find_hash_for_file(checksums_text, filename) -} - /// Parse a checksum file and return the hash for a given filename. /// /// Supports common formats: @@ -145,108 +140,56 @@ pub fn find_hash_for_file(checksums_text: &str, filename: &str) -> Option String { - format!( - "{}/{}", - base.trim_end_matches('/'), - path.trim_start_matches('/') - ) -} +const IMAGE_SOURCES_CONFIG_PATH: &str = "qlean-images.toml"; -async fn fetch_ubuntu_sha256sums(base_url: &str) -> Result { - let primary = join_url(base_url, "SHA256SUMS"); - match fetch_text(&primary).await { - Ok(text) => Ok(text), - Err(primary_err) => { - let fallback = join_url(base_url, "SHA256SUMS.txt"); - fetch_text(&fallback).await.with_context(|| { - format!( - "failed to fetch Ubuntu checksums from {} or {} ({:#})", - primary, fallback, primary_err - ) - }) - } - } +#[derive(Debug, Deserialize)] +struct ImageSourcesConfig { + debian: RemoteImageConfig, + ubuntu: RemoteImageConfig, + fedora: RemoteImageConfig, + arch: RemoteImageConfig, } -#[derive(Debug, Clone)] -struct ResolvedUbuntuCloudImage { - base_url: String, - disk_name: String, - disk_sha256: String, +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +struct RemoteImageConfig { + image_url: String, + checksum_url: String, + checksum_entry: String, + checksum_type: ShaType, } -fn pick_existing_ubuntu_name(checksums: &str, candidates: &[&str]) -> Option { - candidates - .iter() - .find(|name| { - checksums.contains(&format!(" {}", name)) || checksums.contains(&format!(" *{}", name)) - }) - .map(|name| (*name).to_string()) -} - -async fn resolve_ubuntu_noble_cloudimg() -> Result { - let bases = [ - "https://cloud-images.ubuntu.com/releases/noble/release", - "https://cloud-images.ubuntu.com/noble/current", - "https://cloud-images.ubuntu.com/daily/server/releases/noble/release", - ]; - - let mut last_err: Option = None; - - for (idx, base) in bases.iter().enumerate() { - info!("Ubuntu metadata source {}/{}", idx + 1, bases.len()); - - let checksums = match fetch_ubuntu_sha256sums(base).await { - Ok(v) => v, - Err(e) => { - last_err = Some(e); - continue; - } - }; - - let disk_name = match pick_existing_ubuntu_name( - &checksums, - &[ - "ubuntu-24.04-server-cloudimg-amd64.img", - "noble-server-cloudimg-amd64.img", - ], - ) { - Some(v) => v, - None => { - last_err = Some(anyhow::anyhow!( - "Ubuntu SHA256SUMS did not contain amd64 cloud image entry" - )); - continue; - } - }; - - let disk_sha256 = match ubuntu_sha256_for(&checksums, &disk_name) { - Ok(v) => v, - Err(e) => { - last_err = Some(e); - continue; - } - }; - - return Ok(ResolvedUbuntuCloudImage { - base_url: (*base).to_string(), - disk_name, - disk_sha256, - }); +impl ImageSourcesConfig { + fn for_distro(&self, distro: Distro) -> Result<&RemoteImageConfig> { + match distro { + Distro::Debian => Ok(&self.debian), + Distro::Ubuntu => Ok(&self.ubuntu), + Distro::Fedora => Ok(&self.fedora), + Distro::Arch => Ok(&self.arch), + Distro::Custom => bail!("custom images do not use qlean-images.toml"), + } } +} - Err(last_err - .unwrap_or_else(|| anyhow::anyhow!("failed to resolve Ubuntu cloud image metadata"))) +fn image_sources_config_path() -> PathBuf { + PathBuf::from(IMAGE_SOURCES_CONFIG_PATH) } -fn ubuntu_sha256_for(checksums: &str, filename: &str) -> Result { - find_hash_for_file(checksums, filename) - .with_context(|| format!("Ubuntu SHA256SUMS did not contain hash for {}", filename)) +async fn load_image_sources_config() -> Result { + let path = image_sources_config_path(); + let content = tokio::fs::read_to_string(&path) + .await + .with_context(|| { + format!( + "failed to read image source config at {}. Copy or edit qlean-images.toml before creating distro images", + path.display() + ) + })?; + + toml::from_str(&content) + .with_context(|| format!("failed to parse TOML from {}", path.display())) } async fn fetch_text(url: &str) -> Result { - // Keep metadata fetches snappy: if a mirror is slow/hung, we fall back. let client = reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(15)) .timeout(std::time::Duration::from_secs(30)) @@ -267,282 +210,44 @@ async fn fetch_text(url: &str) -> Result { .with_context(|| format!("failed reading body from {}", url)) } -// --------------------------------------------------------------------------- -// Fedora/Arch "latest stable" resolvers -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone)] -struct ResolvedRemote { - /// Candidate URLs to try in order. Mirrors can hang; we retry/fallback. - urls: Vec, - sha256: String, -} - -/// Query endoflife.date for the latest maintained Fedora release number. -/// -/// This endpoint is stable and returns structured JSON. We select the first -/// release entry where `isMaintained=true` and `isEol=false`. -const FALLBACK_FEDORA_RELEASE: &str = "43"; - -async fn resolve_latest_fedora_release() -> Result { - #[derive(Deserialize)] - struct EolResp { - result: EolResult, - } - #[derive(Deserialize)] - struct EolResult { - releases: Vec, - } - #[derive(Deserialize)] - struct EolRelease { - name: String, - #[serde(default, rename = "isEol")] - is_eol: bool, - #[serde(default, rename = "isMaintained")] - is_maintained: bool, - } - - // Public JSON API documented on https://endoflife.date/fedora - let url = "https://endoflife.date/api/v1/products/fedora/"; - match fetch_text(url).await { - Ok(body) => { - let parsed: EolResp = serde_json::from_str(&body) - .with_context(|| "failed to parse Fedora release JSON")?; - - let latest = parsed - .result - .releases - .into_iter() - .find(|r| r.is_maintained && !r.is_eol) - .map(|r| r.name) - .with_context(|| "could not determine latest maintained Fedora release")?; - - Ok(latest) - } - Err(err) => { - warn!( - "failed to query endoflife.date for latest Fedora release ({}); falling back to {}", - err, FALLBACK_FEDORA_RELEASE - ); - Ok(FALLBACK_FEDORA_RELEASE.to_string()) - } - } -} +async fn fetch_expected_hash(config: &RemoteImageConfig) -> Result { + let checksums_text = fetch_text(&config.checksum_url) + .await + .with_context(|| format!("failed to fetch checksum file from {}", config.checksum_url))?; -/// Resolve the latest Fedora Cloud Base Generic qcow2 URL and its SHA256. -/// -/// Implementation strategy: -/// 1) Determine the latest maintained Fedora version. -/// 2) Fetch an HTML directory listing from one of several known mirrors. -/// 3) Parse the directory listing to locate the *exact* qcow2 filename and -/// the corresponding CHECKSUM file name. -/// 4) Download the CHECKSUM file and extract the SHA256 for the qcow2. -async fn resolve_latest_fedora_cloud_qcow2() -> Result { - let ver = resolve_latest_fedora_release().await?; - - // Mirror directory patterns differ. We try a small set of mirrors known to - // provide HTML listings. We keep this list short to reduce fragility. - // Order matters: prefer official redirector first, then a couple of mirrors - // that typically provide directory listings. - let candidates = [ + find_hash_for_file(&checksums_text, &config.checksum_entry).with_context(|| { format!( - "https://download.fedoraproject.org/pub/fedora/linux/releases/{}/Cloud/x86_64/images/", - ver - ), - format!( - "https://ftp2.osuosl.org/pub/fedora/linux/releases/{}/Cloud/x86_64/images/", - ver - ), - format!( - "https://mirrors.oit.uci.edu/fedora/linux/releases/{}/Cloud/x86_64/images/", - ver - ), - format!( - "https://mirrors.telepoint.bg/fedora/releases/{}/Cloud/x86_64/images/", - ver - ), - format!( - "https://mirrors.kernel.org/fedora/releases/{}/Cloud/x86_64/images/", - ver - ), - ]; - - // Collect all mirrors where we can fetch a usable listing. We'll parse once - // and then try downloads across all mirrors. - let mut good_bases: Vec = Vec::new(); - let mut listing_html: Option = None; - for u in &candidates { - match fetch_text(u).await { - Ok(text) => { - if text.contains("Fedora-Cloud-Base-Generic") && text.contains("CHECKSUM") { - good_bases.push(u.clone()); - if listing_html.is_none() { - listing_html = Some(text); - } - } - } - Err(e) => { - debug!("Fedora listing fetch failed for {}: {:#}", u, e); - } - } - } - - anyhow::ensure!( - !good_bases.is_empty(), - "failed to fetch Fedora Cloud images listing from mirrors" - ); - let listing_html = listing_html.unwrap(); - - let (qcow2_name, checksum_name) = parse_fedora_cloud_listing(&listing_html, &ver)?; - - // Build base URLs. Prefer bases where listing worked, but also try the - // full candidate set as a fallback (a mirror may block directory listing - // but still serve the file). - let mut bases = good_bases; - for c in &candidates { - if !bases.iter().any(|b| b == c) { - bases.push(c.clone()); - } - } - - // Fetch CHECKSUM across mirrors too. Relying on a single mirror defeats - // the multi-mirror resilience we provide for the qcow2 download. - let sha256 = fetch_fedora_checksum_sha256(&bases, &checksum_name, &qcow2_name).await?; - - let urls = bases - .into_iter() - .map(|base| format!("{}{}", base, qcow2_name)) - .collect::>(); - - Ok(ResolvedRemote { urls, sha256 }) -} - -async fn fetch_fedora_checksum_sha256( - bases: &[String], - checksum_name: &str, - qcow2_name: &str, -) -> Result { - let mut last_err: Option = None; - - for base in bases { - let checksum_url = format!("{}{}", base, checksum_name); - match fetch_text(&checksum_url).await { - Ok(text) => { - if let Some(sha) = find_hash_for_file(&text, qcow2_name) { - return Ok(sha); - } - - last_err = Some(anyhow::anyhow!( - "checksum file {} did not contain hash for {}", - checksum_url, - qcow2_name - )); - } - Err(e) => { - debug!("Fedora CHECKSUM fetch failed for {}: {:#}", checksum_url, e); - last_err = Some(e); - } - } - } - - Err(last_err.unwrap_or_else(|| anyhow::anyhow!("failed to fetch Fedora CHECKSUM from mirrors"))) -} - -fn parse_fedora_cloud_listing(listing_html: &str, ver: &str) -> Result<(String, String)> { - // Parse filename candidates. - // Listings generally contain only one compose, so the first match is fine. - let qcow2_prefix = format!("Fedora-Cloud-Base-Generic-{}-", ver); - let qcow2_suffix = ".x86_64.qcow2"; - let mut qcow2_name: Option = None; - let mut checksum_name: Option = None; - - for token in listing_html - .split(|c: char| !(c.is_ascii_alphanumeric() || c == '.' || c == '-' || c == '_')) - { - if qcow2_name.is_none() && token.starts_with(&qcow2_prefix) && token.ends_with(qcow2_suffix) - { - qcow2_name = Some(token.to_string()); - } - if checksum_name.is_none() - && token.starts_with(&format!("Fedora-Cloud-{}-", ver)) - && token.ends_with("-x86_64-CHECKSUM") - { - checksum_name = Some(token.to_string()); - } - if qcow2_name.is_some() && checksum_name.is_some() { - break; - } - } - - let qcow2_name = qcow2_name - .with_context(|| "could not locate Fedora Cloud Base Generic qcow2 filename in listing")?; - let checksum_name = checksum_name - .with_context(|| "could not locate Fedora Cloud CHECKSUM filename in listing")?; - - Ok((qcow2_name, checksum_name)) + "checksum file {} did not contain an entry for {}", + config.checksum_url, config.checksum_entry + ) + }) } -/// Resolve the latest Arch cloud image URL and SHA256. -/// -/// Arch publishes stable "latest" URLs plus a sidecar .SHA256 file. -async fn resolve_latest_arch_cloudimg() -> Result { - let bases = [ - "https://mirrors.tuna.tsinghua.edu.cn/archlinux/images/latest", - "https://mirrors.ustc.edu.cn/archlinux/images/latest", - "https://mirrors.sjtug.sjtu.edu.cn/archlinux/images/latest", - "https://fastly.mirror.pkgbuild.com/images/latest", - "https://geo.mirror.pkgbuild.com/images/latest", - "https://mirror.citrahost.com/archlinux/images/latest", - "https://mirrors.teamcloud.am/archlinux/images/latest", - "https://mirror.umd.edu/archlinux/images/latest", - "https://ftp.jaist.ac.jp/pub/Linux/ArchLinux/images/latest", - ]; - let filename = "Arch-Linux-x86_64-cloudimg.qcow2"; - - info!("Resolving Arch cloud image metadata"); - - let mut last_err: Option = None; - let mut selected_base: Option<&str> = None; - let mut sha256: Option = None; - - for (idx, base) in bases.iter().enumerate() { - info!("Arch metadata mirror {}/{}", idx + 1, bases.len()); - let sha_url = format!("{}/{}.SHA256", base, filename); - match fetch_text(&sha_url).await { - Ok(text) => { - if let Some(hash) = find_hash_for_file(&text, filename) { - selected_base = Some(*base); - sha256 = Some(hash); - break; - } - last_err = Some(anyhow::anyhow!("invalid checksum format at {}", sha_url)); - } - Err(e) => { - debug!("Arch metadata fetch failed for {}: {:#}", sha_url, e); - last_err = Some(e); - } - } - } +async fn download_remote_image(name: &str, distro: Distro) -> Result<()> { + let dirs = QleanDirs::new()?; + let image_path = dirs.images.join(name).join(format!("{}.qcow2", name)); - let selected_base = selected_base.ok_or_else(|| { - last_err.unwrap_or_else(|| anyhow::anyhow!("no Arch mirror metadata succeeded")) - })?; - let sha256 = sha256.expect("sha256 must exist when metadata resolves"); + let sources = load_image_sources_config().await?; + let config = sources.for_distro(distro)?; + let expected_hash = fetch_expected_hash(config).await?; - let mut urls = vec![format!("{}/{}", selected_base, filename)]; - for base in bases { - if base != selected_base { - urls.push(format!("{}/{}", base, filename)); - } - } + materialize_source_with_hash( + &ImageSource::Url(config.image_url.clone()), + &image_path, + &expected_hash, + config.checksum_type.clone(), + ) + .await?; - Ok(ResolvedRemote { urls, sha256 }) + Ok(()) } // --------------------------------------------------------------------------- // Streaming hash functions - optimized for release mode performance // --------------------------------------------------------------------------- +// --------------------------------------------------------------------------- + /// Compute SHA-256 hash using streaming approach with sync I/O /// This provides 7-27% better performance than shell commands in release mode pub async fn compute_sha256_streaming(path: &Path) -> Result { @@ -602,22 +307,18 @@ pub async fn compute_sha512_streaming(path: &Path) -> Result { .with_context(|| "hash computation task failed")? } -/// Download file and compute hash in single pass to avoid reading file twice -async fn download_with_hash_impl( +/// Download a remote file and compute its hash in a single pass. +async fn stream_download_with_hash( url: &str, dest_path: &PathBuf, hash_type: ShaType, - slow_link_cutoff: Option<(std::time::Duration, u64)>, ) -> Result { - // Keep a temp file to avoid leaving a partially-downloaded blob behind. let tmp_path = dest_path.with_extension("part"); debug!("Downloading {} to {}", url, dest_path.display()); let client = reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(20)) - // Do NOT set a short global timeout for large images; we handle stalls - // with an idle timeout on the stream. .user_agent("qlean/0.2 (image-download)") .build() .with_context(|| "failed to build HTTP client")?; @@ -635,14 +336,12 @@ async fn download_with_hash_impl( info!("Remote size: {} MiB ({})", total / (1024 * 1024), url); } - // Ensure destination directory exists. if let Some(parent) = tmp_path.parent() { tokio::fs::create_dir_all(parent) .await .with_context(|| format!("failed to create dir {}", parent.display()))?; } - // Start fresh on each attempt. let _ = tokio::fs::remove_file(&tmp_path).await; let mut file = File::create(&tmp_path) @@ -653,12 +352,8 @@ async fn download_with_hash_impl( let idle = std::time::Duration::from_secs(60); let mut downloaded: u64 = 0; let mut last_report: u64 = 0; - // Report download progress in reasonably small increments. On slower links or in CI, - // 64MiB can take long enough that the test runner prints a scary "running over 60 seconds" - // warning with no other output. - let report_step: u64 = 8 * 1024 * 1024; // 8 MiB - let started_at = std::time::Instant::now(); - let mut last_report_at = started_at; + let report_step: u64 = 8 * 1024 * 1024; + let mut last_report_at = std::time::Instant::now(); let hash = match hash_type { ShaType::Sha256 => { @@ -694,17 +389,6 @@ async fn download_with_hash_impl( ); } } - if let Some((slow_link_timeout, min_bytes)) = slow_link_cutoff - && started_at.elapsed() >= slow_link_timeout - && downloaded < min_bytes - { - anyhow::bail!( - "download too slow for {} ({} MiB in {:?}); trying next mirror", - url, - downloaded / (1024 * 1024), - started_at.elapsed() - ); - } h.update(&chunk); file.write_all(&chunk) .await @@ -745,17 +429,6 @@ async fn download_with_hash_impl( ); } } - if let Some((slow_link_timeout, min_bytes)) = slow_link_cutoff - && started_at.elapsed() >= slow_link_timeout - && downloaded < min_bytes - { - anyhow::bail!( - "download too slow for {} ({} MiB in {:?}); trying next mirror", - url, - downloaded / (1024 * 1024), - started_at.elapsed() - ); - } h.update(&chunk); file.write_all(&chunk) .await @@ -767,7 +440,6 @@ async fn download_with_hash_impl( file.flush().await.with_context(|| "failed to flush file")?; - // Atomically move into place. tokio::fs::rename(&tmp_path, dest_path) .await .with_context(|| { @@ -786,86 +458,8 @@ async fn download_with_hash_impl( Ok(hash) } -pub async fn download_with_hash( - url: &str, - dest_path: &PathBuf, - hash_type: ShaType, -) -> Result { - download_with_hash_impl(url, dest_path, hash_type, None).await -} - -/// Try downloading a remote file from multiple candidate URLs. -/// -/// Mirrors can hang mid-transfer; we apply an idle timeout and move on. -pub async fn download_with_hash_multi( - urls: &[String], - dest_path: &PathBuf, - hash_type: ShaType, - expected_hex: Option<&str>, -) -> Result<(String, String)> { - let mut last_err: Option = None; - - for (idx, url) in urls.iter().enumerate() { - // If a cached file exists and matches expected, short-circuit. - if let Some(expected) = expected_hex - && dest_path.exists() - { - // Avoid moving `hash_type` (non-Copy) so it can be reused for mirror retries. - let computed = match &hash_type { - ShaType::Sha256 => compute_sha256_streaming(dest_path).await, - ShaType::Sha512 => compute_sha512_streaming(dest_path).await, - }; - - if let Ok(h) = computed - && h.eq_ignore_ascii_case(expected) - { - debug!("Using cached file at {}", dest_path.display()); - return Ok((h, "(cached)".to_string())); - } - } - - info!("Trying mirror {}/{}", idx + 1, urls.len()); - debug!("Download attempt {}/{}: {}", idx + 1, urls.len(), url); - match download_with_hash_impl( - url, - dest_path, - hash_type.clone(), - Some((std::time::Duration::from_secs(90), 32 * 1024 * 1024)), - ) - .await - { - Ok(h) => { - if let Some(expected) = expected_hex - && !h.eq_ignore_ascii_case(expected) - { - warn!( - "hash mismatch from {}: expected {}, got {}", - url, expected, h - ); - last_err = Some(anyhow::anyhow!( - "hash mismatch from {}: expected {}, got {}", - url, - expected, - h - )); - // continue to next mirror - continue; - } - return Ok((h, url.clone())); - } - Err(e) => { - warn!("download failed for {}: {:#}", url, e); - last_err = Some(e); - // next mirror - } - } - } - - Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all download mirrors failed"))) -} - -/// Download or copy file from ImageSource with hash verification -async fn download_or_copy_with_hash( +/// Materialize a source file into `dest` and verify it against the expected hash. +async fn materialize_source_with_hash( source: &ImageSource, dest: &PathBuf, expected_hash: &str, @@ -873,7 +467,6 @@ async fn download_or_copy_with_hash( ) -> Result<()> { match source { ImageSource::Url(url) => { - // Reuse cached download if it matches, otherwise overwrite. if dest.exists() { let existing = match &hash_type { ShaType::Sha256 => compute_sha256_streaming(dest).await, @@ -886,9 +479,9 @@ async fn download_or_copy_with_hash( } } - let computed = download_with_hash(url, dest, hash_type.clone()).await?; + let computed = stream_download_with_hash(url, dest, hash_type.clone()).await?; anyhow::ensure!( - computed.to_lowercase() == expected_hash.to_lowercase(), + computed.eq_ignore_ascii_case(expected_hash), "hash mismatch: expected {}, got {}", expected_hash, computed @@ -904,7 +497,7 @@ async fn download_or_copy_with_hash( }; anyhow::ensure!( - computed.to_lowercase() == expected_hash.to_lowercase(), + computed.eq_ignore_ascii_case(expected_hash), "hash mismatch: expected {}, got {}", expected_hash, computed @@ -2246,44 +1839,7 @@ pub struct Debian {} impl ImageAction for Debian { async fn download(&self, name: &str) -> Result<()> { - let checksums_url = "https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS"; - let checksums_text = reqwest::get(checksums_url) - .await - .with_context(|| format!("failed to download SHA512SUMS from {}", checksums_url))? - .text() - .await - .with_context(|| format!("failed to read SHA512SUMS text from {}", checksums_url))?; - - let target_filename = format!("{}.qcow2", name); - let expected_sha512 = find_sha512_for_file(&checksums_text, &target_filename) - .with_context(|| { - format!( - "failed to find SHA512 checksum entry for {} in remote SHA512SUMS file", - target_filename - ) - })?; - - let dirs = QleanDirs::new()?; - let image_path = dirs.images.join(name).join(&target_filename); - - let download_url = format!( - "https://cloud.debian.org/images/cloud/trixie/latest/{}.qcow2", - name - ); - - // Single-pass download + hash computation - let computed_sha512 = - download_with_hash(&download_url, &image_path, ShaType::Sha512).await?; - - // Verify the downloaded file matches the expected checksum - anyhow::ensure!( - computed_sha512.to_lowercase() == expected_sha512.to_lowercase(), - "downloaded image checksum mismatch: expected {}, got {}", - expected_sha512, - computed_sha512 - ); - - Ok(()) + download_remote_image(name, Distro::Debian).await } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { @@ -2310,26 +1866,7 @@ pub struct Ubuntu {} impl ImageAction for Ubuntu { async fn download(&self, name: &str) -> Result<()> { - let dirs = QleanDirs::new()?; - let image_dir = dirs.images.join(name); - - let resolved = resolve_ubuntu_noble_cloudimg().await?; - debug!( - "Resolved Ubuntu cloud image from {}: disk={}", - resolved.base_url, resolved.disk_name - ); - - let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - let qcow2_url = join_url(&resolved.base_url, &resolved.disk_name); - download_or_copy_with_hash( - &ImageSource::Url(qcow2_url), - &qcow2_path, - &resolved.disk_sha256, - ShaType::Sha256, - ) - .await?; - - Ok(()) + download_remote_image(name, Distro::Ubuntu).await } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { @@ -2356,31 +1893,7 @@ pub struct Fedora {} impl ImageAction for Fedora { async fn download(&self, name: &str) -> Result<()> { - let dirs = QleanDirs::new()?; - let image_dir = dirs.images.join(name); - - let resolved = resolve_latest_fedora_cloud_qcow2().await?; - debug!( - "Resolved Fedora Cloud qcow2 (sha256={}): {} mirror(s)", - resolved.sha256, - resolved.urls.len() - ); - - let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - let (_hash, used_url) = download_with_hash_multi( - &resolved.urls, - &qcow2_path, - ShaType::Sha256, - Some(&resolved.sha256), - ) - .await - .with_context(|| "failed to download Fedora cloud image from all mirrors")?; - debug!( - "Fedora cloud image downloaded successfully from {}", - used_url - ); - - Ok(()) + download_remote_image(name, Distro::Fedora).await } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { @@ -2407,28 +1920,7 @@ pub struct Arch {} impl ImageAction for Arch { async fn download(&self, name: &str) -> Result<()> { - let dirs = QleanDirs::new()?; - let image_dir = dirs.images.join(name); - - let resolved = resolve_latest_arch_cloudimg().await?; - debug!( - "Resolved Arch cloudimg (sha256={}): {} mirror(s)", - resolved.sha256, - resolved.urls.len() - ); - - let qcow2_path = image_dir.join(format!("{}.qcow2", name)); - let (_hash, used_url) = download_with_hash_multi( - &resolved.urls, - &qcow2_path, - ShaType::Sha256, - Some(&resolved.sha256), - ) - .await - .with_context(|| "failed to download Arch cloud image from all mirrors")?; - debug!("Arch cloud image downloaded successfully from {}", used_url); - - Ok(()) + download_remote_image(name, Distro::Arch).await } async fn extract(&self, name: &str) -> Result<(PathBuf, PathBuf)> { @@ -2468,7 +1960,7 @@ impl ImageAction for Custom { // Download main image file let image_path = image_dir.join(format!("{}.qcow2", name)); - download_or_copy_with_hash( + materialize_source_with_hash( &self.config.image_source, &image_path, &self.config.image_hash, @@ -2481,7 +1973,7 @@ impl ImageAction for Custom { (&self.config.kernel_source, &self.config.kernel_hash) { let kernel_path = image_dir.join("vmlinuz"); - download_or_copy_with_hash( + materialize_source_with_hash( kernel_src, &kernel_path, kernel_hash, @@ -2495,7 +1987,7 @@ impl ImageAction for Custom { (&self.config.initrd_source, &self.config.initrd_hash) { let initrd_path = image_dir.join("initrd.img"); - download_or_copy_with_hash( + materialize_source_with_hash( initrd_src, &initrd_path, initrd_hash, @@ -2732,12 +2224,12 @@ mod tests { } #[test] - fn test_find_sha512_for_exact_filename() { + fn test_find_hash_for_exact_filename() { let checksums = "\ 748f52b959f63352e1e121508cedeae2e66d3e90be00e6420a0b8b9f14a0f84dc54ed801fb5be327866876268b808543465b1613c8649efeeb5f987ff9df1549 debian-13-generic-amd64.json \ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65 debian-13-generic-amd64.qcow2"; - let result = find_sha512_for_file(checksums, "debian-13-generic-amd64.qcow2"); + let result = find_hash_for_file(checksums, "debian-13-generic-amd64.qcow2"); assert_eq!( result, Some("f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea67709154d84220059672758508afbb0691c41ba8aa6d76818d89d65".to_string()) @@ -2762,6 +2254,43 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 assert_eq!(decoded, config); } + #[test] + fn test_image_sources_config_toml_parse() { + let config: ImageSourcesConfig = toml::from_str( + r#" +[debian] +image_url = "https://example.com/debian.qcow2" +checksum_url = "https://example.com/SHA512SUMS" +checksum_entry = "debian.qcow2" +checksum_type = "Sha512" + +[ubuntu] +image_url = "https://example.com/ubuntu.img" +checksum_url = "https://example.com/SHA256SUMS" +checksum_entry = "ubuntu.img" +checksum_type = "Sha256" + +[fedora] +image_url = "https://example.com/fedora.qcow2" +checksum_url = "https://example.com/CHECKSUM" +checksum_entry = "fedora.qcow2" +checksum_type = "Sha256" + +[arch] +image_url = "https://example.com/arch.qcow2" +checksum_url = "https://example.com/arch.SHA256" +checksum_entry = "arch.qcow2" +checksum_type = "Sha256" +"#, + ) + .unwrap(); + + assert_eq!(config.debian.checksum_type, ShaType::Sha512); + assert_eq!(config.ubuntu.checksum_entry, "ubuntu.img"); + assert_eq!(config.fedora.image_url, "https://example.com/fedora.qcow2"); + assert_eq!(config.arch.checksum_url, "https://example.com/arch.SHA256"); + } + #[test] fn test_find_hash_for_file_formats() { // Format 1: " " @@ -2786,17 +2315,6 @@ f0442f3cd0087a609ecd5241109ddef0cbf4a1e05372e13d82c97fc77b35b2d8ecff85aea6770915 ); } - #[test] - fn test_parse_fedora_cloud_listing() { - let html = r#" - Fedora-Cloud-43-1.6-x86_64-CHECKSUM - Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2 - "#; - let (qcow2, checksum) = parse_fedora_cloud_listing(html, "43").unwrap(); - assert_eq!(qcow2, "Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2"); - assert_eq!(checksum, "Fedora-Cloud-43-1.6-x86_64-CHECKSUM"); - } - #[tokio::test] async fn test_streaming_sha256_empty_file() -> Result<()> { let tmp = tempfile::NamedTempFile::new()?; diff --git a/src/utils.rs b/src/utils.rs index 7187510..b017ee3 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -207,8 +207,8 @@ async fn ensure_network() -> Result<()> { .output() .await .context("failed to execute virsh to check qlean network")?; - let stdout = String::from_utf8_lossy(&output.stdout); - let all_networks = stdout.lines().collect::>(); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let all_networks = stdout.lines().map(str::to_owned).collect::>(); let net_exists = all_networks.contains("qlean"); let output = tokio::process::Command::new("virsh") @@ -219,8 +219,8 @@ async fn ensure_network() -> Result<()> { .output() .await .context("failed to execute virsh to check qlean network")?; - let stdout = String::from_utf8_lossy(&output.stdout); - let active_networks = stdout.lines().collect::>(); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let active_networks = stdout.lines().map(str::to_owned).collect::>(); let net_active = active_networks.contains("qlean"); if !net_exists { diff --git a/tests/arch_image.rs b/tests/arch_image.rs index 5c2d163..7c30937 100644 --- a/tests/arch_image.rs +++ b/tests/arch_image.rs @@ -19,9 +19,7 @@ use logging::tracing_subscriber_init; async fn test_arch_image_startup_flow() -> Result<()> { tracing_subscriber_init(); - if !ensure_vm_test_env()? { - return Ok(()); - } + ensure_vm_test_env()?; eprintln!("INFO: host checks passed"); diff --git a/tests/fedora_image.rs b/tests/fedora_image.rs index b0f5775..1620877 100644 --- a/tests/fedora_image.rs +++ b/tests/fedora_image.rs @@ -19,9 +19,7 @@ use logging::tracing_subscriber_init; async fn test_fedora_image_startup_flow() -> Result<()> { tracing_subscriber_init(); - if !ensure_vm_test_env()? { - return Ok(()); - } + ensure_vm_test_env()?; eprintln!("INFO: host checks passed"); diff --git a/tests/support/e2e.rs b/tests/support/e2e.rs index 33667fb..77f30b7 100644 --- a/tests/support/e2e.rs +++ b/tests/support/e2e.rs @@ -4,18 +4,6 @@ use anyhow::{Context, Result, bail}; const QLEAN_BRIDGE_NAME: &str = "qlbr0"; -/// Return true if slow E2E tests are explicitly enabled. -/// -/// These tests are intentionally opt-in because they download large images and -/// boot real VMs. -pub fn e2e_enabled() -> bool { - matches!( - std::env::var("QLEAN_RUN_E2E").as_deref(), - Ok("1") | Ok("true") | Ok("yes") - ) -} - -/// Return `true` if a command exists on PATH. fn has_cmd(cmd: &str) -> bool { match Command::new(cmd).arg("--version").output() { Ok(_) => true, @@ -24,7 +12,6 @@ fn has_cmd(cmd: &str) -> bool { } } -/// Validate mandatory host commands for E2E execution. fn ensure_vm_test_commands() -> Result<()> { if !has_cmd("virsh") { bail!("Missing required command: virsh (libvirt-clients)."); @@ -35,7 +22,6 @@ fn ensure_vm_test_commands() -> Result<()> { Ok(()) } -/// Validate the libvirt system URI before running slow tests. fn ensure_libvirt_system() -> Result<()> { let output = Command::new("virsh") .args(["-c", "qemu:///system", "list", "--all"]) @@ -69,27 +55,16 @@ fn bridge_conf_allows(bridge: &str) -> bool { .any(|line| line == "allow all" || line == format!("allow {bridge}")) } -/// Validate E2E host prerequisites. -/// -/// Returns `Ok(false)` when E2E tests are not enabled (so callers can skip -/// without failing CI). -pub fn ensure_vm_test_env() -> Result { - if !e2e_enabled() { - eprintln!("SKIP: E2E VM tests are disabled. Set QLEAN_RUN_E2E=1 to run them."); - return Ok(false); - } - +pub fn ensure_vm_test_env() -> Result<()> { ensure_vm_test_commands()?; ensure_libvirt_system()?; - // vhost-vsock is required for Qlean's SSH transport. if !Path::new("/dev/vhost-vsock").exists() { bail!( "Missing required device: /dev/vhost-vsock (vhost-vsock is required; no TCP fallback)." ); } - // Qlean expects the libvirt-managed qlbr0 bridge and an allow rule for qemu-bridge-helper. if !has_iface(QLEAN_BRIDGE_NAME) { bail!( "Missing required bridge interface '{}'. Hint: ensure the libvirt network 'qlean' is active (virsh -c qemu:///system net-start qlean).", @@ -118,5 +93,5 @@ Then re-run the test."#, ); } - Ok(true) + Ok(()) } diff --git a/tests/ubuntu_image.rs b/tests/ubuntu_image.rs index e9c214a..fdb18d3 100644 --- a/tests/ubuntu_image.rs +++ b/tests/ubuntu_image.rs @@ -19,9 +19,7 @@ use logging::tracing_subscriber_init; async fn test_ubuntu_image_creation() -> Result<()> { tracing_subscriber_init(); - if !ensure_vm_test_env()? { - return Ok(()); - } + ensure_vm_test_env()?; eprintln!("INFO: host checks passed");