diff --git a/monitor.toml.example b/monitor.toml.example index 4827aeb8..7da6c822 100644 --- a/monitor.toml.example +++ b/monitor.toml.example @@ -52,6 +52,12 @@ base_image_max_age = 86400 # Uncomment to skip cached Servo repo updates. # dont_update_cached_servo_repo = true +# Create libvirt guests for profile templates as “ci-template-.0”. Namespace must not be used by anything else! +# libvirt_template_guest_prefix = "ci-template" + +# Create libvirt guests for image rebuilds as “ci-rebuild-.0”. Namespace must not be used by anything else! +# libvirt_rebuild_guest_prefix = "ci-rebuild" + # Create libvirt guests for runners as “ci-runner-.0”. Namespace must not be used by anything else! # libvirt_runner_guest_prefix = "ci-runner" diff --git a/monitor/settings/src/lib.rs b/monitor/settings/src/lib.rs index dae6c75f..db453684 100644 --- a/monitor/settings/src/lib.rs +++ b/monitor/settings/src/lib.rs @@ -80,6 +80,8 @@ pub struct Toml { pub main_repo_path: String, base_image_max_age: u64, dont_update_cached_servo_repo: Option, + libvirt_template_guest_prefix: Option, + libvirt_rebuild_guest_prefix: Option, libvirt_runner_guest_prefix: Option, pub available_1g_hugepages: usize, pub available_normal_memory: MemorySize, @@ -242,6 +244,18 @@ impl Toml { self.queue_member.unwrap_or(false) } + pub fn libvirt_template_guest_prefix(&self) -> &str { + self.libvirt_template_guest_prefix + .as_deref() + .unwrap_or("ci-template") + } + + pub fn libvirt_rebuild_guest_prefix(&self) -> &str { + self.libvirt_rebuild_guest_prefix + .as_deref() + .unwrap_or("ci-rebuild") + } + pub fn libvirt_runner_guest_prefix(&self) -> &str { self.libvirt_runner_guest_prefix .as_deref() diff --git a/monitor/settings/src/profile.rs b/monitor/settings/src/profile.rs index 9af58f1f..3680fc96 100644 --- a/monitor/settings/src/profile.rs +++ b/monitor/settings/src/profile.rs @@ -1,3 +1,4 @@ +use jane_eyre::eyre::{self, OptionExt}; use serde::{Deserialize, Serialize}; use crate::{TOML, units::MemorySize}; @@ -20,8 +21,24 @@ pub enum ImageType { } impl Profile { - pub fn profile_guest_name(&self) -> String { - format!("{}", self.profile_name) + pub fn snapshot_path_slug(&self, snapshot_name: &str) -> String { + format!("{}@{snapshot_name}", self.profile_name) + } + + pub fn template_guest_name(&self, snapshot_name: &str) -> String { + format!( + "{}-{}@{snapshot_name}", + TOML.libvirt_template_guest_prefix(), + self.profile_name + ) + } + + pub fn rebuild_guest_name(&self, snapshot_name: &str) -> String { + format!( + "{}-{}@{snapshot_name}", + TOML.libvirt_rebuild_guest_prefix(), + self.profile_name + ) } pub fn runner_guest_name(&self, id: usize) -> String { @@ -33,3 +50,23 @@ impl Profile { ) } } + +pub fn parse_template_guest_name(template_guest_name: &str) -> eyre::Result<(&str, &str)> { + let prefix = format!("{}-", TOML.libvirt_template_guest_prefix()); + let (profile_key, snapshot_name) = template_guest_name + .strip_prefix(&prefix) + .ok_or_eyre("Failed to strip template guest prefix")? + .split_once("@") + .ok_or_eyre("Failed to split snapshot path slug into profile key and snapshot name")?; + Ok((profile_key, snapshot_name)) +} + +pub fn parse_rebuild_guest_name(rebuild_guest_name: &str) -> eyre::Result<(&str, &str)> { + let prefix = format!("{}-", TOML.libvirt_rebuild_guest_prefix()); + let (profile_key, snapshot_name) = rebuild_guest_name + .strip_prefix(&prefix) + .ok_or_eyre("Failed to strip rebuild guest prefix")? + .split_once("@") + .ok_or_eyre("Failed to split snapshot path slug into profile key and snapshot name")?; + Ok((profile_key, snapshot_name)) +} diff --git a/monitor/src/image.rs b/monitor/src/image.rs index 64f62e3c..cf93c90b 100644 --- a/monitor/src/image.rs +++ b/monitor/src/image.rs @@ -4,10 +4,10 @@ pub mod windows10; use core::str; use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, ffi::OsStr, - fs::{create_dir_all, read_dir, read_link, remove_file, set_permissions, File}, - io::{ErrorKind, Seek, Write}, + fs::{create_dir_all, read_dir, remove_file, set_permissions, File}, + io::{Seek, Write}, mem::take, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, @@ -17,13 +17,17 @@ use std::{ use bytesize::ByteSize; use chrono::{SecondsFormat, Utc}; -use cmd_lib::{run_cmd, spawn_with_output}; +use cmd_lib::{run_cmd, run_fun, spawn_with_output}; use jane_eyre::eyre::{self, bail, OptionExt}; -use settings::{profile::Profile, TOML}; -use tracing::{debug, error, info, trace, warn}; +use settings::{ + profile::{parse_rebuild_guest_name, parse_template_guest_name, Profile}, + TOML, +}; +use tracing::{debug, error, info, warn}; use crate::{ - policy::{base_images_path, runner_images_path, Policy}, + libvirt::{list_rebuild_guests, list_template_guests}, + policy::{runner_images_path, template_or_rebuild_images_path, Policy}, shell::{log_output_as_info, reflink_or_copy_with_warning}, }; @@ -37,10 +41,39 @@ pub struct Rebuilds { struct Rebuild { thread: JoinHandle>, snapshot_name: String, + guest_name: String, } impl Rebuilds { pub fn run(&mut self, policy: &mut Policy) -> eyre::Result<()> { + // Clean up any dangling resources from past rebuilds. + let current_known_rebuild_guest_names = self + .rebuild_guest_names() + .into_iter() + .map(|(_key, guest_name)| guest_name) + .collect::>(); + for rebuild_guest_name in list_rebuild_guests()? { + if !current_known_rebuild_guest_names.contains(&rebuild_guest_name) { + undefine_libvirt_guest(&rebuild_guest_name)?; + let (profile_key, snapshot_name) = + match parse_rebuild_guest_name(&rebuild_guest_name) { + Ok(result) => result, + Err(error) => { + warn!(?error, "Failed to clean up bad image files"); + continue; + } + }; + let Some(profile) = policy.profile(profile_key) else { + warn!( + ?profile_key, + "Failed to clean up bad image files: Unknown profile" + ); + continue; + }; + delete_template(profile, snapshot_name)?; + } + } + let mut profiles_needing_rebuild = BTreeMap::default(); let mut cached_servo_repo_was_just_updated = false; @@ -118,6 +151,7 @@ impl Rebuilds { Rebuild { thread, snapshot_name: snapshot_name.clone(), + guest_name: profile.rebuild_guest_name(&snapshot_name), }, ); } @@ -142,6 +176,13 @@ impl Rebuilds { Ok(()) } + + pub fn rebuild_guest_names(&self) -> BTreeMap { + self.rebuilds + .iter() + .map(|(profile_key, rebuild)| (profile_key.clone(), rebuild.guest_name.clone())) + .collect() + } } #[tracing::instrument] @@ -170,10 +211,9 @@ fn rebuild_with_rust( ) -> Result<(), eyre::Error> { info!(?snapshot_name, "Starting image rebuild"); - let base_images_path = create_base_images_dir(&profile)?; - undefine_libvirt_guest(&profile.profile_guest_name())?; + let base_images_path = create_template_or_rebuild_images_dir(&profile)?; - match match match &*profile.profile_name { + match match &*profile.profile_name { "servo-macos13" => macos13::rebuild( &base_images_path, &profile, @@ -233,76 +273,27 @@ fn rebuild_with_rust( other => todo!("Rebuild not yet implemented: {other}"), } { result @ Ok(()) => { - prune_images(&profile)?; + prune_templates(&profile)?; result } Err(error) => { warn!(?error, "Image rebuild error"); - delete_image(&profile, snapshot_name); + delete_template(&profile, snapshot_name)?; Err(error) } - } { - result => { - // After a rebuild attempt, the base guest should always use the symlinks to the last known good image. - // On success, these will be the new image files. On failure, these will be the old image files. - match &*profile.profile_name { - "servo-macos13" => { - macos13::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-macos14" => { - macos13::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-macos15" => { - macos13::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-ubuntu2204" => { - ubuntu2204::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-ubuntu2204-bench" => { - ubuntu2204::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-ubuntu2204-rust" => { - ubuntu2204::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-ubuntu2204-wpt" => { - ubuntu2204::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - "servo-windows10" => { - windows10::redefine_base_guest_with_symlinks(&base_images_path, &profile)?; - } - other => { - todo!("Redefining base guest with symlinks not implemented: {other}") - } - } - result - } - } -} - -pub fn prune_images(profile: &Profile) -> eyre::Result<()> { - match &*profile.profile_name { - "servo-macos13" => macos13::prune_images(profile), - "servo-macos14" => macos13::prune_images(profile), - "servo-macos15" => macos13::prune_images(profile), - "servo-ubuntu2204" => ubuntu2204::prune_images(profile), - "servo-ubuntu2204-bench" => ubuntu2204::prune_images(profile), - "servo-ubuntu2204-rust" => ubuntu2204::prune_images(profile), - "servo-ubuntu2204-wpt" => ubuntu2204::prune_images(profile), - "servo-windows10" => windows10::prune_images(profile), - other => todo!("Image pruning not yet implemented: {other}"), } } -pub fn delete_image(profile: &Profile, snapshot_name: &str) { +pub fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::Result<()> { match &*profile.profile_name { - "servo-macos13" => macos13::delete_image(profile, snapshot_name), - "servo-macos14" => macos13::delete_image(profile, snapshot_name), - "servo-macos15" => macos13::delete_image(profile, snapshot_name), - "servo-ubuntu2204" => ubuntu2204::delete_image(profile, snapshot_name), - "servo-ubuntu2204-bench" => ubuntu2204::delete_image(profile, snapshot_name), - "servo-ubuntu2204-rust" => ubuntu2204::delete_image(profile, snapshot_name), - "servo-ubuntu2204-wpt" => ubuntu2204::delete_image(profile, snapshot_name), - "servo-windows10" => windows10::delete_image(profile, snapshot_name), + "servo-macos13" => macos13::delete_template(profile, snapshot_name), + "servo-macos14" => macos13::delete_template(profile, snapshot_name), + "servo-macos15" => macos13::delete_template(profile, snapshot_name), + "servo-ubuntu2204" => ubuntu2204::delete_template(profile, snapshot_name), + "servo-ubuntu2204-bench" => ubuntu2204::delete_template(profile, snapshot_name), + "servo-ubuntu2204-rust" => ubuntu2204::delete_template(profile, snapshot_name), + "servo-ubuntu2204-wpt" => ubuntu2204::delete_template(profile, snapshot_name), + "servo-windows10" => windows10::delete_template(profile, snapshot_name), other => todo!("Image pruning not yet implemented: {other}"), } } @@ -323,20 +314,35 @@ pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Resu pub fn create_runner( profile: &Profile, + snapshot_name: &str, runner_guest_name: &str, runner_id: usize, ) -> eyre::Result { match &*profile.profile_name { - "servo-macos13" => macos13::create_runner(profile, runner_guest_name, runner_id), - "servo-macos14" => macos13::create_runner(profile, runner_guest_name, runner_id), - "servo-macos15" => macos13::create_runner(profile, runner_guest_name, runner_id), - "servo-ubuntu2204" => ubuntu2204::create_runner(profile, runner_guest_name, runner_id), + "servo-macos13" => { + macos13::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } + "servo-macos14" => { + macos13::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } + "servo-macos15" => { + macos13::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } + "servo-ubuntu2204" => { + ubuntu2204::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } "servo-ubuntu2204-bench" => { - ubuntu2204::create_runner(profile, runner_guest_name, runner_id) + ubuntu2204::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } + "servo-ubuntu2204-rust" => { + ubuntu2204::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } + "servo-ubuntu2204-wpt" => { + ubuntu2204::create_runner(profile, snapshot_name, runner_guest_name, runner_id) + } + "servo-windows10" => { + windows10::create_runner(profile, snapshot_name, runner_guest_name, runner_id) } - "servo-ubuntu2204-rust" => ubuntu2204::create_runner(profile, runner_guest_name, runner_id), - "servo-ubuntu2204-wpt" => ubuntu2204::create_runner(profile, runner_guest_name, runner_id), - "servo-windows10" => windows10::create_runner(profile, runner_guest_name, runner_id), other => todo!("Runner creation not yet implemented: {other}"), } } @@ -359,8 +365,8 @@ pub fn destroy_runner( } } -pub(self) fn create_base_images_dir(profile: &Profile) -> eyre::Result { - let base_images_path = base_images_path(profile); +pub(self) fn create_template_or_rebuild_images_dir(profile: &Profile) -> eyre::Result { + let base_images_path = template_or_rebuild_images_path(profile); debug!(?base_images_path, "Creating base images subdirectory"); create_dir_all(&base_images_path)?; @@ -375,67 +381,51 @@ pub(self) fn create_runner_images_dir() -> eyre::Result { Ok(runner_images_path) } -pub(self) fn prune_base_image_files(profile: &Profile, prefix: &str) -> eyre::Result<()> { - let base_images_path = base_images_path(profile); - info!(?base_images_path, "Pruning base image files"); - create_dir_all(&base_images_path)?; - - let matches_prefix = |target: &str| { - target - .strip_prefix(prefix) - .and_then(move |f| f.strip_prefix("@")) - .is_some() - }; - - // Check the symlink target for the most recent successful build. - let mut symlink = match read_link(base_images_path.join(prefix)) { - Ok(result) => Some(result), - Err(error) if error.kind() == ErrorKind::NotFound => None, - Err(other) => Err(other)?, - }; - - // The symlink target should be of the form `{prefix}@{snapshot_name}`. - if let Some(target) = symlink.as_ref() { - let target = target.to_str().ok_or_eyre("Unsupported path")?; - if !matches_prefix(target) { - warn!(target, "Unexpected symlink target format"); - symlink.take(); +pub(self) fn prune_templates(profile: &Profile) -> eyre::Result<()> { + // Build a sorted list of template guest names for this profile. + let mut snapshot_names = vec![]; + for template_guest_name in list_template_guests()? { + if let Ok((profile_key, snapshot_name)) = parse_template_guest_name(&template_guest_name) { + if profile_key == profile.profile_name { + snapshot_names.push(snapshot_name.to_owned()); + } + } else { + undefine_libvirt_guest(&template_guest_name)?; } } + snapshot_names.sort(); + + // Delete all of those templates, except the three most recent. + // Since the snapshot names are RFC 3339 timestamps, we can use the sorted order (until year 10000). + let keep_snapshots = snapshot_names.clone().into_iter().rev().take(3); + let delete_snapshots = snapshot_names.iter().rev().skip(3); + for snapshot_name in delete_snapshots { + delete_template(profile, snapshot_name)?; + } + + // Now delete any files that are not associated with a known snapshot. + let keep_snapshots = keep_snapshots.collect::>(); + let base_images_path = template_or_rebuild_images_path(profile); + info!(?base_images_path, "Pruning base image files"); + create_dir_all(&base_images_path)?; - // Build a sorted list of filenames starting with `{prefix}@`. - let mut filenames = vec![]; for entry in read_dir(&base_images_path)? { let filename = entry?.file_name(); let filename = filename.to_str().ok_or_eyre("Unsupported path")?; - if matches_prefix(filename) { - filenames.push(filename.to_owned()); - } - } - filenames.sort(); - - // Delete all of those files, except the most recent successful build and up to three builds before that. - // Since the snapshot names are RFC 3339 timestamps, we can use the sorted order (until year 10000). - // FIXME: past images may be bad, if the monitor was restarted during an image build. - let mut filenames = filenames.iter().rev(); - while let Some(filename) = filenames.next() { - if let Some(target) = symlink.as_ref() { - if Path::new(filename) == target { - trace!(filename, "Keeping"); - filenames.next(); - filenames.next(); - filenames.next(); - continue; + if let Some((_base, snapshot_name)) = filename.split_once("@") { + if !keep_snapshots.contains(snapshot_name) { + delete_template_or_rebuild_image_file(profile, filename); } + } else { + delete_template_or_rebuild_image_file(profile, filename); } - delete_base_image_file(profile, filename); } Ok(()) } -pub(self) fn delete_base_image_file(profile: &Profile, filename: &str) { - let base_images_path = base_images_path(profile); +pub(self) fn delete_template_or_rebuild_image_file(profile: &Profile, filename: &str) { + let base_images_path = template_or_rebuild_images_path(profile); let path = base_images_path.join(filename); info!(?path, "Deleting"); if let Err(error) = remove_file(&path) { @@ -537,10 +527,20 @@ pub fn start_libvirt_guest(guest_name: &str) -> eyre::Result<()> { pub(self) fn wait_for_guest(guest_name: &str, timeout: Duration) -> eyre::Result<()> { let timeout = timeout.as_secs(); - info!("Waiting for guest to shut down (max {timeout} seconds)"); // normally ~37 seconds + info!("Waiting for guest to shut down (max {timeout} seconds)"); if !run_cmd!(time virsh event --timeout $timeout -- $guest_name lifecycle).is_ok() { bail!("`virsh event` failed or timed out!"); } + for _ in 0..100 { + if run_fun!(virsh domstate -- $guest_name)?.trim_ascii() == "shut off" { + return Ok(()); + } + } + + bail!("Guest did not shut down as expected") +} +pub(self) fn rename_guest(old_guest_name: &str, new_guest_name: &str) -> eyre::Result<()> { + run_cmd!(virsh domrename -- $old_guest_name $new_guest_name)?; Ok(()) } diff --git a/monitor/src/image/macos13.rs b/monitor/src/image/macos13.rs index 76c522ba..e4885f53 100644 --- a/monitor/src/image/macos13.rs +++ b/monitor/src/image/macos13.rs @@ -12,14 +12,15 @@ use jane_eyre::eyre::OptionExt; use settings::profile::Profile; use tracing::warn; -use crate::image::create_base_images_dir; +use crate::data::get_profile_data_path; use crate::image::create_runner_images_dir; -use crate::image::delete_base_image_file; +use crate::image::delete_template_or_rebuild_image_file; use crate::image::libvirt_change_media; -use crate::image::prune_base_image_files; +use crate::image::rename_guest; use crate::image::undefine_libvirt_guest; use crate::image::CdromImage; use crate::policy::runner_image_path; +use crate::policy::template_or_rebuild_image_path; use crate::shell::atomic_symlink; use crate::shell::log_output_as_info; use crate::shell::reflink_or_copy_with_warning; @@ -36,10 +37,11 @@ pub(super) fn rebuild( wait_duration: Duration, ) -> eyre::Result<()> { let base_images_path = base_images_path.as_ref(); - let profile_guest_name = &profile.profile_guest_name(); + let profile_name = &profile.profile_name; + let snapshot_path_slug = &profile.snapshot_path_slug(snapshot_name); + let rebuild_guest_name = &profile.rebuild_guest_name(snapshot_name); - let base_image_symlink_path = base_images_path.join(format!("base.img")); - let initial_contents_path = format!("/var/lib/libvirt/images/{profile_guest_name}.clean.img"); + let initial_contents_path = format!("/var/lib/libvirt/images/{profile_name}.clean.img"); let base_image_path = create_disk_image( base_images_path, snapshot_name, @@ -47,64 +49,49 @@ pub(super) fn rebuild( Path::new(&initial_contents_path), )?; - define_base_guest(profile, &base_image_path, &[])?; + define_base_guest(profile, snapshot_name, &base_image_path, &[])?; let ovmf_vars_clean_path = - format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{profile_guest_name}.clean.fd"); + format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{profile_name}.clean.fd"); let ovmf_vars_path = - format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{profile_guest_name}.fd"); + format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{snapshot_path_slug}.fd"); copy(ovmf_vars_clean_path, ovmf_vars_path)?; - start_libvirt_guest(profile_guest_name)?; - wait_for_guest(profile_guest_name, wait_duration)?; + start_libvirt_guest(rebuild_guest_name)?; + wait_for_guest(rebuild_guest_name, wait_duration)?; - let base_image_filename = Path::new( - base_image_path - .file_name() - .expect("Guaranteed by make_disk_image"), - ); - atomic_symlink(base_image_filename, base_image_symlink_path)?; - - Ok(()) -} - -pub(super) fn redefine_base_guest_with_symlinks( - base_images_path: impl AsRef, - profile: &Profile, -) -> Result<(), eyre::Error> { - let base_images_path = base_images_path.as_ref(); - let base_image_symlink_path = base_images_path.join(format!("base.img")); - undefine_libvirt_guest(&profile.profile_guest_name())?; - define_base_guest(profile, &base_image_symlink_path, &[])?; + let template_guest_name = &profile.template_guest_name(snapshot_name); + rename_guest(rebuild_guest_name, template_guest_name)?; + let snapshot_symlink_path = + get_profile_data_path(&profile.profile_name, Path::new("snapshot"))?; + atomic_symlink(snapshot_name, snapshot_symlink_path)?; Ok(()) } fn define_base_guest( profile: &Profile, + snapshot_name: &str, base_image_path: &dyn AsRef, cdrom_images: &[CdromImage], ) -> eyre::Result<()> { - let profile_guest_name = &profile.profile_guest_name(); + let clean_guest_name = &format!("{}.clean", profile.profile_name); + let rebuild_guest_name = &profile.rebuild_guest_name(snapshot_name); let base_image_path = base_image_path .as_ref() .to_str() .ok_or_eyre("Unsupported path")?; // Clone the hand-made clean guest, since we can’t yet automate the macOS install - run_cmd!(virt-clone --preserve-data --check path_in_use=off -o $profile_guest_name.clean -n $profile_guest_name --nvram /var/lib/libvirt/images/OSX-KVM/OVMF_VARS.$profile_guest_name.fd --skip-copy sda -f $base_image_path --skip-copy sdc)?; - libvirt_change_media(profile_guest_name, cdrom_images)?; + run_cmd!(virt-clone --preserve-data --check path_in_use=off -o $clean_guest_name -n $rebuild_guest_name --nvram /var/lib/libvirt/images/OSX-KVM/OVMF_VARS.$clean_guest_name.fd --skip-copy sda -f $base_image_path --skip-copy sdc)?; + libvirt_change_media(rebuild_guest_name, cdrom_images)?; Ok(()) } -pub(super) fn prune_images(profile: &Profile) -> eyre::Result<()> { - prune_base_image_files(profile, "base.img")?; - +pub(super) fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::Result<()> { + undefine_libvirt_guest(&profile.template_guest_name(snapshot_name))?; + delete_template_or_rebuild_image_file(profile, &format!("base.img@{snapshot_name}")); Ok(()) } -pub(super) fn delete_image(profile: &Profile, snapshot_name: &str) { - delete_base_image_file(profile, &format!("base.img@{snapshot_name}")); -} - pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { monitor::github::register_runner( runner_guest_name, @@ -115,27 +102,28 @@ pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Resu pub fn create_runner( profile: &Profile, + snapshot_name: &str, runner_guest_name: &str, runner_id: usize, ) -> eyre::Result { let pipe = || |reader| log_output_as_info(reader); - let profile_guest_name = &profile.profile_name; + let snapshot_path_slug = &profile.snapshot_path_slug(snapshot_name); + let template_guest_name = &profile.template_guest_name(snapshot_name); // Copy images in the monitor, not with `virt-clone --auto-clone --reflink`, // because the latter can’t be parallelised without causing errors. - let base_images_path = create_base_images_dir(profile)?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); + let template_base_img = template_or_rebuild_image_path(profile, snapshot_name, "base.img"); create_runner_images_dir()?; - let runner_base_image_path = runner_image_path(runner_id, "base.img"); - reflink_or_copy_with_warning(&base_image_symlink_path, &runner_base_image_path)?; + let runner_base_img = runner_image_path(runner_id, "base.img"); + reflink_or_copy_with_warning(&template_base_img, &runner_base_img)?; let ovmf_vars_base_path = - format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{profile_guest_name}.clean.fd"); + format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{snapshot_path_slug}.fd"); let ovmf_vars_path = format!("/var/lib/libvirt/images/OSX-KVM/OVMF_VARS.{runner_guest_name}.fd"); copy(ovmf_vars_base_path, ovmf_vars_path)?; - spawn_with_output!(virt-clone -o $profile_guest_name -n $runner_guest_name --nvram /var/lib/libvirt/images/OSX-KVM/OVMF_VARS.$runner_guest_name.fd --preserve-data --skip-copy sda -f $runner_base_image_path --skip-copy sdc 2>&1)?.wait_with_pipe(&mut pipe())?; + spawn_with_output!(virt-clone -o $template_guest_name -n $runner_guest_name --nvram /var/lib/libvirt/images/OSX-KVM/OVMF_VARS.$runner_guest_name.fd --preserve-data --skip-copy sda -f $runner_base_img --skip-copy sdc 2>&1)?.wait_with_pipe(&mut pipe())?; Ok(runner_guest_name.to_owned()) } diff --git a/monitor/src/image/ubuntu2204.rs b/monitor/src/image/ubuntu2204.rs index a5981720..96f4c99f 100644 --- a/monitor/src/image/ubuntu2204.rs +++ b/monitor/src/image/ubuntu2204.rs @@ -7,18 +7,18 @@ use bytesize::ByteSize; use cmd_lib::run_cmd; use cmd_lib::spawn_with_output; use jane_eyre::eyre; -use jane_eyre::eyre::OptionExt; use settings::profile::Profile; use tracing::info; use tracing::warn; use crate::data::get_profile_configuration_path; -use crate::image::create_base_images_dir; +use crate::data::get_profile_data_path; use crate::image::create_runner_images_dir; -use crate::image::delete_base_image_file; -use crate::image::prune_base_image_files; +use crate::image::delete_template_or_rebuild_image_file; +use crate::image::rename_guest; use crate::image::undefine_libvirt_guest; use crate::policy::runner_image_path; +use crate::policy::template_or_rebuild_image_path; use crate::shell::atomic_symlink; use crate::shell::log_output_as_info; use crate::shell::reflink_or_copy_with_warning; @@ -38,16 +38,14 @@ pub(super) fn rebuild( wait_duration: Duration, ) -> eyre::Result<()> { let base_images_path = base_images_path.as_ref(); - let profile_guest_name = &profile.profile_guest_name(); + let rebuild_guest_name = &profile.rebuild_guest_name(snapshot_name); let profile_configuration_path = get_profile_configuration_path(&profile, None)?; - let config_iso_symlink_path = base_images_path.join(format!("config.iso")); let config_iso_filename = format!("config.iso@{snapshot_name}"); let config_iso_path = base_images_path.join(&config_iso_filename); let config_iso_path = config_iso_path.to_str().expect("Unsupported path"); info!(config_iso_path, "Creating config image file"); run_cmd!(genisoimage -V CIDATA -R -f -o $config_iso_path $profile_configuration_path/user-data $profile_configuration_path/meta-data)?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); let os_image_path = IMAGE_DEPS_DIR .join("ubuntu2204") .join("jammy-server-cloudimg-amd64.raw"); @@ -60,53 +58,33 @@ pub(super) fn rebuild( define_base_guest( profile, + snapshot_name, &base_image_path, &[CdromImage::new("sda", config_iso_path)], )?; - start_libvirt_guest(profile_guest_name)?; - wait_for_guest(profile_guest_name, wait_duration)?; + start_libvirt_guest(rebuild_guest_name)?; + wait_for_guest(rebuild_guest_name, wait_duration)?; - let base_image_filename = Path::new( - base_image_path - .file_name() - .expect("Guaranteed by make_disk_image"), - ); - atomic_symlink(config_iso_filename, config_iso_symlink_path)?; - atomic_symlink(base_image_filename, base_image_symlink_path)?; - - Ok(()) -} - -pub(super) fn redefine_base_guest_with_symlinks( - base_images_path: impl AsRef, - profile: &Profile, -) -> Result<(), eyre::Error> { - let base_images_path = base_images_path.as_ref(); - let config_iso_symlink_path = base_images_path.join(format!("config.iso")); - let config_iso_symlink_path = config_iso_symlink_path - .to_str() - .ok_or_eyre("Unsupported path")?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); - undefine_libvirt_guest(&profile.profile_guest_name())?; - define_base_guest( - profile, - &base_image_symlink_path, - &[CdromImage::new("sda", &config_iso_symlink_path)], - )?; + let template_guest_name = &profile.template_guest_name(snapshot_name); + rename_guest(rebuild_guest_name, template_guest_name)?; + let snapshot_symlink_path = + get_profile_data_path(&profile.profile_name, Path::new("snapshot"))?; + atomic_symlink(snapshot_name, snapshot_symlink_path)?; Ok(()) } fn define_base_guest( profile: &Profile, + snapshot_name: &str, base_image_path: &dyn AsRef, cdrom_images: &[CdromImage], ) -> eyre::Result<()> { - let profile_guest_name = &profile.profile_guest_name(); + let rebuild_guest_name = &profile.rebuild_guest_name(snapshot_name); let guest_xml_path = get_profile_configuration_path(&profile, Path::new("guest.xml"))?; define_libvirt_guest( &profile.profile_name, - profile_guest_name, + rebuild_guest_name, guest_xml_path, &[&"-f", &base_image_path], cdrom_images, @@ -115,40 +93,35 @@ fn define_base_guest( Ok(()) } -pub(super) fn prune_images(profile: &Profile) -> eyre::Result<()> { - prune_base_image_files(profile, "config.iso")?; - prune_base_image_files(profile, "base.img")?; - +pub(super) fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::Result<()> { + undefine_libvirt_guest(&profile.template_guest_name(snapshot_name))?; + delete_template_or_rebuild_image_file(profile, &format!("config.iso@{snapshot_name}")); + delete_template_or_rebuild_image_file(profile, &format!("base.img@{snapshot_name}")); Ok(()) } -pub(super) fn delete_image(profile: &Profile, snapshot_name: &str) { - delete_base_image_file(profile, &format!("config.iso@{snapshot_name}")); - delete_base_image_file(profile, &format!("base.img@{snapshot_name}")); -} - pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { monitor::github::register_runner(runner_guest_name, &profile.github_runner_label, "/a") } pub fn create_runner( profile: &Profile, + snapshot_name: &str, runner_guest_name: &str, runner_id: usize, ) -> eyre::Result { let pipe = || |reader| log_output_as_info(reader); - let profile_guest_name = &profile.profile_guest_name(); + let template_guest_name = &profile.template_guest_name(snapshot_name); // Copy images in the monitor, not with `virt-clone --auto-clone --reflink`, // because the latter can’t be parallelised without causing errors. // TODO copy config.iso? - let base_images_path = create_base_images_dir(profile)?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); + let template_base_img = template_or_rebuild_image_path(profile, snapshot_name, "base.img"); create_runner_images_dir()?; - let runner_base_image_path = runner_image_path(runner_id, "base.img"); - reflink_or_copy_with_warning(&base_image_symlink_path, &runner_base_image_path)?; + let runner_base_img = runner_image_path(runner_id, "base.img"); + reflink_or_copy_with_warning(&template_base_img, &runner_base_img)?; - spawn_with_output!(virt-clone -o $profile_guest_name -n $runner_guest_name --preserve-data -f $runner_base_image_path 2>&1)? + spawn_with_output!(virt-clone -o $template_guest_name -n $runner_guest_name --preserve-data -f $runner_base_img 2>&1)? .wait_with_pipe(&mut pipe())?; Ok(runner_guest_name.to_owned()) diff --git a/monitor/src/image/windows10.rs b/monitor/src/image/windows10.rs index 6b5ec422..07b818de 100644 --- a/monitor/src/image/windows10.rs +++ b/monitor/src/image/windows10.rs @@ -7,18 +7,18 @@ use bytesize::ByteSize; use cmd_lib::run_cmd; use cmd_lib::spawn_with_output; use jane_eyre::eyre; -use jane_eyre::eyre::OptionExt; use settings::profile::Profile; use tracing::info; use tracing::warn; use crate::data::get_profile_configuration_path; -use crate::image::create_base_images_dir; +use crate::data::get_profile_data_path; use crate::image::create_runner_images_dir; -use crate::image::delete_base_image_file; -use crate::image::prune_base_image_files; +use crate::image::delete_template_or_rebuild_image_file; +use crate::image::rename_guest; use crate::image::undefine_libvirt_guest; use crate::policy::runner_image_path; +use crate::policy::template_or_rebuild_image_path; use crate::shell::atomic_symlink; use crate::shell::log_output_as_info; use crate::shell::reflink_or_copy_with_warning; @@ -38,16 +38,14 @@ pub(super) fn rebuild( wait_duration: Duration, ) -> eyre::Result<()> { let base_images_path = base_images_path.as_ref(); - let profile_guest_name = &profile.profile_guest_name(); + let rebuild_guest_name = &profile.rebuild_guest_name(snapshot_name); let profile_configuration_path = get_profile_configuration_path(&profile, None)?; - let config_iso_symlink_path = base_images_path.join(format!("config.iso")); let config_iso_filename = format!("config.iso@{snapshot_name}"); let config_iso_path = base_images_path.join(&config_iso_filename); let config_iso_path = config_iso_path.to_str().expect("Unsupported path"); info!(config_iso_path, "Creating config image file"); run_cmd!(genisoimage -J -f -o $config_iso_path $profile_configuration_path/autounattend.xml)?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); let base_image_path = create_disk_image(base_images_path, snapshot_name, base_image_size, None)?; @@ -62,6 +60,7 @@ pub(super) fn rebuild( define_base_guest( profile, + snapshot_name, &base_image_path, &[ CdromImage::new("sdb", installer_iso_path), @@ -69,64 +68,29 @@ pub(super) fn rebuild( CdromImage::new("sdd", config_iso_path), ], )?; - start_libvirt_guest(profile_guest_name)?; - wait_for_guest(profile_guest_name, wait_duration)?; + start_libvirt_guest(rebuild_guest_name)?; + wait_for_guest(rebuild_guest_name, wait_duration)?; - let base_image_filename = Path::new( - base_image_path - .file_name() - .expect("Guaranteed by make_disk_image"), - ); - atomic_symlink(config_iso_filename, config_iso_symlink_path)?; - atomic_symlink(base_image_filename, base_image_symlink_path)?; - - Ok(()) -} - -pub(super) fn redefine_base_guest_with_symlinks( - base_images_path: impl AsRef, - profile: &Profile, -) -> Result<(), eyre::Error> { - let base_images_path = base_images_path.as_ref(); - let config_iso_symlink_path = base_images_path.join(format!("config.iso")); - let config_iso_symlink_path = config_iso_symlink_path - .to_str() - .ok_or_eyre("Unsupported path")?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); - - let installer_iso_path = IMAGE_DEPS_DIR - .join("windows10") - .join("Win10_22H2_English_x64v1.iso"); - let installer_iso_path = installer_iso_path.to_str().expect("Unsupported path"); - let drivers_iso_path = IMAGE_DEPS_DIR - .join("windows10") - .join("virtio-win-0.1.240.iso"); - let drivers_iso_path = drivers_iso_path.to_str().expect("Unsupported path"); - - undefine_libvirt_guest(&profile.profile_guest_name())?; - define_base_guest( - profile, - &base_image_symlink_path, - &[ - CdromImage::new("sdb", installer_iso_path), - CdromImage::new("sdc", drivers_iso_path), - CdromImage::new("sdd", &config_iso_symlink_path), - ], - )?; + let template_guest_name = &profile.template_guest_name(snapshot_name); + rename_guest(rebuild_guest_name, template_guest_name)?; + let snapshot_symlink_path = + get_profile_data_path(&profile.profile_name, Path::new("snapshot"))?; + atomic_symlink(snapshot_name, snapshot_symlink_path)?; Ok(()) } fn define_base_guest( profile: &Profile, + snapshot_name: &str, base_image_path: &dyn AsRef, cdrom_images: &[CdromImage], ) -> eyre::Result<()> { - let profile_guest_name = &profile.profile_guest_name(); + let rebuild_guest_name = &profile.rebuild_guest_name(snapshot_name); let guest_xml_path = get_profile_configuration_path(&profile, Path::new("guest.xml"))?; define_libvirt_guest( &profile.profile_name, - profile_guest_name, + rebuild_guest_name, guest_xml_path, &[&"-f", &base_image_path], cdrom_images, @@ -135,40 +99,35 @@ fn define_base_guest( Ok(()) } -pub(super) fn prune_images(profile: &Profile) -> eyre::Result<()> { - prune_base_image_files(profile, "config.iso")?; - prune_base_image_files(profile, "base.img")?; - +pub(super) fn delete_template(profile: &Profile, snapshot_name: &str) -> eyre::Result<()> { + undefine_libvirt_guest(&profile.template_guest_name(snapshot_name))?; + delete_template_or_rebuild_image_file(profile, &format!("config.iso@{snapshot_name}")); + delete_template_or_rebuild_image_file(profile, &format!("base.img@{snapshot_name}")); Ok(()) } -pub(super) fn delete_image(profile: &Profile, snapshot_name: &str) { - delete_base_image_file(profile, &format!("config.iso@{snapshot_name}")); - delete_base_image_file(profile, &format!("base.img@{snapshot_name}")); -} - pub fn register_runner(profile: &Profile, runner_guest_name: &str) -> eyre::Result { monitor::github::register_runner(runner_guest_name, &profile.github_runner_label, r"C:\a") } pub fn create_runner( profile: &Profile, + snapshot_name: &str, runner_guest_name: &str, runner_id: usize, ) -> eyre::Result { let pipe = || |reader| log_output_as_info(reader); - let profile_guest_name = &profile.profile_guest_name(); + let template_guest_name = &profile.template_guest_name(snapshot_name); // Copy images in the monitor, not with `virt-clone --auto-clone --reflink`, // because the latter can’t be parallelised without causing errors. // TODO copy config.iso? - let base_images_path = create_base_images_dir(profile)?; - let base_image_symlink_path = base_images_path.join(format!("base.img")); + let template_base_img = template_or_rebuild_image_path(profile, snapshot_name, "base.img"); create_runner_images_dir()?; - let runner_base_image_path = runner_image_path(runner_id, "base.img"); - reflink_or_copy_with_warning(&base_image_symlink_path, &runner_base_image_path)?; + let runner_base_img = runner_image_path(runner_id, "base.img"); + reflink_or_copy_with_warning(&template_base_img, &runner_base_img)?; - spawn_with_output!(virt-clone -o $profile_guest_name -n $runner_guest_name --preserve-data -f $runner_base_image_path 2>&1)? + spawn_with_output!(virt-clone -o $template_guest_name -n $runner_guest_name --preserve-data -f $runner_base_img 2>&1)? .wait_with_pipe(&mut pipe())?; Ok(runner_guest_name.to_owned()) diff --git a/monitor/src/libvirt.rs b/monitor/src/libvirt.rs index b5c48823..231a9090 100644 --- a/monitor/src/libvirt.rs +++ b/monitor/src/libvirt.rs @@ -12,6 +12,30 @@ use tracing::debug; use crate::shell::log_output_as_trace; +pub fn list_template_guests() -> eyre::Result> { + // Output is not filtered by prefix, so we must filter it ourselves. + let prefix = format!("{}-", TOML.libvirt_template_guest_prefix()); + let result = run_fun!(virsh list --name --all)?; + let result = result + .split_terminator('\n') + .filter(|name| name.starts_with(&prefix)) + .map(str::to_owned); + + Ok(result.collect()) +} + +pub fn list_rebuild_guests() -> eyre::Result> { + // Output is not filtered by prefix, so we must filter it ourselves. + let prefix = format!("{}-", TOML.libvirt_rebuild_guest_prefix()); + let result = run_fun!(virsh list --name --all)?; + let result = result + .split_terminator('\n') + .filter(|name| name.starts_with(&prefix)) + .map(str::to_owned); + + Ok(result.collect()) +} + pub fn list_runner_guests() -> eyre::Result> { // Output is not filtered by prefix, so we must filter it ourselves. let prefix = format!("{}-", TOML.libvirt_runner_guest_prefix()); diff --git a/monitor/src/main.rs b/monitor/src/main.rs index c8280477..78f53095 100644 --- a/monitor/src/main.rs +++ b/monitor/src/main.rs @@ -53,7 +53,7 @@ use crate::{ id::IdGen, image::{start_libvirt_guest, Rebuilds}, libvirt::list_runner_guests, - policy::{base_image_path, Override, Policy, RunnerCounts}, + policy::{Override, Policy, RunnerCounts}, runner::{Runners, Status}, }; @@ -504,24 +504,16 @@ fn monitor_thread() -> eyre::Result<()> { }, ) in profile_runner_counts.iter() { - let profile = policy.profile(key).ok_or_eyre("Failed to get profile")?; - let image = policy - .base_image_snapshot(key) - .map(|snapshot| match profile.image_type { - settings::profile::ImageType::Rust => base_image_path(profile, &**snapshot) - .as_os_str() - .to_str() - .expect("Guaranteed by base_image_path()") - .to_owned(), - }); - info!("profile {key}: {healthy}/{target} healthy runners ({idle} idle, {reserved} reserved, {busy} busy, {started_or_crashed} started or crashed, {excess_healthy} excess healthy, {wanted} wanted), image {:?} age {image_age:?}", image); + let snapshot = policy.base_image_snapshot(key); + info!("profile {key}: {healthy}/{target} healthy runners ({idle} idle, {reserved} reserved, {busy} busy, {started_or_crashed} started or crashed, {excess_healthy} excess healthy, {wanted} wanted), snapshot {snapshot:?} age {image_age:?}"); } for (_id, runner) in policy.runners() { runner.log_info(); } - policy.update_screenshots(); - policy.update_ipv4_addresses_for_profile_guests(); + let rebuild_guest_names = image_rebuilds.rebuild_guest_names(); + policy.update_screenshots(&rebuild_guest_names); + policy.update_ipv4_addresses_for_rebuild_guests(&rebuild_guest_names); if TOML.destroy_all_non_busy_runners() { let non_busy_runners = policy @@ -689,14 +681,14 @@ fn monitor_thread() -> eyre::Result<()> { // GET /github-jitconfig request both happen in step (2) without step (1) in between, we won’t know // the IPv4 address, so let’s update the IPv4 addresses before continuing. policy.update_ipv4_addresses_for_runner_guests()?; - policy.update_ipv4_addresses_for_profile_guests(); + policy.update_ipv4_addresses_for_rebuild_guests(&rebuild_guest_names); let result = policy .boot_script_for_runner_guest(remote_addr.clone()) .transpose() .or_else(|| { policy - .boot_script_for_profile_guest(remote_addr) + .boot_script_for_rebuild_guest(remote_addr) .transpose() }) .transpose() diff --git a/monitor/src/policy.rs b/monitor/src/policy.rs index 2cd919c3..eff1534b 100644 --- a/monitor/src/policy.rs +++ b/monitor/src/policy.rs @@ -9,7 +9,7 @@ use std::{ time::{Duration, SystemTime, UNIX_EPOCH}, }; -use cfg_if::cfg_if; +use chrono::DateTime; use itertools::Itertools; use jane_eyre::eyre::{self, bail, Context, OptionExt}; use mktemp::Temp; @@ -265,7 +265,8 @@ impl Policy { profile: &Profile, id: usize, ) -> eyre::Result>> { - if self.base_image_snapshot(&profile.profile_name).is_none() { + let Some(base_image_snapshot) = self.base_image_snapshot(&profile.profile_name).cloned() + else { bail!( "Tried to create runner, but profile has no base image snapshot (profile {})", profile.profile_name @@ -299,7 +300,7 @@ impl Policy { .write_all(github_api_registration.as_bytes())?; } - create_runner(&profile, &runner_guest_name, id) + create_runner(&profile, &base_image_snapshot, &runner_guest_name, id) } } })) @@ -471,26 +472,9 @@ impl Policy { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .wrap_err("Failed to get current time")?; - let creation_time = match profile.image_type { - ImageType::Rust => { - let base_image_path = base_image_path(profile, &**base_image_snapshot); - let Some(mtime) = base_image_mtime(profile, &base_image_path) else { - return Ok(None); - }; - match mtime.duration_since(UNIX_EPOCH) { - Ok(result) => result, - Err(error) => { - debug!( - profile.profile_name, - ?base_image_path, - ?error, - "Failed to calculate image age" - ); - return Ok(None); - } - } - } - }; + let creation_time = DateTime::parse_from_rfc3339(&base_image_snapshot)? + .signed_duration_since(DateTime::UNIX_EPOCH) + .to_std()?; Ok(Some(now - creation_time)) } @@ -506,13 +490,16 @@ impl Policy { Ok(()) } - pub fn update_ipv4_addresses_for_profile_guests(&mut self) { - for (key, profile) in self.profiles.iter() { - let ipv4_address = get_ipv4_address(&profile.profile_guest_name()); - let entry = self.ipv4_addresses.entry(key.clone()).or_default(); + pub fn update_ipv4_addresses_for_rebuild_guests( + &mut self, + rebuild_guest_names: &BTreeMap, + ) { + for (profile_key, guest_name) in rebuild_guest_names { + let ipv4_address = get_ipv4_address(&guest_name); + let entry = self.ipv4_addresses.entry(profile_key.clone()).or_default(); if ipv4_address != *entry { info!( - "IPv4 address changed for profile guest {key}: {:?} -> {:?}", + "IPv4 address changed for profile guest {profile_key}: {:?} -> {:?}", *entry, ipv4_address ); } @@ -520,7 +507,7 @@ impl Policy { } } - pub fn boot_script_for_profile_guest( + pub fn boot_script_for_rebuild_guest( &self, remote_addr: web::auth::RemoteAddr, ) -> eyre::Result> { @@ -571,24 +558,20 @@ impl Policy { .filter(|(_id, runner)| runner.status() == Status::Idle) } - pub fn update_screenshots(&self) { + pub fn update_screenshots(&self, rebuild_guest_names: &BTreeMap) { if let Some(runners) = self.runners.as_ref() { runners.update_screenshots(); } - for (_key, profile) in self.profiles() { - if let Err(error) = self.try_update_screenshot(profile) { - debug!( - profile.profile_name, - ?error, - "Failed to update screenshot for profile guest" - ); + for (profile_key, guest_name) in rebuild_guest_names { + if let Err(error) = self.try_update_screenshot(&profile_key, &guest_name) { + debug!(guest_name, ?error, "Failed to update screenshot for guest"); } } } - fn try_update_screenshot(&self, profile: &Profile) -> eyre::Result<()> { - let output_dir = get_profile_data_path(&profile.profile_name, None)?; - update_screenshot(&profile.profile_guest_name(), &output_dir)?; + fn try_update_screenshot(&self, profile_key: &str, guest_name: &str) -> eyre::Result<()> { + let output_dir = get_profile_data_path(&profile_key, None)?; + update_screenshot(guest_name, &output_dir)?; Ok(()) } @@ -870,7 +853,7 @@ impl Policy { } } -pub fn base_images_path(profile: &Profile) -> PathBuf { +pub fn template_or_rebuild_images_path(profile: &Profile) -> PathBuf { Path::new("/var/lib/libvirt/images/base").join(&profile.profile_name) } @@ -878,15 +861,12 @@ pub fn runner_images_path() -> PathBuf { PathBuf::from("/var/lib/libvirt/images/runner") } -pub fn base_image_path<'snap>( +pub fn template_or_rebuild_image_path( profile: &Profile, - snapshot_name: impl Into>, + snapshot_name: &str, + filename: impl AsRef, ) -> PathBuf { - if let Some(snapshot_name) = snapshot_name.into() { - base_images_path(profile).join(format!("base.img@{snapshot_name}")) - } else { - base_images_path(profile).join("base.img") - } + template_or_rebuild_images_path(profile).join(format!("{}@{snapshot_name}", filename.as_ref())) } pub fn runner_image_path(runner_id: usize, filename: impl AsRef) -> PathBuf { @@ -894,72 +874,26 @@ pub fn runner_image_path(runner_id: usize, filename: impl AsRef) -> PathBuf } fn read_base_image_snapshot(profile: &Profile) -> eyre::Result> { - let path = base_image_path(profile, None); + let path = get_profile_data_path(&profile.profile_name, Path::new("snapshot"))?; if let Ok(path) = read_link(path) { - let path = path.to_str().ok_or_eyre("Symlink target is unsupported")?; - let (_, snapshot_name) = path - .split_once("@") - .ok_or_eyre("Symlink target has no snapshot name")?; + let snapshot_name = path.to_str().ok_or_eyre("Symlink target is unsupported")?; return Ok(Some(snapshot_name.to_owned())); } Ok(None) } -cfg_if! { - if #[cfg(not(test))] { - fn base_image_mtime(profile: &Profile, base_image_path: impl AsRef) -> Option { - let base_image_path = base_image_path.as_ref(); - let metadata = match std::fs::metadata(&base_image_path) { - Ok(result) => result, - Err(error) => { - debug!( - profile.profile_name, - ?base_image_path, - ?error, - "Failed to get file metadata" - ); - return None; - } - }; - - Some(metadata.modified().expect("Guaranteed by platform")) - } - } else { - use std::cell::RefCell; - - thread_local! { - static BASE_IMAGE_MTIMES: RefCell> = RefCell::new(BTreeMap::new()); - } - - fn base_image_mtime(_profile: &Profile, base_image_path: impl AsRef) -> Option { - let base_image_path = base_image_path.as_ref().to_str().expect("Unsupported path"); - - BASE_IMAGE_MTIMES.with_borrow(|mtimes| mtimes.get(base_image_path).copied()) - } - - fn set_base_image_mtime_for_test(base_image_path: &str, mtime: impl Into>) { - BASE_IMAGE_MTIMES.with_borrow_mut(|mtimes| { - if let Some(mtime) = mtime.into() { - mtimes.insert(base_image_path.to_owned(), mtime); - } else { - mtimes.remove(base_image_path); - } - }); - } - } -} - #[cfg(test)] mod test { use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use chrono::{SecondsFormat, Utc}; use jane_eyre::eyre; use monitor::github::{ApiRunner, ApiRunnerLabel}; use settings::{profile::Profile, TOML}; use crate::{ - policy::{set_base_image_mtime_for_test, Override, RunnerChanges}, + policy::{Override, RunnerChanges}, runner::{set_runner_created_time_for_test, Runners, Status}, }; @@ -1089,6 +1023,9 @@ mod test { Runners::new(registrations, guest_names) } + fn snapshot_now_minus_seconds(delta: u64) -> String { + (Utc::now() - Duration::from_secs(delta)).to_rfc3339_opts(SecondsFormat::Nanos, true) + } fn system_time_minus_seconds(delta: u64) -> SystemTime { SystemTime::now() .checked_sub(Duration::from_secs(delta)) @@ -1189,14 +1126,11 @@ mod test { ); // Images need rebuild, because they are too old. - let too_old = system_time_minus_seconds(86500); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/linux/base.img@", too_old); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/macos/base.img@", too_old); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/windows/base.img@", too_old); - policy.set_base_image_snapshot("linux", "")?; - policy.set_base_image_snapshot("macos", "")?; - policy.set_base_image_snapshot("windows", "")?; - policy.set_base_image_snapshot("wpt", "")?; + let too_old = snapshot_now_minus_seconds(86500); + policy.set_base_image_snapshot("linux", &too_old)?; + policy.set_base_image_snapshot("macos", &too_old)?; + policy.set_base_image_snapshot("windows", &too_old)?; + policy.set_base_image_snapshot("wpt", &too_old)?; assert_eq!( policy.compute_runner_changes()?, RunnerChanges { @@ -1212,10 +1146,10 @@ mod test { ); // Empty state. - let fresh = system_time_minus_seconds(0); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/linux/base.img@", fresh); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/macos/base.img@", fresh); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/windows/base.img@", fresh); + let fresh = snapshot_now_minus_seconds(0); + policy.set_base_image_snapshot("linux", &fresh)?; + policy.set_base_image_snapshot("macos", &fresh)?; + policy.set_base_image_snapshot("windows", &fresh)?; assert_eq!( policy.compute_runner_changes()?, RunnerChanges { @@ -1440,15 +1374,11 @@ mod test { ] .into(), )?; - let fresh = system_time_minus_seconds(0); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/linux/base.img@", fresh); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/macos/base.img@", fresh); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/windows/base.img@", fresh); - set_base_image_mtime_for_test("/var/lib/libvirt/images/base/wpt/base.img@", fresh); - policy.set_base_image_snapshot("linux", "")?; - policy.set_base_image_snapshot("macos", "")?; - policy.set_base_image_snapshot("windows", "")?; - policy.set_base_image_snapshot("wpt", "")?; + let fresh = snapshot_now_minus_seconds(0); + policy.set_base_image_snapshot("linux", &fresh)?; + policy.set_base_image_snapshot("macos", &fresh)?; + policy.set_base_image_snapshot("windows", &fresh)?; + policy.set_base_image_snapshot("wpt", &fresh)?; // Proposed create counts should be adjusted for critical runners, regardless of whether // those runners fit the current policy. @@ -1482,21 +1412,10 @@ mod test { ] .into(), )?; - set_base_image_mtime_for_test( - "/var/lib/libvirt/images/base/linux/base.img@", - SystemTime::now(), - ); - set_base_image_mtime_for_test( - "/var/lib/libvirt/images/base/windows/base.img@", - SystemTime::now(), - ); - set_base_image_mtime_for_test( - "/var/lib/libvirt/images/base/macos/base.img@", - SystemTime::now(), - ); - policy.set_base_image_snapshot("linux", "")?; - policy.set_base_image_snapshot("windows", "")?; - policy.set_base_image_snapshot("macos", "")?; + let now = snapshot_now_minus_seconds(0); + policy.set_base_image_snapshot("linux", &now)?; + policy.set_base_image_snapshot("windows", &now)?; + policy.set_base_image_snapshot("macos", &now)?; // If the runners are not yet known, refuse the request. assert!(policy.try_override([].into()).is_err()); @@ -1576,11 +1495,8 @@ mod test { ); // The image is ready. Let’s create runners. - set_base_image_mtime_for_test( - "/var/lib/libvirt/images/base/wpt/base.img@", - SystemTime::now(), - ); - policy.set_base_image_snapshot("wpt", "")?; + let now = snapshot_now_minus_seconds(0); + policy.set_base_image_snapshot("wpt", &now)?; assert_eq!( policy.compute_runner_changes()?, RunnerChanges {