From 8e1ac45ce182e30c0d5dca0e4382b6afcddd3928 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sat, 19 Dec 2020 21:33:42 +0100 Subject: [PATCH 01/21] api: add Singlestep Intercept and Event type --- src/api.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api.rs b/src/api.rs index 15045c72..98f83d62 100644 --- a/src/api.rs +++ b/src/api.rs @@ -314,6 +314,7 @@ pub enum InterceptType { /// Intercept when guest requests an access to a page for which the requested type of access is not granted. For example , guest tries to write on a read only page. Breakpoint, Pagefault, + Singlestep, } /// Various types of events along with their relevant attributes being handled by this driver @@ -351,6 +352,7 @@ pub enum EventType { /// Acsess responsible for thr pagefault access: Access, }, + Singlestep, } ///Types of x86 control registers are listed here From 5e45404d9b61087d1df067553d18a2cdbc77e8c3 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sat, 19 Dec 2020 22:10:41 +0100 Subject: [PATCH 02/21] cargo: update xenctrl --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9197abfb..fc22e401 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ hyper-v = ["winapi", "widestring", "ntapi", "vid-sys"] log = "0.4.8" env_logger = "0.7.1" libc = { version = "0.2.58", optional = true } -xenctrl = { version = "0.4.2", optional = true } +xenctrl = { version = "0.4.4", optional = true } xenstore-rs = { version = "0.3.0", optional = true } xenforeignmemory = { version = "0.1.0", optional = true } xenevtchn = { version = "0.1.2", optional = true } From 26f88fcc2a1b7c435028e41a5c16778f19b3d5b3 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sat, 19 Dec 2020 22:11:41 +0100 Subject: [PATCH 03/21] xen: toggle intercept for singlestep --- src/driver/xen.rs | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 818a6c0b..6d04f627 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -10,8 +10,10 @@ use std::error::Error; use std::io::ErrorKind; use std::mem; use xenctrl::consts::{PAGE_SHIFT, PAGE_SIZE}; -use xenctrl::RING_HAS_UNCONSUMED_REQUESTS; -use xenctrl::{XenControl, XenCr, XenEventType}; +use xenctrl::{ + XenControl, XenCr, XenEventType, RING_HAS_UNCONSUMED_REQUESTS, + XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_OFF, XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_ON, +}; use xenevtchn::XenEventChannel; use xenforeignmemory::XenForeignMem; use xenstore_rs::{XBTransaction, Xs, XsOpenFlags}; @@ -71,6 +73,17 @@ impl Xen { let (_ring_page, back_ring, remote_port) = xc .monitor_enable(cand_domid) .expect("Failed to map event ring page"); + // enable singlestep monitoring + // it will only intercept events when explicitely requested using + // xc_domain_debug_control() + // TODO: call get_vcpu_count() + let domain_info = xc.domain_getinfo(cand_domid).unwrap(); + let vcpu_count = (domain_info.max_vcpu_id + 1).try_into().unwrap(); + for vcpu in 0..vcpu_count { + xc.monitor_singlestep(cand_domid, true).unwrap_or_else(|_| { + panic!("Failed to enable singlestep monitoring on VCPU {}", vcpu) + }); + } let xev = XenEventChannel::new(cand_domid, remote_port).unwrap(); let xen_fgn = XenForeignMem::new().unwrap(); @@ -82,6 +95,7 @@ impl Xen { domid: cand_domid, back_ring, }; + debug!("Initialized {:#?}", xen); xen } @@ -364,7 +378,7 @@ impl Introspectable for Xen { fn toggle_intercept( &mut self, - _vcpu: u16, + vcpu: u16, intercept_type: InterceptType, enabled: bool, ) -> Result<(), Box> { @@ -387,6 +401,15 @@ impl Introspectable for Xen { InterceptType::Breakpoint => { Ok(self.xc.monitor_software_breakpoint(self.domid, enabled)?) } + InterceptType::Singlestep => { + let op: u32 = match enabled { + false => XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_OFF, + true => XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_ON, + }; + Ok(self + .xc + .domain_debug_control(self.domid, op, vcpu.try_into().unwrap())?) + } _ => unimplemented!(), } } @@ -405,6 +428,14 @@ impl Introspectable for Xen { impl Drop for Xen { fn drop(&mut self) { debug!("Closing Xen driver"); + let vcpu_cpunt = self.get_vcpu_count().expect("Failed to get VCPU count"); + for vcpu in 0..vcpu_cpunt { + self.xc + .monitor_singlestep(self.domid, false) + .unwrap_or_else(|_| { + panic!("Failed to disable singlestep monitoring on VCPU {}", vcpu) + }) + } self.xc .monitor_disable(self.domid) .expect("Failed to unmap event ring page"); From 115a333f3884568ec89584278984dcfd6bdf9060 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sat, 19 Dec 2020 22:14:45 +0100 Subject: [PATCH 04/21] xen: handle Singlestep event --- src/driver/xen.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 6d04f627..02b1d00b 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -354,6 +354,7 @@ impl Introspectable for Xen { XenEventType::Breakpoint { insn_len, .. } => { EventType::Breakpoint { gpa: 0, insn_len } } + XenEventType::Singlestep { .. } => EventType::Singlestep, _ => unimplemented!(), }; vcpu = req.vcpu_id.try_into().unwrap(); From 0044483e8856b3908c7b75b06d2778b0e48ece78 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sat, 19 Dec 2020 22:38:58 +0100 Subject: [PATCH 05/21] examples: add singlestep-events --- examples/singlestep-events.rs | 96 +++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 examples/singlestep-events.rs diff --git a/examples/singlestep-events.rs b/examples/singlestep-events.rs new file mode 100644 index 00000000..e44b530a --- /dev/null +++ b/examples/singlestep-events.rs @@ -0,0 +1,96 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use clap::{App, Arg, ArgMatches}; +use colored::*; +use env_logger; + +use microvmi::api::*; + +fn parse_args() -> ArgMatches<'static> { + App::new(file!()) + .version("0.1") + .about("Watches singlestep VMI events") + .arg(Arg::with_name("vm_name").index(1).required(true)) + .get_matches() +} + +fn toggle_singlestep_interception(drv: &mut Box, enabled: bool) { + drv.pause().expect("Failed to pause VM"); + + let intercept = InterceptType::Singlestep; + let status_str = if enabled { "Enabling" } else { "Disabling" }; + println!("{} singlestep events", status_str); + for vcpu in 0..1 { + drv.toggle_intercept(vcpu, intercept, enabled) + .expect(&format!("Failed to enable singlestep")); + } + + drv.resume().expect("Failed to resume VM"); +} + +fn main() { + env_logger::init(); + + let matches = parse_args(); + + let domain_name = matches.value_of("vm_name").unwrap(); + + let init_option = matches + .value_of("kvmi_socket") + .map(|socket| DriverInitParam::KVMiSocket(socket.into())); + // set CTRL-C handler + let running = Arc::new(AtomicBool::new(true)); + let r = running.clone(); + ctrlc::set_handler(move || { + r.store(false, Ordering::SeqCst); + }) + .expect("Error setting Ctrl-C handler"); + + println!("Initialize Libmicrovmi"); + let mut drv: Box = microvmi::init(domain_name, None, init_option); + + //Enable singlestep interception + toggle_singlestep_interception(&mut drv, true); + + println!("Listen for singlestep events..."); + // record elapsed time + let start = Instant::now(); + // listen + let mut i: u64 = 0; + while running.load(Ordering::SeqCst) { + let event = drv.listen(1000).expect("Failed to listen for events"); + match event { + Some(ev) => { + match ev.kind { + EventType::Singlestep {} => (), + _ => panic!("Not singlestep event"), + }; + let ev_nb_output = format!("{}", i).cyan(); + let vcpu_output = format!("VCPU {}", ev.vcpu).yellow(); + let singlestep_output = format!("singlestep occurred!").color("blue"); + println!( + "[{}] {} - {}: ", + ev_nb_output, vcpu_output, singlestep_output + ); + drv.reply_event(ev, EventReplyType::Continue) + .expect("Failed to send event reply"); + i = i + 1; + } + None => println!("No events yet..."), + } + } + let duration = start.elapsed(); + + //disable singlestep interception + toggle_singlestep_interception(&mut drv, false); + + let ev_per_sec = i as f64 / duration.as_secs_f64(); + println!( + "Caught {} events in {:.2} seconds ({:.2} events/sec)", + i, + duration.as_secs_f64(), + ev_per_sec + ); +} From 9bbd1b4b3f61f05397b6d5ab1a3ed64a6a342a7f Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 03:34:14 +0100 Subject: [PATCH 06/21] xen: refactor listen --- src/driver/xen.rs | 156 +++++++++++++++++++++++++++------------------- 1 file changed, 91 insertions(+), 65 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 02b1d00b..27cff801 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -1,12 +1,13 @@ use crate::api::{ - CrType, DriverInitParam, Event, EventType, InterceptType, Introspectable, Registers, - SegmentReg, SystemTableReg, X86Registers, + CrType, DriverInitParam, Event, EventReplyType, EventType, InterceptType, Introspectable, + Registers, SegmentReg, SystemTableReg, X86Registers, }; use libc::{PROT_READ, PROT_WRITE}; use nix::poll::PollFlags; use nix::poll::{poll, PollFd}; use std::convert::TryInto; use std::error::Error; +use std::io::Error as IoError; use std::io::ErrorKind; use std::mem; use xenctrl::consts::{PAGE_SHIFT, PAGE_SIZE}; @@ -18,17 +19,20 @@ use xenevtchn::XenEventChannel; use xenforeignmemory::XenForeignMem; use xenstore_rs::{XBTransaction, Xs, XsOpenFlags}; use xenvmevent_sys::{ - vm_event_back_ring, vm_event_response_t, VM_EVENT_FLAG_VCPU_PAUSED, VM_EVENT_INTERFACE_VERSION, + vm_event_back_ring, vm_event_request_t, vm_event_response_t, VM_EVENT_FLAG_VCPU_PAUSED, + VM_EVENT_INTERFACE_VERSION, }; -#[derive(Debug)] pub struct Xen { xc: XenControl, xev: XenEventChannel, xen_fgn: XenForeignMem, - dom_name: String, + _dom_name: String, domid: u32, back_ring: vm_event_back_ring, + evtchn_pollfd: PollFd, + // VCPU -> vm_event_request_t + vec_events: Vec>, } impl Xen { @@ -73,6 +77,7 @@ impl Xen { let (_ring_page, back_ring, remote_port) = xc .monitor_enable(cand_domid) .expect("Failed to map event ring page"); + // enable singlestep monitoring // it will only intercept events when explicitely requested using // xc_domain_debug_control() @@ -84,19 +89,27 @@ impl Xen { panic!("Failed to enable singlestep monitoring on VCPU {}", vcpu) }); } - let xev = XenEventChannel::new(cand_domid, remote_port).unwrap(); + // init vec events + let mut vec_events: Vec> = Vec::new(); + vec_events.resize(vcpu_count, None); + let xev = XenEventChannel::new(cand_domid, remote_port).unwrap(); + let fd = xev.xenevtchn_fd().unwrap(); + let evtchn_pollfd = PollFd::new(fd, PollFlags::POLLIN | PollFlags::POLLERR); let xen_fgn = XenForeignMem::new().unwrap(); let xen = Xen { xc, xev, xen_fgn, - dom_name: domain_name.to_string(), + _dom_name: domain_name.to_string(), domid: cand_domid, back_ring, + evtchn_pollfd, + vec_events, }; - debug!("Initialized {:#?}", xen); + // TODO: vm_event_request_t (vm_event_st) doesn't derive Debug even when .derive_debug(true) + // debug!("Initialized {:#?}", xen); xen } } @@ -314,67 +327,80 @@ impl Introspectable for Xen { } fn listen(&mut self, timeout: u32) -> Result, Box> { - let fd = self.xev.xenevtchn_fd()?; - let fd_struct = PollFd::new(fd, PollFlags::POLLIN | PollFlags::POLLERR); - let mut fds = [fd_struct]; - let mut vcpu: u16 = 0; - let mut event_type = unsafe { mem::MaybeUninit::::zeroed().assume_init() }; - let poll_result = poll(&mut fds, timeout.try_into().unwrap()).unwrap(); - let mut pending_event_port = -1; - if poll_result == 1 { - pending_event_port = self.xev.xenevtchn_pending()?; - if pending_event_port != -1 { - self.xev - .xenevtchn_unmask(pending_event_port.try_into().unwrap())?; + let mut fds: [PollFd; 1] = [self.evtchn_pollfd]; + let event: Option = match poll(&mut fds, timeout.try_into().unwrap()).unwrap() { + 0 => { + // timeout. no file descriptors were ready + None } - } - let back_ring_ptr = &mut self.back_ring; - let mut flag = false; - if poll_result > 0 - && self.xev.get_bind_port() == pending_event_port - && RING_HAS_UNCONSUMED_REQUESTS!(back_ring_ptr) != 0 - { - flag = true; - let req = self.xc.get_request(back_ring_ptr)?; - if req.version != VM_EVENT_INTERFACE_VERSION { - panic!("version mismatch"); + -1 => { + // failure + return Err(Box::new(IoError::last_os_error())); } - let xen_event_type = (self.xc.get_event_type(req)).unwrap(); - event_type = match xen_event_type { - XenEventType::Cr { cr_type, new, old } => EventType::Cr { - cr_type: match cr_type { - XenCr::Cr0 => CrType::Cr0, - XenCr::Cr3 => CrType::Cr3, - XenCr::Cr4 => CrType::Cr4, - }, - new, - old, - }, - XenEventType::Msr { msr_type, value } => EventType::Msr { msr_type, value }, - XenEventType::Breakpoint { insn_len, .. } => { - EventType::Breakpoint { gpa: 0, insn_len } + 1 => { + // event available + match self.xev.xenevtchn_pending()? { + -1 => { + // no event channel port is pending + // TODO: Err + panic!("No event channel port is pending"); + } + pending_event_port => { + let bind_port = self.xev.get_bind_port(); + if pending_event_port != self.xev.get_bind_port() { + panic!( + "Event received for invalid port {}, expected port {}", + pending_event_port, bind_port + ); + } + // unmask + self.xev.xenevtchn_unmask(pending_event_port.try_into()?)?; + } + }; + let back_ring_ptr = &mut self.back_ring; + match RING_HAS_UNCONSUMED_REQUESTS!(back_ring_ptr) != 0 { + false => None, + true => { + let req = self.xc.get_request(back_ring_ptr)?; + if req.version != VM_EVENT_INTERFACE_VERSION { + panic!("version mismatch"); + } + let xen_event_type = (self.xc.get_event_type(req))?; + let vcpu: u32 = req.vcpu_id; + let event_type: EventType = match xen_event_type { + XenEventType::Cr { cr_type, new, old } => EventType::Cr { + cr_type: match cr_type { + XenCr::Cr0 => CrType::Cr0, + XenCr::Cr3 => CrType::Cr3, + XenCr::Cr4 => CrType::Cr4, + }, + new, + old, + }, + XenEventType::Msr { msr_type, value } => { + EventType::Msr { msr_type, value } + } + XenEventType::Breakpoint { insn_len, .. } => { + EventType::Breakpoint { gpa: 0, insn_len } + } + XenEventType::Singlestep { .. } => EventType::Singlestep, + _ => unimplemented!(), + }; + // associate VCPU => vm_event_request_t + // to find it in reply_event() + let vcpu_index: usize = vcpu.try_into().unwrap(); + self.vec_events[vcpu_index] = Some(req); + Some(Event { + vcpu: vcpu.try_into().unwrap(), + kind: event_type, + }) + } } - XenEventType::Singlestep { .. } => EventType::Singlestep, - _ => unimplemented!(), - }; - vcpu = req.vcpu_id.try_into().unwrap(); - let mut rsp = - unsafe { mem::MaybeUninit::::zeroed().assume_init() }; - rsp.reason = req.reason; - rsp.version = VM_EVENT_INTERFACE_VERSION; - rsp.vcpu_id = req.vcpu_id; - rsp.flags = req.flags & VM_EVENT_FLAG_VCPU_PAUSED; - self.xc.put_response(&mut rsp, &mut self.back_ring)?; - } + } + x => panic!("Unexpected poll return value {}", x), + }; self.xev.xenevtchn_notify()?; - if flag { - Ok(Some(Event { - vcpu, - kind: event_type, - })) - } else { - Ok(None) - } + Ok(event) } fn toggle_intercept( From 135ea9c4d2a1ddd1df1c7344d545d74145210b30 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 03:50:04 +0100 Subject: [PATCH 07/21] xen: impl reply_event --- src/driver/xen.rs | 60 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 27cff801..3c2887f9 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -78,26 +78,12 @@ impl Xen { .monitor_enable(cand_domid) .expect("Failed to map event ring page"); - // enable singlestep monitoring - // it will only intercept events when explicitely requested using - // xc_domain_debug_control() - // TODO: call get_vcpu_count() - let domain_info = xc.domain_getinfo(cand_domid).unwrap(); - let vcpu_count = (domain_info.max_vcpu_id + 1).try_into().unwrap(); - for vcpu in 0..vcpu_count { - xc.monitor_singlestep(cand_domid, true).unwrap_or_else(|_| { - panic!("Failed to enable singlestep monitoring on VCPU {}", vcpu) - }); - } - // init vec events - let mut vec_events: Vec> = Vec::new(); - vec_events.resize(vcpu_count, None); - let xev = XenEventChannel::new(cand_domid, remote_port).unwrap(); let fd = xev.xenevtchn_fd().unwrap(); let evtchn_pollfd = PollFd::new(fd, PollFlags::POLLIN | PollFlags::POLLERR); let xen_fgn = XenForeignMem::new().unwrap(); - let xen = Xen { + + let mut xen = Xen { xc, xev, xen_fgn, @@ -105,9 +91,23 @@ impl Xen { domid: cand_domid, back_ring, evtchn_pollfd, - vec_events, + vec_events: Vec::new(), }; + // enable singlestep monitoring + // it will only intercept events when explicitely requested using + // xc_domain_debug_control() + let vcpu_count = xen.get_vcpu_count().expect("Failed to get VCPU count"); + for vcpu in 0..vcpu_count { + xen.xc + .monitor_singlestep(cand_domid, true) + .unwrap_or_else(|_| { + panic!("Failed to enable singlestep monitoring on VCPU {}", vcpu) + }); + } + + // init vec events + xen.vec_events.resize(vcpu_count.try_into().unwrap(), None); // TODO: vm_event_request_t (vm_event_st) doesn't derive Debug even when .derive_debug(true) // debug!("Initialized {:#?}", xen); xen @@ -184,7 +184,8 @@ impl Introspectable for Xen { fn get_vcpu_count(&self) -> Result> { let domain_info = self.xc.domain_getinfo(self.domid)?; - Ok((domain_info.max_vcpu_id + 1).try_into()?) + let vcpu_count = (domain_info.max_vcpu_id + 1).try_into()?; + Ok(vcpu_count) } fn read_registers(&self, vcpu: u16) -> Result> { @@ -399,10 +400,31 @@ impl Introspectable for Xen { } x => panic!("Unexpected poll return value {}", x), }; - self.xev.xenevtchn_notify()?; + Ok(event) } + fn reply_event( + &mut self, + event: Event, + reply_type: EventReplyType, + ) -> Result<(), Box> { + let add_flags: u32 = match reply_type { + EventReplyType::Continue => VM_EVENT_FLAG_VCPU_PAUSED, + }; + // get the request back + let vcpu_index: usize = event.vcpu.try_into().unwrap(); + let req: vm_event_request_t = mem::replace(&mut self.vec_events[vcpu_index], None).unwrap(); + let mut rsp: vm_event_response_t = + unsafe { mem::MaybeUninit::::zeroed().assume_init() }; + rsp.reason = req.reason; + rsp.version = VM_EVENT_INTERFACE_VERSION; + rsp.vcpu_id = req.vcpu_id; + rsp.flags = req.flags & add_flags; + self.xc.put_response(&mut rsp, &mut self.back_ring)?; + Ok(self.xev.xenevtchn_notify()?) + } + fn toggle_intercept( &mut self, vcpu: u16, From 185ad0ccba717304af0a7effc848f6c7d6cfaa73 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 03:50:20 +0100 Subject: [PATCH 08/21] examples/singlestep-events: intercept on all VCPUs --- examples/singlestep-events.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/singlestep-events.rs b/examples/singlestep-events.rs index e44b530a..a5c6e2e9 100644 --- a/examples/singlestep-events.rs +++ b/examples/singlestep-events.rs @@ -22,7 +22,7 @@ fn toggle_singlestep_interception(drv: &mut Box, enabled: bo let intercept = InterceptType::Singlestep; let status_str = if enabled { "Enabling" } else { "Disabling" }; println!("{} singlestep events", status_str); - for vcpu in 0..1 { + for vcpu in 0..drv.get_vcpu_count().expect("Failed to get VCPU count") { drv.toggle_intercept(vcpu, intercept, enabled) .expect(&format!("Failed to enable singlestep")); } From 59045e960e520c20bdbe8c05aa88edfe8a6ff3c4 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 04:00:42 +0100 Subject: [PATCH 09/21] examples/singlestep-events: display listen error and break the loop --- examples/singlestep-events.rs | 41 ++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/examples/singlestep-events.rs b/examples/singlestep-events.rs index a5c6e2e9..c5e158a6 100644 --- a/examples/singlestep-events.rs +++ b/examples/singlestep-events.rs @@ -60,25 +60,30 @@ fn main() { // listen let mut i: u64 = 0; while running.load(Ordering::SeqCst) { - let event = drv.listen(1000).expect("Failed to listen for events"); - match event { - Some(ev) => { - match ev.kind { - EventType::Singlestep {} => (), - _ => panic!("Not singlestep event"), - }; - let ev_nb_output = format!("{}", i).cyan(); - let vcpu_output = format!("VCPU {}", ev.vcpu).yellow(); - let singlestep_output = format!("singlestep occurred!").color("blue"); - println!( - "[{}] {} - {}: ", - ev_nb_output, vcpu_output, singlestep_output - ); - drv.reply_event(ev, EventReplyType::Continue) - .expect("Failed to send event reply"); - i = i + 1; + match drv.listen(1000) { + Err(error) => { + println!("Error while listening for events: {}. Exiting.", error); + break; } - None => println!("No events yet..."), + Ok(event) => match event { + Some(ev) => { + match ev.kind { + EventType::Singlestep {} => (), + _ => panic!("Not singlestep event"), + }; + let ev_nb_output = format!("{}", i).cyan(); + let vcpu_output = format!("VCPU {}", ev.vcpu).yellow(); + let singlestep_output = format!("singlestep occurred!").color("blue"); + println!( + "[{}] {} - {}: ", + ev_nb_output, vcpu_output, singlestep_output + ); + drv.reply_event(ev, EventReplyType::Continue) + .expect("Failed to send event reply"); + i = i + 1; + } + None => println!("No events yet..."), + }, } } let duration = start.elapsed(); From 1cf4f5688be08be43f96a43f26c32d5d6d72f58f Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 04:05:15 +0100 Subject: [PATCH 10/21] xen: remove unwraps in listen --- src/driver/xen.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 3c2887f9..8faca8ca 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -329,7 +329,7 @@ impl Introspectable for Xen { fn listen(&mut self, timeout: u32) -> Result, Box> { let mut fds: [PollFd; 1] = [self.evtchn_pollfd]; - let event: Option = match poll(&mut fds, timeout.try_into().unwrap()).unwrap() { + let event: Option = match poll(&mut fds, timeout.try_into()?)? { 0 => { // timeout. no file descriptors were ready None @@ -389,10 +389,10 @@ impl Introspectable for Xen { }; // associate VCPU => vm_event_request_t // to find it in reply_event() - let vcpu_index: usize = vcpu.try_into().unwrap(); + let vcpu_index: usize = vcpu.try_into()?; self.vec_events[vcpu_index] = Some(req); Some(Event { - vcpu: vcpu.try_into().unwrap(), + vcpu: vcpu.try_into()?, kind: event_type, }) } @@ -413,7 +413,7 @@ impl Introspectable for Xen { EventReplyType::Continue => VM_EVENT_FLAG_VCPU_PAUSED, }; // get the request back - let vcpu_index: usize = event.vcpu.try_into().unwrap(); + let vcpu_index: usize = event.vcpu.try_into()?; let req: vm_event_request_t = mem::replace(&mut self.vec_events[vcpu_index], None).unwrap(); let mut rsp: vm_event_response_t = unsafe { mem::MaybeUninit::::zeroed().assume_init() }; From 8d0dcae156713d6d8576ab59c9c6660fdb9d66d2 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 12:51:24 +0100 Subject: [PATCH 11/21] kvm: handle unimplemented intercepts in match arm --- src/driver/kvm.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/driver/kvm.rs b/src/driver/kvm.rs index d99c4c12..1f305e43 100644 --- a/src/driver/kvm.rs +++ b/src/driver/kvm.rs @@ -289,6 +289,9 @@ impl Introspectable for Kvm { .kvmi .control_events(vcpu, KVMiInterceptType::Pagefault, enabled)?) } + _ => { + unimplemented!() + } } } From 512506740a6e12f1dbc627464d807eada1be7f4b Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 13:39:47 +0100 Subject: [PATCH 12/21] xen: clean the ring on exit --- src/api.rs | 1 + src/driver/xen.rs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/api.rs b/src/api.rs index 98f83d62..887df9b9 100644 --- a/src/api.rs +++ b/src/api.rs @@ -369,6 +369,7 @@ pub enum CrType { ///This provides an abstraction of event which the hypervisor reports and using which we introspect the guest #[repr(C)] +#[derive(Debug)] pub struct Event { ///vcpu on which the event is detected pub vcpu: u16, diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 8faca8ca..ba1c30db 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -477,6 +477,22 @@ impl Introspectable for Xen { impl Drop for Xen { fn drop(&mut self) { debug!("Closing Xen driver"); + // ensure paused + self.pause().expect("Failed to pause VM"); + // listen for remaining events to clear the ring + let mut cleaned = false; + while !cleaned { + match self.listen(0).expect("Failed to listen for events") { + None => cleaned = true, + Some(e) => { + debug!("cleaning queue: {:?}", e); + // replying continue + self.reply_event(e, EventReplyType::Continue) + .unwrap_or_else(|_| panic!("Failed to reply for event")) + } + } + } + let vcpu_cpunt = self.get_vcpu_count().expect("Failed to get VCPU count"); for vcpu in 0..vcpu_cpunt { self.xc @@ -488,5 +504,7 @@ impl Drop for Xen { self.xc .monitor_disable(self.domid) .expect("Failed to unmap event ring page"); + // resume + self.resume().expect("Failed to resume VM"); } } From 668cc86a591b394f2ccf0b0f3ed9d1b69dc86ad0 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 13:50:57 +0100 Subject: [PATCH 13/21] xen: replace match on booleans by ifs --- src/driver/xen.rs | 76 +++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index ba1c30db..c27cf9a7 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -359,43 +359,40 @@ impl Introspectable for Xen { } }; let back_ring_ptr = &mut self.back_ring; - match RING_HAS_UNCONSUMED_REQUESTS!(back_ring_ptr) != 0 { - false => None, - true => { - let req = self.xc.get_request(back_ring_ptr)?; - if req.version != VM_EVENT_INTERFACE_VERSION { - panic!("version mismatch"); - } - let xen_event_type = (self.xc.get_event_type(req))?; - let vcpu: u32 = req.vcpu_id; - let event_type: EventType = match xen_event_type { - XenEventType::Cr { cr_type, new, old } => EventType::Cr { - cr_type: match cr_type { - XenCr::Cr0 => CrType::Cr0, - XenCr::Cr3 => CrType::Cr3, - XenCr::Cr4 => CrType::Cr4, - }, - new, - old, - }, - XenEventType::Msr { msr_type, value } => { - EventType::Msr { msr_type, value } - } - XenEventType::Breakpoint { insn_len, .. } => { - EventType::Breakpoint { gpa: 0, insn_len } - } - XenEventType::Singlestep { .. } => EventType::Singlestep, - _ => unimplemented!(), - }; - // associate VCPU => vm_event_request_t - // to find it in reply_event() - let vcpu_index: usize = vcpu.try_into()?; - self.vec_events[vcpu_index] = Some(req); - Some(Event { - vcpu: vcpu.try_into()?, - kind: event_type, - }) + if RING_HAS_UNCONSUMED_REQUESTS!(back_ring_ptr) == 0 { + None + } else { + let req = self.xc.get_request(back_ring_ptr)?; + if req.version != VM_EVENT_INTERFACE_VERSION { + panic!("version mismatch"); } + let xen_event_type = (self.xc.get_event_type(req))?; + let vcpu: u32 = req.vcpu_id; + let event_type: EventType = match xen_event_type { + XenEventType::Cr { cr_type, new, old } => EventType::Cr { + cr_type: match cr_type { + XenCr::Cr0 => CrType::Cr0, + XenCr::Cr3 => CrType::Cr3, + XenCr::Cr4 => CrType::Cr4, + }, + new, + old, + }, + XenEventType::Msr { msr_type, value } => EventType::Msr { msr_type, value }, + XenEventType::Breakpoint { insn_len, .. } => { + EventType::Breakpoint { gpa: 0, insn_len } + } + XenEventType::Singlestep { .. } => EventType::Singlestep, + _ => unimplemented!(), + }; + // associate VCPU => vm_event_request_t + // to find it in reply_event() + let vcpu_index: usize = vcpu.try_into()?; + self.vec_events[vcpu_index] = Some(req); + Some(Event { + vcpu: vcpu.try_into()?, + kind: event_type, + }) } } x => panic!("Unexpected poll return value {}", x), @@ -451,9 +448,10 @@ impl Introspectable for Xen { Ok(self.xc.monitor_software_breakpoint(self.domid, enabled)?) } InterceptType::Singlestep => { - let op: u32 = match enabled { - false => XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_OFF, - true => XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_ON, + let op: u32 = if enabled { + XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_ON + } else { + XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_OFF }; Ok(self .xc From e12a4c062f7fe22eab6d0ec5fdc62162c46e09ae Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 13:53:13 +0100 Subject: [PATCH 14/21] xen: refactor building vm_event_response_t struct --- src/driver/xen.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index c27cf9a7..ba589134 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -412,12 +412,13 @@ impl Introspectable for Xen { // get the request back let vcpu_index: usize = event.vcpu.try_into()?; let req: vm_event_request_t = mem::replace(&mut self.vec_events[vcpu_index], None).unwrap(); - let mut rsp: vm_event_response_t = - unsafe { mem::MaybeUninit::::zeroed().assume_init() }; - rsp.reason = req.reason; - rsp.version = VM_EVENT_INTERFACE_VERSION; - rsp.vcpu_id = req.vcpu_id; - rsp.flags = req.flags & add_flags; + let mut rsp = vm_event_response_t { + reason: req.reason, + version: VM_EVENT_INTERFACE_VERSION, + vcpu_id: req.vcpu_id, + flags: req.flags & add_flags, + ..Default::default() + }; self.xc.put_response(&mut rsp, &mut self.back_ring)?; Ok(self.xev.xenevtchn_notify()?) } From 26da7a56ca3abece8a7f75764193d09b5629c3d8 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Sun, 20 Dec 2020 15:14:15 +0100 Subject: [PATCH 15/21] cargo: update xenevtchn --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index fc22e401..e0cc5778 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ libc = { version = "0.2.58", optional = true } xenctrl = { version = "0.4.4", optional = true } xenstore-rs = { version = "0.3.0", optional = true } xenforeignmemory = { version = "0.1.0", optional = true } -xenevtchn = { version = "0.1.2", optional = true } +xenevtchn = { version = "0.1.4", optional = true } xenvmevent-sys = { version = "0.1.3", optional = true } kvmi = { version = "0.2.1", optional = true } fdp = { version = "0.1.0", optional = true } @@ -49,3 +49,4 @@ clap = "2.33.0" colored = "1.9.3" mockall = "0.7.1" test-case = "1.0.0" + From 837accf7eb1de0292287a1d23b7c4735d90aaa72 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Mon, 28 Dec 2020 23:51:04 +0100 Subject: [PATCH 16/21] cargo: upgrade xenctrl to 0.4.5 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e0cc5778..5a4de727 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ hyper-v = ["winapi", "widestring", "ntapi", "vid-sys"] log = "0.4.8" env_logger = "0.7.1" libc = { version = "0.2.58", optional = true } -xenctrl = { version = "0.4.4", optional = true } +xenctrl = { version = "0.4.5", optional = true } xenstore-rs = { version = "0.3.0", optional = true } xenforeignmemory = { version = "0.1.0", optional = true } xenevtchn = { version = "0.1.4", optional = true } From e9ae8258755e16a07fe1067cda50e75023120933 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Mon, 28 Dec 2020 23:53:02 +0100 Subject: [PATCH 17/21] xen: unmap ring buffer on drop --- src/driver/xen.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index ba589134..450a2d73 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -5,8 +5,10 @@ use crate::api::{ use libc::{PROT_READ, PROT_WRITE}; use nix::poll::PollFlags; use nix::poll::{poll, PollFd}; +use nix::sys::mman::munmap; use std::convert::TryInto; use std::error::Error; +use std::ffi::c_void; use std::io::Error as IoError; use std::io::ErrorKind; use std::mem; @@ -19,8 +21,8 @@ use xenevtchn::XenEventChannel; use xenforeignmemory::XenForeignMem; use xenstore_rs::{XBTransaction, Xs, XsOpenFlags}; use xenvmevent_sys::{ - vm_event_back_ring, vm_event_request_t, vm_event_response_t, VM_EVENT_FLAG_VCPU_PAUSED, - VM_EVENT_INTERFACE_VERSION, + vm_event_back_ring, vm_event_request_t, vm_event_response_t, vm_event_sring, + VM_EVENT_FLAG_VCPU_PAUSED, VM_EVENT_INTERFACE_VERSION, }; pub struct Xen { @@ -29,6 +31,7 @@ pub struct Xen { xen_fgn: XenForeignMem, _dom_name: String, domid: u32, + ring_page: *mut vm_event_sring, back_ring: vm_event_back_ring, evtchn_pollfd: PollFd, // VCPU -> vm_event_request_t @@ -74,7 +77,7 @@ impl Xen { } let mut xc = XenControl::new(None, None, 0).unwrap(); - let (_ring_page, back_ring, remote_port) = xc + let (ring_page, back_ring, remote_port) = xc .monitor_enable(cand_domid) .expect("Failed to map event ring page"); @@ -89,6 +92,7 @@ impl Xen { xen_fgn, _dom_name: domain_name.to_string(), domid: cand_domid, + ring_page, back_ring, evtchn_pollfd, vec_events: Vec::new(), @@ -494,12 +498,23 @@ impl Drop for Xen { let vcpu_cpunt = self.get_vcpu_count().expect("Failed to get VCPU count"); for vcpu in 0..vcpu_cpunt { + debug!("disabling singlestep for VCPU {}", vcpu); self.xc .monitor_singlestep(self.domid, false) .unwrap_or_else(|_| { panic!("Failed to disable singlestep monitoring on VCPU {}", vcpu) }) } + // unmap + debug!("unmapping ring buffer"); + unsafe { + munmap( + self.ring_page as *mut c_void, + PAGE_SIZE.try_into().expect("Failed to convert to u32"), + ) + .expect("Failed to unmap ring page") + } + self.xc .monitor_disable(self.domid) .expect("Failed to unmap event ring page"); From e0ba9f7493e3218a6eaa35f201f64f6312ba54a6 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Mon, 28 Dec 2020 23:53:34 +0100 Subject: [PATCH 18/21] xen: rework pause to check if domain is already paused --- src/driver/xen.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index 450a2d73..e0856665 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -468,6 +468,18 @@ impl Introspectable for Xen { fn pause(&mut self) -> Result<(), Box> { debug!("pause"); + // get domain info, check if already paused + let dom_info = self.xc.domain_getinfo(self.domid)?; + if dom_info.domid != self.domid { + // TODO error + panic!("Invalid domid: {}", dom_info.domid); + } + if dom_info.paused() == 1 { + // already paused + // nothing to do here + debug!("already paused"); + return Ok(()); + } Ok(self.xc.domain_pause(self.domid)?) } From dde195ab794e5e3b9001379c0981e433096edc1f Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Tue, 29 Dec 2020 00:02:25 +0100 Subject: [PATCH 19/21] xen: fix disabling singlestep on drop --- src/driver/xen.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/driver/xen.rs b/src/driver/xen.rs index e0856665..245161fd 100644 --- a/src/driver/xen.rs +++ b/src/driver/xen.rs @@ -101,15 +101,11 @@ impl Xen { // enable singlestep monitoring // it will only intercept events when explicitely requested using // xc_domain_debug_control() - let vcpu_count = xen.get_vcpu_count().expect("Failed to get VCPU count"); - for vcpu in 0..vcpu_count { - xen.xc - .monitor_singlestep(cand_domid, true) - .unwrap_or_else(|_| { - panic!("Failed to enable singlestep monitoring on VCPU {}", vcpu) - }); - } + xen.xc + .monitor_singlestep(cand_domid, true) + .unwrap_or_else(|_| panic!("Failed to enable singlestep monitoring")); + let vcpu_count = xen.get_vcpu_count().expect("Failed to get VCPU count"); // init vec events xen.vec_events.resize(vcpu_count.try_into().unwrap(), None); // TODO: vm_event_request_t (vm_event_st) doesn't derive Debug even when .derive_debug(true) @@ -511,12 +507,13 @@ impl Drop for Xen { let vcpu_cpunt = self.get_vcpu_count().expect("Failed to get VCPU count"); for vcpu in 0..vcpu_cpunt { debug!("disabling singlestep for VCPU {}", vcpu); - self.xc - .monitor_singlestep(self.domid, false) - .unwrap_or_else(|_| { - panic!("Failed to disable singlestep monitoring on VCPU {}", vcpu) - }) + self.toggle_intercept(vcpu, InterceptType::Singlestep, false) + .unwrap_or_else(|_| panic!("Failed to disable singlestep on VCPU {}", vcpu)); } + + self.xc + .monitor_singlestep(self.domid, false) + .unwrap_or_else(|_| panic!("Failed to disable singlestep monitoring")); // unmap debug!("unmapping ring buffer"); unsafe { @@ -524,7 +521,7 @@ impl Drop for Xen { self.ring_page as *mut c_void, PAGE_SIZE.try_into().expect("Failed to convert to u32"), ) - .expect("Failed to unmap ring page") + .expect("Failed to unmap ring page"); } self.xc From f3ea0aed3af25d4b7f74dc6f59431d4c1899ce4f Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Tue, 29 Dec 2020 01:41:46 +0100 Subject: [PATCH 20/21] examples/singlestep-events: add counter option --- examples/singlestep-events.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/singlestep-events.rs b/examples/singlestep-events.rs index c5e158a6..70b043fc 100644 --- a/examples/singlestep-events.rs +++ b/examples/singlestep-events.rs @@ -13,6 +13,12 @@ fn parse_args() -> ArgMatches<'static> { .version("0.1") .about("Watches singlestep VMI events") .arg(Arg::with_name("vm_name").index(1).required(true)) + .arg( + Arg::with_name("count") + .help("Listen for events then quit") + .takes_value(true) + .short("c"), + ) .get_matches() } @@ -40,6 +46,11 @@ fn main() { let init_option = matches .value_of("kvmi_socket") .map(|socket| DriverInitParam::KVMiSocket(socket.into())); + let count_opt = matches.value_of("count").map(|counter_str| { + counter_str + .parse::() + .expect("Counter is not a valid number") + }); // set CTRL-C handler let running = Arc::new(AtomicBool::new(true)); let r = running.clone(); @@ -81,6 +92,14 @@ fn main() { drv.reply_event(ev, EventReplyType::Continue) .expect("Failed to send event reply"); i = i + 1; + match count_opt { + None => continue, + Some(counter) => { + if i == counter { + break; + } + } + }; } None => println!("No events yet..."), }, From 0d957f022376df5103c083440568e3c33f32c067 Mon Sep 17 00:00:00 2001 From: Mathieu Tarral Date: Tue, 29 Dec 2020 01:55:38 +0100 Subject: [PATCH 21/21] c_examples/regs-dump: fix compilation --- c_examples/regs-dump.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/c_examples/regs-dump.c b/c_examples/regs-dump.c index 602c1265..9a56a5f1 100644 --- a/c_examples/regs-dump.c +++ b/c_examples/regs-dump.c @@ -14,17 +14,17 @@ void read_registers(void* driver, const char* vm_name) { Registers regs; memset(®s, 0, sizeof(regs)); if (microvmi_read_registers(driver, 0, ®s)) { - printf("rax: 0x%" PRIx64 "\n", regs.x86._0.rax); - printf("rbx: 0x%" PRIx64 "\n", regs.x86._0.rbx); - printf("rcx: 0x%" PRIx64 "\n", regs.x86._0.rcx); - printf("rdx: 0x%" PRIx64 "\n", regs.x86._0.rdx); - printf("rsi: 0x%" PRIx64 "\n", regs.x86._0.rsi); - printf("rdi: 0x%" PRIx64 "\n", regs.x86._0.rdi); - printf("rsp: 0x%" PRIx64 "\n", regs.x86._0.rsp); - printf("rbp: 0x%" PRIx64 "\n", regs.x86._0.rbp); - printf("rip: 0x%" PRIx64 "\n", regs.x86._0.rip); - printf("rflags: 0x%" PRIx64 "\n", regs.x86._0.rflags); - printf("cr3: 0x%" PRIx64 "\n", regs.x86._0.cr3); + printf("rax: 0x%" PRIx64 "\n", regs.x86.rax); + printf("rbx: 0x%" PRIx64 "\n", regs.x86.rbx); + printf("rcx: 0x%" PRIx64 "\n", regs.x86.rcx); + printf("rdx: 0x%" PRIx64 "\n", regs.x86.rdx); + printf("rsi: 0x%" PRIx64 "\n", regs.x86.rsi); + printf("rdi: 0x%" PRIx64 "\n", regs.x86.rdi); + printf("rsp: 0x%" PRIx64 "\n", regs.x86.rsp); + printf("rbp: 0x%" PRIx64 "\n", regs.x86.rbp); + printf("rip: 0x%" PRIx64 "\n", regs.x86.rip); + printf("rflags: 0x%" PRIx64 "\n", regs.x86.rflags); + printf("cr3: 0x%" PRIx64 "\n", regs.x86.cr3); } else { printf("Unable to read registers.\n"); }