From 78dd7b6d1790d064df260a6b13428eb95f09af80 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 19 Apr 2026 22:53:43 +0200 Subject: [PATCH] Add flamegraph diff action for two selected queries Select two queries with and pick "Query flamegraph diff" from the action menu to compare their trace_log flamegraphs. The diff is similar to `difffolded.pl|flamegraph.pl`. Supports CPU, Real, Memory, MemorySample, JemallocSample, MemoryAllocatedWithoutCheck and ProfileEvent trace types, in both TUI and pastila-share modes. Fixes #100. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 105 ++++------------------------------ Cargo.toml | 2 +- src/interpreter/flamegraph.rs | 51 +++++++++-------- src/interpreter/worker.rs | 62 +++++++++++++++++--- src/view/queries_view.rs | 99 ++++++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1dcacac..910941a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,7 +125,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -136,7 +136,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -158,7 +158,7 @@ dependencies = [ "objc2-foundation", "parking_lot", "percent-encoding", - "windows-sys 0.60.2", + "windows-sys 0.52.0", "x11rb", ] @@ -1030,7 +1030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1065,7 +1065,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flamelens" version = "0.4.0" -source = "git+https://github.com/ys-l/flamelens?branch=main#789af11886e8a200ab66ebb2d1ee591abb7ea7a0" +source = "git+https://github.com/azat-rust/flamelens?branch=diff-mode#292bdd09f0db37c3a0e40b2011ae114106dc5694" dependencies = [ "anyhow", "cfg-if", @@ -1665,7 +1665,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2586,7 +2586,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2599,7 +2599,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2827,7 +2827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2941,7 +2941,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3606,24 +3606,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -3657,30 +3639,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3693,12 +3658,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3711,12 +3670,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3729,24 +3682,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3759,12 +3700,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3777,12 +3712,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3795,12 +3724,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3813,12 +3736,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/Cargo.toml b/Cargo.toml index 993eb36..ac8f785 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ clickhouse-rs = { git = "https://github.com/azat-rust/clickhouse-rs", branch = " tokio = { version = "*", default-features = false, features = ["macros"] } console-subscriber = { version = "*", default-features = false, optional = true } # Flamegraphs -flamelens = { git = "https://github.com/ys-l/flamelens", branch = "main", default-features = false } +flamelens = { git = "https://github.com/azat-rust/flamelens", branch = "diff-mode", default-features = false } ratatui = { version = "0.29.0", features = ["unstable-rendered-line-info"] } # Should **only** with the flamelens, since cursive re-export it, while flamelens does not crossterm = { version = "0.28.1", features = ["use-dev-tty"] } diff --git a/src/interpreter/flamegraph.rs b/src/interpreter/flamegraph.rs index 302fa7e..b992b86 100644 --- a/src/interpreter/flamegraph.rs +++ b/src/interpreter/flamegraph.rs @@ -10,8 +10,8 @@ use ratatui::Terminal; use ratatui::backend::CrosstermBackend; use std::io; -pub fn show(block: Columns) -> AppResult<()> { - let data = block +pub fn block_to_folded(block: &Columns) -> String { + block .rows() .map(|x| { [ @@ -21,15 +21,10 @@ pub fn show(block: Columns) -> AppResult<()> { .join(" ") }) .collect::>() - .join("\n"); - - if data.trim().is_empty() { - return Err(Error::msg("Flamegraph is empty").into()); - } - - let flamegraph = FlameGraph::from_string(data, true); - let mut app = App::with_flamegraph("Query", flamegraph); + .join("\n") +} +fn run_flamelens(mut app: App) -> AppResult<()> { let backend = CrosstermBackend::new(io::stderr()); let mut terminal = Terminal::new(backend)?; let timeout = std::time::Duration::from_secs(1); @@ -73,23 +68,33 @@ pub fn show(block: Columns) -> AppResult<()> { Ok(()) } +pub fn show(title: &'static str, data: String) -> AppResult<()> { + if data.trim().is_empty() { + return Err(Error::msg("Flamegraph is empty").into()); + } + + let flamegraph = FlameGraph::from_string(data, true); + run_flamelens(App::with_flamegraph(title, flamegraph)) +} + +/// Show a differential flamegraph: `after` rendered with per-frame coloring +/// against the `before` baseline (handled by flamelens's `diff_mode`). +pub fn show_diff(title: &'static str, before: String, after: String) -> AppResult<()> { + if before.trim().is_empty() && after.trim().is_empty() { + return Err(Error::msg("Flamegraph diff is empty (both queries have no samples)").into()); + } + + let before_fg = FlameGraph::from_string(before, true); + let mut after_fg = FlameGraph::from_string(after, true); + after_fg.set_diff_against(&before_fg); + run_flamelens(App::with_flamegraph(title, after_fg)) +} + pub async fn share( - block: Columns, + data: String, pastila_clickhouse_host: &str, pastila_url: &str, ) -> Result { - let data = block - .rows() - .map(|x| { - [ - x.get::(0).unwrap(), - x.get::(1).unwrap().to_string(), - ] - .join(" ") - }) - .collect::>() - .join("\n"); - if data.trim().is_empty() { return Err(Error::msg("Flamegraph is empty")); } diff --git a/src/interpreter/worker.rs b/src/interpreter/worker.rs index 18fc01e..e44465a 100644 --- a/src/interpreter/worker.rs +++ b/src/interpreter/worker.rs @@ -2,7 +2,7 @@ use crate::{ common::{RelativeDateTime, Stopwatch}, interpreter::{ ContextArc, Query, - clickhouse::{ClickHouse, Columns, TextLogArguments, TraceType}, + clickhouse::{ClickHouse, TextLogArguments, TraceType}, flamegraph, perfetto::PerfettoTraceBuilder, }, @@ -44,6 +44,15 @@ pub enum Event { Option>, Vec, ), + // (type, start time, end time, [query_ids_a = before], [query_ids_b = after]). + // Diff mode is TUI-only (color-coded via flamelens), no share path. + QueryFlameGraphDiff( + TraceType, + DateTime, + Option>, + Vec, + Vec, + ), // [bool (true - show in TUI, false - open in browser), query_ids] LiveQueryFlameGraph(bool, Option>), Summary, @@ -100,6 +109,7 @@ impl Event { Event::ServerFlameGraph(..) => "ServerFlameGraph".to_string(), Event::JemallocFlameGraph(..) => "JemallocFlameGraph".to_string(), Event::QueryFlameGraph(..) => "QueryFlameGraph".to_string(), + Event::QueryFlameGraphDiff(..) => "QueryFlameGraphDiff".to_string(), Event::LiveQueryFlameGraph(..) => "LiveQueryFlameGraph".to_string(), Event::Summary => "Summary".to_string(), Event::KillQuery(..) => "KillQuery".to_string(), @@ -309,14 +319,15 @@ async fn start_tokio(context: ContextArc, receiver: ReceiverArc) { async fn render_or_share_flamegraph( tui: bool, cb_sink: cursive::CbSink, - block: Columns, + title: &'static str, + data: String, pastila_clickhouse_host: String, pastila_url: String, ) -> Result<()> { if tui { cb_sink .send(Box::new(move |siv: &mut cursive::Cursive| { - flamegraph::show(block) + flamegraph::show(title, data) .or_else(|e| { siv.add_layer(views::Dialog::info(e.to_string())); return anyhow::Ok(()); @@ -325,7 +336,7 @@ async fn render_or_share_flamegraph( })) .map_err(|_| anyhow!("Cannot send message to UI"))?; } else { - let url = flamegraph::share(block, &pastila_clickhouse_host, &pastila_url).await?; + let url = flamegraph::share(data, &pastila_clickhouse_host, &pastila_url).await?; let url_clone = url.clone(); cb_sink @@ -776,7 +787,8 @@ async fn process_event(context: ContextArc, event: Event, need_clear: &mut bool) render_or_share_flamegraph( tui, cb_sink, - flamegraph_block, + "Server", + flamegraph::block_to_folded(&flamegraph_block), pastila_clickhouse_host, pastila_url, ) @@ -790,7 +802,8 @@ async fn process_event(context: ContextArc, event: Event, need_clear: &mut bool) render_or_share_flamegraph( tui, cb_sink, - flamegraph_block, + "jemalloc", + flamegraph::block_to_folded(&flamegraph_block), pastila_clickhouse_host, pastila_url, ) @@ -810,13 +823,45 @@ async fn process_event(context: ContextArc, event: Event, need_clear: &mut bool) render_or_share_flamegraph( tui, cb_sink, - flamegraph_block, + "Query", + flamegraph::block_to_folded(&flamegraph_block), pastila_clickhouse_host, pastila_url, ) .await?; *need_clear = true; } + Event::QueryFlameGraphDiff(trace_type, start, end, query_ids_a, query_ids_b) => { + let (block_a, block_b) = tokio::try_join!( + clickhouse.get_flamegraph( + trace_type.clone(), + Some(&query_ids_a), + Some(start), + end, + selected_host.as_ref(), + ), + clickhouse.get_flamegraph( + trace_type, + Some(&query_ids_b), + Some(start), + end, + selected_host.as_ref(), + ), + )?; + let before = flamegraph::block_to_folded(&block_a); + let after = flamegraph::block_to_folded(&block_b); + cb_sink + .send(Box::new(move |siv: &mut cursive::Cursive| { + flamegraph::show_diff("Query diff", before, after) + .or_else(|e| { + siv.add_layer(views::Dialog::info(e.to_string())); + return anyhow::Ok(()); + }) + .unwrap(); + })) + .map_err(|_| anyhow!("Cannot send message to UI"))?; + *need_clear = true; + } Event::LiveQueryFlameGraph(tui, query_ids) => { let flamegraph_block = clickhouse .get_live_query_flamegraph(&query_ids, selected_host.as_ref()) @@ -824,7 +869,8 @@ async fn process_event(context: ContextArc, event: Event, need_clear: &mut bool) render_or_share_flamegraph( tui, cb_sink, - flamegraph_block, + "Query (live)", + flamegraph::block_to_folded(&flamegraph_block), pastila_clickhouse_host, pastila_url, ) diff --git a/src/view/queries_view.rs b/src/view/queries_view.rs index c359484..644c7fe 100644 --- a/src/view/queries_view.rs +++ b/src/view/queries_view.rs @@ -368,6 +368,33 @@ impl QueriesView { return Ok(()); } + fn show_flamegraph_diff(&mut self, trace_type: TraceType) -> Result<()> { + let (groups, min_query_start_microseconds, max_query_end_microseconds) = + self.get_query_id_groups()?; + if groups.len() != 2 { + return Err(Error::msg(format!( + "Flamegraph diff requires exactly 2 queries selected with , got {}", + groups.len() + ))); + } + let mut groups_iter = groups.into_iter(); + let query_ids_a = groups_iter.next().unwrap(); + let query_ids_b = groups_iter.next().unwrap(); + let mut context_locked = self.context.lock().unwrap(); + context_locked.worker.send( + true, + WorkerEvent::QueryFlameGraphDiff( + trace_type, + min_query_start_microseconds, + max_query_end_microseconds, + query_ids_a, + query_ids_b, + ), + ); + + return Ok(()); + } + fn get_selected_query(&self) -> Result { let item_index = self.table.item().ok_or(Error::msg("No query selected"))?; let item = self @@ -441,6 +468,63 @@ impl QueriesView { )); } + /// Group selected queries by their initial_query_id so each logical distributed + /// query becomes a single group of constituent query_ids. Preserves the selection + /// order: the group whose initial_query_id first appears among the selected rows + /// comes first. + fn get_query_id_groups( + &self, + ) -> Result<(Vec>, DateTime, Option>)> { + if self.selected_query_ids.len() < 2 { + return Err(Error::msg( + "Select at least 2 queries with to diff their flamegraphs", + )); + } + + // Dedup initial_query_ids for the selected rows, keeping insertion order so the + // diff is deterministic (first-selected is "before", next "after"). + let mut initial_query_ids: Vec = Vec::new(); + for q in self.items.values() { + let key = query_key(q); + let initial_key = (q.initial_query_id.clone(), q.host_name.clone()); + if (self.selected_query_ids.contains(&initial_key) + || self.selected_query_ids.contains(&key)) + && !initial_query_ids.contains(&q.initial_query_id) + { + initial_query_ids.push(q.initial_query_id.clone()); + } + } + + let mut min_start: Option> = None; + let mut max_end: Option> = None; + let mut groups: Vec> = Vec::with_capacity(initial_query_ids.len()); + for iqid in &initial_query_ids { + let mut group = Vec::new(); + for q in self.items.values() { + if &q.initial_query_id != iqid { + continue; + } + group.push(q.query_id.clone()); + min_start = Some(match min_start { + Some(cur) => cur.min(q.query_start_time_microseconds), + None => q.query_start_time_microseconds, + }); + if !self.is_system_processes { + max_end = Some(match max_end { + Some(cur) => cur.max(q.query_end_time_microseconds), + None => q.query_end_time_microseconds, + }); + } + } + if !group.is_empty() { + groups.push(group); + } + } + + let min_start = min_start.ok_or_else(|| Error::msg("No queries matched selection"))?; + return Ok((groups, min_start, max_end)); + } + pub fn update_limit(&mut self, is_sub: bool) { let new_limit = if is_sub { self.limit.clone().lock().unwrap().saturating_sub(20) @@ -496,6 +580,14 @@ impl QueriesView { Ok(Some(EventResult::consumed())) } + fn action_show_flamegraph_diff( + &mut self, + trace_type: TraceType, + ) -> Result> { + self.show_flamegraph_diff(trace_type)?; + Ok(Some(EventResult::consumed())) + } + fn action_query_profile_events(&mut self) -> Result> { // Check if multiple queries are selected if self.selected_query_ids.len() > 1 { @@ -1167,6 +1259,13 @@ impl QueriesView { add_action!(context, &mut event_view, "Share Query MemoryAllocatedWithoutCheck flamegraph", action_show_flamegraph(false, Some(TraceType::MemoryAllocatedWithoutCheck))); add_action!(context, &mut event_view, "Share Query events flamegraph", action_show_flamegraph(false, Some(TraceType::ProfileEvent))); add_action!(context, &mut event_view, "Share Query live flamegraph", action_show_flamegraph(false, None)); + add_action!(context, &mut event_view, "Query CPU flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::CPU)); + add_action!(context, &mut event_view, "Query Real flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::Real)); + add_action!(context, &mut event_view, "Query memory flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::Memory)); + add_action!(context, &mut event_view, "Query memory sample flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::MemorySample)); + add_action!(context, &mut event_view, "Query jemalloc sample flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::JemallocSample)); + add_action!(context, &mut event_view, "Query MemoryAllocatedWithoutCheck flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::MemoryAllocatedWithoutCheck)); + add_action!(context, &mut event_view, "Query events flamegraph diff (select 2 with )", action_show_flamegraph_diff(TraceType::ProfileEvent)); add_action!(context, &mut event_view, "EXPLAIN INDEXES", 'I', action_explain_indexes); add_action!(context, &mut event_view, "EXPLAIN PIPELINE graph=1 (share)", 'G', action_explain_pipeline_graph); add_action!(context, &mut event_view, "KILL query", 'K', action_kill_query);