From 90f8c601c6415b4e64b4f996e9a6cf4c9cf2a67d Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:33:08 -0400 Subject: [PATCH 1/4] feat(viz): KPI overview row, icicle charts & layout uirevision (plotly.rs #423) Adopt dathere/plotly branch `feature/trace-type-additions` (PR plotly/plotly.rs#423, a strict superset of the prior pin) and use the new API in `viz`: - viz smart now leads with a full-width KPI overview row of plotly `Indicator` tiles: a built-in completeness gauge plus the headline numeric measures (sum for extensive, mean for intensive). A measure becomes a gauge when the data dictionary supplies a validated `x-qsv.gauge_range` (drawn only when the range actually contains the observed value; a range also forces the mean so it lands in-scale) and a "vs target" delta when it supplies `x-qsv.target` (a semantically-justified goal, never a fabricated prior-period baseline). Renders in both the inline and typed-grid paths and stays invisible to --max-charts, the panel-count notices, and the render-path thresholds (it is a domain band, not a cartesian subplot). HTML-only. - Add `viz icicle` subcommand and `--hierarchy-style icicle`, a level-aligned sibling of treemap/sunburst. - Set layout-level `uirevision` centrally in apply_theme so user zoom/pan/ selection/legend state survives the Plotly.newPlot re-renders that inject the fullscreen modebar button and recolor on theme toggle. describegpt emission of gauge_range/target is deferred; until then the hints are consumed from any dictionary that carries them, with completeness + number tiles working with no dictionary. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 36 ++--- Cargo.toml | 14 +- docs/help/viz.md | 5 +- src/cmd/viz.rs | 337 +++++++++++++++++++++++++++++++++++++++++++--- tests/test_viz.rs | 102 ++++++++++++-- 5 files changed, 443 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a05cdf42db..70c59e252a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -341,7 +341,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -352,7 +352,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2355,7 +2355,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2567,7 +2567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5483,7 +5483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5828,7 +5828,7 @@ dependencies = [ [[package]] name = "plotly" version = "0.14.1" -source = "git+https://github.com/dathere/plotly?rev=8d56b22#8d56b22d5a3f145f57462b79444157cad93736fc" +source = "git+https://github.com/dathere/plotly?branch=feature%2Ftrace-type-additions#554f6609e2861ce82b481e5beb8d6e1fed50ba3f" dependencies = [ "askama", "async-trait", @@ -5851,7 +5851,7 @@ dependencies = [ [[package]] name = "plotly_derive" version = "0.14.1" -source = "git+https://github.com/dathere/plotly?rev=8d56b22#8d56b22d5a3f145f57462b79444157cad93736fc" +source = "git+https://github.com/dathere/plotly?branch=feature%2Ftrace-type-additions#554f6609e2861ce82b481e5beb8d6e1fed50ba3f" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5862,7 +5862,7 @@ dependencies = [ [[package]] name = "plotly_static" version = "0.1.0" -source = "git+https://github.com/dathere/plotly?rev=8d56b22#8d56b22d5a3f145f57462b79444157cad93736fc" +source = "git+https://github.com/dathere/plotly?branch=feature%2Ftrace-type-additions#554f6609e2861ce82b481e5beb8d6e1fed50ba3f" dependencies = [ "anyhow", "base64 0.22.1", @@ -7094,7 +7094,7 @@ dependencies = [ "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7945,7 +7945,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8013,7 +8013,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8616,7 +8616,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8741,7 +8741,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9008,10 +9008,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9024,7 +9024,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook 0.3.18", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9060,7 +9060,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10338,7 +10338,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 49db43deae..aadf442cda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -462,12 +462,14 @@ sled = { git = "https://github.com/dathere/sled", branch = "v0.34.7-drop-fxhash" # use our patched fork of whatlang to bump hashbrown until our PR is merged whatlang = { git = "https://github.com/jqnatividad/whatlang-rs", branch = "bump-hashbrown-to-16" } -# Our Treemap/Sunburst PR has been merged (https://github.com/plotly/plotly.rs/pull/406) -# Pinned to the MapLibre-migration integration HEAD, which adds the MapLibre `map`-subplot -# traces qsv's `viz` maps now use: ChoroplethMap, DensityMap, and ScatterMap (replacing the -# deprecated *mapbox traces). Tracks upstream PRs #410 (Choropleth), #415/#416 (DensityMapbox -# hover+color / DensityMap) and the ScatterMap follow-up. -plotly = { git = "https://github.com/dathere/plotly", rev = "8d56b22" } +# Pinned to our stacked feature branch `feature/trace-type-additions` (PR +# https://github.com/plotly/plotly.rs/pull/423), a strict superset of the earlier pins: +# - MapLibre `map`-subplot traces qsv's `viz` maps use (ChoroplethMap, DensityMap, +# ScatterMap; upstream PRs #406/#410/#415/#416) — now merged into plotly:main +# - plotly.js 3.7 attribute backfill (legendrank/zorder/fillgradient/... ; PR #421) +# - new trace types Indicator, Histogram2d, Icicle + layout-level `uirevision` (PR #423) +# When #421/#423 merge upstream, repoint this to a plotly.rs release/main rev. +plotly = { git = "https://github.com/dathere/plotly", branch = "feature/trace-type-additions" } # TRACKING: remove this patch when upstream resolves it (see exit conditions below). # diff --git a/docs/help/viz.md b/docs/help/viz.md index 42155b61dc..0400f25f24 100644 --- a/docs/help/viz.md +++ b/docs/help/viz.md @@ -60,6 +60,8 @@ treemap Part-to-whole hierarchy as nested tiles. --cols = 2+ dimension columns (levels), optional --value and --agg. sunburst Part-to-whole hierarchy as concentric rings (same inputs as treemap). Better for deeper hierarchies. +icicle Part-to-whole hierarchy as stacked bars per level (same inputs as + treemap). Level-aligned; good for deep, wide hierarchies. map Geographic point map (or --density heatmap) on tile basemaps. Pick the coordinate columns with the lat/lon options below. geo Geographic point map on a projection basemap (coastlines/land/ @@ -313,6 +315,7 @@ qsv viz geo [options] qsv viz choropleth [options] qsv viz treemap [options] qsv viz sunburst [options] +qsv viz icicle [options] qsv viz --help ``` @@ -393,7 +396,7 @@ qsv viz --help |  `‑‑no‑nulls`  | flag | Omit the "(NULL)" bar (empty cells) from frequency bar charts. By default `viz smart` shows a "(NULL)" bar, like `qsv frequency`. | | |  `‑‑no‑other`  | flag | Omit the "Other (N)" aggregate bar from frequency bar charts. It collects the categories beyond --limit (N = how many distinct categories were rolled up) and is shown by default. | | |  `‑‑smarter`  | flag | Before building the dashboard, run `qsv moarstats --advanced` to enrich the stats cache with distribution-shape statistics (bimodality, entropy, skewness, outlier share). This unlocks histograms for bimodal columns, frequency bars for concentrated high-cardinality columns, and skew/outlier hints on box panels. Costs one extra pass over the data and writes .stats.csv, its sidecars, and an .idx index (like running `qsv moarstats` manually). On geocode-enabled builds it also enriches map point hovers with the US FIPS code and annotates the spatial-extent summary with the country's continent; the county is always shown in map hovers, with or without --smarter. Only affects `smart`. Applied only with default parsing; inputs using --no-headers or a custom --delimiter fall back to the standard dashboard. | | -|  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure). Only affects `smart`. | | +|  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst, icicle. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure); icicle is an opt-in level-aligned alternative. Only affects `smart`. | | |  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. Only affects `smart`. | | |  `‑‑dictionary‑context`  | string | Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to describegpt as --context-file when `--dictionary infer` generates the dictionary. Better context yields better role/concept/label/grain tags, hence a better dashboard. Ignored unless `--dictionary infer` is used (it does not apply when reading an existing dictionary file). Only affects `smart`. | | |  `‑‑dict‑info`  | flag | When a usable Data Dictionary is available (per --dictionary), add a "Data Dictionary" link beneath the dashboard title and an info icon on each panel title: hovering shows that column's dictionary description; clicking opens a human-friendly rendering of the dictionary in a side drawer NEXT TO the plots (embedded in the dashboard file - no extra file is written), scrolled to and highlighting that column's entry. The drawer is open by default on load and can be dismissed with its close button or Esc. The dictionary page carries a role-tinted table of contents and per-column "View chart" links back to the panels. The drawer's popout button opens the same document in its own browser tab instead (needs a browser that allows user-initiated pop-ups). HTML output only; ignored with a note when no dictionary is available or when exporting an image. Only affects `smart`. | | diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 94da32767b..72aebaba3c 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -46,6 +46,8 @@ Chart types (subcommands): columns (levels), optional --value and --agg. sunburst Part-to-whole hierarchy as concentric rings (same inputs as treemap). Better for deeper hierarchies. + icicle Part-to-whole hierarchy as stacked bars per level (same inputs as + treemap). Level-aligned; good for deep, wide hierarchies. map Geographic point map (or --density heatmap) on tile basemaps. Pick the coordinate columns with the lat/lon options below. geo Geographic point map on a projection basemap (coastlines/land/ @@ -205,6 +207,7 @@ Usage: qsv viz choropleth [options] qsv viz treemap [options] qsv viz sunburst [options] + qsv viz icicle [options] qsv viz --help viz options: @@ -433,10 +436,11 @@ smart options: fall back to the standard dashboard. --hierarchy-style For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). - One of: auto (default), treemap, sunburst. auto follows best - practice — a treemap for a shallow 2-level hierarchy (accurate + One of: auto (default), treemap, sunburst, icicle. auto follows + best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent - child structure). Only affects `smart`. + child structure); icicle is an opt-in level-aligned alternative. + Only affects `smart`. --dictionary Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric @@ -588,16 +592,17 @@ use plotly::layout::update_menu::UpdateMenuDirection; use plotly::layout::update_menu::{Button, ButtonMethod, UpdateMenu, UpdateMenuType}; use plotly::{ Bar, BoxPlot, Candlestick, Choropleth, ChoroplethMap, Configuration, Contour, DensityMap, - HeatMap, Histogram, Ohlc, Pie, Plot, Sankey, Scatter, Scatter3D, ScatterGeo, ScatterMap, - ScatterPolar, Sunburst, Trace, Treemap, Violin, + HeatMap, Histogram, Icicle, Indicator, Ohlc, Pie, Plot, Sankey, Scatter, Scatter3D, ScatterGeo, + ScatterMap, ScatterPolar, Sunburst, Trace, Treemap, Violin, box_plot::{BoxPoints, QuartileMethod}, choropleth::{LocationMode, Marker as ChoroplethMarker}, color::NamedColor, common::{ - Anchor, ColorBar, ColorScale, ColorScalePalette, ErrorData, ErrorType, Fill, Font, + Anchor, ColorBar, ColorScale, ColorScalePalette, Domain, ErrorData, ErrorType, Fill, Font, HoverInfo, Line, Marker, Mode, Orientation, Pattern, PatternShape, TextPosition, TickMode, Title, }, + indicator::{Delta, Gauge, GaugeAxis, IndicatorTitle, Mode as IndicatorMode, Number}, layout::{ Annotation, Axis, AxisType, Center, GeoFitBounds, GeoResolution, HoverMode, Layout, LayoutGeo, LayoutMap, LayoutScene, MapBounds, MapStyle, Margin, ModeBar, Projection, @@ -605,7 +610,7 @@ use plotly::{ }, sankey::{Arrangement, Link, Node}, sunburst::InsideTextOrientation, - traces::scatter_map::Cluster, + traces::{icicle::BranchValues as IcicleBranchValues, scatter_map::Cluster}, treemap::{BranchValues, Marker as TreemapMarker, Pad}, violin::{MeanLine, SpanMode, ViolinBox, ViolinPoints}, }; @@ -990,6 +995,12 @@ const PALETTE: [&str; 8] = [ const FONT_FAMILY: &str = "Helvetica Neue, Helvetica, Arial, sans-serif"; const INK: &str = "#2A3F5F"; const PAPER_BG: &str = "#FFFFFF"; +/// Stable `layout.uirevision` token. Plotly preserves user UI state (zoom, pan, box/lasso +/// selection, legend visibility) across `Plotly.react`/`Plotly.newPlot` calls as long as this +/// value is unchanged. qsv re-renders every graph div via `newPlot` to inject the fullscreen +/// modebar button and to recolor on theme toggle; without a constant `uirevision` those +/// re-renders reset the user's view. Applied centrally in `apply_theme`. +const VIZ_UIREVISION: &str = "qsv-viz"; const GRID_COLOR: &str = "#ECECEC"; const AXIS_LINE: &str = "#BCC4CE"; @@ -1160,6 +1171,7 @@ struct Args { cmd_geo: bool, cmd_treemap: bool, cmd_sunburst: bool, + cmd_icicle: bool, cmd_choropleth: bool, arg_input: Option, flag_x: Option, @@ -1540,7 +1552,10 @@ fn build_plot(args: &Args, out_format: OutFormat, progress: &ProgressBar) -> Cli // treemap / sunburst are domain-based (no cartesian x/y axes), like pie — they own their whole // `Plot` as well. - if matches!(chart_kind(args), Chart::Treemap | Chart::Sunburst) { + if matches!( + chart_kind(args), + Chart::Treemap | Chart::Sunburst | Chart::Icicle + ) { return build_hierarchy_plot(args); } @@ -1654,6 +1669,7 @@ enum Chart { Choropleth, Treemap, Sunburst, + Icicle, } fn chart_kind(args: &Args) -> Chart { @@ -1695,6 +1711,8 @@ fn chart_kind(args: &Args) -> Chart { Chart::Treemap } else if args.cmd_sunburst { Chart::Sunburst + } else if args.cmd_icicle { + Chart::Icicle } else { unreachable!("docopt guarantees exactly one chart subcommand") } @@ -4929,7 +4947,8 @@ fn build_hierarchy_plot(args: &Args) -> CliResult { }, Some(other) => { return fail_incorrectusage_clierror!( - "treemap/sunburst only supports additive --agg (count or sum); got '{other}'." + "treemap/sunburst/icicle only supports additive --agg (count or sum); got \ + '{other}'." ); }, }; @@ -4939,6 +4958,8 @@ fn build_hierarchy_plot(args: &Args) -> CliResult { let top_n = args.flag_limit.max(1); let style = if args.cmd_sunburst { HierStyle::Sunburst + } else if args.cmd_icicle { + HierStyle::Icicle } else { HierStyle::Treemap }; @@ -5958,6 +5979,9 @@ impl Args { /// overrides at the call sites are gated on `theme.is_none()` so the template can /// drive backgrounds, fonts and axis chrome. fn apply_theme(layout: Layout, theme: Option) -> Layout { + // Preserve user UI state (zoom/pan/selection/legend) across the `Plotly.newPlot` + // re-renders qsv triggers for the fullscreen-modebar button and theme toggle. + let layout = layout.ui_revision(VIZ_UIREVISION); match theme { Some(t) => layout.template(t.build()), None => layout, @@ -7222,8 +7246,9 @@ fn resolve_hierarchy_style(flag: Option<&str>, depth: usize) -> CliResult Ok(auto_hierarchy_style(depth)), Some("treemap") => Ok(HierStyle::Treemap), Some("sunburst") => Ok(HierStyle::Sunburst), + Some("icicle") => Ok(HierStyle::Icicle), Some(other) => fail_incorrectusage_clierror!( - "Unknown --hierarchy-style '{other}'. Use auto, treemap, or sunburst." + "Unknown --hierarchy-style '{other}'. Use auto, treemap, sunburst, or icicle." ), } } @@ -8772,9 +8797,29 @@ struct GeoMeta { enum HierStyle { Treemap, Sunburst, + Icicle, +} + +/// One KPI "big number" tile in a `PanelKind::KpiRow`. `value` is the headline number (a dataset +/// summary or an aggregated measure); `format` is a plotly d3 value-format string. `gauge` carries +/// a validated `[min, max]` domain (drawn as an angular gauge) and `target` a delta reference +/// (drawn as a "vs target" delta); both are optional and gate the indicator `mode`. +#[derive(Clone, Debug)] +struct KpiTile { + label: String, + value: f64, + format: String, + gauge: Option<[f64; 2]>, + target: Option, } enum PanelKind { + /// A leading full-width row of KPI "big number" tiles (plotly `Indicator` traces) summarizing + /// the dataset — record count, field count, completeness, and the headline numeric measures. + /// A tile shows a gauge when its dictionary supplies a validated `gauge_range`, a "vs target" + /// delta when it supplies a `target`, else a plain number. Rendered domain-positioned so N + /// tiles split the row width. + KpiRow { tiles: Vec }, /// Box plot drawn from precomputed quartiles (no raw data). BoxStats { q1: f64, @@ -9293,6 +9338,15 @@ struct DictRow { concept: String, label: String, description: String, + /// Optional `[min, max]` domain for a KPI gauge tile (`x-qsv.gauge_range`), set only for + /// numeric measures whose concept implies a canonical scale (percent, ratio, rating, score). + /// The KPI builder still validates that the observed value falls inside the range before + /// drawing a gauge (else it falls back to a plain number tile). + gauge_range: Option<[f64; 2]>, + /// Optional KPI delta reference (`x-qsv.target`): a semantically-justified TARGET/goal (e.g. + /// 100% completeness, 0 error-rate), never a fabricated prior-period baseline. Rendered as a + /// "vs target" delta. + target: Option, } /// A parsed describegpt Data Dictionary: per-column semantic rows keyed by field name, plus the @@ -9777,6 +9831,21 @@ fn parse_dictionary_semantics(json_text: &str) -> Option { .unwrap_or_default() .to_string() }; + // KPI hints: `x-qsv.gauge_range` (a 2-element numeric array) and `x-qsv.target` (a + // number). Optional; absent on legacy dictionaries and on non-measure fields. + let xq_num = |k: &str| { + xq.and_then(|x| x.get(k)) + .and_then(serde_json::Value::as_f64) + }; + let xq_range = |k: &str| -> Option<[f64; 2]> { + let a = xq + .and_then(|x| x.get(k)) + .and_then(serde_json::Value::as_array)?; + match a.as_slice() { + [lo, hi] => Some([lo.as_f64()?, hi.as_f64()?]), + _ => None, + } + }; let label = prop .get("title") .and_then(serde_json::Value::as_str) @@ -9795,6 +9864,8 @@ fn parse_dictionary_semantics(json_text: &str) -> Option { concept: from_xq("concept"), label, description, + gauge_range: xq_range("gauge_range"), + target: xq_num("target"), }, ); } @@ -9852,6 +9923,9 @@ fn parse_dictionary_semantics(json_text: &str) -> Option { concept: from_field("concept"), label: from_field("label"), description: from_field("description"), + // legacy plain-json dictionaries don't carry KPI gauge/target hints + gauge_range: None, + target: None, }, ); } @@ -14398,6 +14472,143 @@ fn outlier_marker_geo() -> Marker { outlier_marker_map().symbol(plotly::common::MarkerSymbol::X) } +/// Maximum number of tiles in the leading `viz smart` KPI overview row (dataset-summary tiles + +/// headline measures). Keeps the row readable and each gauge/number a legible size. +const KPI_MAX_TILES: usize = 6; + +/// Choose a plotly d3 value-format for a KPI number: SI-suffixed for large magnitudes ("1.05M", +/// "258k"), a grouped integer for whole numbers, else two decimals. +fn kpi_number_format(value: f64) -> String { + let a = value.abs(); + if a >= 10_000.0 { + ".3~s".to_string() + } else if value.fract() == 0.0 { + ",.0f".to_string() + } else { + ",.2f".to_string() + } +} + +/// Build one KPI `Indicator` trace for a tile, positioned by paper-domain `[x0,x1] × [y0,y1]`. +/// A validated `gauge` adds an angular gauge, a `target` adds a "vs target" delta; the `mode` +/// follows whichever are present (always including the big number). +fn kpi_indicator(tile: &KpiTile, x_domain: [f64; 2], y_domain: [f64; 2]) -> Box { + let mode = match (tile.gauge.is_some(), tile.target.is_some()) { + (true, true) => IndicatorMode::NumberAndDeltaAndGauge, + (true, false) => IndicatorMode::NumberAndGauge, + (false, true) => IndicatorMode::NumberAndDelta, + (false, false) => IndicatorMode::Number, + }; + let mut ind = Indicator::new(tile.value) + .mode(mode) + .domain(Domain::new().x(&x_domain).y(&y_domain)) + .title(IndicatorTitle::new().text(tile.label.clone())) + .number(Number::new().value_format(tile.format.clone())); + if let Some([g_lo, g_hi]) = tile.gauge { + ind = ind.gauge(Gauge::new().axis(GaugeAxis::new().range([g_lo, g_hi]))); + } + if let Some(target) = tile.target { + // a "vs target" delta — the reference is a semantically-justified goal from the dictionary, + // never a fabricated prior-period baseline. + ind = ind.delta(Delta::new().reference(target)); + } + ind +} + +/// Build the leading KPI-tile row for `viz smart`: dataset-summary tiles (record count, field +/// count, a completeness gauge) followed by the headline numeric measures (capped at +/// `KPI_MAX_TILES`). A measure tile becomes a gauge when its dictionary supplies a `gauge_range` +/// that CONTAINS the observed value (else it falls back to a plain number — never a misleading +/// gauge), and a "vs target" delta when the dictionary supplies a `target` (a semantically +/// justified goal, never a fabricated prior-period baseline). Returns `None` for empty stats. +fn build_kpi_row( + n: u64, + stats: &[crate::cmd::stats::StatsData], + panels: &[Panel], + dict: Option<&DictData>, +) -> Option { + let ncols = stats.len(); + if ncols == 0 { + return None; + } + let mut tiles: Vec = Vec::with_capacity(KPI_MAX_TILES); + // Record/field counts are intentionally omitted — they duplicate the "Rows"/"Columns" lines in + // the dashboard's metadata header. The KPI row leads with data-quality and measure signals the + // header doesn't carry. + // + // completeness = non-null cells / total cells — an inherently 0–100% scale, so a built-in + // gauge that needs no LLM hint. + if n > 0 { + let total_cells = n as f64 * ncols as f64; + let total_null: f64 = stats.iter().map(|s| s.nullcount as f64).sum(); + let completeness = (1.0 - total_null / total_cells).clamp(0.0, 1.0); + tiles.push(KpiTile { + label: "Completeness".to_string(), + value: completeness, + format: ".1%".to_string(), + gauge: Some([0.0, 1.0]), + target: None, + }); + } + + // headline measures: the numeric columns the dashboard already treats as continuous measures + // (i.e. those that earned a box/violin/histogram distribution panel), summed (extensive) or + // averaged (intensive), up to the remaining tile budget. Keying off the built panels keeps the + // KPI measures consistent with the charts and works at any data size (no re-classification). + for p in panels { + if tiles.len() >= KPI_MAX_TILES { + break; + } + if !matches!( + p.kind, + PanelKind::BoxStats { .. } + | PanelKind::BoxRaw { .. } + | PanelKind::BoxOutliers { .. } + | PanelKind::Violin { .. } + | PanelKind::Histogram { .. } + ) { + continue; + } + let Some(s) = stats.iter().find(|s| s.field == p.name) else { + continue; + }; + if !matches!(s.r#type.as_str(), "Integer" | "Float") { + continue; + } + let row = dict.and_then(|d| d.rows.get(&s.field)); + let label = match row { + Some(r) if !r.label.is_empty() => r.label.clone(), + _ => s.field.clone(), + }; + // A `gauge_range` describes a bounded per-record quantity (a 0–5 rating, a 0–1 ratio), so + // its headline value is the MEAN — summing it would blow past the gauge scale. Otherwise + // fall back to the extensive/intensive keyword heuristic. + let has_range = row.and_then(|r| r.gauge_range).is_some(); + let intensive = has_range || is_intensive_measure(&label, &s.field); + let Some(value) = (if intensive { s.mean } else { s.sum }).filter(|v| v.is_finite()) else { + continue; + }; + // gauge only when a range is supplied AND actually contains the value (guardrail against a + // misleading gauge); delta only when a finite target is supplied. + let gauge = row + .and_then(|r| r.gauge_range) + .filter(|[lo, hi]| lo < hi && value >= *lo && value <= *hi); + let target = row.and_then(|r| r.target).filter(|t| t.is_finite()); + tiles.push(KpiTile { + label: format!("{} {label}", if intensive { "Mean" } else { "Total" }), + value, + format: kpi_number_format(value), + gauge, + target, + }); + } + + Some(Panel::new( + "Overview".to_string(), + PanelKind::KpiRow { tiles }, + )) +} + /// Build the `viz smart` auto-dashboard from the dataset's statistics + frequency data. /// Classifies columns into panels, then renders either a single-`Plot` subplot grid (≤8 panels, /// or any image export) or a self-contained inline-div HTML page (>8 panels, HTML output). @@ -15344,7 +15555,7 @@ fn build_smart( .as_deref() .map(str::to_ascii_lowercase) .as_deref(), - Some("treemap" | "sunburst") + Some("treemap" | "sunburst" | "icicle") ); // Mutual exclusivity with the directed-flow Sankey panel: a 2-level treemap of exactly // the pair already drawn as a Sankey is redundant, so skip it (the flow view conveys @@ -15371,8 +15582,8 @@ fn build_smart( "viz smart: skipping the '{title}' hierarchy panel — its dimensions are \ statistically independent (max Cramér's V={assoc:.2} < \ {HIER_MIN_ASSOCIATION_CRAMERS_V:.2}); the per-column frequency bars \ - convey the same information. Use --hierarchy-style treemap|sunburst to \ - force it." + convey the same information. Use --hierarchy-style \ + treemap|sunburst|icicle to force it." )); } else if let Some((labels, parents, values, ids)) = hierarchy_arrays(&leaves, depth, HIER_TOP_N_PER_LEVEL, HIER_MAX_NODES, "All") @@ -15524,6 +15735,7 @@ fn build_smart( } panels = kept; } + if !sentinel_suspects.is_empty() { viz_note(&format!( "viz smart: {} column(s) declared a `measure` in the data dictionary were typed \ @@ -15597,7 +15809,8 @@ fn build_smart( .iter() .filter_map(|p| match p.kind { PanelKind::FreqBar { idx } => Some(idx), - PanelKind::BoxStats { .. } + PanelKind::KpiRow { .. } + | PanelKind::BoxStats { .. } | PanelKind::BoxRaw { .. } | PanelKind::BoxOutliers { .. } | PanelKind::Violin { .. } @@ -15804,6 +16017,18 @@ fn build_smart( && panels .iter() .any(|p| matches!(p.kind, PanelKind::Geo { .. })); + + // Leading KPI overview row, prepended LAST (after every panel-count-dependent decision — the + // `--max-charts` ranking, the "charting N columns" note, the box-overlay demotion, and the + // render-path choice) so it stays invisible to them and simply lands at index 0, on top of the + // dashboard. HTML only: Indicator tiles are domain-positioned and never enter a static image. + if !out_format.is_image() { + let n = *nrows.get_or_insert_with(|| util::count_rows(&count_conf).unwrap_or(0)); + if let Some(panel) = build_kpi_row(n, &stats, &panels, dict_data.as_ref()) { + panels.insert(0, panel); + } + } + progress.set_message("Rendering dashboard…"); if inline { Ok(SmartRender::Inline(render_smart_inline( @@ -15817,7 +16042,13 @@ fn build_smart( dict_page.as_deref(), metadata_html.as_deref(), ))) - } else if panels.len() > MAX_SUBPLOTS || has_geo_image { + } else if panels + .iter() + .filter(|p| !matches!(p.kind, PanelKind::KpiRow { .. })) + .count() + > MAX_SUBPLOTS + || has_geo_image + { // static image export of >MAX_SUBPLOTS panels (or any panel count with a geo subplot): // plotly's typed Layout only has xaxis1..xaxis8 and a single typed `geo`, so assemble the // grid as raw JSON with domain-positioned xaxis9+ and (for geo panels) geo/geo2+ subplots. @@ -16254,7 +16485,8 @@ fn panel_trace( // 3D scene, or domain-based treemap/sunburst) that can't share the typed x/y subplot grid, // so they are rendered entirely by `smart_inline_panel_plot` and never reach this // assembler. - PanelKind::Map { .. } + PanelKind::KpiRow { .. } + | PanelKind::Map { .. } | PanelKind::Geo { .. } | PanelKind::Choropleth { .. } | PanelKind::ChoroplethMap { .. } @@ -16317,7 +16549,8 @@ fn smart_grid_parts( PanelKind::CorrHeatmap { labels, .. } | PanelKind::AssocHeatmap { labels, .. } => { Some(labels.as_slice()) }, - PanelKind::BoxStats { .. } + PanelKind::KpiRow { .. } + | PanelKind::BoxStats { .. } | PanelKind::BoxRaw { .. } | PanelKind::BoxOutliers { .. } | PanelKind::Violin { .. } @@ -16424,6 +16657,26 @@ fn smart_grid_parts( let yref = axis_ref('y', pos); let color = PALETTE[n % PALETTE.len()]; + // KPI row: a full-width band of domain-positioned Indicator tiles (no cartesian axes). + // Sub-divide this panel's paper x-domain into N tiles, emit each as an Indicator, then skip + // the cartesian axis assignment below. No section title — the tiles are self-labeling. + if let PanelKind::KpiRow { tiles } = &panel.kind { + let geom = &geoms[n]; + let x0 = geom.x_domain.first().copied().unwrap_or(0.0); + let x1 = geom.x_domain.get(1).copied().unwrap_or(1.0); + let y0 = geom.y_domain.first().copied().unwrap_or(0.0); + let y1 = geom.y_domain.get(1).copied().unwrap_or(1.0); + let count = tiles.len().max(1) as f64; + let width = (x1 - x0) / count; + let gutter = width * 0.06; + for (i, tile) in tiles.iter().enumerate() { + let lo = x0 + i as f64 * width + gutter / 2.0; + let hi = x0 + (i as f64 + 1.0) * width - gutter / 2.0; + traces.push(kpi_indicator(tile, [lo, hi], [y0, y1])); + } + continue; + } + // geo panels use a `geo{pos}` projection subplot (no cartesian x/y axes), only reachable on // the raw-JSON static-export path. Build the ScatterGeo trace, fit the projection to the // data extent, and emit the domain-positioned geo layout object as raw JSON (the typed @@ -17391,6 +17644,35 @@ fn smart_inline_panel_plot( // hierarchy (treemap/sunburst) panels are domain-based — no cartesian x/y axes — so, like the // map/geo/3D panels above, they own their whole Plot here. + // KPI row: a full-width band of domain-positioned `Indicator` tiles (no cartesian axes), so it + // owns its whole Plot here like the hierarchy/sankey panels below. + if let PanelKind::KpiRow { tiles } = &panel.kind { + let mut plot = Plot::new(); + let count = tiles.len().max(1) as f64; + // even horizontal split with a small inter-tile gutter so adjacent gauges/numbers breathe + let gutter = 0.02_f64; + for (i, tile) in tiles.iter().enumerate() { + let lo = (i as f64 / count) + gutter / 2.0; + let hi = ((i + 1) as f64 / count) - gutter / 2.0; + plot.add_trace(kpi_indicator(tile, [lo, hi], [0.0, 1.0])); + } + // no panel title — the tiles are self-labeling (each Indicator carries its own title), so + // reclaim the top band the title would occupy. Just the vertical hover modebar. + let mut layout = Layout::new() + .show_legend(false) + .height(row_height) + .margin(Margin::new().top(20).bottom(20).left(20).right(20).pad(4)) + .mode_bar(ModeBar::new().orientation(Orientation::Vertical)); + if !themed { + layout = layout + .font(Font::new().family(FONT_FAMILY).color(INK).size(12)) + .paper_background_color(PAPER_BG); + } + plot.set_layout(apply_theme(layout, theme)); + plot.set_configuration(Configuration::new().responsive(true)); + return plot; + } + if let PanelKind::Hierarchy { style, labels, @@ -18434,6 +18716,15 @@ fn hierarchy_trace( // ring's spoke, so deep-path sectors stay legible instead of clipping tangential text. .inside_text_orientation(InsideTextOrientation::Radial) .max_depth(SUNBURST_MAXDEPTH), + // an icicle is the rectangular sibling of the sunburst: the same rolled-up subtree totals + // laid out as stacked bars per level (top-down by default), which keeps deep paths and many + // small leaves more legible than a treemap's nested rectangles while preserving strict + // level alignment for cross-branch comparison. Same `label+value+percent parent` tile text. + HierStyle::Icicle => Icicle::new(labels.to_vec(), parents.to_vec()) + .ids(ids.to_vec()) + .values(values.to_vec()) + .branch_values(IcicleBranchValues::Total) + .text_info("label+value+percent parent"), } } @@ -18762,7 +19053,8 @@ struct SubplotGeometry { /// multi-column grid below. fn is_overview_panel(kind: &PanelKind) -> bool { match kind { - PanelKind::CorrHeatmap { .. } + PanelKind::KpiRow { .. } + | PanelKind::CorrHeatmap { .. } | PanelKind::AssocHeatmap { .. } | PanelKind::TopRelationships { .. } | PanelKind::ScatterPair { .. } @@ -22422,6 +22714,15 @@ mod tests { resolve_hierarchy_style(Some("sunburst"), 2).unwrap(), HierStyle::Sunburst ); + // icicle is an explicit-only override (auto never picks it) + assert_eq!( + resolve_hierarchy_style(Some("icicle"), 3).unwrap(), + HierStyle::Icicle + ); + assert_eq!( + resolve_hierarchy_style(Some("ICICLE"), 2).unwrap(), + HierStyle::Icicle + ); assert!(resolve_hierarchy_style(Some("pie"), 2).is_err()); } diff --git a/tests/test_viz.rs b/tests/test_viz.rs index 61dbe36e3d..10f53435ff 100644 --- a/tests/test_viz.rs +++ b/tests/test_viz.rs @@ -5793,10 +5793,11 @@ fn viz_smart_compressed_map_figure_payload() { let out = wrk.output(&mut cmd); assert!(out.status.success()); let html = String::from_utf8_lossy(&out.stdout); - // the map panel is the leading full-width panel (qsv-viz-panel-0): payload + deferred render - assert!(html.contains("id=\"qsv-viz-panel-0-fig\"")); - assert!(html.contains("qsvNewPlotGz(\"qsv-viz-panel-0\")")); - let figure = inflate_gz_payload(&html, "qsv-viz-panel-0-fig"); + // panel-0 is the leading KPI overview row; the map is the next full-width panel + // (qsv-viz-panel-1): payload + deferred render + assert!(html.contains("id=\"qsv-viz-panel-1-fig\"")); + assert!(html.contains("qsvNewPlotGz(\"qsv-viz-panel-1\")")); + let figure = inflate_gz_payload(&html, "qsv-viz-panel-1-fig"); assert!(figure.contains(r#""type":"scattermap""#)); assert!(figure.contains(r#""cluster":{"enabled":false,"maxzoom":17.0}"#)); // coordinates ride as little-endian float32 typed arrays, not decimal-text JSON @@ -5883,11 +5884,12 @@ fn viz_cdn_keeps_gz_prelude_for_compressed_map_figures() { assert!(html.contains(r#" diff --git a/examples/viz/gen_gallery.py b/examples/viz/gen_gallery.py index 2c2e213000..0676f28169 100755 --- a/examples/viz/gen_gallery.py +++ b/examples/viz/gen_gallery.py @@ -494,6 +494,11 @@ "rings (maxdepth) so labels stay legible instead of crowding a ~100-sector outer ring; click a " "sector to drill in and the deeper ring's labels grow back. Hover always shows value + percent.", False, ["sunburst", "sales_sample.csv", "--cols", "region,product_category,payment_method"]), + ("icicle", "Same three-level hierarchy (region -> product_category -> payment_method) as a rectangular " + "icicle: parents on the left, children fanning right, each rectangle sized by row count. The flat " + "left-to-right layout keeps deep labels readable where a sunburst's outer ring would crowd; click a " + "rectangle to zoom into that branch. Hover shows label + value + percent of parent.", + False, ["icicle", "sales_sample.csv", "--cols", "region,product_category,payment_method"]), ("map", "Earthquake points on token-free OpenStreetMap tiles; marker color = magnitude, size = depth.", False, ["map", "quakes.csv", "--lat", "lat", "--lon", "lon", "--color", "magnitude", "--size", "depth_km"]), ("map (density)", "DensityMap heatmap of the same points on a light Carto basemap.", diff --git a/examples/viz/smart_allegheny_dogs.html b/examples/viz/smart_allegheny_dogs.html index 7ba34dda97..50d00319df 100644 --- a/examples/viz/smart_allegheny_dogs.html +++ b/examples/viz/smart_allegheny_dogs.html @@ -4,7 +4,7 @@ allegheny_dog_licenses.csv — data overview - +