diff --git a/Cargo.lock b/Cargo.lock index a05cdf42db..70c59e252a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -341,7 +341,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -352,7 +352,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2355,7 +2355,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2567,7 +2567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5483,7 +5483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5828,7 +5828,7 @@ dependencies = [ [[package]] name = "plotly" version = "0.14.1" -source = "git+https://github.com/dathere/plotly?rev=8d56b22#8d56b22d5a3f145f57462b79444157cad93736fc" +source = "git+https://github.com/dathere/plotly?branch=feature%2Ftrace-type-additions#554f6609e2861ce82b481e5beb8d6e1fed50ba3f" dependencies = [ "askama", "async-trait", @@ -5851,7 +5851,7 @@ dependencies = [ [[package]] name = "plotly_derive" version = "0.14.1" -source = "git+https://github.com/dathere/plotly?rev=8d56b22#8d56b22d5a3f145f57462b79444157cad93736fc" +source = "git+https://github.com/dathere/plotly?branch=feature%2Ftrace-type-additions#554f6609e2861ce82b481e5beb8d6e1fed50ba3f" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5862,7 +5862,7 @@ dependencies = [ [[package]] name = "plotly_static" version = "0.1.0" -source = "git+https://github.com/dathere/plotly?rev=8d56b22#8d56b22d5a3f145f57462b79444157cad93736fc" +source = "git+https://github.com/dathere/plotly?branch=feature%2Ftrace-type-additions#554f6609e2861ce82b481e5beb8d6e1fed50ba3f" dependencies = [ "anyhow", "base64 0.22.1", @@ -7094,7 +7094,7 @@ dependencies = [ "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7945,7 +7945,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8013,7 +8013,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8616,7 +8616,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8741,7 +8741,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9008,10 +9008,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9024,7 +9024,7 @@ dependencies = [ "parking_lot", "rustix 1.1.4", "signal-hook 0.3.18", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9060,7 +9060,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10338,7 +10338,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 49db43deae..aadf442cda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -462,12 +462,14 @@ sled = { git = "https://github.com/dathere/sled", branch = "v0.34.7-drop-fxhash" # use our patched fork of whatlang to bump hashbrown until our PR is merged whatlang = { git = "https://github.com/jqnatividad/whatlang-rs", branch = "bump-hashbrown-to-16" } -# Our Treemap/Sunburst PR has been merged (https://github.com/plotly/plotly.rs/pull/406) -# Pinned to the MapLibre-migration integration HEAD, which adds the MapLibre `map`-subplot -# traces qsv's `viz` maps now use: ChoroplethMap, DensityMap, and ScatterMap (replacing the -# deprecated *mapbox traces). Tracks upstream PRs #410 (Choropleth), #415/#416 (DensityMapbox -# hover+color / DensityMap) and the ScatterMap follow-up. -plotly = { git = "https://github.com/dathere/plotly", rev = "8d56b22" } +# Pinned to our stacked feature branch `feature/trace-type-additions` (PR +# https://github.com/plotly/plotly.rs/pull/423), a strict superset of the earlier pins: +# - MapLibre `map`-subplot traces qsv's `viz` maps use (ChoroplethMap, DensityMap, +# ScatterMap; upstream PRs #406/#410/#415/#416) — now merged into plotly:main +# - plotly.js 3.7 attribute backfill (legendrank/zorder/fillgradient/... ; PR #421) +# - new trace types Indicator, Histogram2d, Icicle + layout-level `uirevision` (PR #423) +# When #421/#423 merge upstream, repoint this to a plotly.rs release/main rev. +plotly = { git = "https://github.com/dathere/plotly", branch = "feature/trace-type-additions" } # TRACKING: remove this patch when upstream resolves it (see exit conditions below). # diff --git a/docs/help/viz.md b/docs/help/viz.md index 42155b61dc..0400f25f24 100644 --- a/docs/help/viz.md +++ b/docs/help/viz.md @@ -60,6 +60,8 @@ treemap Part-to-whole hierarchy as nested tiles. --cols = 2+ dimension columns (levels), optional --value and --agg. sunburst Part-to-whole hierarchy as concentric rings (same inputs as treemap). Better for deeper hierarchies. +icicle Part-to-whole hierarchy as stacked bars per level (same inputs as + treemap). Level-aligned; good for deep, wide hierarchies. map Geographic point map (or --density heatmap) on tile basemaps. Pick the coordinate columns with the lat/lon options below. geo Geographic point map on a projection basemap (coastlines/land/ @@ -313,6 +315,7 @@ qsv viz geo [options] qsv viz choropleth [options] qsv viz treemap [options] qsv viz sunburst [options] +qsv viz icicle [options] qsv viz --help ``` @@ -393,7 +396,7 @@ qsv viz --help |  `‑‑no‑nulls`  | flag | Omit the "(NULL)" bar (empty cells) from frequency bar charts. By default `viz smart` shows a "(NULL)" bar, like `qsv frequency`. | | |  `‑‑no‑other`  | flag | Omit the "Other (N)" aggregate bar from frequency bar charts. It collects the categories beyond --limit (N = how many distinct categories were rolled up) and is shown by default. | | |  `‑‑smarter`  | flag | Before building the dashboard, run `qsv moarstats --advanced` to enrich the stats cache with distribution-shape statistics (bimodality, entropy, skewness, outlier share). This unlocks histograms for bimodal columns, frequency bars for concentrated high-cardinality columns, and skew/outlier hints on box panels. Costs one extra pass over the data and writes .stats.csv, its sidecars, and an .idx index (like running `qsv moarstats` manually). On geocode-enabled builds it also enriches map point hovers with the US FIPS code and annotates the spatial-extent summary with the country's continent; the county is always shown in map hovers, with or without --smarter. Only affects `smart`. Applied only with default parsing; inputs using --no-headers or a custom --delimiter fall back to the standard dashboard. | | -|  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure). Only affects `smart`. | | +|  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst, icicle. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure); icicle is an opt-in level-aligned alternative. Only affects `smart`. | | |  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. Only affects `smart`. | | |  `‑‑dictionary‑context`  | string | Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to describegpt as --context-file when `--dictionary infer` generates the dictionary. Better context yields better role/concept/label/grain tags, hence a better dashboard. Ignored unless `--dictionary infer` is used (it does not apply when reading an existing dictionary file). Only affects `smart`. | | |  `‑‑dict‑info`  | flag | When a usable Data Dictionary is available (per --dictionary), add a "Data Dictionary" link beneath the dashboard title and an info icon on each panel title: hovering shows that column's dictionary description; clicking opens a human-friendly rendering of the dictionary in a side drawer NEXT TO the plots (embedded in the dashboard file - no extra file is written), scrolled to and highlighting that column's entry. The drawer is open by default on load and can be dismissed with its close button or Esc. The dictionary page carries a role-tinted table of contents and per-column "View chart" links back to the panels. The drawer's popout button opens the same document in its own browser tab instead (needs a browser that allows user-initiated pop-ups). HTML output only; ignored with a note when no dictionary is available or when exporting an image. Only affects `smart`. | | diff --git a/examples/viz/gallery.html b/examples/viz/gallery.html index 5a3e580d7b..3e78ece159 100644 --- a/examples/viz/gallery.html +++ b/examples/viz/gallery.html @@ -32,7 +32,7 @@

qsv viz — chart gallery

Every chart type produced by qsv viz, from the sample datasets in examples/viz/. Generated with the viz feature; each chart is fully interactive.

-
Jump to a chart37 charts + data dictionary
+
Jump to a chart38 charts + data dictionary
smart dashboard (--smarter, geospatial)One command, 13 auto-chosen panels — nearly every panel type at once on a synthetic catalog of Japanese earthquakes. Things the raw table hides but the dashboard makes obvious: depth_km is bimodal (two populations — shallow interplate quakes ~20 km and the deep Wadati-Benioff slab ~450 km — so --smarter draws a histogram, not a box that would average the peaks away); the points trace Japan's subduction arcs on the map; and a prefecture choropleth bins each quake into the GeoJSON region that contains it (point-in-polygon, no geocoding). Most of this catalog is offshore Pacific seismicity, so under the default 10 km snap cap the far-offshore quakes are dropped (287 of 417 here — the panel title reports it) and the on-land/near-coast prefectures are colored; raise --snap-max-dist to snap distant quakes to the nearest prefecture instead, or --no-snap to drop every offshore point. magnitude vs felt_reports is almost perfectly correlated (r=0.95); magnitude and felt_reports are right-skewed with flagged outliers; and the magnitude-over-time trend spikes during a September aftershock sequence. --bivariate adds an NMI association heatmap spanning every column type, not just the continuous-numeric ones the Pearson heatmap covers — it surfaces depth_km and felt_reports as strongly associated with occurrence_date (NMI=0.93 and 0.89), a temporal-clustering signal (the same September aftershock sequence) a Pearson matrix restricted to numeric pairs alone cannot express against a date column. Coordinate columns are shown on the map only, not re-charted as distributions. Rendered with the built-in plotly_dark theme.
qsv viz smart seismic_events.csv --smarter --bivariate --theme plotly_dark \
   --grid-cols 3 --geojson japan_prefectures.geojson -o smart_dashboard_smarter_geospatial.html
@@ -74,19 +74,21 @@

qsv viz — chart gallery

--agg sum -o treemap.html
sunburstThree-level hierarchy (region -> product_category -> payment_method) as concentric rings, sized by row count; inner rings are parents, outer rings their children. Opens at two rings (maxdepth) so labels stay legible instead of crowding a ~100-sector outer ring; click a sector to drill in and the deeper ring's labels grow back. Hover always shows value + percent.
qsv viz sunburst sales_sample.csv --cols region,product_category,payment_method \
   -o sunburst.html
+
icicleSame three-level hierarchy (region -> product_category -> payment_method) as a rectangular icicle: parents on the left, children fanning right, each rectangle sized by row count. The flat left-to-right layout keeps deep labels readable where a sunburst's outer ring would crowd; click a rectangle to zoom into that branch. Hover shows label + value + percent of parent.
qsv viz icicle sales_sample.csv --cols region,product_category,payment_method \
+  -o icicle.html
mapEarthquake points on token-free OpenStreetMap tiles; marker color = magnitude, size = depth.
qsv viz map quakes.csv --lat lat --lon lon --color magnitude \
-  --size depth_km -o map.html
+ --size depth_km -o map.html
map (density)DensityMap heatmap of the same points on a light Carto basemap.
qsv viz map quakes.csv --lat lat --lon lon --density --style carto-positron \
-  -o map_density.html
+ -o map_density.html
geoSame earthquakes on an offline natural-earth projection (no tiles, no token); marker color = magnitude. viz smart auto-uses this projection for global-extent coordinates.
qsv viz geo quakes.csv --lat lat --lon lon --color magnitude \
-  --projection natural-earth -o geo.html
+ --projection natural-earth -o geo.html
choroplethFilled-region map coloring countries by GDP, matched by ISO-3 code on a token-free projection basemap. Use --location-mode usa-states / country-names / geojson-id for other region keys, --map for a MapLibre tile basemap, or --geocode to derive codes from lat/lon or place names.
qsv viz choropleth country_stats.csv --locations iso3 --value gdp_usd_tn \
-  --color-scale viridis -o choropleth.html
+ --color-scale viridis -o choropleth.html
choropleth (US states)Same chart, --location-mode usa-states: state codes matched to Plotly's built-in US-state geometry on the token-free albers-usa projection (CONUS + Alaska/Hawaii insets) — no GeoJSON needed. States are colored by renewable-electricity share.
qsv viz choropleth us_state_stats.csv --locations state --value renewable_electricity_pct \
-  --location-mode usa-states -o choropleth_us_states.html
+ --location-mode usa-states -o choropleth_us_states.html
choropleth (MapLibre + GeoJSON)--map draws the filled regions on an interactive MapLibre tile basemap (token-free carto-positron) instead of a projection. The regions come from a custom GeoJSON (--geojson local file or URL) matched to the data by --feature-id-key — here the near-rectangular western states, colored by installed wind capacity. The view auto-centers and zooms to the GeoJSON extent (shown full-width so the computed zoom frames the regions as the CLI does — a tile map's zoom is fixed, so a narrow grid cell would crop it).
qsv viz choropleth western_states.csv --locations state --value wind_capacity_gw \
   --geojson western_states.geojson --feature-id-key id --map --style carto-positron \
-  -o choropleth_maplibre_geojson.html
+ -o choropleth_maplibre_geojson.html
smart dashboard (time-series)Auto dashboard for stock_prices: a time-series trend panel (the first numeric column over the date) leads, alongside box-plot summaries of the OHLC columns.
qsv viz smart stock_prices.csv --max-charts 8 -o smart_dashboard_time_series.html
smart dashboard (per-US-state choropleth)`viz smart` reverse-geocodes each point; because every city resolves to a US state, it adds a per-US-state choropleth (cities-per-state, albers-usa) beside the point map, alongside the usual box plots, frequency bars and the strongest-pair scatter. (The point map's spatial extent caption counts the data's bounding-box corners, which spill into neighboring countries and ocean — the choropleth instead resolves each city to its own state.) No flags, no LLM — the state fill is derived purely from the lat/lon columns.
qsv viz smart us_cities.csv -o smart_dashboard_per_us_state_choropleth.html
smart dashboard (--dictionary infer, treemap)Auto dashboard for customer_spend with a describegpt-inferred Data Dictionary (--dictionary infer) guiding panel selection & field labels. Two categorical dimensions (plan, region) form a shallow part-to-whole hierarchy, auto-rendered as a TREEMAP (area = size). Requires a local LLM; the committed HTML is reused on regen.
qsv viz smart customer_spend.csv --dictionary infer -o smart_dashboard_dictionary_infer_treemap.html
@@ -103,7 +105,7 @@

qsv viz — chart gallery

--geojson pittsburgh-neighborhoods --dataset-pid https://data.wprdc.org/dataset/pittsburgh-311-dataPittsburgh 311 smart visual data dictionary screenshot diff --git a/examples/viz/gen_gallery.py b/examples/viz/gen_gallery.py index 2c2e213000..8561292a08 100755 --- a/examples/viz/gen_gallery.py +++ b/examples/viz/gen_gallery.py @@ -61,6 +61,11 @@ # so a normal `gen_gallery.py` run stays LLM-free and deterministic. To refresh them in-place, run # with QSV_VIZ_REGEN_LLM=1 and your local LLM up (LM Studio / Ollama) — main() then treats this set # as empty so these `--dictionary infer` figures are regenerated and re-cdnified like the others. +# NOTE: because they are reused as-is, these figures intentionally LAG new smart-dashboard features +# until an LLM-backed regen — e.g. they do not yet show the KPI/Completeness overview row that the +# deterministic dashboards above carry. This is deferred on purpose: they will be regenerated in one +# pass once describegpt emits the gauge_range/target dictionary hints, so the KPI row lands with its +# gauges/deltas rather than as bare number tiles now. PREGENERATED = { "smart_dict_treemap.html", "smart_dict_sunburst.html", @@ -494,6 +499,11 @@ "rings (maxdepth) so labels stay legible instead of crowding a ~100-sector outer ring; click a " "sector to drill in and the deeper ring's labels grow back. Hover always shows value + percent.", False, ["sunburst", "sales_sample.csv", "--cols", "region,product_category,payment_method"]), + ("icicle", "Same three-level hierarchy (region -> product_category -> payment_method) as a rectangular " + "icicle: parents on the left, children fanning right, each rectangle sized by row count. The flat " + "left-to-right layout keeps deep labels readable where a sunburst's outer ring would crowd; click a " + "rectangle to zoom into that branch. Hover shows label + value + percent of parent.", + False, ["icicle", "sales_sample.csv", "--cols", "region,product_category,payment_method"]), ("map", "Earthquake points on token-free OpenStreetMap tiles; marker color = magnitude, size = depth.", False, ["map", "quakes.csv", "--lat", "lat", "--lon", "lon", "--color", "magnitude", "--size", "depth_km"]), ("map (density)", "DensityMap heatmap of the same points on a light Carto basemap.", diff --git a/examples/viz/smart_allegheny_dogs.html b/examples/viz/smart_allegheny_dogs.html index 7ba34dda97..50d00319df 100644 --- a/examples/viz/smart_allegheny_dogs.html +++ b/examples/viz/smart_allegheny_dogs.html @@ -4,7 +4,7 @@ allegheny_dog_licenses.csv — data overview - +