From 1b81233d2888edfad0188491413c99de138d9a2a Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:50:45 -0400 Subject: [PATCH 01/12] feat(describegpt): emit x-qsv.gauge_range KPI hint for canonical-scale measures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teaches the `--dictionary --infer-content-type` pass to propose a `gauge_range` [min, max] for numeric MEASURE columns on a canonical scale (percent, ratio, rating, bounded index), which `viz smart --dictionary` reads as `x-qsv.gauge_range` to draw a gauge KPI tile instead of a bare number. Follows the existing propose-then-verify discipline (same as null_values): - LlmDictField carries the raw model proposal; parse_llm_dictionary_response only shape-validates ([min,max] of two finite numbers, min --- resources/describegpt_defaults.toml | 14 +- src/cmd/describegpt.rs | 34 +++- src/cmd/describegpt/dictionary.rs | 283 ++++++++++++++++++++++++++++ src/cmd/describegpt/formatters.rs | 75 ++++++++ 4 files changed, 403 insertions(+), 3 deletions(-) diff --git a/resources/describegpt_defaults.toml b/resources/describegpt_defaults.toml index 977b0dec51..6a227995a0 100644 --- a/resources/describegpt_defaults.toml +++ b/resources/describegpt_defaults.toml @@ -140,6 +140,18 @@ For each field, provide: none do. Do NOT invent tokens. NOTE: "id.surrogate_key" is RESERVED and set deterministically by qsv for unique-key columns - do not use it yourself. Allowed Concept tokens: {{ concept_vocab }} +- Gauge Range (OPTIONAL, numeric MEASURE fields only): if - and ONLY if - this field is a numeric + "measure" that rides a well-known CANONICAL scale with a fixed, meaningful full-scale maximum, + add a "gauge_range" as a two-number array [min, max] giving that scale's bounds. Use it for a + percentage (`[0, 100]`), a probability or ratio (`[0, 1]`), a 1-to-5 or 0-to-10 rating/score + (`[0, 5]`, `[0, 10]`), or a bounded index. The range MUST contain the field's observed Min and + Max (from the Summary Statistics) - a dashboard draws the value against this scale, so a range + the data spills past is worse than none. Do NOT invent a range for an OPEN-ENDED measure whose + maximum is just "however big the data happens to get" - revenue, counts, amounts, durations, + distances, weights - OMIT "gauge_range" entirely for those. When in doubt, omit it: qsv drops + any range that does not contain the observed data, and a missing gauge_range simply renders as a + plain number instead of a dial. Never emit a "target", baseline, or goal value - you cannot know + a business goal from the data. {%- endif %} {%- if infer_content_type %} @@ -199,7 +211,7 @@ automatically, so propose a numeric placeholder whenever the field's meaning gen one. Do not invent one to be thorough. {%- endif %} -Return the results in JSON format where each field name is a key, and the value is an object with "label"{% if infer_content_type %},{% else %} and{% endif %} "description"{% if infer_content_type %}, "content_type", "role" and "concept"{% endif %}{% if infer_null_values %} and "null_values"{% endif %} properties{% if infer_content_type %}, plus a top-level "grain" string and an optional "relationships" array{% endif %}: +Return the results in JSON format where each field name is a key, and the value is an object with "label"{% if infer_content_type %},{% else %} and{% endif %} "description"{% if infer_content_type %}, "content_type", "role", "concept" and (for canonical-scale numeric measures only) an optional "gauge_range"{% endif %}{% if infer_null_values %} and "null_values"{% endif %} properties{% if infer_content_type %}, plus a top-level "grain" string and an optional "relationships" array{% endif %}: {% raw %}{ "field_name_1": { "label": "Human-friendly label", diff --git a/src/cmd/describegpt.rs b/src/cmd/describegpt.rs index 88b3963cbe..b1bd618b7e 100644 --- a/src/cmd/describegpt.rs +++ b/src/cmd/describegpt.rs @@ -7628,6 +7628,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }]; let first = build_first_pass_dictionary_json_string(&args, &entries); sleep(Duration::from_millis(10)); @@ -7829,6 +7830,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, dictionary::DictionaryEntry { name: "category|raw".to_string(), @@ -7850,6 +7852,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, ]; @@ -7933,6 +7936,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, dictionary::DictionaryEntry { name: "Status".to_string(), @@ -7967,6 +7971,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, ]; @@ -8066,6 +8071,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, dictionary::DictionaryEntry { name: "Status".to_string(), @@ -8087,6 +8093,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, ]; @@ -8161,6 +8168,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }]; let shared = SharedRenderCtx::new(&args, model, base_url, PromptType::Dictionary); @@ -8249,6 +8257,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, // Text column with only `min_length` retained. dictionary::DictionaryEntry { @@ -8271,6 +8280,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, // Text column with only `max_length` retained. dictionary::DictionaryEntry { @@ -8293,6 +8303,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, ]; @@ -8366,6 +8377,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }]; let shared = SharedRenderCtx::new(&args, model, base_url, PromptType::Dictionary); @@ -8526,6 +8538,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, dictionary::DictionaryEntry { name: "category".to_string(), @@ -8547,6 +8560,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, ]; @@ -8612,6 +8626,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, // datetime with an inferred format (contains colons) over an RFC3339 min/max. dictionary::DictionaryEntry { @@ -8634,6 +8649,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, // bare `date` token (no inferred fmt) — Min/Max stay as-is. dictionary::DictionaryEntry { @@ -8656,6 +8672,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, // non-date content type — Min/Max untouched even though numeric. dictionary::DictionaryEntry { @@ -8678,6 +8695,7 @@ p_fewshot_examples = "" null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }, ]; @@ -8759,6 +8777,11 @@ p_fewshot_examples = "" !off.contains("Concept:") && !off.contains("\"concept\"") && !off.contains("\"grain\""), "flag-off prompt must not mention Concept/Role/grain:\n{off}" ); + // gauge_range (the measure gauge hint for viz) rides the same flag. + assert!( + !off.contains("gauge_range") && !off.contains("Gauge Range"), + "flag-off prompt must not mention gauge_range:\n{off}" + ); assert!( off.contains("\"label\" and \"description\" properties"), "flag-off prompt must keep the legacy properties sentence:\n{off}" @@ -8779,9 +8802,16 @@ p_fewshot_examples = "" ); assert!( on.contains( - "\"label\", \"description\", \"content_type\", \"role\" and \"concept\" properties" + "\"content_type\", \"role\", \"concept\" and (for canonical-scale numeric \ + measures only) an optional \"gauge_range\" properties" ), - "flag-on prompt must list content_type/role/concept in the properties sentence:\n{on}" + "flag-on prompt must list content_type/role/concept/gauge_range in the properties \ + sentence:\n{on}" + ); + // The Gauge Range instruction is injected when the flag is on. + assert!( + on.contains("- Gauge Range (OPTIONAL, numeric MEASURE fields only)"), + "flag-on prompt must include the Gauge Range instruction:\n{on}" ); // Concept + Role instructions and vocabularies are injected when the flag is on. assert!( diff --git a/src/cmd/describegpt/dictionary.rs b/src/cmd/describegpt/dictionary.rs index c1bef72563..ac9a1eefdd 100644 --- a/src/cmd/describegpt/dictionary.rs +++ b/src/cmd/describegpt/dictionary.rs @@ -532,6 +532,12 @@ pub(super) struct LlmDictField { /// (which alone can see the column's type and observed values) decides which /// are verifiable. Empty unless the dictionary prompt asked for them. pub(super) null_values: Vec, + /// RAW `[min, max]` gauge scale proposed by the LLM for a numeric measure. + /// `None` unless the dictionary prompt asked for it (under `--infer-content-type`) + /// and the model returned a well-formed 2-number array with `min < max`. The + /// role/stats VERIFICATION happens later in `combine_dictionary_entries`, not + /// here — this struct only carries the model's raw proposal. + pub(super) gauge_range: Option<[f64; 2]>, } pub(crate) struct StatsRecord { @@ -632,6 +638,16 @@ pub(super) struct DictionaryEntry { /// `#[serde(default)]` for cache backward-compatibility. #[serde(default)] pub(super) null_candidates: Vec, + /// Optional `[min, max]` canonical scale for a KPI gauge on a numeric MEASURE + /// (e.g. `[0, 100]` for a percent, `[0, 5]` for a rating). The LLM proposes it + /// only under `--infer-content-type`; `combine_dictionary_entries` then keeps it + /// ONLY when the column's finalized `role` is `measure` AND the observed + /// `[min, max]` lies inside the proposed range — an unfalsifiable or + /// out-of-scale range is dropped, mirroring the propose-then-verify discipline + /// of `null_values`. Consumed by `viz smart --dictionary` to draw a gauge tile + /// (`x-qsv.gauge_range`). `#[serde(default)]` for cache backward-compatibility. + #[serde(default)] + pub(super) gauge_range: Option<[f64; 2]>, } /// Parse the `stats` CSV into structured records, returning the records plus @@ -1034,6 +1050,9 @@ pub(super) fn generate_code_based_dictionary( // deterministic seed, since proposing a sentinel requires judgment. null_values: Vec::new(), null_candidates: Vec::new(), + // Proposed by the LLM pass (measure gauges) and verified against the + // observed min/max in `combine_dictionary_entries`; no deterministic seed. + gauge_range: None, }); } @@ -1254,12 +1273,16 @@ pub(super) fn combine_dictionary_entries( entry.content_type = merge_content_type(&entry.content_type, &llm.content_type, false); entry.concept = merge_concept(&entry.concept, &llm.concept, false); entry.role = merge_role(&entry.role, &llm.role, false); + entry.gauge_range = llm.gauge_range; } if infer_content_type { if entry.content_type.is_empty() { entry.content_type = "unknown".to_string(); } coerce_role_concept(entry); + // AFTER role is finalized: keep the proposed gauge scale only if the + // column is a measure and its observed range fits inside it. + verify_gauge_range(entry); } } code_entries @@ -1368,6 +1391,39 @@ fn coerce_role_concept(entry: &mut DictionaryEntry) { } } +/// Verify (and otherwise drop) a proposed `gauge_range`. A KPI gauge is only +/// meaningful, and only non-misleading, when: +/// +/// 1. the column's FINALIZED `role` is `measure` (gauges are for quantities you aggregate — not +/// dimensions, identifiers, or timestamps), and +/// 2. the column's OBSERVED `[min, max]` lies inside the proposed `[lo, hi]`. +/// +/// Requirement 2 is the guardrail against a misleading gauge: `viz` renders the +/// measure's value against this scale, so a range that does not contain the data +/// would draw a needle pinned past the end. The shape (two finite numbers, `lo < +/// hi`) was already validated in `parse_llm_dictionary_response`; this is where the +/// SEMANTIC check happens, because only here are the merged role and the observed +/// min/max both known. Anything that fails is reset to `None` — an unverifiable +/// gauge is simply not emitted (byte-identical to a run that never proposed one). +/// +/// Must be called AFTER `coerce_role_concept`, so `role` is final. +fn verify_gauge_range(entry: &mut DictionaryEntry) { + let Some([lo, hi]) = entry.gauge_range else { + return; + }; + let keep = if entry.role == "measure" + && let Ok(obs_min) = entry.min.parse::() + && let Ok(obs_max) = entry.max.parse::() + { + obs_min.is_finite() && obs_max.is_finite() && obs_min >= lo && obs_max <= hi + } else { + false + }; + if !keep { + entry.gauge_range = None; + } +} + /// Two-pass-aware merge: seed `code_entries` with the BASELINE LLM Label / Description / /// Content Type (from the first pass) and then overlay the REFINE pass's LLM fields on top. /// If the refine pass omits a field, the baseline Label / Description / Content Type are @@ -1398,6 +1454,7 @@ pub(super) fn combine_dictionary_entries_with_baseline( merge_content_type(&entry.content_type, &baseline.content_type, false); entry.concept = merge_concept(&entry.concept, &baseline.concept, false); entry.role = merge_role(&entry.role, &baseline.role, false); + entry.gauge_range = baseline.gauge_range; } // Stage 2: overlay refine-pass LLM values where present. Omitted fields keep their // baseline values from stage 1 — this is the whole point of the baseline merge. @@ -1412,6 +1469,11 @@ pub(super) fn combine_dictionary_entries_with_baseline( merge_content_type(&entry.content_type, &refine.content_type, true); entry.concept = merge_concept(&entry.concept, &refine.concept, true); entry.role = merge_role(&entry.role, &refine.role, true); + // A refine pass that re-proposes a gauge scale overrides the baseline; one + // that omits it keeps the baseline value (mirrors the label/description rule). + if refine.gauge_range.is_some() { + entry.gauge_range = refine.gauge_range; + } } // Stage 3: same final "unknown"/fallback coercion as `combine_dictionary_entries` so // the two-pass output matches single-pass invariants. @@ -1420,6 +1482,7 @@ pub(super) fn combine_dictionary_entries_with_baseline( entry.content_type = "unknown".to_string(); } coerce_role_concept(entry); + verify_gauge_range(entry); } } code_entries @@ -1816,6 +1879,28 @@ pub(super) fn parse_llm_dictionary_response( }) .unwrap_or_default(); + // `gauge_range` rides the same `infer_content_type` gate as role/concept + // (a gauge only makes sense once the field is classed as a measure). We + // accept only a well-formed `[min, max]` array of two FINITE numbers with + // `min < max`; anything else (missing key, wrong arity, NaN/inf, min>=max) + // yields `None`. This is purely SHAPE validation — whether the range is + // appropriate (role == measure, observed data lies inside it) is decided + // later in `combine_dictionary_entries`, the only place that sees the + // finalized role and the column's observed min/max. + let gauge_range = if infer_content_type { + field_map + .get("gauge_range") + .and_then(|v| v.as_array()) + .and_then(|a| match a.as_slice() { + [lo, hi] => Some((lo.as_f64()?, hi.as_f64()?)), + _ => None, + }) + .filter(|(lo, hi)| lo.is_finite() && hi.is_finite() && lo < hi) + .map(|(lo, hi)| [lo, hi]) + } else { + None + }; + result.insert( field_name.clone(), LlmDictField { @@ -1825,6 +1910,7 @@ pub(super) fn parse_llm_dictionary_response( concept, role, null_values, + gauge_range, }, ); } @@ -1933,6 +2019,7 @@ mod tests { role: String::new(), null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, } } @@ -2347,6 +2434,7 @@ mod tests { fn llm_nulls(tokens: &[&str]) -> LlmDictField { LlmDictField { null_values: tokens.iter().map(ToString::to_string).collect(), + gauge_range: None, ..Default::default() } } @@ -2728,6 +2816,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code_entries, &llm, true); @@ -2738,6 +2827,97 @@ mod tests { ); } + #[test] + fn parse_gauge_range_shape_validation() { + // Only a well-formed [min, max] of two finite numbers with min < max survives. + let names: Vec = ["a", "b", "c", "d", "e", "f"] + .iter() + .map(ToString::to_string) + .collect(); + let resp = r#"{ + "a": {"label":"A","description":"d","gauge_range":[0,100]}, + "b": {"label":"B","description":"d","gauge_range":[100,0]}, + "c": {"label":"C","description":"d","gauge_range":[5]}, + "d": {"label":"D","description":"d"}, + "e": {"label":"E","description":"d","gauge_range":["x","y"]}, + "f": {"label":"F","description":"d","gauge_range":[0,0]} + }"#; + let got = parse_llm_dictionary_response(resp, &names, true).unwrap(); + assert_eq!(got["a"].gauge_range, Some([0.0, 100.0])); + assert_eq!(got["b"].gauge_range, None, "min >= max rejected"); + assert_eq!(got["c"].gauge_range, None, "wrong arity rejected"); + assert_eq!(got["d"].gauge_range, None, "missing key -> None"); + assert_eq!(got["e"].gauge_range, None, "non-numeric rejected"); + assert_eq!(got["f"].gauge_range, None, "min == max rejected"); + } + + #[test] + fn parse_gauge_range_gated_off_without_infer_content_type() { + let names = vec!["a".to_string()]; + let resp = r#"{"a":{"label":"A","description":"d","gauge_range":[0,100]}}"#; + let got = parse_llm_dictionary_response(resp, &names, false).unwrap(); + assert_eq!(got["a"].gauge_range, None); + } + + /// A numeric MEASURE whose observed [min, max] fits inside the proposed scale keeps it. + #[test] + fn combine_keeps_gauge_range_for_measure_when_observed_fits() { + let mut e = blank_entry("score"); + e.r#type = "Float".to_string(); + e.min = "10".to_string(); + e.max = "90".to_string(); + let mut llm = HashMap::new(); + llm.insert( + "score".to_string(), + LlmDictField { + role: "measure".to_string(), + gauge_range: Some([0.0, 100.0]), + ..Default::default() + }, + ); + let out = combine_dictionary_entries(vec![e], &llm, true); + assert_eq!(out[0].role, "measure"); + assert_eq!(out[0].gauge_range, Some([0.0, 100.0])); + } + + /// A range the observed data spills past is DROPPED (a misleading gauge is worse than none). + #[test] + fn combine_drops_gauge_range_when_observed_exceeds_scale() { + let mut e = blank_entry("pct"); + e.r#type = "Float".to_string(); + e.min = "10".to_string(); + e.max = "90".to_string(); // spills past [0, 50] + let mut llm = HashMap::new(); + llm.insert( + "pct".to_string(), + LlmDictField { + role: "measure".to_string(), + gauge_range: Some([0.0, 50.0]), + ..Default::default() + }, + ); + let out = combine_dictionary_entries(vec![e], &llm, true); + assert_eq!(out[0].gauge_range, None); + } + + /// A gauge on a non-measure (dimension) is DROPPED regardless of the numbers. + #[test] + fn combine_drops_gauge_range_for_non_measure() { + let e = blank_entry("category"); // String -> coerces to dimension + let mut llm = HashMap::new(); + llm.insert( + "category".to_string(), + LlmDictField { + role: "dimension".to_string(), + gauge_range: Some([0.0, 100.0]), + ..Default::default() + }, + ); + let out = combine_dictionary_entries(vec![e], &llm, true); + assert_eq!(out[0].role, "dimension"); + assert_eq!(out[0].gauge_range, None); + } + #[test] fn parse_llm_response_drops_out_of_vocab_content_type() { // An out-of-vocabulary token is left empty by parsing; combine_dictionary_entries @@ -2845,6 +3025,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); // infer_content_type = false: pure copy, no "unknown" coercion. @@ -2876,6 +3057,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); llm.insert( @@ -2887,6 +3069,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); // "omitted" is intentionally absent from the LLM map. @@ -2914,6 +3097,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); llm.insert( @@ -2925,6 +3109,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code_entries, &llm, true); @@ -3196,6 +3381,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); baseline.insert( @@ -3207,6 +3393,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); @@ -3221,6 +3408,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); @@ -3240,6 +3428,88 @@ mod tests { assert_eq!(combined[1].content_type, "address_street"); } + /// Two-pass: the refine prompt never asks for `gauge_range`, so the refine pass + /// re-emits the measure field (label/description) but with NO gauge — exactly what + /// the pipeline produces. The first-pass gauge scale MUST survive that overlay + /// (this is the line most at risk from a future refactor of the baseline merge). + #[test] + fn two_pass_refine_omitting_gauge_range_preserves_the_baseline() { + let mut score = blank_entry("score"); + score.r#type = "Float".to_string(); + score.min = "12".to_string(); + score.max = "88".to_string(); + let code_entries = vec![score]; + + let mut baseline = HashMap::new(); + baseline.insert( + "score".to_string(), + LlmDictField { + label: "Score".to_string(), + description: "First-pass description.".to_string(), + role: "measure".to_string(), + gauge_range: Some([0.0, 100.0]), + ..Default::default() + }, + ); + + // Refine RETURNS the field (present in the map) but omits gauge_range — the + // realistic case, distinct from the field being absent entirely. + let mut refine = HashMap::new(); + refine.insert( + "score".to_string(), + LlmDictField { + label: "Score".to_string(), + description: "Refined description.".to_string(), + role: "measure".to_string(), + gauge_range: None, + ..Default::default() + }, + ); + + let combined = + combine_dictionary_entries_with_baseline(code_entries, &baseline, &refine, true); + assert_eq!(combined[0].description, "Refined description."); + assert_eq!( + combined[0].gauge_range, + Some([0.0, 100.0]), + "baseline gauge scale must survive a refine pass that omits it" + ); + } + + /// Two-pass: a refine pass that DOES re-propose a gauge scale overrides the baseline + /// (and is still verified against the observed range). + #[test] + fn two_pass_refine_restating_gauge_range_overrides_the_baseline() { + let mut score = blank_entry("score"); + score.r#type = "Float".to_string(); + score.min = "1".to_string(); + score.max = "4".to_string(); + let code_entries = vec![score]; + + let mut baseline = HashMap::new(); + baseline.insert( + "score".to_string(), + LlmDictField { + role: "measure".to_string(), + gauge_range: Some([0.0, 100.0]), + ..Default::default() + }, + ); + let mut refine = HashMap::new(); + refine.insert( + "score".to_string(), + LlmDictField { + role: "measure".to_string(), + gauge_range: Some([0.0, 5.0]), + ..Default::default() + }, + ); + + let combined = + combine_dictionary_entries_with_baseline(code_entries, &baseline, &refine, true); + assert_eq!(combined[0].gauge_range, Some([0.0, 5.0])); + } + #[test] fn combine_with_baseline_rejects_refine_supplied_unique_id() { // The deterministic "unique_id" stamp (cardinality == rowcount) must survive @@ -3265,6 +3535,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); refine.insert( @@ -3277,6 +3548,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); @@ -3311,6 +3583,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); @@ -3324,6 +3597,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); @@ -3352,6 +3626,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); @@ -3591,6 +3866,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code, &llm, true); @@ -3612,6 +3888,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code, &llm, true); @@ -3633,6 +3910,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code, &llm, true); @@ -3654,6 +3932,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let mut refine = HashMap::new(); @@ -3666,6 +3945,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries_with_baseline(code, &baseline, &refine, true); @@ -4207,6 +4487,7 @@ mod tests { concept: String::new(), role: "dimension".to_string(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code, &llm, true); @@ -4229,6 +4510,7 @@ mod tests { concept: "unknown".to_string(), role: "dimension".to_string(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code, &llm, true); @@ -4249,6 +4531,7 @@ mod tests { concept: String::new(), role: String::new(), null_values: Vec::new(), + gauge_range: None, }, ); let combined = combine_dictionary_entries(code, &llm, true); diff --git a/src/cmd/describegpt/formatters.rs b/src/cmd/describegpt/formatters.rs index 66d9af0da3..d87b754439 100644 --- a/src/cmd/describegpt/formatters.rs +++ b/src/cmd/describegpt/formatters.rs @@ -431,6 +431,13 @@ fn build_x_qsv( if !entry.concept.is_empty() { x_qsv.insert("concept".to_string(), Value::String(entry.concept.clone())); } + // KPI gauge scale for a measure, already role/stats-verified in + // `combine_dictionary_entries` (so it is `Some` only when safe to draw). + // Emitted as the `[min, max]` array `viz smart --dictionary` reads back as + // `x-qsv.gauge_range`. Absent otherwise, keeping no-flag runs byte-identical. + if let Some([lo, hi]) = entry.gauge_range { + x_qsv.insert("gauge_range".to_string(), json!([lo, hi])); + } } // Null sentinels. Deliberately NOT gated on the flag: emission keys off the lists being // non-empty, and they are only populated when a response actually supplied `null_values`. @@ -1097,6 +1104,7 @@ mod tests { role: String::new(), null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, } } @@ -1365,6 +1373,72 @@ mod tests { ); } + #[test] + fn jsonschema_x_qsv_carries_gauge_range_for_measure() { + // A verified gauge_range emits as the 2-element numeric array `[min, max]` + // that `viz smart --dictionary` reads back from `x-qsv.gauge_range`. + let mut rating = sample_entry("rating", ""); + rating.r#type = "Float".to_string(); + rating.role = "measure".to_string(); + rating.gauge_range = Some([0.0, 5.0]); + let schema = format_dictionary_jsonschema( + std::slice::from_ref(&rating), + "test.csv", + 10, + 5, + 25, + true, + false, + false, + None, + ); + let gr = &schema["properties"]["rating"]["x-qsv"]["gauge_range"]; + assert_eq!(gr, &json!([0.0, 5.0])); + // Exactly the shape viz's `xq_range` matches: a 2-element numeric array. + assert_eq!(gr.as_array().map(Vec::len), Some(2)); + + // flag off: gauge_range absent (legacy schema stays byte-identical). + let off = format_dictionary_jsonschema( + std::slice::from_ref(&rating), + "test.csv", + 10, + 5, + 25, + false, + false, + false, + None, + ); + assert!( + off["properties"]["rating"]["x-qsv"] + .get("gauge_range") + .is_none(), + "gauge_range leaked when flag off" + ); + + // None gauge_range is omitted even with the flag on. + let mut plain = sample_entry("plain_measure", ""); + plain.r#type = "Float".to_string(); + plain.role = "measure".to_string(); + let schema3 = format_dictionary_jsonschema( + std::slice::from_ref(&plain), + "test.csv", + 10, + 5, + 25, + true, + false, + false, + None, + ); + assert!( + schema3["properties"]["plain_measure"]["x-qsv"] + .get("gauge_range") + .is_none(), + "gauge_range emitted when None" + ); + } + #[test] fn tsv_header_unchanged_when_flag_off() { let entries = vec![sample_entry("col", "email")]; @@ -1456,6 +1530,7 @@ mod tests { null_values: Vec::new(), null_candidates: Vec::new(), + gauge_range: None, }; let schema = format_dictionary_jsonschema( std::slice::from_ref(&entry), From e15b55ee6fd0afa7adc219a510e5fa9277f6c2eb Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:09:21 -0400 Subject: [PATCH 02/12] fix(describegpt): scope gauge_range to renderable continuous measures (roborev #3606) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Medium: viz's build_kpi_row only turns a column into a gauge KPI tile when it earned a box/violin/histogram distribution panel, which low-cardinality integer ratings never do — so the prompt's 1-to-5 / 0-to-10 rating examples advertised gauges viz cannot render. Narrowed the prompt to CONTINUOUS canonical scales (percentage, probability/ratio, bounded continuous index) and added an explicit clause telling the model NOT to gauge discrete/low-cardinality ratings. (Gauging panel-less measures in viz is left as a follow-up enhancement.) Low: verify_gauge_range now also requires the column's qsv type to be Integer/Float, so a String column the LLM mis-tags role:measure whose lexicographic min/max happen to parse as floats can no longer keep/emit a gauge. New test combine_drops_gauge_range_for_non_numeric_type; prompt-gating assertion updated to the new bullet wording. Co-Authored-By: Claude Opus 4.8 (1M context) --- resources/describegpt_defaults.toml | 27 ++++++++++-------- src/cmd/describegpt.rs | 2 +- src/cmd/describegpt/dictionary.rs | 43 ++++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/resources/describegpt_defaults.toml b/resources/describegpt_defaults.toml index 6a227995a0..a24d4f9a8c 100644 --- a/resources/describegpt_defaults.toml +++ b/resources/describegpt_defaults.toml @@ -140,18 +140,23 @@ For each field, provide: none do. Do NOT invent tokens. NOTE: "id.surrogate_key" is RESERVED and set deterministically by qsv for unique-key columns - do not use it yourself. Allowed Concept tokens: {{ concept_vocab }} -- Gauge Range (OPTIONAL, numeric MEASURE fields only): if - and ONLY if - this field is a numeric - "measure" that rides a well-known CANONICAL scale with a fixed, meaningful full-scale maximum, +- Gauge Range (OPTIONAL, CONTINUOUS numeric MEASURE fields only): if - and ONLY if - this field is + a CONTINUOUS numeric "measure" (a quantity that varies smoothly across many values, charted as a + distribution) that rides a well-known CANONICAL scale with a fixed, meaningful full-scale maximum, add a "gauge_range" as a two-number array [min, max] giving that scale's bounds. Use it for a - percentage (`[0, 100]`), a probability or ratio (`[0, 1]`), a 1-to-5 or 0-to-10 rating/score - (`[0, 5]`, `[0, 10]`), or a bounded index. The range MUST contain the field's observed Min and - Max (from the Summary Statistics) - a dashboard draws the value against this scale, so a range - the data spills past is worse than none. Do NOT invent a range for an OPEN-ENDED measure whose - maximum is just "however big the data happens to get" - revenue, counts, amounts, durations, - distances, weights - OMIT "gauge_range" entirely for those. When in doubt, omit it: qsv drops - any range that does not contain the observed data, and a missing gauge_range simply renders as a - plain number instead of a dial. Never emit a "target", baseline, or goal value - you cannot know - a business goal from the data. + percentage (`[0, 100]`), a probability or ratio (`[0, 1]`), or a bounded continuous index/score. + The range MUST contain the field's observed Min and Max (from the Summary Statistics) - a + dashboard draws the value against this scale, so a range the data spills past is worse than none. + Do NOT add a gauge_range to: + - an OPEN-ENDED measure whose maximum is just "however big the data happens to get" - revenue, + counts, amounts, durations, distances, weights; or + - a DISCRETE / low-cardinality rating with only a handful of distinct levels (a 1-to-5 star + rating, a 0-to-10 score with few values) - the dashboard treats those as categories, not + continuous measures, so a gauge would never render for them. + OMIT "gauge_range" entirely in those cases. When in doubt, omit it: qsv drops any range that does + not contain the observed data, and a missing gauge_range simply renders as a plain number instead + of a dial. Never emit a "target", baseline, or goal value - you cannot know a business goal from + the data. {%- endif %} {%- if infer_content_type %} diff --git a/src/cmd/describegpt.rs b/src/cmd/describegpt.rs index b1bd618b7e..26b2f2e806 100644 --- a/src/cmd/describegpt.rs +++ b/src/cmd/describegpt.rs @@ -8810,7 +8810,7 @@ p_fewshot_examples = "" ); // The Gauge Range instruction is injected when the flag is on. assert!( - on.contains("- Gauge Range (OPTIONAL, numeric MEASURE fields only)"), + on.contains("- Gauge Range (OPTIONAL, CONTINUOUS numeric MEASURE fields only)"), "flag-on prompt must include the Gauge Range instruction:\n{on}" ); // Concept + Role instructions and vocabularies are injected when the flag is on. diff --git a/src/cmd/describegpt/dictionary.rs b/src/cmd/describegpt/dictionary.rs index ac9a1eefdd..687d9e9d6e 100644 --- a/src/cmd/describegpt/dictionary.rs +++ b/src/cmd/describegpt/dictionary.rs @@ -1394,24 +1394,29 @@ fn coerce_role_concept(entry: &mut DictionaryEntry) { /// Verify (and otherwise drop) a proposed `gauge_range`. A KPI gauge is only /// meaningful, and only non-misleading, when: /// -/// 1. the column's FINALIZED `role` is `measure` (gauges are for quantities you aggregate — not +/// 1. the column's qsv `type` is numeric (`Integer`/`Float`) — a gauge scale on a String/Date +/// column is nonsense even if the LLM mis-tagged it `role: measure` and its lexicographic +/// min/max happen to parse as floats, and +/// 2. the column's FINALIZED `role` is `measure` (gauges are for quantities you aggregate — not /// dimensions, identifiers, or timestamps), and -/// 2. the column's OBSERVED `[min, max]` lies inside the proposed `[lo, hi]`. +/// 3. the column's OBSERVED `[min, max]` lies inside the proposed `[lo, hi]`. /// -/// Requirement 2 is the guardrail against a misleading gauge: `viz` renders the +/// Requirement 3 is the guardrail against a misleading gauge: `viz` renders the /// measure's value against this scale, so a range that does not contain the data /// would draw a needle pinned past the end. The shape (two finite numbers, `lo < /// hi`) was already validated in `parse_llm_dictionary_response`; this is where the -/// SEMANTIC check happens, because only here are the merged role and the observed -/// min/max both known. Anything that fails is reset to `None` — an unverifiable -/// gauge is simply not emitted (byte-identical to a run that never proposed one). +/// SEMANTIC check happens, because only here are the merged role, the column type, +/// and the observed min/max all known. Anything that fails is reset to `None` — an +/// unverifiable gauge is simply not emitted (byte-identical to a run that never +/// proposed one). /// /// Must be called AFTER `coerce_role_concept`, so `role` is final. fn verify_gauge_range(entry: &mut DictionaryEntry) { let Some([lo, hi]) = entry.gauge_range else { return; }; - let keep = if entry.role == "measure" + let keep = if matches!(entry.r#type.as_str(), "Integer" | "Float") + && entry.role == "measure" && let Ok(obs_min) = entry.min.parse::() && let Ok(obs_max) = entry.max.parse::() { @@ -2918,6 +2923,30 @@ mod tests { assert_eq!(out[0].gauge_range, None); } + /// A String column the LLM mis-tags `role: measure` must NOT keep a gauge, even when + /// its lexicographic min/max happen to parse as floats (roborev #3606). + #[test] + fn combine_drops_gauge_range_for_non_numeric_type() { + let mut e = blank_entry("code"); // r#type stays "String" + e.min = "10".to_string(); // numeric-looking strings that parse as f64 + e.max = "90".to_string(); + let mut llm = HashMap::new(); + llm.insert( + "code".to_string(), + LlmDictField { + role: "measure".to_string(), + gauge_range: Some([0.0, 100.0]), + ..Default::default() + }, + ); + let out = combine_dictionary_entries(vec![e], &llm, true); + assert_eq!(out[0].r#type, "String"); + assert_eq!( + out[0].gauge_range, None, + "a non-numeric column must not emit a gauge even if role==measure" + ); + } + #[test] fn parse_llm_response_drops_out_of_vocab_content_type() { // An out-of-vocabulary token is left empty by parsing; combine_dictionary_entries From daaee4e9f142f87703408e8a22a7456ab1112b33 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:49:30 -0400 Subject: [PATCH 03/12] feat(viz): render KPI tile labels as wrapped subtitles below each indicator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Long auto-generated KPI labels (e.g. "Mean Profit Margin Percentage") rendered as the plotly Indicator's built-in title — a large font above the number — and overran narrow tiles, colliding with neighbours. Move the label out of the indicator title into a smaller, word-wrapped subtitle annotation centered BELOW each tile: kpi_indicator now owns only the number/gauge/delta and reserves the bottom band of its domain, and both render paths (typed grid + inline) draw the label via kpi_label_annotation. wrap_kpi_label greedily wraps on word boundaries (never hard-splitting a word); tile width (tile count) sets the wrap threshold. Verified in-browser on the gemma-inferred sales_sample gauge dashboard: labels sit cleanly below each tile, and the longest wraps to two lines with no overlap. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cmd/viz.rs | 124 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 9 deletions(-) diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 9a7d6c9d39..1f7f6badb5 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -602,7 +602,7 @@ use plotly::{ HoverInfo, Line, Marker, Mode, Orientation, Pattern, PatternShape, TextPosition, TickMode, Title, }, - indicator::{Delta, Gauge, GaugeAxis, IndicatorTitle, Mode as IndicatorMode, Number}, + indicator::{Delta, Gauge, GaugeAxis, Mode as IndicatorMode, Number}, layout::{ Annotation, Axis, AxisType, Center, GeoFitBounds, GeoResolution, HoverMode, Layout, LayoutGeo, LayoutMap, LayoutScene, MapBounds, MapStyle, Margin, ModeBar, Projection, @@ -14491,7 +14491,11 @@ fn kpi_number_format(value: f64) -> String { /// Build one KPI `Indicator` trace for a tile, positioned by paper-domain `[x0,x1] × [y0,y1]`. /// A validated `gauge` adds an angular gauge, a `target` adds a "vs target" delta; the `mode` -/// follows whichever are present (always including the big number). +/// follows whichever are present (always including the big number). The tile LABEL is NOT set as +/// the indicator's built-in title (which renders large, above the number, and overruns narrow +/// tiles for long auto-generated labels) — the caller instead draws it as a smaller, word-wrapped +/// subtitle annotation BELOW the tile via `kpi_label_annotation`, so the indicator here owns only +/// the number/gauge/delta and leaves the bottom band of its domain for that label. fn kpi_indicator(tile: &KpiTile, x_domain: [f64; 2], y_domain: [f64; 2]) -> Box { let mode = match (tile.gauge.is_some(), tile.target.is_some()) { (true, true) => IndicatorMode::NumberAndDeltaAndGauge, @@ -14502,7 +14506,6 @@ fn kpi_indicator(tile: &KpiTile, x_domain: [f64; 2], y_domain: [f64; 2]) -> Box< let mut ind = Indicator::new(tile.value) .mode(mode) .domain(Domain::new().x(&x_domain).y(&y_domain)) - .title(IndicatorTitle::new().text(tile.label.clone())) .number(Number::new().value_format(tile.format.clone())); if let Some([g_lo, g_hi]) = tile.gauge { ind = ind.gauge(Gauge::new().axis(GaugeAxis::new().range([g_lo, g_hi]))); @@ -14515,6 +14518,54 @@ fn kpi_indicator(tile: &KpiTile, x_domain: [f64; 2], y_domain: [f64; 2]) -> Box< ind } +/// Greedy word-wrap a KPI tile label into `
`-separated lines of at most `max_chars` +/// characters, so a long auto-generated label (e.g. "Mean Profit Margin Percentage") renders as a +/// compact multi-line subtitle under its tile instead of overrunning the tile width and colliding +/// with its neighbours. A single word longer than `max_chars` is left intact (never hard-split +/// mid-word — a broken token reads worse than a slightly-wide one). +fn wrap_kpi_label(text: &str, max_chars: usize) -> String { + let max = max_chars.max(1); + let mut lines: Vec = Vec::new(); + let mut cur = String::new(); + for word in text.split_whitespace() { + if cur.is_empty() { + cur.push_str(word); + } else if cur.chars().count() + 1 + word.chars().count() <= max { + cur.push(' '); + cur.push_str(word); + } else { + lines.push(std::mem::take(&mut cur)); + cur.push_str(word); + } + } + if !cur.is_empty() { + lines.push(cur); + } + lines.join("
") +} + +/// The word-wrapped subtitle label for one KPI tile, centered under the tile at paper-y `y_top` +/// (anchored at its top so it hangs down into the band reserved below the indicator). Smaller than +/// the number it captions; `font` follows the dashboard's themed/non-themed convention. +fn kpi_label_annotation( + label: &str, + x_center: f64, + y_top: f64, + max_chars: usize, + font: Font, +) -> Annotation { + Annotation::new() + .text(wrap_kpi_label(label, max_chars)) + .x(x_center) + .y(y_top) + .x_ref("paper") + .y_ref("paper") + .x_anchor(Anchor::Center) + .y_anchor(Anchor::Top) + .show_arrow(false) + .font(font) +} + /// Build the leading KPI-tile row for `viz smart`: dataset-summary tiles (record count, field /// count, a completeness gauge) followed by the headline numeric measures (capped at /// `KPI_MAX_TILES`). A measure tile becomes a gauge when its dictionary supplies a `gauge_range` @@ -16669,7 +16720,8 @@ fn smart_grid_parts( // KPI row: a full-width band of domain-positioned Indicator tiles (no cartesian axes). // Sub-divide this panel's paper x-domain into N tiles, emit each as an Indicator, then skip - // the cartesian axis assignment below. No section title — the tiles are self-labeling. + // the cartesian axis assignment below. No section title — each tile carries its own + // word-wrapped subtitle label below the number (`kpi_label_annotation`). if let PanelKind::KpiRow { tiles } = &panel.kind { let geom = &geoms[n]; let x0 = geom.x_domain.first().copied().unwrap_or(0.0); @@ -16679,10 +16731,23 @@ fn smart_grid_parts( let count = tiles.len().max(1) as f64; let width = (x1 - x0) / count; let gutter = width * 0.06; + // reserve the bottom slice of the row band for the subtitle labels, so the indicator's + // number/gauge sits above and the (smaller, wrapped) label hangs below it. + let label_band = (y1 - y0) * 0.26; + let ind_y0 = y0 + label_band; + // narrower tiles (more of them) wrap sooner; the smaller subtitle font fits ~2 lines. + let max_chars = ((170.0 / count) as usize).clamp(14, 40); for (i, tile) in tiles.iter().enumerate() { let lo = x0 + i as f64 * width + gutter / 2.0; let hi = x0 + (i as f64 + 1.0) * width - gutter / 2.0; - traces.push(kpi_indicator(tile, [lo, hi], [y0, y1])); + traces.push(kpi_indicator(tile, [lo, hi], [ind_y0, y1])); + annotations.push(kpi_label_annotation( + &tile.label, + f64::midpoint(lo, hi), + ind_y0, + max_chars, + ann_font(10), + )); } continue; } @@ -17661,18 +17726,36 @@ fn smart_inline_panel_plot( let count = tiles.len().max(1) as f64; // even horizontal split with a small inter-tile gutter so adjacent gauges/numbers breathe let gutter = 0.02_f64; + // reserve the bottom slice for each tile's word-wrapped subtitle label; the indicator's + // number/gauge sits above it. + let label_band = 0.26_f64; + let max_chars = ((170.0 / count) as usize).clamp(14, 40); + let ann_font = |size: usize| { + let f = Font::new().size(size); + if themed { f } else { f.family(FONT_FAMILY) } + }; + let mut kpi_labels: Vec = Vec::with_capacity(tiles.len()); for (i, tile) in tiles.iter().enumerate() { let lo = (i as f64 / count) + gutter / 2.0; let hi = ((i + 1) as f64 / count) - gutter / 2.0; - plot.add_trace(kpi_indicator(tile, [lo, hi], [0.0, 1.0])); + plot.add_trace(kpi_indicator(tile, [lo, hi], [label_band, 1.0])); + kpi_labels.push(kpi_label_annotation( + &tile.label, + f64::midpoint(lo, hi), + label_band, + max_chars, + ann_font(10), + )); } - // no panel title — the tiles are self-labeling (each Indicator carries its own title), so - // reclaim the top band the title would occupy. Just the vertical hover modebar. + // no panel title — each tile carries its own word-wrapped subtitle label below its number + // (`kpi_label_annotation`), so reclaim the top band a title would occupy. Vertical hover + // modebar only. let mut layout = Layout::new() .show_legend(false) .height(row_height) .margin(Margin::new().top(20).bottom(20).left(20).right(20).pad(4)) - .mode_bar(ModeBar::new().orientation(Orientation::Vertical)); + .mode_bar(ModeBar::new().orientation(Orientation::Vertical)) + .annotations(kpi_labels); if !themed { layout = layout .font(Font::new().family(FONT_FAMILY).color(INK).size(12)) @@ -22707,6 +22790,29 @@ mod tests { assert_eq!(auto_hierarchy_style(4), HierStyle::Sunburst); } + #[test] + fn wrap_kpi_label_wraps_on_word_boundaries() { + // short labels stay on one line (no
injected), so exact-text consumers/tests still see + // the whole label. + assert_eq!(wrap_kpi_label("Completeness", 40), "Completeness"); + assert_eq!( + wrap_kpi_label("Mean Customer Rating", 40), + "Mean Customer Rating" + ); + // a long label wraps at the last word boundary that fits within max_chars. + assert_eq!( + wrap_kpi_label("Mean Profit Margin Percentage", 20), + "Mean Profit Margin
Percentage" + ); + // a single word longer than max_chars is left intact rather than hard-split mid-word. + assert_eq!( + wrap_kpi_label("Supercalifragilistic", 8), + "Supercalifragilistic" + ); + // multiple wraps. + assert_eq!(wrap_kpi_label("a b c d e", 3), "a b
c d
e"); + } + #[test] fn resolve_hierarchy_style_flag() { assert_eq!( From 79f0325d7d0c045207c653b4ad2c2e0700f26e8a Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:59:32 -0400 Subject: [PATCH 04/12] fix(viz): enlarge KPI subtitle labels (13px) below each indicator The previous KPI subtitle labels were too small (10px) to read comfortably. Keep them below the indicator (not as titles) but bump the font to 13px, widen the reserved bottom band to 30% to fit two wrapped lines, and tighten the wrap threshold so long labels still break cleanly. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cmd/viz.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 1f7f6badb5..eac10aa7fb 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -16731,12 +16731,12 @@ fn smart_grid_parts( let count = tiles.len().max(1) as f64; let width = (x1 - x0) / count; let gutter = width * 0.06; - // reserve the bottom slice of the row band for the subtitle labels, so the indicator's - // number/gauge sits above and the (smaller, wrapped) label hangs below it. - let label_band = (y1 - y0) * 0.26; + // reserve the bottom slice of the row band for the labels, so the indicator's + // number/gauge sits above and the wrapped label sits below it. + let label_band = (y1 - y0) * 0.30; let ind_y0 = y0 + label_band; - // narrower tiles (more of them) wrap sooner; the smaller subtitle font fits ~2 lines. - let max_chars = ((170.0 / count) as usize).clamp(14, 40); + // narrower tiles (more of them) wrap sooner; the label font fits ~2 lines in the band. + let max_chars = ((150.0 / count) as usize).clamp(12, 36); for (i, tile) in tiles.iter().enumerate() { let lo = x0 + i as f64 * width + gutter / 2.0; let hi = x0 + (i as f64 + 1.0) * width - gutter / 2.0; @@ -16746,7 +16746,7 @@ fn smart_grid_parts( f64::midpoint(lo, hi), ind_y0, max_chars, - ann_font(10), + ann_font(13), )); } continue; @@ -17726,10 +17726,10 @@ fn smart_inline_panel_plot( let count = tiles.len().max(1) as f64; // even horizontal split with a small inter-tile gutter so adjacent gauges/numbers breathe let gutter = 0.02_f64; - // reserve the bottom slice for each tile's word-wrapped subtitle label; the indicator's + // reserve the bottom slice for each tile's word-wrapped label; the indicator's // number/gauge sits above it. - let label_band = 0.26_f64; - let max_chars = ((170.0 / count) as usize).clamp(14, 40); + let label_band = 0.30_f64; + let max_chars = ((150.0 / count) as usize).clamp(12, 36); let ann_font = |size: usize| { let f = Font::new().size(size); if themed { f } else { f.family(FONT_FAMILY) } @@ -17744,7 +17744,7 @@ fn smart_inline_panel_plot( f64::midpoint(lo, hi), label_band, max_chars, - ann_font(10), + ann_font(13), )); } // no panel title — each tile carries its own word-wrapped subtitle label below its number From 672787e259577f507e03575d740fe1e311df17b1 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:09:19 -0400 Subject: [PATCH 05/12] fix(viz): tighten the gap between KPI numbers and their labels The label sat at the very bottom of the reserved band, a full band below the indicator's number (which has its own bottom padding), reading as too detached. Anchor the label up inside that padding (14% of the band above the reserved line) in both render paths so it sits snug under the number. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cmd/viz.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index eac10aa7fb..856e350f03 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -16735,6 +16735,10 @@ fn smart_grid_parts( // number/gauge sits above and the wrapped label sits below it. let label_band = (y1 - y0) * 0.30; let ind_y0 = y0 + label_band; + // the indicator leaves padding below its number, so anchor the label UP inside that + // padding (above `ind_y0`) to sit snug under the number rather than a band away from + // it. + let label_y = ind_y0 + (y1 - y0) * 0.14; // narrower tiles (more of them) wrap sooner; the label font fits ~2 lines in the band. let max_chars = ((150.0 / count) as usize).clamp(12, 36); for (i, tile) in tiles.iter().enumerate() { @@ -16744,7 +16748,7 @@ fn smart_grid_parts( annotations.push(kpi_label_annotation( &tile.label, f64::midpoint(lo, hi), - ind_y0, + label_y, max_chars, ann_font(13), )); @@ -17729,6 +17733,9 @@ fn smart_inline_panel_plot( // reserve the bottom slice for each tile's word-wrapped label; the indicator's // number/gauge sits above it. let label_band = 0.30_f64; + // the indicator leaves padding below its number, so anchor the label UP inside that padding + // (above `label_band`) to sit snug under the number rather than a band away from it. + let label_y = label_band + 0.14; let max_chars = ((150.0 / count) as usize).clamp(12, 36); let ann_font = |size: usize| { let f = Font::new().size(size); @@ -17742,7 +17749,7 @@ fn smart_inline_panel_plot( kpi_labels.push(kpi_label_annotation( &tile.label, f64::midpoint(lo, hi), - label_band, + label_y, max_chars, ann_font(13), )); From 159fbf763dd8d6b1051547d346f89afd8c310b90 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:18:49 -0400 Subject: [PATCH 06/12] fix(viz): keep raised KPI label off the gauge; per-tile label position (roborev #3610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit raised every KPI label uniformly into the indicator domain, which is safe for number-only tiles (empty padding below the centered number) but risks overlapping gauge tiles — the always-present Completeness gauge and any dictionary-backed measure gauge sit lower in the domain. Compute label_y per tile in both render paths: number-only tiles keep the raised position (snug under the number), gauge tiles anchor at the reserved line, fully below the indicator domain so the label can never land on the dial. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cmd/viz.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 856e350f03..35a657e014 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -16735,16 +16735,22 @@ fn smart_grid_parts( // number/gauge sits above and the wrapped label sits below it. let label_band = (y1 - y0) * 0.30; let ind_y0 = y0 + label_band; - // the indicator leaves padding below its number, so anchor the label UP inside that - // padding (above `ind_y0`) to sit snug under the number rather than a band away from - // it. - let label_y = ind_y0 + (y1 - y0) * 0.14; + // a NUMBER-only tile centers its value high in the domain, leaving empty padding below, + // so raise its label up into that padding to sit snug under the number. A GAUGE tile + // fills the domain with the dial + a low-sitting number, so keep its label at `ind_y0` + // (fully below the indicator domain) — raising it there would overlap the gauge. + let number_label_y = ind_y0 + (y1 - y0) * 0.14; // narrower tiles (more of them) wrap sooner; the label font fits ~2 lines in the band. let max_chars = ((150.0 / count) as usize).clamp(12, 36); for (i, tile) in tiles.iter().enumerate() { let lo = x0 + i as f64 * width + gutter / 2.0; let hi = x0 + (i as f64 + 1.0) * width - gutter / 2.0; traces.push(kpi_indicator(tile, [lo, hi], [ind_y0, y1])); + let label_y = if tile.gauge.is_some() { + ind_y0 + } else { + number_label_y + }; annotations.push(kpi_label_annotation( &tile.label, f64::midpoint(lo, hi), @@ -17733,9 +17739,11 @@ fn smart_inline_panel_plot( // reserve the bottom slice for each tile's word-wrapped label; the indicator's // number/gauge sits above it. let label_band = 0.30_f64; - // the indicator leaves padding below its number, so anchor the label UP inside that padding - // (above `label_band`) to sit snug under the number rather than a band away from it. - let label_y = label_band + 0.14; + // a NUMBER-only tile centers its value high in the domain, leaving empty padding below, so + // raise its label up into that padding to sit snug under the number. A GAUGE tile fills the + // domain with the dial + a low-sitting number, so keep its label at `label_band` (fully + // below the indicator domain) — raising it there would overlap the gauge. + let number_label_y = label_band + 0.14; let max_chars = ((150.0 / count) as usize).clamp(12, 36); let ann_font = |size: usize| { let f = Font::new().size(size); @@ -17746,6 +17754,11 @@ fn smart_inline_panel_plot( let lo = (i as f64 / count) + gutter / 2.0; let hi = ((i + 1) as f64 / count) - gutter / 2.0; plot.add_trace(kpi_indicator(tile, [lo, hi], [label_band, 1.0])); + let label_y = if tile.gauge.is_some() { + label_band + } else { + number_label_y + }; kpi_labels.push(kpi_label_annotation( &tile.label, f64::midpoint(lo, hi), From b4a40aefb2af05068600182ca6d2d911ac4a2e93 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:36:37 -0400 Subject: [PATCH 07/12] fix(viz): align all KPI labels on one baseline under the numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-tile label split (roborev #3610) dropped gauge-tile labels to a lower line, creating an uneven, distracting baseline across the KPI row. Visual review shows the gauge dial is a semicircle in the TOP of the domain with its number below it, so the label band directly under the number is clear of the dial — the overlap #3610 guarded against does not occur at the label's actual position. Anchor every label (number-only AND gauge) at the same raised y for one clean baseline snug under the numbers, with no dial overlap. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cmd/viz.rs | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 35a657e014..0fbca99407 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -16735,22 +16735,17 @@ fn smart_grid_parts( // number/gauge sits above and the wrapped label sits below it. let label_band = (y1 - y0) * 0.30; let ind_y0 = y0 + label_band; - // a NUMBER-only tile centers its value high in the domain, leaving empty padding below, - // so raise its label up into that padding to sit snug under the number. A GAUGE tile - // fills the domain with the dial + a low-sitting number, so keep its label at `ind_y0` - // (fully below the indicator domain) — raising it there would overlap the gauge. - let number_label_y = ind_y0 + (y1 - y0) * 0.14; + // Every tile — number-only OR gauge — puts empty space directly below its number: the + // gauge dial is a semicircle in the TOP of the domain and its number sits below it, so + // the label band under the number is clear of the dial. Anchor ALL labels at the same + // raised y so they share one clean baseline snug under the numbers (no per-tile step). + let label_y = ind_y0 + (y1 - y0) * 0.14; // narrower tiles (more of them) wrap sooner; the label font fits ~2 lines in the band. let max_chars = ((150.0 / count) as usize).clamp(12, 36); for (i, tile) in tiles.iter().enumerate() { let lo = x0 + i as f64 * width + gutter / 2.0; let hi = x0 + (i as f64 + 1.0) * width - gutter / 2.0; traces.push(kpi_indicator(tile, [lo, hi], [ind_y0, y1])); - let label_y = if tile.gauge.is_some() { - ind_y0 - } else { - number_label_y - }; annotations.push(kpi_label_annotation( &tile.label, f64::midpoint(lo, hi), @@ -17739,11 +17734,11 @@ fn smart_inline_panel_plot( // reserve the bottom slice for each tile's word-wrapped label; the indicator's // number/gauge sits above it. let label_band = 0.30_f64; - // a NUMBER-only tile centers its value high in the domain, leaving empty padding below, so - // raise its label up into that padding to sit snug under the number. A GAUGE tile fills the - // domain with the dial + a low-sitting number, so keep its label at `label_band` (fully - // below the indicator domain) — raising it there would overlap the gauge. - let number_label_y = label_band + 0.14; + // Every tile — number-only OR gauge — puts empty space directly below its number: the + // gauge dial is a semicircle in the TOP of the domain and its number sits below it, so the + // label band under the number is clear of the dial. Anchor ALL labels at the same raised y + // so they share one clean baseline snug under the numbers (no per-tile step). + let label_y = label_band + 0.14; let max_chars = ((150.0 / count) as usize).clamp(12, 36); let ann_font = |size: usize| { let f = Font::new().size(size); @@ -17754,11 +17749,6 @@ fn smart_inline_panel_plot( let lo = (i as f64 / count) + gutter / 2.0; let hi = ((i + 1) as f64 / count) - gutter / 2.0; plot.add_trace(kpi_indicator(tile, [lo, hi], [label_band, 1.0])); - let label_y = if tile.gauge.is_some() { - label_band - } else { - number_label_y - }; kpi_labels.push(kpi_label_annotation( &tile.label, f64::midpoint(lo, hi), From 7c478773e4039e894bb4f1577edccf654ec6567f Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:09:53 -0400 Subject: [PATCH 08/12] docs(viz,describegpt): document the KPI overview row & gauge_range/target hints The KPI overview row (Completeness gauge + headline measure tiles), and the x-qsv.gauge_range / x-qsv.target dictionary hints that turn a measure tile into a gauge or a "vs target" delta, were undocumented. Add them to the viz `smart` Overview-panels list and the --dictionary help, and note gauge_range emission under describegpt --infer-content-type (clarifying target is hand-authored, not inferred). Regenerated docs/help/{viz,describegpt}.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/help/describegpt.md | 2 +- docs/help/viz.md | 10 +++++++++- src/cmd/describegpt.rs | 7 +++++++ src/cmd/viz.rs | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/help/describegpt.md b/docs/help/describegpt.md index 4e388bee89..fd5cbc16b9 100644 --- a/docs/help/describegpt.md +++ b/docs/help/describegpt.md @@ -226,7 +226,7 @@ qsv describegpt --help |--------|------|-------------|--------| |  `‑‑num‑examples`  | integer | The number of Example values to include in the dictionary. | `5` | |  `‑‑truncate‑str`  | integer | The maximum length of an Example value in the dictionary. An ellipsis is appended to the truncated value. If zero, no truncation is performed. | `25` | -|  `‑‑infer‑content‑type`  | flag | Also have the LLM classify each field's semantic "Content Type", mapped to a curated, documented vocabulary (e.g. email, city, category, name, credit card, etc.) see . Adds a "Content Type" column/field to the Data Dictionary output. Fields where cardinality equals the row count (i.e. every row has a distinct non-null value - primary keys, surrogate keys, sequence numbers) are deterministically classified as "unique_id", overriding any token the LLM returned for that field. For Date/DateTime fields, the LLM also infers the column's strftime date format (e.g. "date:%m/%d/%Y"); the Markdown, JSON & JSON Schema dictionaries then render Min/Max AND Examples in that inferred format so they match how the dates actually appear in the data, instead of qsv's normalized form. (TSV output keeps Min/Max & Examples in qsv's raw normalized form.) | | +|  `‑‑infer‑content‑type`  | flag | Also have the LLM classify each field's semantic "Content Type", mapped to a curated, documented vocabulary (e.g. email, city, category, name, credit card, etc.) see . Adds a "Content Type" column/field to the Data Dictionary output. Fields where cardinality equals the row count (i.e. every row has a distinct non-null value - primary keys, surrogate keys, sequence numbers) are deterministically classified as "unique_id", overriding any token the LLM returned for that field. For Date/DateTime fields, the LLM also infers the column's strftime date format (e.g. "date:%m/%d/%Y"); the Markdown, JSON & JSON Schema dictionaries then render Min/Max AND Examples in that inferred format so they match how the dates actually appear in the data, instead of qsv's normalized form. (TSV output keeps Min/Max & Examples in qsv's raw normalized form.) For a CONTINUOUS numeric measure on a canonical scale (a percentage, a 0-1 ratio/probability, a bounded index), the LLM also proposes an "x-qsv.gauge_range" [min, max]; qsv keeps it only when the field is a numeric measure AND the observed data lies within it, so that a "viz smart" dictionary-driven dashboard draws that KPI tile as a GAUGE. (A KPI "vs target" delta uses "x-qsv.target", which is a GOAL you hand-author - never inferred.) | | |  `‑‑infer‑null‑values`  | flag | Also have the LLM propose each field's null sentinels - literal values that stand in for "missing" (e.g. NULL, N/A, -999, 9999-12-31). Emitted into the JSON Schema dictionary's per-property "x-qsv" object, split into two lists that carry different warranties.
  • "null_values" - proposed by the LLM AND independently confirmed by qsv to occur as a literal value in that String column. Every observed casing is listed.
  • "null_candidates" - proposed by the LLM but NOT confirmable by any scan; numeric/date placeholders that parse as valid values of the column's own type (-999 is a legal integer), or tokens never seen in the data. Each carries "confirm_required": true.
A proposal that is IMPOSSIBLE is dropped rather than reported: a non-numeric token in a numeric column (no cell could hold it, or the column would not have typed as numeric), and an echo of the null placeholder the Frequency Distribution displays for cells qsv already counts as empty. A confirmed "null_values" entry means only that the literal is PRESENT in the column; the judgment that it MEANS "missing" remains the LLM's. This complements "qsv denull", which reports a narrower set - only the columns that would promote to a numeric type once their sentinels are blanked. A purely categorical column (status = ok/pending/NULL) is outside denull's remit. Sentinels are REPORTED, never applied - qsv does not modify your data on an LLM's say-so. Only "qsv denull --apply" masks values, and only in the columns it independently confirmed. Only rendered by "--format JSONSchema"; the other formats ignore them. | | |  `‑‑two‑pass`  | flag | Run a second LLM call that takes the full first-pass Data Dictionary as JSON context and refines each field's Label, Description and (when --infer-content-type is set) Content Type using cross-field awareness. The LLM can then relate fields that belong together (e.g. street_no + street_name + city + state + zip describing a single mailing address; first_name + last_name naming a single person; lat + lng forming a coordinate pair). The refined dictionary becomes the emitted output and is also what downstream Description, Tags and Prompt inference phases see as dictionary context. Roughly doubles dictionary LLM cost and latency, so opt-in. Most useful when combined with --infer-content-type. Allowed with the --dictionary, --all and --prompt inference flags. Mutually exclusive with --prepare-context and --process-response (MCP sampling is single-turn per inference phase). | | |  `‑‑addl‑cols`  | flag | Add additional columns to the dictionary from the Summary Statistics. | | diff --git a/docs/help/viz.md b/docs/help/viz.md index 0400f25f24..12cf8f544c 100644 --- a/docs/help/viz.md +++ b/docs/help/viz.md @@ -84,6 +84,13 @@ points overlaid by a size heuristic, see --box-points) - ID-like (near-unique) and all-empty columns are skipped Overview panels (each leads the dashboard on its own full-width row): +- KPI overview row (leads every dashboard): a strip of "big number" tiles - a +Completeness gauge (share of non-empty cells across the dataset, a built-in +0-100% scale needing no hint) followed by the headline numeric measures +(summed for extensive quantities, averaged for intensive ones). A measure +tile becomes a GAUGE when the dictionary supplies a validated +`x-qsv.gauge_range` that contains the value, and gains a "vs target" DELTA +when it supplies an `x-qsv.target` (see --dictionary). Omitted for image exports. - correlation heatmap, when 2+ continuous numeric columns exist (one extra data pass for Pearson correlations). If the strongest pair is at least moderately correlated, a drill-down is added beside it: a scatter (or a 2D @@ -397,7 +404,8 @@ qsv viz --help |  `‑‑no‑other`  | flag | Omit the "Other (N)" aggregate bar from frequency bar charts. It collects the categories beyond --limit (N = how many distinct categories were rolled up) and is shown by default. | | |  `‑‑smarter`  | flag | Before building the dashboard, run `qsv moarstats --advanced` to enrich the stats cache with distribution-shape statistics (bimodality, entropy, skewness, outlier share). This unlocks histograms for bimodal columns, frequency bars for concentrated high-cardinality columns, and skew/outlier hints on box panels. Costs one extra pass over the data and writes .stats.csv, its sidecars, and an .idx index (like running `qsv moarstats` manually). On geocode-enabled builds it also enriches map point hovers with the US FIPS code and annotates the spatial-extent summary with the country's continent; the county is always shown in map hovers, with or without --smarter. Only affects `smart`. Applied only with default parsing; inputs using --no-headers or a custom --delimiter fall back to the standard dashboard. | | |  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst, icicle. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure); icicle is an opt-in level-aligned alternative. Only affects `smart`. | | -|  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. Only affects `smart`. | | +|  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. The dictionary also drives the KPI overview row via two optional per-field hints in a property's "x-qsv" object (edit them in the saved schema to fine-tune):
  • "gauge_range": [min, max] on a continuous numeric measure renders its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent). Ignored unless the observed data lies within the range, so a mis-scaled range can't draw a misleading dial. `--dictionary infer` emits this for canonical-scale measures.
  • "target": on a measure renders a "vs target" DELTA against that goal (value minus target). This is a GOAL you supply, never a fabricated prior-period baseline, so `--dictionary infer` never emits it - hand-author it in the schema.
Only affects `smart`. | | +|  `‑,`
`‑`  | flag | its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent). Ignored unless the observed data lies within the range, so a mis-scaled range can't draw a misleading dial. `--dictionary infer` emits this for canonical-scale measures.
  • "target": on a measure renders a "vs target" DELTA against that goal (value minus target). This is a GOAL you supply, never a fabricated prior-period baseline, so `--dictionary infer` never emits it - hand-author it in the schema.
Only affects `smart`. | | |  `‑‑dictionary‑context`  | string | Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to describegpt as --context-file when `--dictionary infer` generates the dictionary. Better context yields better role/concept/label/grain tags, hence a better dashboard. Ignored unless `--dictionary infer` is used (it does not apply when reading an existing dictionary file). Only affects `smart`. | | |  `‑‑dict‑info`  | flag | When a usable Data Dictionary is available (per --dictionary), add a "Data Dictionary" link beneath the dashboard title and an info icon on each panel title: hovering shows that column's dictionary description; clicking opens a human-friendly rendering of the dictionary in a side drawer NEXT TO the plots (embedded in the dashboard file - no extra file is written), scrolled to and highlighting that column's entry. The drawer is open by default on load and can be dismissed with its close button or Esc. The dictionary page carries a role-tinted table of contents and per-column "View chart" links back to the panels. The drawer's popout button opens the same document in its own browser tab instead (needs a browser that allows user-initiated pop-ups). HTML output only; ignored with a note when no dictionary is available or when exporting an image. Only affects `smart`. | | |  `‑‑dataset‑pid`  | string | A persistent identifier (PID) for the dataset - typically a full URL such as a DOI () or other citable link. When set, a clickable "PID" row is added to the metadata table at the top of the dashboard, using the value verbatim as the link target. HTML output only. Only affects `smart`. | | diff --git a/src/cmd/describegpt.rs b/src/cmd/describegpt.rs index 26b2f2e806..2b9878c327 100644 --- a/src/cmd/describegpt.rs +++ b/src/cmd/describegpt.rs @@ -171,6 +171,13 @@ describegpt options: then render Min/Max AND Examples in that inferred format so they match how the dates actually appear in the data, instead of qsv's normalized form. (TSV output keeps Min/Max & Examples in qsv's raw normalized form.) + For a CONTINUOUS numeric measure on a canonical scale (a percentage, a + 0-1 ratio/probability, a bounded index), the LLM also proposes an + "x-qsv.gauge_range" [min, max]; qsv keeps it only when the field is a + numeric measure AND the observed data lies within it, so that a + "viz smart" dictionary-driven dashboard draws that KPI tile as a GAUGE. + (A KPI "vs target" delta uses "x-qsv.target", which is a GOAL you + hand-author - never inferred.) --infer-null-values Also have the LLM propose each field's null sentinels - literal values that stand in for "missing" (e.g. NULL, N/A, -999, 9999-12-31). Emitted into the JSON Schema dictionary's per-property "x-qsv" object, diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 0fbca99407..f052825af0 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -69,6 +69,13 @@ auto-picks panels, so no --x/--y is needed: - ID-like (near-unique) and all-empty columns are skipped Overview panels (each leads the dashboard on its own full-width row): + - KPI overview row (leads every dashboard): a strip of "big number" tiles - a + Completeness gauge (share of non-empty cells across the dataset, a built-in + 0-100% scale needing no hint) followed by the headline numeric measures + (summed for extensive quantities, averaged for intensive ones). A measure + tile becomes a GAUGE when the dictionary supplies a validated + `x-qsv.gauge_range` that contains the value, and gains a "vs target" DELTA + when it supplies an `x-qsv.target` (see --dictionary). Omitted for image exports. - correlation heatmap, when 2+ continuous numeric columns exist (one extra data pass for Pearson correlations). If the strongest pair is at least moderately correlated, a drill-down is added beside it: a scatter (or a 2D @@ -459,6 +466,18 @@ smart options: file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. + The dictionary also drives the KPI overview row via two optional + per-field hints in a property's "x-qsv" object (edit them in the saved + schema to fine-tune): + - "gauge_range": [min, max] on a continuous numeric measure renders + its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a + ratio, [0,100] for a percent). Ignored unless the observed data lies + within the range, so a mis-scaled range can't draw a misleading dial. + `--dictionary infer` emits this for canonical-scale measures. + - "target": on a measure renders a "vs target" DELTA against + that goal (value minus target). This is a GOAL you supply, never a + fabricated prior-period baseline, so `--dictionary infer` never emits + it - hand-author it in the schema. Only affects `smart`. --dictionary-context Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to From 409087fb787778582e52fbe6b4b51e9648dbf9f0 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:16:33 -0400 Subject: [PATCH 09/12] docs(viz): reword KPI hint bullets as prose to fix bogus help-table row (roborev #3613) The nested `- "gauge_range"` / `- "target"` bullets inside the --dictionary Options help began with a dash, so the docopt-based help generator parsed them as an option definition and emitted a spurious `-, -` row in docs/help/viz.md. Reword both hints as flowing prose (no leading dashes) and regenerate the help; the --dictionary row is now a single clean entry that still documents both hints. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/help/viz.md | 3 +-- src/cmd/viz.rs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/help/viz.md b/docs/help/viz.md index 12cf8f544c..5a3d0a8121 100644 --- a/docs/help/viz.md +++ b/docs/help/viz.md @@ -404,8 +404,7 @@ qsv viz --help |  `‑‑no‑other`  | flag | Omit the "Other (N)" aggregate bar from frequency bar charts. It collects the categories beyond --limit (N = how many distinct categories were rolled up) and is shown by default. | | |  `‑‑smarter`  | flag | Before building the dashboard, run `qsv moarstats --advanced` to enrich the stats cache with distribution-shape statistics (bimodality, entropy, skewness, outlier share). This unlocks histograms for bimodal columns, frequency bars for concentrated high-cardinality columns, and skew/outlier hints on box panels. Costs one extra pass over the data and writes .stats.csv, its sidecars, and an .idx index (like running `qsv moarstats` manually). On geocode-enabled builds it also enriches map point hovers with the US FIPS code and annotates the spatial-extent summary with the country's continent; the county is always shown in map hovers, with or without --smarter. Only affects `smart`. Applied only with default parsing; inputs using --no-headers or a custom --delimiter fall back to the standard dashboard. | | |  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst, icicle. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure); icicle is an opt-in level-aligned alternative. Only affects `smart`. | | -|  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. The dictionary also drives the KPI overview row via two optional per-field hints in a property's "x-qsv" object (edit them in the saved schema to fine-tune):
  • "gauge_range": [min, max] on a continuous numeric measure renders its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent). Ignored unless the observed data lies within the range, so a mis-scaled range can't draw a misleading dial. `--dictionary infer` emits this for canonical-scale measures.
  • "target": on a measure renders a "vs target" DELTA against that goal (value minus target). This is a GOAL you supply, never a fabricated prior-period baseline, so `--dictionary infer` never emits it - hand-author it in the schema.
Only affects `smart`. | | -|  `‑,`
`‑`  | flag | its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent). Ignored unless the observed data lies within the range, so a mis-scaled range can't draw a misleading dial. `--dictionary infer` emits this for canonical-scale measures.
  • "target": on a measure renders a "vs target" DELTA against that goal (value minus target). This is a GOAL you supply, never a fabricated prior-period baseline, so `--dictionary infer` never emits it - hand-author it in the schema.
Only affects `smart`. | | +|  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. The dictionary also drives the KPI overview row via two optional per-field hints in a property's "x-qsv" object (edit them in the saved schema to fine-tune). A "gauge_range" of [min, max] on a continuous numeric measure renders its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent); qsv keeps it only when the observed data lies within the range, so a mis-scaled range can't draw a misleading dial, and "infer" emits it for canonical-scale measures. A "target" number on a measure renders a "vs target" DELTA against that goal (value minus target) - a GOAL you supply, never a fabricated prior-period baseline, so "infer" never emits it; hand-author it. Only affects `smart`. | | |  `‑‑dictionary‑context`  | string | Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to describegpt as --context-file when `--dictionary infer` generates the dictionary. Better context yields better role/concept/label/grain tags, hence a better dashboard. Ignored unless `--dictionary infer` is used (it does not apply when reading an existing dictionary file). Only affects `smart`. | | |  `‑‑dict‑info`  | flag | When a usable Data Dictionary is available (per --dictionary), add a "Data Dictionary" link beneath the dashboard title and an info icon on each panel title: hovering shows that column's dictionary description; clicking opens a human-friendly rendering of the dictionary in a side drawer NEXT TO the plots (embedded in the dashboard file - no extra file is written), scrolled to and highlighting that column's entry. The drawer is open by default on load and can be dismissed with its close button or Esc. The dictionary page carries a role-tinted table of contents and per-column "View chart" links back to the panels. The drawer's popout button opens the same document in its own browser tab instead (needs a browser that allows user-initiated pop-ups). HTML output only; ignored with a note when no dictionary is available or when exporting an image. Only affects `smart`. | | |  `‑‑dataset‑pid`  | string | A persistent identifier (PID) for the dataset - typically a full URL such as a DOI () or other citable link. When set, a clickable "PID" row is added to the metadata table at the top of the dashboard, using the value verbatim as the link target. HTML output only. Only affects `smart`. | | diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index f052825af0..c4d0631df3 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -468,16 +468,14 @@ smart options: Generation/read failures soft-fall back to the stats-only dashboard. The dictionary also drives the KPI overview row via two optional per-field hints in a property's "x-qsv" object (edit them in the saved - schema to fine-tune): - - "gauge_range": [min, max] on a continuous numeric measure renders - its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a - ratio, [0,100] for a percent). Ignored unless the observed data lies - within the range, so a mis-scaled range can't draw a misleading dial. - `--dictionary infer` emits this for canonical-scale measures. - - "target": on a measure renders a "vs target" DELTA against - that goal (value minus target). This is a GOAL you supply, never a - fabricated prior-period baseline, so `--dictionary infer` never emits - it - hand-author it in the schema. + schema to fine-tune). A "gauge_range" of [min, max] on a continuous + numeric measure renders its KPI tile as a GAUGE on that canonical scale + (e.g. [0,1] for a ratio, [0,100] for a percent); qsv keeps it only when + the observed data lies within the range, so a mis-scaled range can't draw + a misleading dial, and "infer" emits it for canonical-scale measures. + A "target" number on a measure renders a "vs target" DELTA against that + goal (value minus target) - a GOAL you supply, never a fabricated + prior-period baseline, so "infer" never emits it; hand-author it. Only affects `smart`. --dictionary-context Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to From b27cace60b33494575566792d0ed8972a90482e1 Mon Sep 17 00:00:00 2001 From: Joel Natividad <1980690+jqnatividad@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:36:35 -0400 Subject: [PATCH 10/12] feat(viz): move Completeness to the header; add KPI gauge/delta gallery example Completeness was a permanent KPI gauge tile that read ~100% for most datasets and crowded the overview row. Move it to a quiet "Completeness:" line in the dashboard header table (between Columns: and Compiled:), and drop it from the KPI row - which now leads purely with the headline MEASURES and returns None when a dataset has no measure to headline (e.g. a map-only dataset, so its map reclaims panel-0). build_kpi_row no longer needs the row count. Add a deterministic gallery example (examples/viz/sales_kpi_dict.schema.json + gen_gallery.py entry -> smart_sales_kpi.html) that showcases both dictionary KPI hints on sales_sample: x-qsv.gauge_range [0,1] draws Discount % / Profit Margin % as GAUGES, and x-qsv.target 0.25 on Profit Margin adds a "vs target" DELTA (mean 0.21 -> red -0.042). Hand-authored, so it regenerates without a live LLM. Regenerated the gallery (39 figures) and the affected smart dashboards (their completeness now shows in the header). Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/viz/gallery.html | 62 +-- examples/viz/gen_gallery.py | 12 + examples/viz/smart_allegheny_dogs.html | 45 +- examples/viz/smart_geo_outliers.html | 5 +- examples/viz/smart_nyc311.html | 163 ++++---- examples/viz/smart_sales.html | 5 +- examples/viz/smart_sales_kpi.html | 544 +++++++++++++++++++++++++ examples/viz/smart_smarter.html | 5 +- examples/viz/smart_timeseries.html | 5 +- examples/viz/smart_us_choropleth.html | 5 +- src/cmd/viz.rs | 54 +-- tests/test_viz.rs | 25 +- 12 files changed, 741 insertions(+), 189 deletions(-) create mode 100644 examples/viz/smart_sales_kpi.html diff --git a/examples/viz/gallery.html b/examples/viz/gallery.html index 3e78ece159..0a10374f3c 100644 --- a/examples/viz/gallery.html +++ b/examples/viz/gallery.html @@ -32,63 +32,65 @@

qsv viz — chart gallery

Every chart type produced by qsv viz, from the sample datasets in examples/viz/. Generated with the viz feature; each chart is fully interactive.

-
Jump to a chart38 charts + data dictionary
smart (--smarter, geospatial) dashboardsmart (geographic outliers) dashboardsmart dashboardsmart (--smarter) dashboardbarlinescatterscatter (bubble)scatter3dhistogramboxbox (grouped)violinpie (donut)heatmap (correlation)scatter (correlated pair)contourheatmap (pivot)candlestickohlcsankeyradartreemapsunbursticiclemapmap (density)geochoroplethchoropleth (US states)choropleth (MapLibre + GeoJSON)smart (time-series) dashboardsmart (per-US-state choropleth) dashboardsmart (--dictionary infer, treemap) dashboardsmart (--dictionary infer, sunburst) dashboardsmart (--dictionary infer, world choropleth) dashboardsmart (--smarter, committed --dictionary, NYC 311 metro choropleth) dashboardsmart (--smarter, curated --dictionary, region-code zip choropleth) dashboardPittsburgh 311 smart visual data dictionary
+
Jump to a chart39 charts + data dictionary
smart (--smarter, geospatial) dashboardsmart (geographic outliers) dashboardsmart dashboardsmart (KPI gauges & target delta) dashboardsmart (--smarter) dashboardbarlinescatterscatter (bubble)scatter3dhistogramboxbox (grouped)violinpie (donut)heatmap (correlation)scatter (correlated pair)contourheatmap (pivot)candlestickohlcsankeyradartreemapsunbursticiclemapmap (density)geochoroplethchoropleth (US states)choropleth (MapLibre + GeoJSON)smart (time-series) dashboardsmart (per-US-state choropleth) dashboardsmart (--dictionary infer, treemap) dashboardsmart (--dictionary infer, sunburst) dashboardsmart (--dictionary infer, world choropleth) dashboardsmart (--smarter, committed --dictionary, NYC 311 metro choropleth) dashboardsmart (--smarter, curated --dictionary, region-code zip choropleth) dashboardPittsburgh 311 smart visual data dictionary
smart dashboard (--smarter, geospatial)One command, 13 auto-chosen panels — nearly every panel type at once on a synthetic catalog of Japanese earthquakes. Things the raw table hides but the dashboard makes obvious: depth_km is bimodal (two populations — shallow interplate quakes ~20 km and the deep Wadati-Benioff slab ~450 km — so --smarter draws a histogram, not a box that would average the peaks away); the points trace Japan's subduction arcs on the map; and a prefecture choropleth bins each quake into the GeoJSON region that contains it (point-in-polygon, no geocoding). Most of this catalog is offshore Pacific seismicity, so under the default 10 km snap cap the far-offshore quakes are dropped (287 of 417 here — the panel title reports it) and the on-land/near-coast prefectures are colored; raise --snap-max-dist to snap distant quakes to the nearest prefecture instead, or --no-snap to drop every offshore point. magnitude vs felt_reports is almost perfectly correlated (r=0.95); magnitude and felt_reports are right-skewed with flagged outliers; and the magnitude-over-time trend spikes during a September aftershock sequence. --bivariate adds an NMI association heatmap spanning every column type, not just the continuous-numeric ones the Pearson heatmap covers — it surfaces depth_km and felt_reports as strongly associated with occurrence_date (NMI=0.93 and 0.89), a temporal-clustering signal (the same September aftershock sequence) a Pearson matrix restricted to numeric pairs alone cannot express against a date column. Coordinate columns are shown on the map only, not re-charted as distributions. Rendered with the built-in plotly_dark theme.
qsv viz smart seismic_events.csv --smarter --bivariate --theme plotly_dark \
   --grid-cols 3 --geojson japan_prefectures.geojson -o smart_dashboard_smarter_geospatial.html
smart dashboard (geographic outliers)Delivery stops clustered in metro Denver with four bad-geocode strays. Points far from the cluster centroid (beyond the Tukey far-out fence of their distances) are flagged as geographic outliers: drawn as distinct amber markers, drawn outside the purple (filled) spatial-extent box, and excluded from the auto-zoom — so the default view stays tight on the core cluster. A second, dashed-magenta no-fill box marks the full extent (core + outliers); use the Core extent / Full extent buttons at the top-left of the map to jump between the tight core view and the full spread (where the strays and the magenta box become visible). In the full qsv viz smart HTML output the spatial-extent label calls them out — Colorado, United States — 4 outliers (Wyoming, Kansas & Nebraska) — while strays within the core's own jurisdiction are folded back in silently instead. Each stop also carries delivery attributes (packages, weight_kg, distance_km, delivery_minutes, a vehicle class and a delivered_date), so beyond the map the auto-profiler fills the dashboard out with box plots, frequency bars, a correlation heatmap, the strongest-pair scatter (packages vs weight_kg) and a delivered-over-time trend — all without --smarter.
qsv viz smart delivery_stops.csv -o smart_dashboard_geographic_outliers.html
smart dashboardAuto-profiled overview: correlation heatmap + box plots + frequency bars, led by a drill-down sunburst. `viz smart` now SKIPS an auto hierarchy when the candidate dimensions are statistically independent (nesting them would just replicate each level's marginal); sales_sample's region/payment_method/product_category are independent, so `--hierarchy-style sunburst` is passed to deliberately showcase the interactive sunburst.
qsv viz smart sales_sample.csv --hierarchy-style sunburst --max-charts 8 \
   -o smart_dashboard.html
+
smart dashboard (KPI gauges & target delta)The KPI overview row driven by a hand-authored `--dictionary` (sales_kpi_dict.schema.json). Two optional `x-qsv` hints turn plain measure tiles into richer KPIs: a `gauge_range` of [0,1] draws Discount % and Profit Margin % as GAUGES on their canonical ratio scale, and a `target` of 0.25 on Profit Margin adds a "vs target" DELTA (the mean is 0.21, so a red -0.042 below the goal). qsv keeps a gauge only when the data lies within its range, so a mis-scaled range can't mislead; `gauge_range` is what `--dictionary infer` emits for canonical-scale ratios, while `target` is a business goal you hand-author (never LLM-inferred). Overall dataset completeness rides quietly in the header table, not as a tile.
qsv viz smart sales_sample.csv --dictionary sales_kpi_dict.schema.json \
+  -o smart_dashboard_kpi_gauges_target_delta.html
smart dashboard (--smarter)Same auto-profiler with `--smarter`, which runs `qsv moarstats --advanced` itself to enrich the stats cache in one step: the bimodal monthly_spend column renders as a histogram (a box plot would hide its two peaks), and the skewed account_age_days box is annotated with its skew direction and outlier share.
qsv viz smart customer_spend.csv --smarter --max-charts 8 -o smart_dashboard_smarter.html
barRevenue by region (aggregated sum).
qsv viz bar sales_sample.csv --x region --y revenue --agg sum \
-  -o bar.html
-
lineClosing price over time.
qsv viz line stock_prices.csv --x date --y close -o line.html
-
scatterUnits sold vs revenue.
qsv viz scatter sales_sample.csv --x units_sold --y revenue -o scatter.html
+ -o bar.html
+
lineClosing price over time.
qsv viz line stock_prices.csv --x date --y close -o line.html
+
scatterUnits sold vs revenue.
qsv viz scatter sales_sample.csv --x units_sold --y revenue -o scatter.html
scatter (bubble)Units vs revenue; marker size = shipping cost, color = profit margin %.
qsv viz scatter sales_sample.csv --x units_sold --y revenue --size shipping_cost \
-  --color profit_margin_pct -o scatter_bubble.html
+ --color profit_margin_pct -o scatter_bubble.html
scatter3dUnits vs revenue vs shipping cost in 3D; marker color = profit margin %.
qsv viz scatter3d sales_sample.csv --x units_sold --y revenue \
-  --z shipping_cost --color profit_margin_pct -o scatter3d.html
-
histogramDistribution of unit price.
qsv viz histogram sales_sample.csv --x unit_price -o histogram.html
-
boxSpread of revenue (Tukey whiskers; points beyond the fences shown as outliers).
qsv viz box sales_sample.csv --y revenue -o box.html
+ --z shipping_cost --color profit_margin_pct -o scatter3d.html
+
histogramDistribution of unit price.
qsv viz histogram sales_sample.csv --x unit_price -o histogram.html
+
boxSpread of revenue (Tukey whiskers; points beyond the fences shown as outliers).
qsv viz box sales_sample.csv --y revenue -o box.html
box (grouped)Revenue spread per region — real Tukey whiskers + every (jittered) point overlaid (--box-points all).
qsv viz box sales_sample.csv --y revenue --x region --box-points all \
-  -o box_grouped.html
-
violinRevenue distribution per region — a KDE density silhouette around an inner quartile box + mean line, revealing shape (modes, shoulders) a box hides. viz smart auto-picks this for columns in the bimodality ambiguity band (--violin auto).
qsv viz violin sales_sample.csv --y revenue --x region -o violin.html
+ -o box_grouped.html
+
violinRevenue distribution per region — a KDE density silhouette around an inner quartile box + mean line, revealing shape (modes, shoulders) a box hides. viz smart auto-picks this for columns in the bimodality ambiguity band (--violin auto).
qsv viz violin sales_sample.csv --y revenue --x region -o violin.html
pie (donut)Revenue share by product category.
qsv viz pie sales_sample.csv --x product_category --y revenue \
-  --donut -o pie_donut.html
-
heatmap (correlation)Pearson correlation matrix over numeric columns.
qsv viz heatmap sales_sample.csv -o heatmap_correlation.html
+ --donut -o pie_donut.html
+
heatmap (correlation)Pearson correlation matrix over numeric columns.
qsv viz heatmap sales_sample.csv -o heatmap_correlation.html
scatter (correlated pair)The most strongly correlated numeric pair (discount_pct vs profit_margin_pct, r=-0.99). viz smart auto-adds this as a drill-down beside the correlation heatmap.
qsv viz scatter sales_sample.csv --x discount_pct --y profit_margin_pct \
-  -o scatter_correlated_pair.html
+ -o scatter_correlated_pair.html
contour2D density of units sold vs revenue (binned into a 20x20 grid). viz smart uses this instead of the pair scatter for large datasets, where a scatter would overplot.
qsv viz contour sales_sample.csv --x units_sold --y revenue --bins 20 \
-  -o contour.html
+ -o contour.html
heatmap (pivot)Region x category grid of revenue.
qsv viz heatmap sales_sample.csv --x region --y product_category \
-  --z revenue -o heatmap_pivot.html
+ --z revenue -o heatmap_pivot.html
candlestickOHLC price action.
qsv viz candlestick stock_prices.csv --x date --ohlc-open open \
-  --high high --low low --close close -o candlestick.html
+ --high high --low low --close close -o candlestick.html
ohlcOpen-high-low-close bars.
qsv viz ohlc stock_prices.csv --x date --ohlc-open open --high high \
-  --low low --close close -o ohlc.html
+ --low low --close close -o ohlc.html
sankeyWeb session funnel (duplicate edges aggregated).
qsv viz sankey web_flows.csv --source source --target target \
-  --value sessions -o sankey.html
+ --value sessions -o sankey.html
radarMulti-axis brand comparison (per-axis mean per series).
qsv viz radar product_ratings.csv --cols battery,camera,performance,display,value,design \
-  --series brand -o radar.html
+ --series brand -o radar.html
treemapPart-to-whole spend by plan then region, sized by summed monthly_spend. Rounded tiles + white separators come from the treemap-specific marker; non-numeric/negative measure cells are rejected so proportions can't silently misstate.
qsv viz treemap customer_spend.csv --cols plan,region --value monthly_spend \
-  --agg sum -o treemap.html
+ --agg sum -o treemap.html
sunburstThree-level hierarchy (region -> product_category -> payment_method) as concentric rings, sized by row count; inner rings are parents, outer rings their children. Opens at two rings (maxdepth) so labels stay legible instead of crowding a ~100-sector outer ring; click a sector to drill in and the deeper ring's labels grow back. Hover always shows value + percent.
qsv viz sunburst sales_sample.csv --cols region,product_category,payment_method \
-  -o sunburst.html
+ -o sunburst.html
icicleSame three-level hierarchy (region -> product_category -> payment_method) as a rectangular icicle: parents on the left, children fanning right, each rectangle sized by row count. The flat left-to-right layout keeps deep labels readable where a sunburst's outer ring would crowd; click a rectangle to zoom into that branch. Hover shows label + value + percent of parent.
qsv viz icicle sales_sample.csv --cols region,product_category,payment_method \
-  -o icicle.html
+ -o icicle.html
mapEarthquake points on token-free OpenStreetMap tiles; marker color = magnitude, size = depth.
qsv viz map quakes.csv --lat lat --lon lon --color magnitude \
-  --size depth_km -o map.html
+ --size depth_km -o map.html
map (density)DensityMap heatmap of the same points on a light Carto basemap.
qsv viz map quakes.csv --lat lat --lon lon --density --style carto-positron \
-  -o map_density.html
+ -o map_density.html
geoSame earthquakes on an offline natural-earth projection (no tiles, no token); marker color = magnitude. viz smart auto-uses this projection for global-extent coordinates.
qsv viz geo quakes.csv --lat lat --lon lon --color magnitude \
-  --projection natural-earth -o geo.html
+ --projection natural-earth -o geo.html
choroplethFilled-region map coloring countries by GDP, matched by ISO-3 code on a token-free projection basemap. Use --location-mode usa-states / country-names / geojson-id for other region keys, --map for a MapLibre tile basemap, or --geocode to derive codes from lat/lon or place names.
qsv viz choropleth country_stats.csv --locations iso3 --value gdp_usd_tn \
-  --color-scale viridis -o choropleth.html
+ --color-scale viridis -o choropleth.html
choropleth (US states)Same chart, --location-mode usa-states: state codes matched to Plotly's built-in US-state geometry on the token-free albers-usa projection (CONUS + Alaska/Hawaii insets) — no GeoJSON needed. States are colored by renewable-electricity share.
qsv viz choropleth us_state_stats.csv --locations state --value renewable_electricity_pct \
-  --location-mode usa-states -o choropleth_us_states.html
+ --location-mode usa-states -o choropleth_us_states.html
choropleth (MapLibre + GeoJSON)--map draws the filled regions on an interactive MapLibre tile basemap (token-free carto-positron) instead of a projection. The regions come from a custom GeoJSON (--geojson local file or URL) matched to the data by --feature-id-key — here the near-rectangular western states, colored by installed wind capacity. The view auto-centers and zooms to the GeoJSON extent (shown full-width so the computed zoom frames the regions as the CLI does — a tile map's zoom is fixed, so a narrow grid cell would crop it).
qsv viz choropleth western_states.csv --locations state --value wind_capacity_gw \
   --geojson western_states.geojson --feature-id-key id --map --style carto-positron \
-  -o choropleth_maplibre_geojson.html
+ -o choropleth_maplibre_geojson.html
smart dashboard (time-series)Auto dashboard for stock_prices: a time-series trend panel (the first numeric column over the date) leads, alongside box-plot summaries of the OHLC columns.
qsv viz smart stock_prices.csv --max-charts 8 -o smart_dashboard_time_series.html
smart dashboard (per-US-state choropleth)`viz smart` reverse-geocodes each point; because every city resolves to a US state, it adds a per-US-state choropleth (cities-per-state, albers-usa) beside the point map, alongside the usual box plots, frequency bars and the strongest-pair scatter. (The point map's spatial extent caption counts the data's bounding-box corners, which spill into neighboring countries and ocean — the choropleth instead resolves each city to its own state.) No flags, no LLM — the state fill is derived purely from the lat/lon columns.
qsv viz smart us_cities.csv -o smart_dashboard_per_us_state_choropleth.html
smart dashboard (--dictionary infer, treemap)Auto dashboard for customer_spend with a describegpt-inferred Data Dictionary (--dictionary infer) guiding panel selection & field labels. Two categorical dimensions (plan, region) form a shallow part-to-whole hierarchy, auto-rendered as a TREEMAP (area = size). Requires a local LLM; the committed HTML is reused on regen.
qsv viz smart customer_spend.csv --dictionary infer -o smart_dashboard_dictionary_infer_treemap.html
@@ -105,9 +107,8 @@

qsv viz — chart gallery

--geojson pittsburgh-neighborhoods --dataset-pid https://data.wprdc.org/dataset/pittsburgh-311-dataPittsburgh 311 smart visual data dictionary screenshot diff --git a/examples/viz/gen_gallery.py b/examples/viz/gen_gallery.py index 8561292a08..8b94b5ee5b 100755 --- a/examples/viz/gen_gallery.py +++ b/examples/viz/gen_gallery.py @@ -44,6 +44,7 @@ # rather than reconstructed as a lossy uniform sub-grid. Keyed by figure title -> iframe filename. SMART_IFRAME = { "smart dashboard": "smart_sales.html", + "smart dashboard (KPI gauges & target delta)": "smart_sales_kpi.html", "smart dashboard (--smarter)": "smart_smarter.html", "smart dashboard (--smarter, geospatial)": "smart_geospatial.html", "smart dashboard (geographic outliers)": "smart_geo_outliers.html", @@ -437,6 +438,17 @@ "sales_sample's region/payment_method/product_category are independent, so " "`--hierarchy-style sunburst` is passed to deliberately showcase the interactive sunburst.", True, ["smart", "sales_sample.csv", "--hierarchy-style", "sunburst", "--max-charts", "8"]), + ("smart dashboard (KPI gauges & target delta)", + "The KPI overview row driven by a hand-authored `--dictionary` " + "(sales_kpi_dict.schema.json). Two optional `x-qsv` hints turn plain measure tiles into " + "richer KPIs: a `gauge_range` of [0,1] draws Discount % and Profit Margin % as GAUGES on " + "their canonical ratio scale, and a `target` of 0.25 on Profit Margin adds a \"vs target\" " + "DELTA (the mean is 0.21, so a red -0.042 below the goal). qsv keeps a gauge only when the " + "data lies within its range, so a mis-scaled range can't mislead; `gauge_range` is what " + "`--dictionary infer` emits for canonical-scale ratios, while `target` is a business goal you " + "hand-author (never LLM-inferred). Overall dataset completeness rides quietly in the header " + "table, not as a tile.", + True, ["smart", "sales_sample.csv", "--dictionary", "sales_kpi_dict.schema.json"]), ("smart dashboard (--smarter)", "Same auto-profiler with `--smarter`, which runs `qsv moarstats --advanced` itself to enrich " "the stats cache in one step: the bimodal monthly_spend column renders as a histogram (a box " diff --git a/examples/viz/smart_allegheny_dogs.html b/examples/viz/smart_allegheny_dogs.html index 50d00319df..ff875b8947 100644 --- a/examples/viz/smart_allegheny_dogs.html +++ b/examples/viz/smart_allegheny_dogs.html @@ -48,22 +48,23 @@

allegheny_dog_licenses.csv — data overview

Description:This dog‑license dataset contains 50 013 distinct records, each identified by a unique integer key. The majority of licenses are lifetime and spayed or neutered, with the most common type being *Dog Lifetime Spayed Female* (≈33 % of records). The breed field is highly heterogeneous, but “Other” accounts for more than half the entries, indicating many unclassified or mixed breeds. Coat color is similarly diverse; “Other” colors comprise a third of the records, while common single‑tone or simple pattern colors such as black, brown, and white dominate the remainder. Dog names are largely unique – over 90 % of names fall into an “Other” bucket that aggregates all uncommon names, with a handful of popular choices like *Bella*, *Lucy*, and *Daisy*. All licenses expire in the year 2099, while their effective dates span from early 2003 to mid‑2026, revealing a long‑term dataset with an uneven temporal distribution. Owner ZIP codes cluster in the 150 00–479 09 range, but a majority of records are labeled “Other”, reflecting many unique or unrecorded ZIP codes. Rows:50,013 Columns:8 -Compiled:2026-07-11 19:54 UTC +Completeness:100.0% +Compiled:2026-07-11 23:34 UTC
-
+
-
+
@@ -71,63 +72,55 @@

allegheny_dog_licenses.csv — data overview

-
-
-
-
-
-
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/examples/viz/smart_geo_outliers.html b/examples/viz/smart_geo_outliers.html index d1ba4344e9..b7ca3f6735 100644 --- a/examples/viz/smart_geo_outliers.html +++ b/examples/viz/smart_geo_outliers.html @@ -47,14 +47,15 @@

delivery_stops.csv — data overview

- + +
Rows:200
Columns:10
Compiled:2026-07-11 19:54 UTC
Completeness:100.0%
Compiled:2026-07-11 23:34 UTC
diff --git a/examples/viz/smart_nyc311.html b/examples/viz/smart_nyc311.html index 5e31e6784f..6da1e57b98 100644 --- a/examples/viz/smart_nyc311.html +++ b/examples/viz/smart_nyc311.html @@ -48,31 +48,32 @@

nyc_311.csv — data overview

Description:This dataset contains 10 000 unique New York City 311 complaint records collected between January 2010 and December 2020. Each record is identified by a system‑generated integer key with no duplicates. The complaints span a wide range of incident types—noise, heat/hot water, illegal parking and other miscellaneous issues—with the “Other” category comprising more than half of all complaints. The majority of incidents are reported in boroughs such as Brooklyn, Queens and Manhattan, with ZIP codes and street names predominating the location fields. Complaint creation dates cluster around the mid‑2010s, while many closure dates are missing (about 3 % of records). Resolution narratives and additional details appear in free‑text fields, often containing long descriptions or agency‐specific language. Geographic coordinates (latitude/longitude and State Plane X/Y) are available for most records, enabling spatial analysis across the five boroughs. Data entry is highly variable: many “Other” values appear in fields with large cardinality, and several address components are missing. Rows:10,000 Columns:41 -Compiled:2026-07-11 19:54 UTC +Completeness:68.4% +Compiled:2026-07-11 23:34 UTC
-
+
+
Spatial extent: New Jersey & New York, United States — 9 outliers (Pennsylvania) · North America
-
Spatial extent: New Jersey & New York, United States — 9 outliers (Pennsylvania) · North America
-
+
@@ -80,15 +81,15 @@

nyc_311.csv — data overview

-
-
+
+
@@ -96,15 +97,15 @@

nyc_311.csv — data overview

-
+
@@ -112,263 +113,255 @@

nyc_311.csv — data overview

-
-
-
-
-
-
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
diff --git a/examples/viz/smart_sales.html b/examples/viz/smart_sales.html index d29a40f4da..ac353d4479 100644 --- a/examples/viz/smart_sales.html +++ b/examples/viz/smart_sales.html @@ -47,14 +47,15 @@

sales_sample.csv — data overview

- + +
Rows:500
Columns:13
Compiled:2026-07-11 19:54 UTC
Completeness:100.0%
Compiled:2026-07-11 23:34 UTC
diff --git a/examples/viz/smart_sales_kpi.html b/examples/viz/smart_sales_kpi.html new file mode 100644 index 0000000000..9443f3c251 --- /dev/null +++ b/examples/viz/smart_sales_kpi.html @@ -0,0 +1,544 @@ + + + + + +sales_sample.csv — data overview + + + + + + +

sales_sample.csv — data overview

+ + + + + + +
Rows:500
Columns:13
Completeness:100.0%
Compiled:2026-07-11 23:34 UTC
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
+ + + + + + + diff --git a/examples/viz/smart_smarter.html b/examples/viz/smart_smarter.html index 9dbb30386d..92f3cc7a11 100644 --- a/examples/viz/smart_smarter.html +++ b/examples/viz/smart_smarter.html @@ -47,14 +47,15 @@

customer_spend.csv — data overview

- + +
Rows:300
Columns:5
Compiled:2026-07-11 19:54 UTC
Completeness:100.0%
Compiled:2026-07-11 23:34 UTC
diff --git a/examples/viz/smart_timeseries.html b/examples/viz/smart_timeseries.html index cf9ecc4afd..b3e54152cd 100644 --- a/examples/viz/smart_timeseries.html +++ b/examples/viz/smart_timeseries.html @@ -47,14 +47,15 @@

stock_prices.csv — data overview

- + +
Rows:205
Columns:6
Compiled:2026-07-11 19:54 UTC
Completeness:100.0%
Compiled:2026-07-11 23:34 UTC
diff --git a/examples/viz/smart_us_choropleth.html b/examples/viz/smart_us_choropleth.html index 3c3f1bffe6..ccead9fe6c 100644 --- a/examples/viz/smart_us_choropleth.html +++ b/examples/viz/smart_us_choropleth.html @@ -47,14 +47,15 @@

us_cities.csv — data overview

- + +
Rows:54
Columns:6
Compiled:2026-07-11 19:54 UTC
Completeness:100.0%
Compiled:2026-07-11 23:34 UTC
diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index c4d0631df3..664c09b0c3 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -14583,42 +14583,27 @@ fn kpi_label_annotation( .font(font) } -/// Build the leading KPI-tile row for `viz smart`: dataset-summary tiles (record count, field -/// count, a completeness gauge) followed by the headline numeric measures (capped at +/// Build the leading KPI-tile row for `viz smart`: the headline numeric measures (capped at /// `KPI_MAX_TILES`). A measure tile becomes a gauge when its dictionary supplies a `gauge_range` /// that CONTAINS the observed value (else it falls back to a plain number — never a misleading /// gauge), and a "vs target" delta when the dictionary supplies a `target` (a semantically -/// justified goal, never a fabricated prior-period baseline). Returns `None` for empty stats. +/// justified goal, never a fabricated prior-period baseline). Returns `None` when no measure tile +/// results (dataset completeness now lives in the dashboard header, not as a KPI tile, so the row +/// is worth drawing only when there is at least one measure to headline). fn build_kpi_row( - n: u64, stats: &[crate::cmd::stats::StatsData], panels: &[Panel], dict: Option<&DictData>, ) -> Option { - let ncols = stats.len(); - if ncols == 0 { + if stats.is_empty() { return None; } let mut tiles: Vec = Vec::with_capacity(KPI_MAX_TILES); - // Record/field counts are intentionally omitted — they duplicate the "Rows"/"Columns" lines in - // the dashboard's metadata header. The KPI row leads with data-quality and measure signals the - // header doesn't carry. + // Record/field counts and overall completeness are intentionally NOT tiles — record/field + // counts duplicate the "Rows"/"Columns" header lines, and completeness (near 100% for most + // datasets) now rides quietly in the header table below "Columns:". The KPI row leads purely + // with the headline MEASURES. // - // completeness = non-null cells / total cells — an inherently 0–100% scale, so a built-in - // gauge that needs no LLM hint. - if n > 0 { - let total_cells = n as f64 * ncols as f64; - let total_null: f64 = stats.iter().map(|s| s.nullcount as f64).sum(); - let completeness = (1.0 - total_null / total_cells).clamp(0.0, 1.0); - tiles.push(KpiTile { - label: "Completeness".to_string(), - value: completeness, - format: ".1%".to_string(), - gauge: Some([0.0, 1.0]), - target: None, - }); - } - // headline measures: the numeric columns the dashboard already treats as continuous measures // (i.e. those that earned a box/violin/histogram distribution panel), summed (extensive) or // averaged (intensive), up to the remaining tile budget. Keying off the built panels keeps the @@ -14671,6 +14656,11 @@ fn build_kpi_row( }); } + // No measures to headline → no KPI row (completeness alone no longer justifies one). + if tiles.is_empty() { + return None; + } + Some(Panel::new( "Overview".to_string(), PanelKind::KpiRow { tiles }, @@ -16059,6 +16049,19 @@ fn build_smart( "Columns:{}\n", HumanCount(stats.len() as u64) )); + // Completeness: share of non-empty cells across the whole dataset. A quiet header stat + // rather than a KPI gauge — it's near 100% for most datasets, so a permanent gauge tile + // just crowded the overview row. + let ncols = stats.len(); + if n > 0 && ncols > 0 { + let total_cells = n as f64 * ncols as f64; + let total_null: f64 = stats.iter().map(|s| s.nullcount as f64).sum(); + let completeness = (1.0 - total_null / total_cells).clamp(0.0, 1.0); + rows.push_str(&format!( + "Completeness:{:.1}%\n", + completeness * 100.0 + )); + } // Compiled: dashboard build timestamp (makes smart HTML output non-deterministic by // design). rows.push_str(&format!( @@ -16091,8 +16094,7 @@ fn build_smart( // render-path choice) so it stays invisible to them and simply lands at index 0, on top of the // dashboard. HTML only: Indicator tiles are domain-positioned and never enter a static image. if !out_format.is_image() { - let n = *nrows.get_or_insert_with(|| util::count_rows(&count_conf).unwrap_or(0)); - if let Some(panel) = build_kpi_row(n, &stats, &panels, dict_data.as_ref()) { + if let Some(panel) = build_kpi_row(&stats, &panels, dict_data.as_ref()) { panels.insert(0, panel); } } diff --git a/tests/test_viz.rs b/tests/test_viz.rs index 0ade4a8851..fa575cfb51 100644 --- a/tests/test_viz.rs +++ b/tests/test_viz.rs @@ -5793,11 +5793,11 @@ fn viz_smart_compressed_map_figure_payload() { let out = wrk.output(&mut cmd); assert!(out.status.success()); let html = String::from_utf8_lossy(&out.stdout); - // panel-0 is the leading KPI overview row; the map is the next full-width panel - // (qsv-viz-panel-1): payload + deferred render - assert!(html.contains("id=\"qsv-viz-panel-1-fig\"")); - assert!(html.contains("qsvNewPlotGz(\"qsv-viz-panel-1\")")); - let figure = inflate_gz_payload(&html, "qsv-viz-panel-1-fig"); + // this dataset has no headline measure (val is low-cardinality), so there is no KPI overview + // row; the map leads as panel-0: payload + deferred render + assert!(html.contains("id=\"qsv-viz-panel-0-fig\"")); + assert!(html.contains("qsvNewPlotGz(\"qsv-viz-panel-0\")")); + let figure = inflate_gz_payload(&html, "qsv-viz-panel-0-fig"); assert!(figure.contains(r#""type":"scattermap""#)); assert!(figure.contains(r#""cluster":{"enabled":false,"maxzoom":17.0}"#)); // coordinates ride as little-endian float32 typed arrays, not decimal-text JSON @@ -5885,11 +5885,11 @@ fn viz_cdn_keeps_gz_prelude_for_compressed_map_figures() { assert!(html.contains(r#"