diff --git a/src/cli.rs b/src/cli.rs index ed3d5fd..76ed779 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -81,6 +81,54 @@ pub enum Commands { /// used when `--expected-error` is set. #[arg(long, default_value_t = 200)] ee_bins: usize, + + /// Maximum number of distinct quality values for the data to be judged + /// "binned" (NovaSeq/NextSeq collapse Phred to a handful of levels). + /// Continuous Illumina/HiFi data has dozens of distinct values. + #[arg(long, default_value_t = 8)] + binned_threshold: usize, + + /// Also print a human-readable metrics report to stderr (total + /// sequences, quality/EE ranges, whether quality scores are binned and + /// their levels). Stdout stays clean JSON for piping. + #[arg(long)] + report: bool, + }, + + /// Merge per-sample `summary` JSONs into a run-level quality/binning report + /// + /// Unions the observed quality-value distribution across all samples so rare + /// bins missed by any single low-count sample (e.g. a seldom-called Q2) are + /// recovered at the run level. Optionally validates the run against a + /// user-declared bin scheme via `--expected-bins`. + SummaryMerge { + /// Per-sample `summary` JSON files (gzip ok; `-` reads stdin). + inputs: Vec, + + /// Write JSON output to this file instead of stdout. + #[arg(long, short = 'o')] + output: Option, + + /// Output compact (minified) JSON instead of pretty-printed. + #[arg(long)] + compact: bool, + + /// Maximum number of distinct quality values for the run to be judged + /// "binned". Matches the `summary` default. + #[arg(long, default_value_t = 8)] + binned_threshold: usize, + + /// Also print a human-readable run-level report to stderr. Stdout stays + /// clean JSON for piping. + #[arg(long)] + report: bool, + + /// Declared bin levels to validate the run against, e.g. `2,12,24,40`. + /// When set, observed ⊆ expected is "consistent"; any observed value + /// outside the declared set is a violation (a wrong or silently changed + /// scheme). Declared-but-unobserved bins are reported as informational. + #[arg(long, value_delimiter = ',')] + expected_bins: Vec, }, /// Dereplicate sequences from a FASTQ file diff --git a/src/main.rs b/src/main.rs index ebfe1ac..a490161 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,7 +50,7 @@ use remove_bimera::{BimeraParams, Method, remove_bimera_denovo}; use remove_primers::{RemovePrimersParams, iupac_reverse_complement, remove_primers}; use sequence_table::{HashAlgo, OrderBy, SequenceTable, make_sequence_table}; use serde::Serialize; -use summary::{ComplexityConfig, ExpectedErrorConfig, SummaryConfig, process}; +use summary::{ComplexityConfig, ExpectedErrorConfig, SummaryConfig, judge_binned, process}; use taxonomy::{ SpeciesHit, SpeciesOptions, SpeciesRef, TaxonomyOptions, TaxonomyRef, assign_species, assign_taxonomy, @@ -210,6 +210,8 @@ fn main() -> io::Result<()> { complexity_bins, expected_error, ee_bins, + binned_threshold, + report, } => { let pool = rayon::ThreadPoolBuilder::new() .num_threads(threads) @@ -263,6 +265,19 @@ fn main() -> io::Result<()> { max: Vec, } + /// Quality-value distribution metrics: whether the FASTQ's quality + /// scores are binned and, if so, to what levels. Always present. + #[derive(Serialize)] + struct QualityMetricsOutput { + distinct_quality_values: Vec, + quality_value_counts: Vec, + min_quality: u8, + max_quality: u8, + is_binned: bool, + #[serde(skip_serializing_if = "Option::is_none")] + binned_scores: Option>, + } + #[derive(Serialize)] struct SummaryOutput { sample: String, @@ -281,6 +296,8 @@ fn main() -> io::Result<()> { /// `--expected-error` was requested. #[serde(skip_serializing_if = "Option::is_none")] expected_error: Option, + /// Quality-value distribution + binning verdict. Always present. + quality_metrics: QualityMetricsOutput, } let sample = sample_name.unwrap_or_else(|| fastq_stem(&input)); @@ -303,6 +320,44 @@ fn main() -> io::Result<()> { q75: m.q75, max: m.max, }); + let metrics = summary.quality_metrics(binned_threshold); + + if report { + let coverage = summary.reads_per_position(); + let read_len = coverage.len(); + let binned = match &metrics.binned_scores { + Some(levels) => { + let levels: Vec = levels.iter().map(|q| q.to_string()).collect(); + format!("yes ({} levels: {})", levels.len(), levels.join(", ")) + } + None => "no".to_string(), + }; + eprintln!("=== FASTQ quality metrics: {sample} ==="); + eprintln!("total sequences: {}", summary.total_reads); + eprintln!("read length: {read_len} cycles"); + eprintln!( + "quality range: Q{}–Q{} ({} distinct values)", + metrics.min_quality, + metrics.max_quality, + metrics.distinct_quality_values.len() + ); + eprintln!("binned: {binned}"); + if let Some(ee) = &expected_error_out + && let (Some(&min), Some(&max)) = (ee.min.last(), ee.max.last()) + { + eprintln!("expected error: {min:.3}–{max:.3} (cumulative, final position)"); + } + } + + let quality_metrics = QualityMetricsOutput { + distinct_quality_values: metrics.distinct_quality_values, + quality_value_counts: metrics.quality_value_counts, + min_quality: metrics.min_quality, + max_quality: metrics.max_quality, + is_binned: metrics.is_binned, + binned_scores: metrics.binned_scores, + }; + let out = SummaryOutput { sample, total_reads: summary.total_reads, @@ -312,6 +367,7 @@ fn main() -> io::Result<()> { quality_histogram, complexity: complexity_out, expected_error: expected_error_out, + quality_metrics, }; let tagged = Tagged::new("summary", out); @@ -328,6 +384,243 @@ fn main() -> io::Result<()> { } } + Commands::SummaryMerge { + inputs, + output, + compact, + binned_threshold, + report, + expected_bins, + } => { + use std::collections::BTreeMap; + + /// Only the fields we need out of a `summary` JSON. + #[derive(serde::Deserialize)] + struct SummaryJson { + #[serde(default)] + sample: Option, + #[serde(default)] + total_reads: u64, + #[serde(default)] + quality_metrics: Option, + } + #[derive(serde::Deserialize)] + struct QualityMetricsJson { + distinct_quality_values: Vec, + quality_value_counts: Vec, + } + + if inputs.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "summary-merge: no input summary JSON files given", + )); + } + + // Union of Q -> summed base count across all samples. + let mut union: BTreeMap = BTreeMap::new(); + let mut total_reads: u64 = 0; + let mut n_samples: u64 = 0; + // Per-sample distinct sets, to flag samples missing run-level bins. + let mut per_sample: Vec<(String, Vec)> = Vec::new(); + + for path in &inputs { + let parsed: SummaryJson = read_tagged_json(path, &["summary"]).with_path(path)?; + let qm = match parsed.quality_metrics { + Some(qm) => qm, + None => { + eprintln!( + "warning: {} has no quality_metrics block; skipping", + path.display() + ); + continue; + } + }; + if qm.distinct_quality_values.len() != qm.quality_value_counts.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{}: quality_metrics distinct/counts length mismatch", + path.display() + ), + )); + } + let sample = parsed.sample.unwrap_or_else(|| fastq_stem(path)); + for (&q, &c) in qm + .distinct_quality_values + .iter() + .zip(qm.quality_value_counts.iter()) + { + *union.entry(q).or_insert(0) += c; + } + per_sample.push((sample, qm.distinct_quality_values)); + total_reads += parsed.total_reads; + n_samples += 1; + } + + if union.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "summary-merge: no usable quality_metrics found in any input", + )); + } + + let distinct: Vec = union.keys().copied().collect(); + let counts: Vec = union.values().copied().collect(); + let run_set: std::collections::BTreeSet = distinct.iter().copied().collect(); + let min_quality = *distinct.first().unwrap(); + let max_quality = *distinct.last().unwrap(); + let is_binned = judge_binned(distinct.len(), binned_threshold); + let binned_scores = is_binned.then(|| distinct.clone()); + + // Per-sample flags: which run-level bins this sample never emitted. + #[derive(Serialize)] + struct SampleFlag { + sample: String, + distinct_quality_values: Vec, + missing_from_run: Vec, + } + let samples: Vec = per_sample + .into_iter() + .map(|(sample, s_distinct)| { + let s_set: std::collections::BTreeSet = + s_distinct.iter().copied().collect(); + let missing_from_run = run_set.difference(&s_set).copied().collect::>(); + SampleFlag { + sample, + distinct_quality_values: s_distinct, + missing_from_run, + } + }) + .collect(); + + // Optional validation against a declared bin scheme. + #[derive(Serialize)] + struct ExpectedValidation { + consistent: bool, + observed_not_in_expected: Vec, + expected_not_observed: Vec, + } + let expected_validation = (!expected_bins.is_empty()).then(|| { + let exp: std::collections::BTreeSet = expected_bins.iter().copied().collect(); + let observed_not_in_expected = + run_set.difference(&exp).copied().collect::>(); + let expected_not_observed = exp.difference(&run_set).copied().collect::>(); + ExpectedValidation { + consistent: observed_not_in_expected.is_empty(), + observed_not_in_expected, + expected_not_observed, + } + }); + + if report { + let binned = match &binned_scores { + Some(levels) => { + let levels: Vec = levels.iter().map(|q| q.to_string()).collect(); + format!("yes ({} levels: {})", levels.len(), levels.join(", ")) + } + None => "no".to_string(), + }; + eprintln!("=== Run-level FASTQ quality metrics ==="); + eprintln!("samples: {n_samples}"); + eprintln!("total sequences: {total_reads}"); + eprintln!( + "quality range: Q{min_quality}–Q{max_quality} ({} distinct values)", + distinct.len() + ); + eprintln!("binned: {binned}"); + if let Some(v) = &expected_validation { + if v.consistent { + eprintln!("expected scheme: CONSISTENT"); + } else { + let bad: Vec = v + .observed_not_in_expected + .iter() + .map(|q| format!("Q{q}")) + .collect(); + eprintln!( + "expected scheme: VIOLATION (observed {} not in declared scheme)", + bad.join(", ") + ); + } + if !v.expected_not_observed.is_empty() { + let missing: Vec = v + .expected_not_observed + .iter() + .map(|q| format!("Q{q}")) + .collect(); + eprintln!( + " declared but unobserved: {}", + missing.join(", ") + ); + } + } + let odd: Vec<&SampleFlag> = samples + .iter() + .filter(|s| !s.missing_from_run.is_empty()) + .collect(); + if !odd.is_empty() { + eprintln!("samples missing run-level bins:"); + for s in odd { + let m: Vec = + s.missing_from_run.iter().map(|q| format!("Q{q}")).collect(); + eprintln!(" {}: missing {}", s.sample, m.join(", ")); + } + } + } + + #[derive(Serialize)] + struct QualityMetricsOut { + distinct_quality_values: Vec, + quality_value_counts: Vec, + min_quality: u8, + max_quality: u8, + is_binned: bool, + #[serde(skip_serializing_if = "Option::is_none")] + binned_scores: Option>, + } + #[derive(Serialize)] + struct MergeOut { + n_samples: u64, + total_reads: u64, + quality_metrics: QualityMetricsOut, + #[serde(skip_serializing_if = "Vec::is_empty")] + expected_bins: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + expected_validation: Option, + samples: Vec, + } + + let out = MergeOut { + n_samples, + total_reads, + quality_metrics: QualityMetricsOut { + distinct_quality_values: distinct, + quality_value_counts: counts, + min_quality, + max_quality, + is_binned, + binned_scores, + }, + expected_bins, + expected_validation, + samples, + }; + + let tagged = Tagged::new("summary-merge", out); + let json = if compact { + serde_json::to_string(&tagged) + } else { + serde_json::to_string_pretty(&tagged) + } + .map_err(io::Error::other)?; + + match output { + Some(path) => std::fs::write(&path, &json)?, + None => println!("{json}"), + } + } + Commands::Derep { input, sample_name, diff --git a/src/summary.rs b/src/summary.rs index 24c691f..dfe2c1b 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -91,6 +91,13 @@ fn seq_complexity(seq: &[u8], k: usize) -> f64 { h.exp() } +/// Whether a quality-value distribution with `distinct_len` distinct levels is +/// judged "binned": 2..=`binned_threshold` distinct levels. Shared by the +/// per-sample `summary` and the run-level `summary-merge` so both use one rule. +pub fn judge_binned(distinct_len: usize, binned_threshold: usize) -> bool { + distinct_len >= 2 && distinct_len <= binned_threshold +} + pub struct QualitySummary { pub total_reads: u64, sums: Vec, @@ -278,6 +285,47 @@ impl QualitySummary { (max_q, trimmed) } + /// Distinct quality values observed anywhere in the file, ascending, each + /// with its total read-base count summed over all positions. Derived from + /// the per-position histogram — no extra pass over the reads. + pub fn quality_value_counts(&self) -> Vec<(u8, u64)> { + let mut totals = [0u64; MAX_QUAL + 1]; + for row in &self.hist { + for (q, &n) in row.iter().enumerate() { + totals[q] += n; + } + } + totals + .iter() + .enumerate() + .filter(|&(_, &n)| n > 0) + .map(|(q, &n)| (q as u8, n)) + .collect() + } + + /// Summary of the quality-value distribution used to judge whether the + /// data is binned. `binned_threshold` is the maximum number of distinct Q + /// values for the data to be judged binned (real binned schemes have 2–8 + /// levels; continuous Illumina/HiFi data has dozens). The full distinct set + /// and counts are always returned so borderline cases can be judged by eye. + pub fn quality_metrics(&self, binned_threshold: usize) -> QualityMetrics { + let counts = self.quality_value_counts(); + let distinct: Vec = counts.iter().map(|&(q, _)| q).collect(); + let value_counts: Vec = counts.iter().map(|&(_, n)| n).collect(); + let min_quality = distinct.first().copied().unwrap_or(0); + let max_quality = distinct.last().copied().unwrap_or(0); + let is_binned = judge_binned(distinct.len(), binned_threshold); + let binned_scores = is_binned.then(|| distinct.clone()); + QualityMetrics { + distinct_quality_values: distinct, + quality_value_counts: value_counts, + min_quality, + max_quality, + is_binned, + binned_scores, + } + } + /// The per-read complexity histogram, if it was requested. /// Returns `(kmer_size, bins, counts)` where `counts[bin]` covers the /// effective-k-mer-count range `[bin, bin+1) · 4^kmer_size / bins`. @@ -341,6 +389,21 @@ impl QualitySummary { } } +/// Summary of the quality-value distribution (see `quality_metrics`), used to +/// answer whether a FASTQ's quality scores are binned and to what levels. +pub struct QualityMetrics { + /// Distinct Q values observed anywhere in the file, ascending. + pub distinct_quality_values: Vec, + /// Total read-base count for each distinct value (parallel to above). + pub quality_value_counts: Vec, + pub min_quality: u8, + pub max_quality: u8, + /// True when the number of distinct Q values is small enough to be binned. + pub is_binned: bool, + /// The bin levels (the distinct set) when `is_binned`, else `None`. + pub binned_scores: Option>, +} + /// Per-position cumulative expected-error metrics (see `expected_error_metrics`). pub struct ExpectedErrorMetrics { pub mean: Vec, @@ -456,6 +519,56 @@ mod tests { assert!((m.max[3] - 0.4).abs() < 1e-12); } + /// Phred+33 char for a given Q value. + fn qchar(q: u8) -> u8 { + q + 33 + } + + #[test] + fn judge_binned_rule() { + assert!(!judge_binned(0, 8)); // no data + assert!(!judge_binned(1, 8)); // constant quality — not binned + assert!(judge_binned(2, 8)); + assert!(judge_binned(8, 8)); + assert!(!judge_binned(9, 8)); // above threshold — continuous + } + + #[test] + fn binned_data_detected() { + let mut s = QualitySummary::with_config(SummaryConfig::default()); + // Only four distinct quality levels, NovaSeq-style {2,12,23,37}. + let quals = [qchar(2), qchar(12), qchar(23), qchar(37)]; + s.add_record(b"ACGT", &quals, 33); + s.add_record(b"ACGT", &[qchar(37), qchar(37), qchar(2), qchar(12)], 33); + let m = s.quality_metrics(8); + assert!(m.is_binned); + assert_eq!(m.distinct_quality_values, vec![2, 12, 23, 37]); + assert_eq!(m.binned_scores, Some(vec![2, 12, 23, 37])); + assert_eq!(m.min_quality, 2); + assert_eq!(m.max_quality, 37); + } + + #[test] + fn continuous_data_not_binned() { + let mut s = QualitySummary::with_config(SummaryConfig::default()); + // Ten distinct quality levels — above the default threshold of 8. + let quals: Vec = (20u8..30).map(qchar).collect(); + s.add_record(&vec![b'A'; quals.len()], &quals, 33); + let m = s.quality_metrics(8); + assert!(!m.is_binned); + assert_eq!(m.binned_scores, None); + assert_eq!(m.distinct_quality_values.len(), 10); + } + + #[test] + fn quality_value_counts_sum_matches_total_bases() { + let mut s = QualitySummary::with_config(SummaryConfig::default()); + s.add_record(b"ACGT", &[qchar(10), qchar(20), qchar(20), qchar(30)], 33); + s.add_record(b"AC", &[qchar(20), qchar(30)], 33); + let total_bases: u64 = s.quality_value_counts().iter().map(|&(_, n)| n).sum(); + assert_eq!(total_bases, 6); + } + #[test] fn complexity_of_a_homopolymer_is_one() { // A single 2-mer ("AA") repeated -> one distinct k-mer -> exp(0) = 1.