diff --git a/.github/workflows/concordance.yml b/.github/workflows/concordance.yml index 466cae9..532237c 100644 --- a/.github/workflows/concordance.yml +++ b/.github/workflows/concordance.yml @@ -52,8 +52,13 @@ jobs: key: ${{ runner.os }}-cargo-concordance-${{ hashFiles('**/Cargo.lock') }} restore-keys: ${{ runner.os }}-cargo- - - name: Build (release) - run: cargo build --release + # Built with `--features wfa` because the WFA concordance steps below drive + # the experimental backend via `ALIGN_BACKEND=wfa2`. The feature is + # off-by-default (issue #63), so without it those steps would hit the + # NW-only stub. The resulting binary still defaults to NW for the plain + # concordance runs. + - name: Build (release, with WFA backend) + run: cargo build --release --features wfa # ---------------- Illumina (in-repo MiSeq tutorial pair) ---------------- - name: Illumina — run dada2-rs pipeline diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..28b48fe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,56 @@ +# Changelog + +All notable changes to this project are documented here. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project aims to +follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (pre-1.0, so +minor versions may carry breaking changes). + +## [0.2.0] - Unreleased + +### Added +- `summary` subcommand: per-position quality metrics, cumulative expected-error + metrics, and an optional per-read sequence-complexity histogram (#8). +- `sample` and `errors-from-sample` subcommands for subsampling input FASTQs and + learning an error model from the subsample (usable for bootstrapping). +- `--failed-uniques` diagnostic on `dada` / `dada-pseudo` / `dada-pooled`: + emits a TSV of uniques that failed to seed a partition (#60). +- `chimera-diagnostics` for higher-order (trimera) screening, with + nearest-parent and flank gates (#54). +- Experimental **WFA alignment backend** (`--align-backend wfa2`) with a + `--wfa-max-edits` edit-budget cap (#49, #51). ASV-equivalent to Needleman- + Wunsch on tested Illumina and PacBio HiFi data, but not byte-identical. +- `merge-pairs` records input-file provenance and warns on mismatch (#10). +- `just` / `make` task runners for build, install, test, and docs (#46). + +### Changed +- The experimental WFA backend is now gated behind an **off-by-default `wfa` + Cargo feature** (#63). Default builds — and the published crate — are + Needleman-Wunsch only; selecting `--align-backend wfa2` without the feature + errors rather than silently falling back to NW. Building WFA requires a source + checkout (`cargo build --features wfa`) because it depends on a git crate that + cannot ship on crates.io. +- Denoising/error-learning verbose logs now echo the active alignment backend + (#51). +- Trimmed the published crate: development, benchmarking, concordance data + (multi-MB PacBio FASTQs), examples, notes, and CI/infra files are excluded. + +### Performance +- Sparse k-mer-8 screen, gated to k ≥ 8, cutting resident memory in the + high-k regime (#43). +- `dada-pooled` streams dereplication into merging instead of holding all + dereps in memory, and uses an integer merge-quality accumulator (#39, #41). +- Various k-mer-vector memory reductions in the alignment screen (#32). + +### CI +- R DADA2 concordance guardrail for Illumina and PacBio, with abundance/recall/ + precision gates (#35). + +## [0.1.0] - 2026-06-03 + +- Initial crates.io release: Rust ports of the core DADA2 pipeline steps + (`filter-and-trim`, `derep`, `learn-errors`, `dada`, `merge-pairs`, + `remove-bimera-denovo`, `assign-taxonomy` / `assign-species`, sequence-table + helpers). + +[0.2.0]: https://github.com/HPCBio/dada2-rs/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/HPCBio/dada2-rs/releases/tag/v0.1.0 diff --git a/Cargo.lock b/Cargo.lock index 1a8a93d..b8d4a69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,7 +226,7 @@ dependencies = [ [[package]] name = "dada2-rs" -version = "0.1.1" +version = "0.2.0" dependencies = [ "clap", "flate2", @@ -327,9 +327,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "matrixmultiply" diff --git a/Cargo.toml b/Cargo.toml index 8716ebf..84b9bcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dada2-rs" -version = "0.1.1" +version = "0.2.0" edition = "2024" # Raised from 1.87 to 1.91 by the experimental WFA backend (wfa2lib-rs MSRV). # Revisit if the WFA backend is dropped. @@ -12,7 +12,23 @@ readme = "README.md" authors = ["Chris Fields "] keywords = ["bioinformatics", "amplicon", "dada2", "metagenomics", "asv"] categories = ["science", "command-line-utilities"] -exclude = ["**/__pycache__/", "*.pyc"] +# Keep the published crate lean: development, benchmarking, concordance data +# (multi-MB PacBio FASTQs), examples, notes, and CI/infra files are not needed by +# `cargo install` consumers. `data/`, `tmp/` are already gitignored. +exclude = [ + "**/__pycache__/", + "*.pyc", + "/dev", + "/examples", + "/notes", + "/data", + "/tmp", + "/.github", + "/.githooks", + "/.readthedocs.yaml", + "/AGENTS.md", + "/Cross.toml", +] [dependencies] noodles = { version = "0.91", features = ["fastq", "fasta", "bgzf"] } @@ -44,6 +60,15 @@ wfa2lib-rs = { git = "https://github.com/HPCBio/wfa2lib-rs", rev = "4cfeda86a3e1 # runtime and the published crate is Needleman-Wunsch only. wfa = ["dep:wfa2lib-rs"] +[lints.rust] +# The publish script (scripts/publish-crate.sh) strips the `wfa` feature from the +# manifest for the crates.io release, since it depends on a git crate. Declaring +# `feature="wfa"` as a known cfg value keeps the `#[cfg(feature = "wfa")]` code in +# nwalign.rs from emitting `unexpected_cfgs` warnings when built without the +# feature defined (e.g. `cargo install dada2-rs`). Harmless when the feature is +# present in the manifest. +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("wfa"))'] } + [dev-dependencies] serde_json = "1" diff --git a/Makefile b/Makefile index 8399d10..7e9bab4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,9 @@ SCRIPTS = $(wildcard scripts/*.R) $(wildcard scripts/*.py) .PHONY: all build build-native build-arm64 install uninstall \ check fmt fmt-check clippy test clean docs-serve docs-build \ - check-build-sync help + publish-crate check-build-sync help + +PUBLISH_ARGS ?= all: build @@ -85,6 +87,10 @@ docs-build: clean: cargo clean +## publish-crate: publish NW-only crate to crates.io (strips WFA git dep; PUBLISH_ARGS=--dry-run to test) +publish-crate: + ./scripts/publish-crate.sh $(PUBLISH_ARGS) + ## check-build-sync: fail if justfile recipes and Makefile targets have drifted check-build-sync: ./scripts/check-build-sync.sh diff --git a/justfile b/justfile index e2ea041..de9f950 100644 --- a/justfile +++ b/justfile @@ -88,6 +88,11 @@ docs-build: clean: cargo clean +# publish an NW-only crate to crates.io (strips the WFA git dep; see issue #63). +# Pass --dry-run to verify without uploading: `just publish-crate --dry-run`. +publish-crate *ARGS: + ./scripts/publish-crate.sh {{ARGS}} + # fail if justfile recipes and Makefile targets have drifted apart check-build-sync: ./scripts/check-build-sync.sh diff --git a/scripts/publish-crate.sh b/scripts/publish-crate.sh new file mode 100755 index 0000000..7462112 --- /dev/null +++ b/scripts/publish-crate.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# publish-crate.sh — publish dada2-rs to crates.io as a Needleman-Wunsch-only crate. +# +# The experimental WFA backend depends on `wfa2lib-rs` via a *git* dependency, +# which crates.io rejects outright (issue #63). This script temporarily strips +# that git dependency and its off-by-default `wfa` feature from Cargo.toml, runs +# `cargo publish`, then restores the original manifest. The published crate is +# therefore NW-only; WFA stays a source-checkout developer build +# (`cargo build --features wfa`). +# +# Usage: +# scripts/publish-crate.sh --dry-run # verify packaging without uploading +# scripts/publish-crate.sh # real publish (needs `cargo login`) +# +# Any extra arguments are passed straight through to `cargo publish`. +set -euo pipefail +cd "$(dirname "$0")/.." + +MANIFEST="Cargo.toml" +BACKUP="$(mktemp)" + +restore() { + if [ -f "$BACKUP" ]; then + cp "$BACKUP" "$MANIFEST" + rm -f "$BACKUP" + echo "==> Restored original $MANIFEST" + fi +} +trap restore EXIT +cp "$MANIFEST" "$BACKUP" + +echo "==> Stripping the experimental WFA git dependency + feature for publishing" +python3 - "$MANIFEST" <<'PY' +import re, sys +path = sys.argv[1] +s = open(path).read() +before = s +# Remove the git `wfa2lib-rs` dependency line... +s = re.sub(r'(?m)^wfa2lib-rs\s*=.*\n', '', s) +# ...and the off-by-default `wfa` feature that referenced it. +s = re.sub(r'(?m)^wfa\s*=\s*\[.*\]\s*\n', '', s) +if s == before: + sys.exit("ERROR: expected WFA dependency/feature lines were not found; " + "manifest layout may have changed — update publish-crate.sh.") +open(path, "w").write(s) +PY + +echo "==> cargo publish --allow-dirty $*" +# --allow-dirty because we have just edited Cargo.toml in place (and callers may +# publish with other working-tree changes present). +cargo publish --allow-dirty "$@"