From 42ba40395a44e0130877bbdb5ba9df425fac345b Mon Sep 17 00:00:00 2001 From: zackees Date: Sun, 10 May 2026 07:37:00 -0700 Subject: [PATCH] feat(library-selection): #218 add bench-fastled-examples Phase 7 warm-cache harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a real-FastLED-matrix bench under `bench/fastled-examples/`, complementing the synthetic per-crate `resolve_warm` from #216. For each of a curated subset of FastLED examples (Blink, Pacifica, Animartrix, Audio, BlurBenchmark, Chromancer), runs `resolve_cached` cold + warm against a fresh `KvStore` and reports timings. Asserts `from_cache=true` on the warm pass so silent re-misses surface immediately. Local numbers (Windows release, FastLED master, 6 examples × synthetic 6-lib Teensyduino-class framework): | example | cold (ms) | warm (ms) | speedup | |---------------|----------:|----------:|--------:| | Blink | 923.58 | 11.36 | 81.3x | | Pacifica | 915.98 | 12.64 | 72.4x | | Animartrix | 970.14 | 11.76 | 82.5x | | Audio | 830.51 | 11.74 | 70.7x | | BlurBenchmark | 827.46 | 10.48 | 79.0x | | Chromancer | 844.13 | 10.89 | 77.5x | Warm path comfortably clears AC#5 (≤ +50 ms over current fbuild) at ~11 ms per example. The asymmetry reflects the cost of walking FastLED src/ (~1000 files) cold vs. a `KvStore::get` + bincode decode warm. CI lane added as `workflow_dispatch`-only (needs FastLED checkout step). No threshold gate yet — capture a stable cross-runner baseline first. Closes #218. Refs #205 Phase 7. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/bench-205.yml | 44 +++++ Cargo.lock | 12 ++ Cargo.toml | 1 + bench/README.md | 11 +- bench/fastled-examples/Cargo.toml | 20 ++ bench/fastled-examples/README.md | 147 +++++++++------ bench/fastled-examples/src/README.md | 4 + bench/fastled-examples/src/main.rs | 266 +++++++++++++++++++++++++++ 8 files changed, 442 insertions(+), 63 deletions(-) create mode 100644 bench/fastled-examples/Cargo.toml create mode 100644 bench/fastled-examples/src/README.md create mode 100644 bench/fastled-examples/src/main.rs diff --git a/.github/workflows/bench-205.yml b/.github/workflows/bench-205.yml index de7bf3f1..277d675e 100644 --- a/.github/workflows/bench-205.yml +++ b/.github/workflows/bench-205.yml @@ -55,3 +55,47 @@ jobs: bench-${{ matrix.bench }}.txt if-no-files-found: warn retention-days: 14 + + fastled-examples: + name: Bench (fastled-examples — workflow_dispatch only) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v3 + - name: Setup soldr + uses: zackees/setup-soldr@v0 + with: + cache: true + build-cache: true + target-cache: true + + - name: Checkout FastLED + uses: actions/checkout@v6 + with: + repository: FastLED/FastLED + path: external/fastled + # Pinned to a FastLED release tag for reproducibility. Bump in + # lockstep with the sample-numbers table in + # bench/fastled-examples/README.md when retaking baseline. + ref: "3.10.3" + + - name: Run bench-fastled-examples + env: + FASTLED_DIR: ${{ github.workspace }}/external/fastled + run: | + mkdir -p bench/fastled-examples + soldr cargo run --release -p fbuild-bench-fastled-examples -- \ + --json bench/fastled-examples/report.json | tee bench/fastled-examples/report.md + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: bench-fastled-examples + path: | + bench/fastled-examples/report.md + bench/fastled-examples/report.json + if-no-files-found: warn + retention-days: 30 diff --git a/Cargo.lock b/Cargo.lock index aa99b2c1..3941b79d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -841,6 +841,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fbuild-bench-fastled-examples" +version = "2.2.3" +dependencies = [ + "fbuild-library-select", + "fbuild-packages", + "fbuild-test-support", + "serde_json", + "tempfile", + "zccache-artifact", +] + [[package]] name = "fbuild-build" version = "2.2.3" diff --git a/Cargo.toml b/Cargo.toml index 769157e3..85d67a13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/fbuild-test-support", "crates/fbuild-header-scan", "crates/fbuild-library-select", + "bench/fastled-examples", ] [workspace.package] diff --git a/bench/README.md b/bench/README.md index 91102efe..6654d3c2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -19,10 +19,13 @@ uv run soldr cargo bench -p fbuild-header-scan --bench scan_throughput ## Subdirectories -- [`fastled-examples/`](fastled-examples/README.md) — reserved for the - real-FastLED warm-cache library-selection matrix (`FastLED/fbuild#205` - AC#5, P-01) once `~/dev/fastled` is wired in. The synthetic warm-path - baseline already lives in `crates/fbuild-library-select/benches/resolve_warm.rs`. +- [`fastled-examples/`](fastled-examples/README.md) — real-FastLED + warm-cache library-selection matrix (`FastLED/fbuild#205` AC#5, P-01). + Discovers examples under `$FASTLED_DIR` (default `~/dev/fastled`), + runs the resolver cold + warm per example, and reports timings. + Run with `uv run soldr cargo run --release -p fbuild-bench-fastled-examples`. + The synthetic warm-path baseline lives in + `crates/fbuild-library-select/benches/resolve_warm.rs`. Other end-to-end matrices (whole-build wall-clock, deploy+flash latency, emulator boot) may join this directory in the future. Each subdirectory diff --git a/bench/fastled-examples/Cargo.toml b/bench/fastled-examples/Cargo.toml new file mode 100644 index 00000000..9beba0a0 --- /dev/null +++ b/bench/fastled-examples/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "fbuild-bench-fastled-examples" +description = "Warm-cache library-selection bench across the FastLED examples matrix (#205 Phase 7)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[[bin]] +name = "bench-fastled-examples" +path = "src/main.rs" + +[dependencies] +fbuild-library-select = { path = "../../crates/fbuild-library-select" } +fbuild-packages = { path = "../../crates/fbuild-packages" } +fbuild-test-support = { path = "../../crates/fbuild-test-support" } +zccache-artifact = { workspace = true } +tempfile = { workspace = true } +serde_json = { workspace = true } diff --git a/bench/fastled-examples/README.md b/bench/fastled-examples/README.md index ba958fab..bbc3d49f 100644 --- a/bench/fastled-examples/README.md +++ b/bench/fastled-examples/README.md @@ -1,69 +1,98 @@ # bench/fastled-examples -Warm-cache library-selection benchmarks across the FastLED examples matrix. -This is the harness referenced by `FastLED/fbuild#205` for acceptance -criterion **AC#5 / P-01**: - -> Warm library-selection on FastLED examples matrix `<= current fbuild -> + 50 ms`. - -## Status: empty placeholder - -There is no harness in this directory yet — the real per-board, per-example -matrix needs a checked-out FastLED tree (`~/dev/fastled`) and orchestrator -wiring that routes through `resolve_cached`. That work is tracked -separately. The synthetic warm baseline (`MiniFramework`-backed cache hit, -no real FastLED) already exists at -[`../../crates/fbuild-library-select/benches/resolve_warm.rs`](../../crates/fbuild-library-select/benches/resolve_warm.rs) -and is the first-pass regression guard for the cache-hit path. - -Phase 4 K/V memoization itself shipped in PR #212, so the warm path is -real and measurable today; what is missing here is the multi-board, -real-sketch matrix that AC#5 requires. - -## The plan once the FastLED tree is wired in - -1. Iterate the FastLED examples tree (`~/dev/fastled/examples/**`) under - each supported board: at minimum `teensyLC`, `teensy30`, `teensy41`, - `stm32f103c8`, `esp32-s3`, `uno`, `ws2812`. The matrix expands with - board coverage. -2. For each `(example, board)` pair, run the resolver twice: - - **Cold pass.** Empty `~/.zccache/`. Captures the K/V miss path and - the underlying scan + walk + LDF cost. This is the P-02 lane. - - **Warm pass.** Populated `~/.zccache/`. Captures the K/V hit path, - where the only work should be cache lookup + result deserialization. - This is the P-01 lane. -3. Diff the warm scan time against a captured baseline checked into - `tasks/baseline-205.md`. CI fails the job if any `(example, board)` - regresses the warm path by more than 50 ms vs. that baseline (the - `#205` AC#5 threshold). -4. Emit a structured JSON report (`bench/fastled-examples/report.json`) - that future PR comments can diff. Format TBD with the harness. - -## Running the synthetic mini benches today - -The closest signal available right now without a FastLED checkout is the -per-crate cold and warm criterion benches against `MiniFramework`: +Warm-cache library-selection benchmark across a curated FastLED examples +matrix. This is the AC#5 / P-01 measurement for +[`FastLED/fbuild#205`](https://github.com/FastLED/fbuild/issues/205). + +## What it measures + +For each example sketch under `$FASTLED_DIR/examples/`, runs the +`fbuild_library_select::cache::resolve_cached` resolver twice against a +fresh `KvStore`: + +- **Cold** — empty cache. Wall-clock includes the scanner walk over the + FastLED `src/` tree (~1000 files), the 2-pass LDF reconciliation, and + the cache write. This dominates total time. +- **Warm** — cache pre-populated. Wall-clock includes the cache-key + compute (sorted seed/header content hashing, bounded by `cache_key` + itself) and the bincode decode of the cached `Selection`. + `from_cache = true` is asserted so silent re-misses surface + immediately. + +The framework library set is a synthetic Teensyduino-class stub built +via `MiniFramework`. The bench measures resolver throughput, not the +correctness of which libraries get selected — that is the acceptance-test +layer (`crates/fbuild-build/tests/teensylc_acceptance.rs`). + +## Running + +`FASTLED_DIR` is required — there is no implicit default, since the +correct path is host-dependent (CI uses `external/fastled` from the +workflow checkout, developers use whatever convention they like) and a +silent fallback would mask configuration mistakes. ```bash -uv run soldr cargo bench -p fbuild-library-select --bench resolve_cold -uv run soldr cargo bench -p fbuild-library-select --bench resolve_warm +FASTLED_DIR=/path/to/fastled \ + uv run soldr cargo run --release -p fbuild-bench-fastled-examples + +# Emit a JSON report alongside stdout +FASTLED_DIR=/path/to/fastled \ + uv run soldr cargo run --release -p fbuild-bench-fastled-examples \ + -- --json bench/fastled-examples/report.json ``` -Those benches drive a synthetic ~30-library Teensyduino-class tree built -from `fbuild-test-support`'s `MiniFramework` rather than real FastLED -sketches. They are useful regression guards for the resolver and its -cache layer respectively, but they do **not** satisfy AC#5 on their own. +If any example fails to measure (missing sketch, KvStore error, warm +miss) the binary exits non-zero rather than skipping the row. CI must +treat a partial matrix as a failure, not a pass. + +## Sample numbers + +Captured 2026-05-10 on Windows / Ryzen workstation, FastLED `main`, +release build: + +| example | cold (ms) | warm (ms) | speedup | +|---------------|----------:|----------:|--------:| +| Blink | 923.58 | 11.36 | 81.3x | +| Pacifica | 915.98 | 12.64 | 72.4x | +| Animartrix | 970.14 | 11.76 | 82.5x | +| Audio | 830.51 | 11.74 | 70.7x | +| BlurBenchmark | 827.46 | 10.48 | 79.0x | +| Chromancer | 844.13 | 10.89 | 77.5x | + +The warm path comfortably clears AC#5 (≤ +50 ms over current fbuild) at +~11 ms per example. The ~75x speedup reflects the cost asymmetry between +walking the FastLED `src/` tree (~1000 files) and a `KvStore` +get + bincode decode of a serialized `Selection`. + +## Curated example set + +The harness intentionally runs a small representative subset rather than +all 80+ examples. Adding more is cheap — see `EXAMPLES` in `src/main.rs`. +The current set spans: + +- Trivial single-strip sketches (`Blink`) +- Animation-heavy sketches (`Pacifica`, `Animartrix`) +- I/O-heavy sketches (`Audio`) +- Throughput stress sketches (`BlurBenchmark`, `Chromancer`) + +## CI + +The `fastled-examples` job in `.github/workflows/bench-205.yml` is +`workflow_dispatch`-only because it requires a FastLED checkout. CI +checks out FastLED at a pinned release tag (currently `3.10.3`) so +measurements are reproducible, then runs the bench and uploads the JSON +report as an artifact. Bumping the pin is a deliberate baseline event — +update both the workflow `ref:` and the sample-numbers table above in +lockstep. + +There is no automatic CI gate on the warm timings yet — first capture a +stable cross-runner baseline, then a follow-up adds the threshold gate. ## Cross-links -- Issue: `FastLED/fbuild#205` -- Phase 4 K/V memoization (shipped in #212): - [`../../tasks/zccache-kv-design.md`](../../tasks/zccache-kv-design.md) +- Issue: [`FastLED/fbuild#205`](https://github.com/FastLED/fbuild/issues/205) +- This harness: [`FastLED/fbuild#218`](https://github.com/FastLED/fbuild/issues/218) +- Per-crate synthetic warm bench: + [`crates/fbuild-library-select/benches/resolve_warm.rs`](../../crates/fbuild-library-select/benches/resolve_warm.rs) - Subsystem architecture: - [`../../docs/architecture/library-selection.md`](../../docs/architecture/library-selection.md) -- Foundation baseline that the warm threshold compares against: - [`../../tasks/baseline-205.md`](../../tasks/baseline-205.md) -- Per-crate cold + warm benches (different scope, same subsystem): - [`../../crates/fbuild-library-select/benches/README.md`](../../crates/fbuild-library-select/benches/README.md), - [`../../crates/fbuild-header-scan/benches/README.md`](../../crates/fbuild-header-scan/benches/README.md) + [`docs/architecture/library-selection.md`](../../docs/architecture/library-selection.md) diff --git a/bench/fastled-examples/src/README.md b/bench/fastled-examples/src/README.md new file mode 100644 index 00000000..a9e5304a --- /dev/null +++ b/bench/fastled-examples/src/README.md @@ -0,0 +1,4 @@ +# bench/fastled-examples/src + +Source for the `bench-fastled-examples` binary. See the parent +[`README.md`](../README.md) for what the harness does and how to run it. diff --git a/bench/fastled-examples/src/main.rs b/bench/fastled-examples/src/main.rs new file mode 100644 index 00000000..38b56c9c --- /dev/null +++ b/bench/fastled-examples/src/main.rs @@ -0,0 +1,266 @@ +//! Warm-cache library-selection bench across a curated FastLED examples matrix. +//! +//! This is the AC#5 / P-01 measurement for FastLED/fbuild#205: for each +//! example sketch under `$FASTLED_DIR/examples/`, runs the resolver cold +//! (empty `KvStore`) and warm (cache pre-populated by the cold call) and +//! reports the timings as a Markdown table. +//! +//! ## Inputs +//! +//! - `FASTLED_DIR` (env, required) — root of a FastLED checkout. Must +//! contain `src/` and `examples//.ino`. No default — see +//! `resolve_fastled_dir` for why. +//! - Curated `EXAMPLES` list below — a representative spread, not the +//! full ~80-example tree. Each is a single-`.ino` Arduino sketch. +//! +//! ## What it measures +//! +//! For each `(example, framework_lib_set)` pair: +//! +//! - **Cold**: open a fresh `KvStore`, call `resolve_cached(...)`. Wall-clock +//! includes scanner walk, LDF reconciliation, and cache write. +//! - **Warm**: call `resolve_cached(...)` again against the same `KvStore`. +//! Wall-clock includes cache-key compute (sorted seed/header hashing — +//! bounded by `cache_key` itself) and bincode decode of the cached +//! `Selection`. Asserts `from_cache = true` so silent re-misses surface +//! immediately. +//! +//! The framework library set is a synthetic Teensyduino-style stub built +//! via `MiniFramework`. The bench measures resolver throughput, not +//! whether the right libs are selected — that's the acceptance-test layer +//! (`tests/teensylc_acceptance.rs`). +//! +//! ## CLI +//! +//! ```text +//! bench-fastled-examples [--json ] +//! ``` +//! +//! `--json ` writes a structured report alongside the stdout table +//! for diffing in PR comments. +//! +//! Refs: #205 Phase 7 (AC#5), #218. + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use fbuild_library_select::cache::{resolve_cached, CacheKeyInputs}; +use fbuild_library_select::CachedSelection; +use fbuild_packages::library::framework_library::discover_framework_libraries; +use fbuild_packages::library::FrameworkLibrary; +use fbuild_test_support::MiniFramework; +use zccache_artifact::KvStore; + +/// Curated subset that spans the simple/complex spectrum without dragging +/// in every one of the ~80 FastLED examples. Bench iteration time at six +/// examples is a few seconds — adding more is cheap if needed. +const EXAMPLES: &[&str] = &[ + "Blink", + "Pacifica", + "Animartrix", + "Audio", + "BlurBenchmark", + "Chromancer", +]; + +/// Synthetic Teensyduino-class framework lib names. We only need names +/// here — the resolver attributes by include-dir prefix, and these libs +/// don't need to be functionally selected for the timing to be meaningful +/// (the cost is in the walker/LDF, not the lib count). +const FRAMEWORK_LIBS: &[&str] = &["SPI", "Wire", "EEPROM", "OctoWS2811", "Audio", "RadioHead"]; + +struct Row { + example: String, + cold_ms: f64, + warm_ms: f64, + selected: Vec, + hit: bool, +} + +fn main() { + if let Err(err) = run() { + eprintln!("bench-fastled-examples: {err}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + let json_out = parse_json_flag(&args); + + let fastled_dir = resolve_fastled_dir()?; + let fastled_src = fastled_dir.join("src"); + if !fastled_src.is_dir() { + return Err(format!( + "FastLED checkout not found: {} (FASTLED_DIR must point at a tree with src/ + examples/)", + fastled_dir.display() + ) + .into()); + } + + let mut mf = MiniFramework::new(); + for name in FRAMEWORK_LIBS { + mf.add_library(name).done(); + } + let libraries = discover_framework_libraries(&mf.libraries_dir()); + let framework_root = mf.framework_root().to_path_buf(); + + println!("# bench/fastled-examples warm-cache report"); + println!(); + println!("- FastLED: `{}`", fastled_dir.display()); + println!("- Framework lib set: {} synthetic libs", libraries.len()); + println!(); + println!("| example | cold (ms) | warm (ms) | speedup | selected | hit |"); + println!("|---|---:|---:|---:|---:|---|"); + + let mut rows: Vec = Vec::new(); + for example in EXAMPLES { + let row = measure_example( + example, + &fastled_dir, + &fastled_src, + &framework_root, + &libraries, + )?; + let speedup = if row.warm_ms > 0.0 { + row.cold_ms / row.warm_ms + } else { + f64::INFINITY + }; + println!( + "| {} | {:.2} | {:.2} | {:.1}x | {} | yes |", + row.example, + row.cold_ms, + row.warm_ms, + speedup, + row.selected.len(), + ); + rows.push(row); + } + + if let Some(path) = json_out { + write_json_report(&path, &fastled_dir, &rows)?; + println!(); + println!("JSON report written to `{}`", path.display()); + } + + Ok(()) +} + +fn measure_example( + name: &str, + fastled_dir: &Path, + fastled_src: &Path, + framework_root: &Path, + libraries: &[FrameworkLibrary], +) -> Result> { + let ino_path = fastled_dir + .join("examples") + .join(name) + .join(format!("{name}.ino")); + if !ino_path.is_file() { + return Err(format!("missing sketch {ino_path:?}").into()); + } + + let stage = tempfile::tempdir()?; + let stage_src = stage.path().join("src"); + std::fs::create_dir_all(&stage_src)?; + let main_cpp = stage_src.join("main.cpp"); + std::fs::write(&main_cpp, std::fs::read(&ino_path)?)?; + + let seeds = vec![main_cpp]; + let search_paths = vec![stage_src, fastled_src.to_path_buf()]; + + let kv_dir = tempfile::tempdir()?; + let kv = KvStore::open(kv_dir.path().join("kv"))?; + + let inputs = CacheKeyInputs { + toolchain_triple: "teensy-arm-none-eabi", + framework_install_path: framework_root, + framework_version: "bench-fastled-examples-v1", + }; + + let (cold, cold_ms) = timed(|| resolve_cached(&seeds, &search_paths, libraries, &inputs, &kv))?; + if cold.from_cache { + return Err("cold call unexpectedly hit the cache".into()); + } + + let (warm, warm_ms) = timed(|| resolve_cached(&seeds, &search_paths, libraries, &inputs, &kv))?; + if !warm.from_cache { + return Err(format!("warm call unexpectedly missed the cache for `{name}`").into()); + } + + Ok(Row { + example: name.to_string(), + cold_ms, + warm_ms, + selected: warm.selection.required_libraries.clone(), + hit: true, + }) +} + +fn timed(f: F) -> Result<(CachedSelection, f64), E> +where + F: FnOnce() -> Result, +{ + let t0 = Instant::now(); + let res = f()?; + Ok((res, t0.elapsed().as_secs_f64() * 1000.0)) +} + +fn write_json_report( + path: &Path, + fastled_dir: &Path, + rows: &[Row], +) -> Result<(), Box> { + let entries: Vec<_> = rows + .iter() + .map(|r| { + serde_json::json!({ + "example": r.example, + "cold_ms": r.cold_ms, + "warm_ms": r.warm_ms, + "selected": r.selected, + "hit": r.hit, + }) + }) + .collect(); + let body = serde_json::json!({ + "fastled_dir": fastled_dir.to_string_lossy(), + "rows": entries, + }); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, serde_json::to_string_pretty(&body)?)?; + Ok(()) +} + +fn parse_json_flag(args: &[String]) -> Option { + let mut iter = args.iter().skip(1); + while let Some(arg) = iter.next() { + if arg == "--json" { + return iter.next().map(PathBuf::from); + } + if let Some(rest) = arg.strip_prefix("--json=") { + return Some(PathBuf::from(rest)); + } + } + None +} + +/// Read `FASTLED_DIR` from the environment. No fallback default: the +/// value depends on the host (CI uses a workspace-relative checkout, +/// developers use whatever convention they like) and silently +/// substituting a workstation-specific path would mask configuration +/// errors and leak the previous developer's layout into reports. +fn resolve_fastled_dir() -> Result> { + match std::env::var("FASTLED_DIR") { + Ok(s) if !s.is_empty() => Ok(PathBuf::from(s)), + _ => Err( + "FASTLED_DIR is not set. Point it at a FastLED checkout root (a directory \ + containing `src/` and `examples/`)." + .into(), + ), + } +}