From 7dfa2220201ef9f38e6aa5ad5fed1069c742b43c Mon Sep 17 00:00:00 2001 From: zackees Date: Sat, 25 Apr 2026 02:43:21 -0700 Subject: [PATCH] feat(library-selection): #205 Phase 7 perf benchmarks (P-02, P-03) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two `criterion` benchmarks that capture baseline numbers for issue #205's Phase 7 perf gates. This PR lands the harness; CI gating against thresholds is a follow-up once we have a stable measurement on a known runner spec. ## Benchmarks - `crates/fbuild-header-scan/benches/scan_throughput.rs` Measures `scan()` throughput on three input sizes (tiny/medium/large) with adversary fixtures (raw strings, comments, identifiers ending in `R`/`L`, char literals). Target per #205 P-03: >= 50 MB/s single-thread. - `crates/fbuild-library-select/benches/resolve_cold.rs` End-to-end `resolve()` walk on a synthetic 30-library framework tree with a 5-level transitive include chain. Target per #205 P-02: <= 200 ms cold for a typical teensy41 project. Uses `MiniFramework` from `fbuild-test-support` so the bench is hermetic. Both use `harness = false` and depend on the new workspace `criterion = "0.5"` dev-dep. Run with: ```bash uv run soldr cargo bench -p fbuild-header-scan --bench scan_throughput uv run soldr cargo bench -p fbuild-library-select --bench resolve_cold ``` ## Out of scope (still tracked) - P-01 (warm matrix) — gated on Phase 4 cache (zccache#130). - P-04 (cache-hit round-trip) — same. - CI gating against the captured thresholds — follow-up once runner variance is characterized. ## Verification - `uv run soldr cargo build --release -p fbuild-header-scan -p fbuild-library-select --benches` — green. - `uv run soldr cargo clippy --workspace --all-targets -- -D warnings` — green. - `uv run soldr cargo fmt --all --check` — clean. Refs: FastLED/fbuild#205, FastLED/fbuild#202, FastLED/fbuild#204 Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 135 ++++++++++++++++++ Cargo.toml | 1 + crates/fbuild-header-scan/Cargo.toml | 5 + crates/fbuild-header-scan/benches/README.md | 20 +++ .../benches/scan_throughput.rs | 59 ++++++++ crates/fbuild-library-select/Cargo.toml | 6 + .../fbuild-library-select/benches/README.md | 23 +++ .../benches/resolve_cold.rs | 92 ++++++++++++ 8 files changed, 341 insertions(+) create mode 100644 crates/fbuild-header-scan/benches/README.md create mode 100644 crates/fbuild-header-scan/benches/scan_throughput.rs create mode 100644 crates/fbuild-library-select/benches/README.md create mode 100644 crates/fbuild-library-select/benches/resolve_cold.rs diff --git a/Cargo.lock b/Cargo.lock index 8cf019a9..2c17b486 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -318,6 +324,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.59" @@ -348,6 +360,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.0" @@ -449,6 +488,40 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -474,6 +547,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -894,6 +973,7 @@ dependencies = [ name = "fbuild-header-scan" version = "2.2.3" dependencies = [ + "criterion", "tempfile", ] @@ -901,8 +981,10 @@ dependencies = [ name = "fbuild-library-select" version = "2.2.3" dependencies = [ + "criterion", "fbuild-header-scan", "fbuild-packages", + "fbuild-test-support", "tempfile", "tracing", "walkdir", @@ -1215,6 +1297,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1254,6 +1347,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "http" version = "1.4.0" @@ -1550,12 +1649,32 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1929,6 +2048,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "parking_lot_core" version = "0.9.12" @@ -2927,6 +3052,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" diff --git a/Cargo.toml b/Cargo.toml index 7425f0ca..9abc3294 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,7 @@ tokio-tungstenite = "0.24" async-trait = "0.1" dashmap = "6" blake3 = "1" +criterion = { version = "0.5", default-features = false, features = ["html_reports"] } mimalloc = "0.1" object = { version = "0.36", default-features = false, features = ["read", "std", "elf", "write"] } rusqlite = { version = "0.31", features = ["bundled"] } diff --git a/crates/fbuild-header-scan/Cargo.toml b/crates/fbuild-header-scan/Cargo.toml index 8d29825a..75869aa7 100644 --- a/crates/fbuild-header-scan/Cargo.toml +++ b/crates/fbuild-header-scan/Cargo.toml @@ -9,4 +9,9 @@ license.workspace = true [dependencies] [dev-dependencies] +criterion = { workspace = true } tempfile = { workspace = true } + +[[bench]] +name = "scan_throughput" +harness = false diff --git a/crates/fbuild-header-scan/benches/README.md b/crates/fbuild-header-scan/benches/README.md new file mode 100644 index 00000000..8b2de944 --- /dev/null +++ b/crates/fbuild-header-scan/benches/README.md @@ -0,0 +1,20 @@ +# benches + +Criterion micro-benchmarks for `fbuild-header-scan`. + +`scan_throughput.rs` measures `scan()` single-thread throughput (MB/s) over +three synthetic C++ fixtures: **tiny** (~64 B, per-call overhead), +**medium** (100 KB), and **large** (2 MB, stand-in for a Teensy-core-sized +translation unit). The fixtures exercise the scanner's adversary paths +(comments, string / raw-string literals containing fake `#include`s, +identifiers ending in `R` / `L`). + +Per FastLED/fbuild#205 P-03 the aspirational threshold is **≥ 50 MB/s +single-thread**. This bench captures the baseline; it is not yet a CI +gate (Phase 7 will wire that up in a follow-up). + +Run: + +```bash +uv run soldr cargo bench -p fbuild-header-scan --bench scan_throughput +``` diff --git a/crates/fbuild-header-scan/benches/scan_throughput.rs b/crates/fbuild-header-scan/benches/scan_throughput.rs new file mode 100644 index 00000000..85c48038 --- /dev/null +++ b/crates/fbuild-header-scan/benches/scan_throughput.rs @@ -0,0 +1,59 @@ +//! Criterion benchmark for `fbuild_header_scan::scan` throughput. +//! +//! P-03 of FastLED/fbuild#205: capture single-thread MB/s on three input +//! sizes (tiny / medium / large) so future PRs can regress against a +//! recorded baseline. The aspirational threshold is ≥ 50 MB/s +//! single-thread; this harness records the number but does not gate CI. + +use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; +use fbuild_header_scan::scan; + +/// Generate a synthetic C++ source string at least `target_bytes` long. +/// +/// The template intentionally exercises the scanner's adversary paths: +/// angled + quoted `#include`, line and multi-line block comments +/// containing fake `#include`s, string and raw-string literals with +/// embedded `#include` payloads, identifiers ending in `R` / `L` +/// (which must NOT be treated as raw-string prefixes), and a char +/// literal containing `#`. Repeated until we hit the byte budget. +fn fixture(target_bytes: usize) -> String { + let template = "\ + #include \n\ + // comment with #include \n\ + const char* s = \"#include \";\n\ + const char* r = R\"(#include )\";\n\ + auto FooR = 0; // identifier ending in R, NOT a raw string\n\ + auto FooL = 1; // identifier ending in L, NOT a wide-string prefix\n\ + /* block\n #include \n*/\n\ + char c = '#';\n\ + #include \"b.h\"\n\ + "; + let mut s = String::with_capacity(target_bytes + template.len()); + while s.len() < target_bytes { + s.push_str(template); + } + s +} + +fn bench_scanner(c: &mut Criterion) { + let mut group = c.benchmark_group("scan"); + for (name, size) in [ + ("tiny", 64usize), + ("medium", 100 * 1024), + ("large", 2 * 1024 * 1024), + ] { + let src = fixture(size); + let actual_len = src.len(); + group.throughput(Throughput::Bytes(actual_len as u64)); + group.bench_function(name, |b| { + b.iter(|| { + let refs = scan(black_box(&src)); + black_box(refs); + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_scanner); +criterion_main!(benches); diff --git a/crates/fbuild-library-select/Cargo.toml b/crates/fbuild-library-select/Cargo.toml index ad6d8ad5..b740050f 100644 --- a/crates/fbuild-library-select/Cargo.toml +++ b/crates/fbuild-library-select/Cargo.toml @@ -14,3 +14,9 @@ walkdir = { workspace = true } [dev-dependencies] tempfile = { workspace = true } +criterion = { workspace = true } +fbuild-test-support = { path = "../fbuild-test-support" } + +[[bench]] +name = "resolve_cold" +harness = false diff --git a/crates/fbuild-library-select/benches/README.md b/crates/fbuild-library-select/benches/README.md new file mode 100644 index 00000000..3d711942 --- /dev/null +++ b/crates/fbuild-library-select/benches/README.md @@ -0,0 +1,23 @@ +# fbuild-library-select benches + +Criterion benchmarks for the PlatformIO-LDF-style library resolver. + +## resolve_cold + +End-to-end cold-path measurement of `resolve()` against a synthetic +~30-library framework tree (Teensyduino-class) built with `MiniFramework`. A +5-deep transitive include chain forces the two-pass LDF reconciliation; the +remaining libraries are unreferenced and must be rejected — that doubles as a +guard against the #204 over-selection regression. Walks the tempdir on every +iteration since no cache sits in front of `resolve()` today (Phase 4 +memoization waits on zccache#130). + +The Phase 7 P-02 threshold from FastLED/fbuild#205 is **≤ 200 ms cold for a +typical teensy41 project**. This bench captures the baseline; future PRs gate +against it. + +Run: + +```bash +uv run soldr cargo bench -p fbuild-library-select --bench resolve_cold +``` diff --git a/crates/fbuild-library-select/benches/resolve_cold.rs b/crates/fbuild-library-select/benches/resolve_cold.rs new file mode 100644 index 00000000..4beeb128 --- /dev/null +++ b/crates/fbuild-library-select/benches/resolve_cold.rs @@ -0,0 +1,92 @@ +//! P-02 cold-resolver benchmark. +//! +//! Phase 7 of sets a ≤ 200 ms +//! cold budget for `resolve()` on a typical teensy41-class project. This +//! benchmark builds a synthetic ~30-library framework tree (matches +//! Teensyduino's library count) on a `MiniFramework` tempdir and walks it +//! end-to-end on every iteration. No cache layer sits in front of `resolve()` +//! today (Phase 4 cache memoization waits on zccache#130), so each iteration +//! is genuinely cold from the resolver's point of view — the OS page cache +//! warms up after the first run, but the resolver still re-canonicalizes +//! every include dir and re-walks every seed. +//! +//! Run with: +//! +//! ```text +//! uv run soldr cargo bench -p fbuild-library-select --bench resolve_cold +//! ``` +//! +//! Future PRs gate against the baseline number this captures. + +use std::path::PathBuf; + +use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; +use fbuild_library_select::resolve; +use fbuild_packages::library::framework_library::discover_framework_libraries; +use fbuild_packages::library::FrameworkLibrary; +use fbuild_test_support::MiniFramework; + +/// Approximate Teensyduino library count. Exact value isn't load-bearing — we +/// just want the fixture to be in the right order of magnitude so the bench +/// surfaces the same allocation/IO patterns a real teensy41 project hits. +const LIB_COUNT: usize = 30; + +/// Length of the transitive include chain rooted at `Lib00`. The other +/// `LIB_COUNT - CHAIN_LEN` libraries are unreferenced and must NOT be selected +/// — that's the #204 regression case the resolver is built around. Keeping +/// some unselected libs in the fixture means the bench measures the cost of +/// rejecting them too, not just the cost of the selected closure. +const CHAIN_LEN: usize = 5; + +fn build_fixture() -> ( + MiniFramework, + Vec, + Vec, + Vec, +) { + let mut mf = MiniFramework::new(); + for i in 0..LIB_COUNT { + let name = format!("Lib{i:02}"); + let next = if i + 1 < CHAIN_LEN { + Some(format!("Lib{:02}", i + 1)) + } else { + None + }; + let header = if let Some(n) = &next { + format!("#pragma once\n#include <{n}.h>\n") + } else { + "#pragma once\n".to_string() + }; + let cpp = format!("#include <{name}.h>\nvoid {name}_func() {{}}\n"); + mf.add_library(&name).header(&header).cpp(&cpp).done(); + } + mf.sketch("#include \nvoid setup() {}\nvoid loop() {}\n"); + + let libs = discover_framework_libraries(&mf.libraries_dir()); + let seeds = mf.project_seeds(); + let search_paths = mf.project_search_paths(); + (mf, seeds, search_paths, libs) +} + +fn bench_resolve(c: &mut Criterion) { + let (mf, seeds, search_paths, libs) = build_fixture(); + let mut group = c.benchmark_group("resolve"); + group.throughput(Throughput::Elements(libs.len() as u64)); + group.bench_function("cold_30_libs_chain_5", |b| { + b.iter(|| { + let sel = resolve( + black_box(&seeds), + black_box(&search_paths), + black_box(&libs), + ); + black_box(sel); + }); + }); + group.finish(); + // Keep `mf` alive until after the bench so the temp dir doesn't get + // cleaned out from under `resolve()` mid-run. + drop(mf); +} + +criterion_group!(benches, bench_resolve); +criterion_main!(benches);