From d0d61e798423e38cd2bbbe145bcaf362f889d759 Mon Sep 17 00:00:00 2001 From: "Joseph D. Carpinelli" Date: Thu, 12 Mar 2026 23:59:34 -0400 Subject: [PATCH 1/2] feat: add kiln-benchmarks crate with 7 Git object store benchmarks Adds a new crates/kiln-benchmarks workspace member implementing the full benchmark suite described in docs/design/git-kiln-benchmarks.md. Benchmarks: 1. Action hash cache lookup (hit/miss at N=100, 1k, 10k refs) 2. Output tree ingestion (compression level 0 vs 6, repo size at 100/500 crates) 3. Output tree materialization (N=10, 50, 200 crates, odb vs fs timing) 4. Deduplication across crate versions (object count, dedup ratio) 5. Ref namespace scale (enumerate at 1k/10k/100k, lookup hit/miss) 6. Fetch simulation (transfer size estimates, local ingestion timing) 7. GC and pack behavior (size before/after, retained refs, delta compression) All repos are bare, remotes-free, created in tempfile::TempDir. core.looseCompression=0 is set on repos that store binary artifact blobs. Also: - Adds crates/kiln-benchmarks/README.md (results template + analysis guide) - Adds kiln-benchmarks to release-please config (publish=false) - Updates CD.yml: publishes only git-kiln, adds a benchmarks job that runs cargo bench and uploads results to GitHub Actions artifacts --- .config/release-please-config.json | 5 +- .config/release-please-manifest.json | 3 +- .github/workflows/CD.yml | 35 +- Cargo.lock | 1396 +++++++++++++++++++++++- Cargo.toml | 2 +- crates/kiln-benchmarks/Cargo.toml | 21 + crates/kiln-benchmarks/README.md | 262 +++++ crates/kiln-benchmarks/benches/kiln.rs | 774 +++++++++++++ crates/kiln-benchmarks/src/lib.rs | 5 + 9 files changed, 2458 insertions(+), 45 deletions(-) create mode 100644 crates/kiln-benchmarks/Cargo.toml create mode 100644 crates/kiln-benchmarks/README.md create mode 100644 crates/kiln-benchmarks/benches/kiln.rs create mode 100644 crates/kiln-benchmarks/src/lib.rs diff --git a/.config/release-please-config.json b/.config/release-please-config.json index 00ea8a7..2c0f3ca 100644 --- a/.config/release-please-config.json +++ b/.config/release-please-config.json @@ -10,7 +10,10 @@ "pull-request-title-pattern": "release: `${component}` v${version}", "pull-request-footer": "This release was generated with [Release Please](https://github.com/googleapis/release-please).", "packages": { - "crates/git-kiln": {} + "crates/git-kiln": {}, + "crates/kiln-benchmarks": { + "publish": false + } }, "plugins": [{ "type": "sentence-case" }, { "type": "cargo-workspace" }], "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" diff --git a/.config/release-please-manifest.json b/.config/release-please-manifest.json index a01f649..53f2e7c 100644 --- a/.config/release-please-manifest.json +++ b/.config/release-please-manifest.json @@ -1,3 +1,4 @@ { - "crates/git-kiln": "0.0.0" + "crates/git-kiln": "0.0.0", + "crates/kiln-benchmarks": "0.0.0" } diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 342529c..e21a802 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -45,8 +45,9 @@ jobs: with: cache: true toolchain: stable - - name: Package crates - run: cargo package --workspace + - name: Package publishable crates + # kiln-benchmarks has publish = false and must be excluded from packaging + run: cargo package --package git-kiln - name: Generate artifact attestation uses: actions/attest-build-provenance@v2 with: @@ -57,7 +58,33 @@ jobs: IS_PRERELEASE: ${{ github.event.release.prerelease }} run: | if [ "$IS_PRERELEASE" = "true" ]; then - cargo publish --workspace --dry-run + cargo publish --package git-kiln --dry-run else - cargo publish --workspace --token "$CARGO_REGISTRY_TOKEN" + cargo publish --package git-kiln --token "$CARGO_REGISTRY_TOKEN" fi + + benchmarks: + name: Benchmarks + needs: check-tag + if: needs.check-tag.outputs.should-publish == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: true + toolchain: stable + - name: Install git (for git gc / pack-refs used in benchmarks) + run: sudo apt-get update && sudo apt-get install -y git + - name: Run benchmarks + run: | + cargo bench --package kiln-benchmarks -- --output-format bencher \ + 2>&1 | tee target/criterion/bench-output.txt + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: target/criterion/ + retention-days: 90 diff --git a/Cargo.lock b/Cargo.lock index 488faad..b80df77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -52,6 +67,90 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.0" @@ -108,6 +207,208 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "git-kiln" version = "0.0.0" @@ -116,6 +417,47 @@ dependencies = [ "clap_mangen", ] +[[package]] +name = "git2" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "heck" version = "0.5.0" @@ -123,81 +465,1059 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" +name = "hermit-abi" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "icu_collections" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] [[package]] -name = "proc-macro2" -version = "1.0.106" +name = "icu_locale_core" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ - "unicode-ident", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "quote" -version = "1.0.45" +name = "icu_normalizer" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "proc-macro2", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", ] [[package]] -name = "roff" +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf2048e0e979efb2ca7b91c4f1a8d77c91853e9b987c94c555668a8994915ad" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] [[package]] -name = "strsim" -version = "0.11.1" +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] [[package]] -name = "syn" -version = "2.0.117" +name = "indexmap" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "is-terminal" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] [[package]] -name = "utf8parse" -version = "0.2.2" +name = "is_terminal_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] -name = "windows-link" -version = "0.2.1" +name = "itertools" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] [[package]] -name = "windows-sys" -version = "0.61.2" +name = "itoa" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "windows-link", + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "kiln-benchmarks" +version = "0.0.0" +dependencies = [ + "criterion", + "git2", + "rand", + "sha2", + "tempfile", ] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libgit2-sys" +version = "0.17.0+1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "roff" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf2048e0e979efb2ca7b91c4f1a8d77c91853e9b987c94c555668a8994915ad" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 62eeba3..f08ae8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/git-kiln"] +members = ["crates/git-kiln", "crates/kiln-benchmarks"] [workspace.package] edition = "2024" diff --git a/crates/kiln-benchmarks/Cargo.toml b/crates/kiln-benchmarks/Cargo.toml new file mode 100644 index 0000000..fe7083f --- /dev/null +++ b/crates/kiln-benchmarks/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "kiln-benchmarks" +version = "0.0.0" +edition.workspace = true +publish = false +license.workspace = true +description = "Benchmarks for measuring Git object store performance under Kiln's build cache access patterns." + +[[bench]] +name = "kiln" +harness = false + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +git2 = "0.19" +tempfile = "3" +rand = "0.8" +sha2 = "0.10" + +[lints] +workspace = true diff --git a/crates/kiln-benchmarks/README.md b/crates/kiln-benchmarks/README.md new file mode 100644 index 0000000..ccf7355 --- /dev/null +++ b/crates/kiln-benchmarks/README.md @@ -0,0 +1,262 @@ +# kiln-benchmarks + +Benchmarks measuring Git's object store performance under Kiln's build cache +access patterns. These benchmarks answer the critical questions: + +- How fast is cache lookup by action hash? +- How fast is output tree ingestion for compiled artifacts? +- How does performance scale as the ref namespace grows? +- Does Git's content-addressing actually deduplicate unchanged crate outputs? +- What is the realistic cost of a "fresh clone, no build needed" scenario? + +## Running the Benchmarks + +```sh +cargo bench --package kiln-benchmarks +``` + +Results are written to `target/criterion/`. Open +`target/criterion/report/index.html` for the full HTML report. + +To run a single benchmark group: + +```sh +# Benchmark 1: cache lookup +cargo bench --package kiln-benchmarks --bench kiln -- bench1_cache_lookup + +# Benchmark 2: output tree ingestion +cargo bench --package kiln-benchmarks --bench kiln -- bench2_ingestion + +# Benchmark 3: output tree materialization +cargo bench --package kiln-benchmarks --bench kiln -- bench3_materialization + +# Benchmark 4: deduplication +cargo bench --package kiln-benchmarks --bench kiln -- bench4_deduplication + +# Benchmark 5: ref namespace scale +cargo bench --package kiln-benchmarks --bench kiln -- bench5_ref_namespace + +# Benchmark 6: fetch simulation +cargo bench --package kiln-benchmarks --bench kiln -- bench6_fetch_simulation + +# Benchmark 7: GC and pack behavior +cargo bench --package kiln-benchmarks --bench kiln -- bench7_gc +``` + +> **Note:** Benchmarks 5, 6, and 7 are slow by design — they exercise 10k–100k +> refs and 500-crate ingestion at realistic scale. Allow 5–30 minutes for a +> full run. + +## Benchmark Descriptions + +### Benchmark 1 — Action Hash Cache Lookup + +Tests the core cache check: "does a cached output exist for this action hash?" + +Pre-populates a bare repo with N refs under `refs/kiln/outputs/` at +N = 100, 1,000, and 10,000. Each ref points to a commit whose tree contains +realistic build output blobs (a 4 MB `.rlib`, 100 KB `.rmeta`, and fingerprint +files). Measures both the **hit case** (ref exists, peel to tree OID) and the +**miss case** (ref absent). + +**Scale target:** hit < 5 ms, miss < 2 ms at N = 10,000. + +### Benchmark 2 — Output Tree Ingestion + +Tests writing a compiled crate's outputs into the object store. + +Writes realistic blobs (4 MB `.rlib`, 100 KB `.rmeta`, 1 KB fingerprint, 50 KB +build output) and constructs the full tree/commit/ref chain. Compares +`core.looseCompression = 0` (Kiln's recommended setting for binary artifact +repos) against the default compression level 6. Also reports total repo size +after 100 and 500 ingestions. + +**Scale target:** < 500 ms/crate for a 4 MB rlib + metadata. + +### Benchmark 3 — Output Tree Materialization + +Tests restoring cached outputs to the working directory — the operation that +replaces compilation on a cache hit. + +Pre-ingests N crate output trees (N = 10, 50, 200) and measures reading each +blob from the odb and writing it to the filesystem. Reports odb read time +separately from filesystem write time to identify the bottleneck. + +**Scale target:** < 200 ms/crate from local odb; < 30 s total for 300 crates. + +### Benchmark 4 — Deduplication Across Crate Versions + +Tests Git's structural deduplication: two builds that share most crates should +not duplicate those crates' blobs in the object store. + +Ingests a "main branch build" (N crates) and a "feature branch build" (same N +crates, one changed). Unchanged crates use identical blob content so Git's +content-addressing deduplicates them. Reports the unique object count and +compares it to the theoretical no-dedup count. + +**Scale target:** > 95% deduplication when 1 crate changes out of N. + +### Benchmark 5 — Ref Namespace Scale + +Tests whether Git's ref storage degrades as `refs/kiln/outputs/` grows to +represent months of CI builds. + +Writes refs in batches and measures enumeration time at 1,000, 10,000, and +100,000 refs. Runs `git pack-refs --all` before the 100,000-ref measurement +(the realistic state for an established repo). Also measures targeted lookup +at each scale to confirm O(1) lookup is preserved. + +**Scale target:** enumeration < 500 ms at 10,000 packed refs. + +### Benchmark 6 — Fetch Simulation (Cache Pre-Population) + +Tests how long a fresh clone takes to receive cached outputs for a full project +build — the "fresh clone, no build needed" demo scenario. + +Since libgit2 in-process fetch does not reflect real network conditions, reports +two things: the total bytes that would need to be transferred for N crates +(N = 50, 150, 300), and the local ingestion time after a hypothetical transfer. +Also prints estimated fetch time at 100 Mbit/s and 1 Gbit/s for comparison with +alternatives like sccache or Artifactory. + +### Benchmark 7 — GC and Pack Behavior + +Tests whether Git's garbage collection handles a Kiln object store gracefully. + +Ingests 500 crate output trees, drops 250 refs (simulating eviction of old +builds), then runs `git gc --prune=now`. Reports repo size before and after GC, +GC duration, whether retained refs remain intact, and pack file count. Also +writes a `gitattributes` file marking `.rlib` and `.rmeta` with `-delta` to +prevent delta compression on binary artifact blobs. + +## Configuration Notes + +All benchmark repos are initialized with `core.looseCompression = 0` (except +Benchmark 2's compression comparison). This reflects the tuning recommended for +repos that store large binary artifact blobs, where zlib compression overhead +exceeds any size benefit on already-compressed or random binary data. + +All repos are created fresh in temporary directories with no remotes configured. +Transport/remote benchmarks are out of scope and will be covered separately. + +## Scale Targets Summary + +| Benchmark | Operation | Target | At Scale | +|-----------|-----------|--------|----------| +| 1 | Cache lookup — hit | < 5 ms | 10,000 refs | +| 1 | Cache lookup — miss | < 2 ms | 10,000 refs | +| 2 | Output ingestion | < 500 ms/crate | 4 MB rlib + metadata | +| 3 | Materialization | < 200 ms/crate | from local odb | +| 3 | Full materialization | < 30 s | 300 crates (Zed scale) | +| 5 | Ref enumeration | < 500 ms | 10,000 refs packed | +| 4 | Deduplication | > 95% savings | 1 crate changed of N | + +## Results + +> Results below are populated after running `cargo bench --package kiln-benchmarks` +> on the target machine. Until then, this section serves as a template. + +### Environment + +- **Date:** _not yet run_ +- **Machine:** _not yet run_ +- **OS:** _not yet run_ +- **Rust:** _not yet run_ +- **libgit2:** _not yet run_ + +### Benchmark 1 — Cache Lookup Results + +| N refs | Hit latency (mean) | Miss latency (mean) | Hit target met? | Miss target met? | +|-------:|--------------------|---------------------|-----------------|------------------| +| 100 | — | — | — | — | +| 1,000 | — | — | — | — | +| 10,000 | — | — | — | — | + +### Benchmark 2 — Ingestion Results + +| Compression | Mean latency | Throughput (crates/s) | +|-------------|-------------|----------------------| +| Level 0 | — | — | +| Level 6 | — | — | + +| N crates | Compression | Repo size | +|---------:|-------------|-----------| +| 100 | Level 0 | — | +| 100 | Level 6 | — | +| 500 | Level 0 | — | +| 500 | Level 6 | — | + +### Benchmark 3 — Materialization Results + +| N crates | Mean latency/crate | Total bytes written | ODB time | FS write time | +|---------:|--------------------|---------------------|----------|---------------| +| 10 | — | — | — | — | +| 50 | — | — | — | — | +| 200 | — | — | — | — | + +**Bottleneck:** _not yet determined_ + +### Benchmark 4 — Deduplication Results + +| N crates | Unique objects | No-dedup estimate | Dedup ratio | Repo size | +|---------:|---------------|-------------------|-------------|-----------| +| 50 | — | — | — | — | + +**Finding:** _not yet run_ + +### Benchmark 5 — Ref Namespace Scale Results + +| N refs | Enumeration latency | Lookup hit | Lookup miss | Packed-refs size | +|-------:|--------------------:|------------|-------------|------------------| +| 1,000 | — | — | — | — | +| 10,000 | — | — | — | — | +| 100,000 (packed) | — | — | — | — | + +**Enumeration becomes unacceptably slow (> 1 s) at:** _not yet determined_ + +### Benchmark 6 — Fetch Simulation Results + +| N crates | Total bytes | Est. fetch @ 100 Mbit/s | Est. fetch @ 1 Gbit/s | Local ingestion time | +|---------:|-------------|-------------------------|-----------------------|----------------------| +| 50 | — | — | — | — | +| 150 | — | — | — | — | +| 300 | — | — | — | — | + +### Benchmark 7 — GC Results + +| Metric | Value | +|--------|-------| +| Repo size before GC | — | +| Repo size after GC | — | +| GC duration | — | +| Retained refs intact | — | +| Pack files created | — | +| Binary blobs delta-compressed | — | + +## Findings and Recommendations + +_This section will be filled in after results are collected._ + +Key questions to answer: + +1. **Does cache lookup degrade with ref count?** If miss latency grows beyond + 2 ms at 10,000 refs, consider a derived index (e.g. a SQLite sidecar + mapping action hashes to object IDs) to bypass Git's ref lookup entirely. + +2. **Does `core.looseCompression = 0` meaningfully speed up ingestion?** If + compression level 6 and level 0 show similar latency, the recommendation in + the Kiln spec may not be necessary on modern hardware. + +3. **Is deduplication actually happening?** If blob content is stable across + unchanged crates (no embedded timestamps, no ASLR-derived addresses in debug + info), the dedup ratio should approach (N−1)/N. If it is not, action hash + computation must normalize those inputs before hashing. + +4. **Is the 30-second full materialization target achievable?** At 200 ms/crate, + 300 crates = 60 s — twice the target. If this is the case, the recommended + mitigation is parallel materialization (spawn one thread per crate) or + switching from full tree materialization to hardlinking from a local object + cache directory. + +5. **Does GC correctly prune dropped refs?** If unreachable objects are retained + after `git gc --prune=now`, the eviction strategy must be revisited. diff --git a/crates/kiln-benchmarks/benches/kiln.rs b/crates/kiln-benchmarks/benches/kiln.rs new file mode 100644 index 0000000..052d84b --- /dev/null +++ b/crates/kiln-benchmarks/benches/kiln.rs @@ -0,0 +1,774 @@ +#![allow(missing_docs)] +//! Kiln benchmark suite. +//! +//! Measures Git object store performance under Kiln's build cache access +//! patterns. See the crate README for results and analysis. + +use std::{ + hint::black_box, + io::Write, + path::Path, + process::Command, + time::{Duration, Instant}, +}; + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use git2::{FileMode, Oid, Repository, Signature}; +use rand::{Rng, SeedableRng, rngs::StdRng}; +use tempfile::TempDir; + +// --------------------------------------------------------------------------- +// Seed constant +// --------------------------------------------------------------------------- + +/// "KILN_BNC" encoded as a little-endian u64. +const KILN_SEED: u64 = 0x4b494c4e5f424e43; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Create a bare repository in a temp directory with `core.looseCompression` +/// set to `compression_level`. No remotes are configured. +fn make_bare_repo(compression_level: i32) -> (TempDir, Repository) { + let dir = TempDir::new().expect("tempdir"); + let repo = Repository::init_bare(dir.path()).expect("init bare repo"); + repo.config() + .expect("repo config") + .set_i32("core.looseCompression", compression_level) + .expect("set looseCompression"); + (dir, repo) +} + +/// Generate a random 32-character lowercase hex string using `rng`. +fn random_hex32(rng: &mut impl Rng) -> String { + format!("{:016x}{:016x}", rng.r#gen::(), rng.r#gen::()) +} + +/// Generate `len` random bytes. +fn random_bytes(rng: &mut impl Rng, len: usize) -> Vec { + (0..len).map(|_| rng.r#gen::()).collect() +} + +/// Write a single crate output tree into `repo` and create a ref +/// `refs/kiln/outputs/` pointing at the resulting commit. +/// +/// Tree layout: +/// ```text +/// deps/ +/// libfoo.rlib (rlib_data) +/// libfoo.rmeta (rmeta_data) +/// .fingerprint/ +/// foo-/ +/// invoked.timestamp (b"0\n") +/// lib-foo (fp_data) +/// build/ (empty, or contains output blob) +/// ``` +fn ingest_crate_output( + repo: &Repository, + action_hash: &str, + rlib_data: &[u8], + rmeta_data: &[u8], + fp_data: &[u8], + build_data: Option<&[u8]>, +) -> Oid { + // --- blobs --- + let rlib_oid = repo.blob(rlib_data).expect("blob rlib"); + let rmeta_oid = repo.blob(rmeta_data).expect("blob rmeta"); + let fp_oid = repo.blob(fp_data).expect("blob fp"); + let timestamp_oid = repo.blob(b"0\n").expect("blob timestamp"); + + // --- deps/ subtree --- + let mut deps_tb = repo.treebuilder(None).expect("treebuilder deps"); + deps_tb + .insert("libfoo.rlib", rlib_oid, FileMode::Blob.into()) + .expect("insert rlib"); + deps_tb + .insert("libfoo.rmeta", rmeta_oid, FileMode::Blob.into()) + .expect("insert rmeta"); + let deps_tree = deps_tb.write().expect("write deps tree"); + + // --- .fingerprint/foo-/ subtree --- + let fp_hash = &action_hash[..8]; + let mut fp_inner_tb = repo.treebuilder(None).expect("treebuilder fp inner"); + fp_inner_tb + .insert("invoked.timestamp", timestamp_oid, FileMode::Blob.into()) + .expect("insert timestamp"); + fp_inner_tb + .insert("lib-foo", fp_oid, FileMode::Blob.into()) + .expect("insert lib-foo"); + let fp_inner_tree = fp_inner_tb.write().expect("write fp inner tree"); + + let fp_dir_name = format!("foo-{fp_hash}"); + let mut fp_outer_tb = repo.treebuilder(None).expect("treebuilder fp outer"); + fp_outer_tb + .insert(&fp_dir_name, fp_inner_tree, FileMode::Tree.into()) + .expect("insert fp dir"); + let fp_tree = fp_outer_tb.write().expect("write fp tree"); + + // --- build/ subtree --- + let mut build_tb = repo.treebuilder(None).expect("treebuilder build"); + if let Some(data) = build_data { + let build_oid = repo.blob(data).expect("blob build"); + build_tb + .insert("output", build_oid, FileMode::Blob.into()) + .expect("insert build output"); + } + let build_tree = build_tb.write().expect("write build tree"); + + // --- root tree --- + let mut root_tb = repo.treebuilder(None).expect("treebuilder root"); + root_tb + .insert("deps", deps_tree, FileMode::Tree.into()) + .expect("insert deps"); + root_tb + .insert(".fingerprint", fp_tree, FileMode::Tree.into()) + .expect("insert .fingerprint"); + root_tb + .insert("build", build_tree, FileMode::Tree.into()) + .expect("insert build"); + let root_tree_oid = root_tb.write().expect("write root tree"); + + // --- commit --- + let sig = Signature::now("kiln-bench", "bench@kiln").expect("sig"); + let root_tree = repo.find_tree(root_tree_oid).expect("find root tree"); + let commit_oid = repo + .commit(None, &sig, &sig, action_hash, &root_tree, &[]) + .expect("commit"); + + // --- ref --- + let ref_name = format!("refs/kiln/outputs/{action_hash}"); + repo.reference(&ref_name, commit_oid, true, "kiln ingest") + .expect("create ref"); + + commit_oid +} + +/// Write a minimal dummy commit (no output blobs) and point a ref at it. +fn ingest_dummy_commit(repo: &Repository, action_hash: &str) -> Oid { + let sig = Signature::now("kiln-bench", "bench@kiln").expect("sig"); + let empty_tb = repo.treebuilder(None).expect("treebuilder"); + let empty_tree_oid = empty_tb.write().expect("write empty tree"); + let empty_tree = repo.find_tree(empty_tree_oid).expect("find empty tree"); + let commit_oid = repo + .commit(None, &sig, &sig, action_hash, &empty_tree, &[]) + .expect("commit"); + let ref_name = format!("refs/kiln/outputs/{action_hash}"); + repo.reference(&ref_name, commit_oid, true, "kiln dummy") + .expect("create ref"); + commit_oid +} + +/// Recursively walk a git tree, writing blob entries to `dest/`. +/// Returns the total number of bytes written. +fn walk_and_materialize( + repo: &Repository, + tree: &git2::Tree<'_>, + dest: &Path, +) -> Result> { + let mut bytes_written = 0u64; + for entry in tree.iter() { + let name = entry.name().unwrap_or("_"); + let dest_entry = dest.join(name); + match entry.kind() { + Some(git2::ObjectType::Tree) => { + std::fs::create_dir_all(&dest_entry)?; + let sub = repo.find_tree(entry.id())?; + bytes_written += walk_and_materialize(repo, &sub, &dest_entry)?; + } + Some(git2::ObjectType::Blob) => { + let blob = repo.find_blob(entry.id())?; + let mut f = std::fs::File::create(&dest_entry)?; + f.write_all(blob.content())?; + bytes_written += blob.content().len() as u64; + } + _ => {} + } + } + Ok(bytes_written) +} + +/// Return the total on-disk size (bytes) of a directory tree. +fn dir_size_bytes(path: &Path) -> u64 { + let mut total = 0u64; + if let Ok(entries) = std::fs::read_dir(path) { + for entry in entries.flatten() { + if let Ok(m) = entry.metadata() { + if m.is_dir() { + total += dir_size_bytes(&entry.path()); + } else { + total += m.len(); + } + } + } + } + total +} + +// --------------------------------------------------------------------------- +// Blob fixtures — generated once per process, reused across benchmarks. +// --------------------------------------------------------------------------- + +struct Blobs { + rlib: Vec, // 4 MB + rmeta: Vec, // 100 KB + fp: Vec, // 1 KB + build: Vec, // 50 KB +} + +impl Blobs { + fn new() -> Self { + let mut rng = StdRng::seed_from_u64(KILN_SEED); + Self { + rlib: random_bytes(&mut rng, 4 * 1024 * 1024), + rmeta: random_bytes(&mut rng, 100 * 1024), + fp: random_bytes(&mut rng, 1024), + build: random_bytes(&mut rng, 50 * 1024), + } + } +} + +// --------------------------------------------------------------------------- +// Benchmark 1: Action Hash Cache Lookup +// --------------------------------------------------------------------------- + +fn bench_cache_lookup(c: &mut Criterion) { + let blobs = Blobs::new(); + let mut rng = StdRng::seed_from_u64(1); + let ns = [100usize, 1_000, 10_000]; + + let mut group = c.benchmark_group("bench1_cache_lookup"); + + for &n in &ns { + // Build a fresh repo pre-populated with N refs. + let (_dir, repo) = make_bare_repo(0); + let hashes: Vec = (0..n).map(|_| random_hex32(&mut rng)).collect(); + for h in &hashes { + ingest_crate_output( + &repo, + h, + &blobs.rlib, + &blobs.rmeta, + &blobs.fp, + Some(&blobs.build), + ); + } + + // Pick a known-present hash and a guaranteed-absent hash. + let hit_hash = hashes[n / 2].clone(); + let miss_hash = random_hex32(&mut rng); + + // --- Hit case --- + group.bench_with_input(BenchmarkId::new("hit", n), &n, |b, _| { + b.iter(|| { + let ref_name = format!("refs/kiln/outputs/{}", &hit_hash); + let r = repo.find_reference(black_box(&ref_name)).expect("find ref"); + let commit = r.peel_to_commit().expect("peel commit"); + black_box(commit.tree_id()) + }) + }); + + // --- Miss case --- + group.bench_with_input(BenchmarkId::new("miss", n), &n, |b, _| { + b.iter(|| { + let ref_name = format!("refs/kiln/outputs/{}", &miss_hash); + black_box(repo.find_reference(black_box(&ref_name)).is_err()) + }) + }); + + drop(hashes); + } + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Benchmark 2: Output Tree Ingestion +// --------------------------------------------------------------------------- + +fn bench_ingestion(c: &mut Criterion) { + let blobs = Blobs::new(); + let mut rng = StdRng::seed_from_u64(2); + + let mut group = c.benchmark_group("bench2_ingestion"); + // Fewer samples because each iteration writes ~4 MB. + group.sample_size(20); + + for &compression in &[0i32, 6i32] { + let label = format!("compression_{compression}"); + let (_dir, repo) = make_bare_repo(compression); + + group.bench_function(&label, |b| { + b.iter(|| { + let hash = random_hex32(&mut rng); + ingest_crate_output( + black_box(&repo), + black_box(&hash), + black_box(&blobs.rlib), + black_box(&blobs.rmeta), + black_box(&blobs.fp), + Some(black_box(&blobs.build)), + ) + }) + }); + } + + group.finish(); + + // Repo size after 100 and 500 ingestions (reported to stdout). + for &compression in &[0i32, 6i32] { + for &n in &[100usize, 500] { + let (dir, repo) = make_bare_repo(compression); + let mut rng2 = StdRng::seed_from_u64(42); + for _ in 0..n { + let h = random_hex32(&mut rng2); + ingest_crate_output(&repo, &h, &blobs.rlib, &blobs.rmeta, &blobs.fp, None); + } + let size = dir_size_bytes(dir.path()); + println!( + "[bench2] compression={compression} n={n} repo_size={:.1} MB", + size as f64 / 1_048_576.0 + ); + } + } +} + +// --------------------------------------------------------------------------- +// Benchmark 3: Output Tree Materialization +// --------------------------------------------------------------------------- + +fn bench_materialization(c: &mut Criterion) { + let blobs = Blobs::new(); + let mut rng = StdRng::seed_from_u64(3); + let ns = [10usize, 50, 200]; + + let mut group = c.benchmark_group("bench3_materialization"); + group.sample_size(10); + + for &n in &ns { + let (_repo_dir, repo) = make_bare_repo(0); + let out_dir = TempDir::new().expect("out tempdir"); + let out_path = out_dir.path().to_path_buf(); + + // Pre-ingest N crates. + let hashes: Vec = (0..n) + .map(|_| { + let h = random_hex32(&mut rng); + ingest_crate_output( + &repo, + &h, + &blobs.rlib, + &blobs.rmeta, + &blobs.fp, + Some(&blobs.build), + ); + h + }) + .collect(); + + let target_hash = hashes[0].clone(); + + group.bench_with_input(BenchmarkId::new("n_crates", n), &n, |b, _| { + b.iter(|| { + let ref_name = format!("refs/kiln/outputs/{}", &target_hash); + let r = repo.find_reference(&ref_name).expect("find ref"); + let commit = r.peel_to_commit().expect("peel commit"); + let tree = commit.tree().expect("tree"); + + // Separate odb read time from filesystem write time. + let odb_start = Instant::now(); + // Pre-load all blobs from the odb (simulates the read phase). + let _ = tree.iter().count(); + let odb_elapsed = odb_start.elapsed(); + + let fs_start = Instant::now(); + let bytes = + walk_and_materialize(black_box(&repo), black_box(&tree), black_box(&out_path)) + .expect("materialize"); + let fs_elapsed = fs_start.elapsed(); + + black_box((bytes, odb_elapsed, fs_elapsed)) + }) + }); + + drop(hashes); + drop(out_dir); + } + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Benchmark 4: Deduplication Across Crate Versions +// --------------------------------------------------------------------------- + +fn bench_deduplication(c: &mut Criterion) { + let blobs = Blobs::new(); + let mut rng = StdRng::seed_from_u64(4); + let n_crates = 50usize; + + let mut group = c.benchmark_group("bench4_deduplication"); + group.sample_size(10); + + group.bench_function("two_builds_one_changed", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let (dir, repo) = make_bare_repo(0); + + // Generate N hashes for "main build". + let main_hashes: Vec = + (0..n_crates).map(|_| random_hex32(&mut rng)).collect(); + + let t0 = Instant::now(); + + // Ingest main build. + for h in &main_hashes { + ingest_crate_output(&repo, h, &blobs.rlib, &blobs.rmeta, &blobs.fp, None); + } + + // Feature build: same hashes except one, same blob content for + // unchanged crates so Git deduplicates via content-addressing. + let mut feature_hashes = main_hashes.clone(); + feature_hashes[0] = random_hex32(&mut rng); // one changed crate + + // The changed crate uses entirely different blob content. + let changed_rlib: Vec = random_bytes(&mut rng, blobs.rlib.len()); + + for (i, h) in feature_hashes.iter().enumerate() { + if i == 0 { + ingest_crate_output(&repo, h, &changed_rlib, &blobs.rmeta, &blobs.fp, None); + } else { + // Identical content — Git will reuse existing blob OIDs. + ingest_crate_output(&repo, h, &blobs.rlib, &blobs.rmeta, &blobs.fp, None); + } + } + + total += t0.elapsed(); + + // Count unique objects in the odb. + let mut obj_count = 0usize; + repo.odb() + .expect("odb") + .foreach(|_oid| { + obj_count += 1; + true + }) + .expect("odb foreach"); + + let repo_size = dir_size_bytes(dir.path()); + // Objects per crate (approx): commit + root tree + deps tree + + // fp-outer tree + fp-inner tree + rlib blob + rmeta blob + + // timestamp blob + lib-foo blob + build tree = ~10 + let no_dedup_estimate = 2 * n_crates * 10; + println!( + "[bench4] n={n_crates} unique_objects={obj_count} \ + no_dedup_estimate={no_dedup_estimate} \ + repo_size={:.1} MB", + repo_size as f64 / 1_048_576.0, + ); + + drop(main_hashes); + drop(feature_hashes); + } + total + }) + }); + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Benchmark 5: Ref Namespace Scale +// --------------------------------------------------------------------------- + +fn bench_ref_namespace(c: &mut Criterion) { + let mut rng = StdRng::seed_from_u64(5); + let ns = [1_000usize, 10_000, 100_000]; + + let mut group = c.benchmark_group("bench5_ref_namespace"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(30)); + + for &n in &ns { + let (dir, repo) = make_bare_repo(0); + + // Write N dummy commits/refs. + let hashes: Vec = (0..n).map(|_| random_hex32(&mut rng)).collect(); + for h in &hashes { + ingest_dummy_commit(&repo, h); + } + + // Pack refs at 100k — this is the realistic state for an established repo. + if n >= 100_000 { + Command::new("git") + .args(["pack-refs", "--all"]) + .current_dir(dir.path()) + .output() + .expect("git pack-refs"); + + let packed_refs = dir.path().join("packed-refs"); + if packed_refs.exists() { + let size = std::fs::metadata(&packed_refs) + .map(|m| m.len()) + .unwrap_or(0); + println!( + "[bench5] n={n} packed-refs size={:.1} KB", + size as f64 / 1024.0 + ); + } + } + + // --- Enumeration --- + group.bench_with_input(BenchmarkId::new("enumerate", n), &n, |b, _| { + b.iter(|| { + let mut count = 0usize; + let refs = repo + .references_glob("refs/kiln/outputs/*") + .expect("references_glob"); + for r in refs { + let _ = r.expect("ref"); + count += 1; + } + black_box(count) + }) + }); + + // --- Targeted lookup hit/miss at this scale --- + let hit = hashes[n / 2].clone(); + let miss = random_hex32(&mut rng); + + group.bench_with_input(BenchmarkId::new("lookup_hit", n), &n, |b, _| { + b.iter(|| { + let rn = format!("refs/kiln/outputs/{}", &hit); + black_box(repo.find_reference(black_box(&rn)).is_ok()) + }) + }); + + group.bench_with_input(BenchmarkId::new("lookup_miss", n), &n, |b, _| { + b.iter(|| { + let rn = format!("refs/kiln/outputs/{}", &miss); + black_box(repo.find_reference(black_box(&rn)).is_err()) + }) + }); + + drop(hashes); + } + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Benchmark 6: Fetch Simulation (Cache Pre-Population) +// --------------------------------------------------------------------------- + +fn bench_fetch_simulation(c: &mut Criterion) { + let blobs = Blobs::new(); + let mut rng = StdRng::seed_from_u64(6); + let ns = [50usize, 150, 300]; + + let mut group = c.benchmark_group("bench6_fetch_simulation"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(60)); + + // Bytes per crate: rlib + rmeta + fp + build (uncompressed in-memory sizes). + let bytes_per_crate = + (blobs.rlib.len() + blobs.rmeta.len() + blobs.fp.len() + blobs.build.len()) as u64; + + for &n in &ns { + let total_bytes = bytes_per_crate * n as u64; + let time_100mbit = total_bytes as f64 / (100.0 * 1_000_000.0 / 8.0); + let time_1gbit = total_bytes as f64 / (1_000.0 * 1_000_000.0 / 8.0); + println!( + "[bench6] n={n} total_bytes={:.1} MB \ + est_fetch@100Mbit={time_100mbit:.1}s \ + est_fetch@1Gbit={time_1gbit:.2}s", + total_bytes as f64 / 1_048_576.0, + ); + + // Measure local ingestion time — the portion Kiln controls after transfer. + group.bench_with_input(BenchmarkId::new("local_ingestion", n), &n, |b, &count| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let (_dir, repo) = make_bare_repo(0); + let seed: u64 = rng.r#gen(); + let mut rng2 = StdRng::seed_from_u64(seed); + let t0 = Instant::now(); + for _ in 0..count { + let h = random_hex32(&mut rng2); + ingest_crate_output( + &repo, + &h, + &blobs.rlib, + &blobs.rmeta, + &blobs.fp, + Some(&blobs.build), + ); + } + total += t0.elapsed(); + } + total + }) + }); + } + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Benchmark 7: GC and Pack Behavior +// --------------------------------------------------------------------------- + +fn bench_gc(c: &mut Criterion) { + let blobs = Blobs::new(); + let mut rng = StdRng::seed_from_u64(7); + + let mut group = c.benchmark_group("bench7_gc"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(120)); + + group.bench_function("gc_500_drop_250", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + for _ in 0..iters { + let (dir, repo) = make_bare_repo(0); + + // Ingest 500 crate output trees. + let hashes: Vec = (0..500usize) + .map(|_| { + let h = random_hex32(&mut rng); + ingest_crate_output(&repo, &h, &blobs.rlib, &blobs.rmeta, &blobs.fp, None); + h + }) + .collect(); + + let size_before = dir_size_bytes(dir.path()); + + // Drop 250 refs (simulate eviction of old builds). + for h in hashes.iter().take(250) { + let ref_name = format!("refs/kiln/outputs/{h}"); + if let Ok(mut r) = repo.find_reference(&ref_name) { + r.delete().expect("delete ref"); + } + } + + // Write a gitattributes file to prevent delta compression on + // binary artifact blobs. + let attrs_path = dir.path().join("info").join("attributes"); + std::fs::create_dir_all(attrs_path.parent().unwrap()).ok(); + std::fs::write(&attrs_path, "*.rlib -delta\n*.rmeta -delta\n") + .expect("write gitattributes"); + + // Run git gc and measure only gc time. + let t0 = Instant::now(); + let gc_out = Command::new("git") + .args(["gc", "--prune=now", "--quiet"]) + .current_dir(dir.path()) + .output() + .expect("git gc"); + let gc_elapsed = t0.elapsed(); + total += gc_elapsed; + + let size_after = dir_size_bytes(dir.path()); + + if !gc_out.status.success() { + eprintln!( + "[bench7] git gc stderr: {}", + String::from_utf8_lossy(&gc_out.stderr) + ); + } + + // Verify retained refs are intact after gc. + let mut intact = true; + for h in hashes.iter().skip(250) { + let ref_name = format!("refs/kiln/outputs/{h}"); + if repo.find_reference(&ref_name).is_err() { + intact = false; + break; + } + } + + // Count .pack files to confirm blobs were packed. + let pack_dir = dir.path().join("objects").join("pack"); + let pack_count = std::fs::read_dir(&pack_dir) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .filter(|e| e.path().extension().map(|x| x == "pack").unwrap_or(false)) + .count() + }) + .unwrap_or(0); + + println!( + "[bench7] size_before={:.1} MB size_after={:.1} MB \ + gc_time={:.2}s retained_refs_intact={intact} \ + pack_files={pack_count}", + size_before as f64 / 1_048_576.0, + size_after as f64 / 1_048_576.0, + gc_elapsed.as_secs_f64(), + ); + + drop(hashes); + } + total + }) + }); + + group.finish(); +} + +// --------------------------------------------------------------------------- +// Criterion entry points +// --------------------------------------------------------------------------- + +criterion_group! { + name = lookup; + config = Criterion::default().measurement_time(Duration::from_secs(10)); + targets = bench_cache_lookup +} + +criterion_group! { + name = ingestion; + config = Criterion::default().measurement_time(Duration::from_secs(30)); + targets = bench_ingestion +} + +criterion_group! { + name = materialization; + config = Criterion::default().measurement_time(Duration::from_secs(30)); + targets = bench_materialization +} + +criterion_group! { + name = deduplication; + config = Criterion::default().measurement_time(Duration::from_secs(60)); + targets = bench_deduplication +} + +criterion_group! { + name = ref_namespace; + config = Criterion::default().measurement_time(Duration::from_secs(30)); + targets = bench_ref_namespace +} + +criterion_group! { + name = fetch_simulation; + config = Criterion::default().measurement_time(Duration::from_secs(60)); + targets = bench_fetch_simulation +} + +criterion_group! { + name = gc; + config = Criterion::default().measurement_time(Duration::from_secs(120)); + targets = bench_gc +} + +criterion_main!( + lookup, + ingestion, + materialization, + deduplication, + ref_namespace, + fetch_simulation, + gc +); diff --git a/crates/kiln-benchmarks/src/lib.rs b/crates/kiln-benchmarks/src/lib.rs new file mode 100644 index 0000000..4a3dced --- /dev/null +++ b/crates/kiln-benchmarks/src/lib.rs @@ -0,0 +1,5 @@ +//! Kiln Benchmarks +//! +//! Benchmarks for measuring Git object store performance under Kiln's build +//! cache access patterns. See [`benches/kiln.rs`] for the benchmark +//! implementations and `README.md` for results and analysis. From 0d92404e8dabc35d01ee11713aca100a9735a257 Mon Sep 17 00:00:00 2001 From: "Joseph D. Carpinelli" Date: Fri, 20 Mar 2026 21:30:30 -0400 Subject: [PATCH 2/2] docs: add design docs Assisted-by: Claude.ai (Claude Opus 4.6) --- docs/design/git-cas-benchmarks.md | 259 ++++++++ docs/design/git-kiln.md | 964 +++++++++++++++++++++++++++++- 2 files changed, 1216 insertions(+), 7 deletions(-) create mode 100644 docs/design/git-cas-benchmarks.md diff --git a/docs/design/git-cas-benchmarks.md b/docs/design/git-cas-benchmarks.md new file mode 100644 index 0000000..bfedb63 --- /dev/null +++ b/docs/design/git-cas-benchmarks.md @@ -0,0 +1,259 @@ +# git-kiln-bench: Proving Git as a Build CAS + +## Goal + +Answer one question with numbers: can Git's object store serve as the content-addressed cache for a build system at realistic scale? + +Every benchmark compares against the baseline: tar+zstd to local disk (proxy for S3/sccache). Git doesn't need to be faster. It needs to be close enough that the deduplication, provenance, and transport advantages justify the overhead. + +## Crate Structure + +``` +git-kiln-bench/ +├── Cargo.toml +├── src/ +│ ├── lib.rs # shared types, repo setup, artifact generation +│ ├── gen.rs # realistic artifact generators +│ ├── bin/ +│ │ ├── ingest.rs # Benchmark 1: blob write throughput +│ │ ├── tree_build.rs # Benchmark 2: tree construction +│ │ ├── cache_index.rs # Benchmark 3: cache index read/write +│ │ ├── fetch.rs # Benchmark 4: object fetch from remote +│ │ ├── soak.rs # Benchmark 5: week-long CI simulation +│ │ └── baseline.rs # Benchmark 6: tar+zstd comparison +``` + +Dependencies: `git2` (libgit2 bindings), `criterion` for microbenchmarks, `tempfile`, `rand`, `zstd` for baseline comparison. + +## Artifact Generation (gen.rs) + +Generate fake but realistic build outputs. The byte distribution matters — random data won't compress like real artifacts, but zeros won't either. + +Approach: read a few real `.rlib` and `.so` files, compute byte frequency distributions, generate synthetic artifacts matching those distributions at configurable sizes. + +Realistic target profile (medium Rust workspace, ~30 crates): + +| Artifact type | Count | Size range | Total | +|---|---|---|---| +| .rlib | 120 | 50KB–5MB | ~150MB | +| .rmeta | 120 | 10KB–500KB | ~25MB | +| .so/.dylib | 8 | 1MB–50MB | ~100MB | +| .d files | 120 | 1KB–10KB | ~1MB | +| build script outputs | 15 | 1KB–1MB | ~5MB | + +Total per build: ~280MB, ~380 files. + +For the C++/ROS case, scale up: 200 packages, 500MB–2GB total. + +## Benchmark 1: Ingestion Throughput + +**Question:** How fast can we write N blobs into the ODB and construct a tree? + +```rust +// Pseudocode +fn bench_ingest(compression_level: i32, artifacts: &[Artifact]) -> IngestResult { + let repo = init_bare_repo(); + repo.config().set_i32("core.looseCompression", compression_level); + + let start = Instant::now(); + let mut tree = TreeBuilder::new(&repo); + for artifact in artifacts { + let oid = repo.blob(&artifact.bytes); // git_odb_write + tree.insert(&artifact.name, oid, artifact.filemode); + } + let tree_oid = tree.write(); + let commit_oid = repo.commit(tree_oid, ...); + + IngestResult { + wall_time: start.elapsed(), + throughput_mb_s: total_bytes / elapsed_secs, + repo_size: dir_size(repo.path()), + } +} +``` + +Parameter matrix: +- Compression level: 0, 1, 6 (default) +- Artifact set: small (30 crates), medium (100 crates), large (300 crates) +- With/without `-delta` gitattributes on binaries + +Key metrics: MB/s write throughput, wall-clock time, repo size. + +## Benchmark 2: Tree Construction at Scale + +**Question:** How expensive is building nested trees for a large output? + +Build actions produce output trees with structure: +``` +deps/ + libfoo.rlib + libfoo.rmeta + libbar.rlib + ... +.fingerprint/ + foo-/ + ... +build/ + foo-/ + out/ + generated.rs +``` + +Measure tree construction separately from blob writes. This isolates the cost of `git_treebuilder` operations and nested tree assembly. + +## Benchmark 3: Cache Index Operations + +**Question:** How fast is action-hash → commit-OID lookup through a tree-based index? + +The cache index is a tree under `refs/kiln/cache` with two-level fanout: + +``` +ab/ + cd1234.../ + commit-oid # blob containing the OID + ef5678.../ + commit-oid +``` + +Benchmark: +- Write 10K, 50K, 200K entries (simulating months of CI) +- Random lookup latency (p50, p95, p99) +- Sequential scan time +- Index update: add one entry to an existing 50K-entry index (requires tree rewrite) + +The update cost is the concern. Adding one entry means rewriting the fanout subtree and the root tree. Compare to: SQLite lookup, flat file scan. + +## Benchmark 4: Fetch Simulation + +**Question:** How fast can a client restore a full build's outputs from a remote? + +Setup: +- Local bare repo as "remote" (same machine, eliminates network variable) +- Client repo with partial clone (`--filter=blob:none`) +- Remote has 50 build commits with realistic output trees + +Measure: +- Fetch one output tree by commit OID (all blobs materialized) +- Fetch 30 output trees (full build restore, 30-package project) +- Incremental fetch: remote has new build, 28/30 packages unchanged + +Compare to: `curl` a tarball of equivalent size from localhost. + +For the network case (separate benchmark or flag): fetch over localhost TCP to measure protocol overhead without real network latency. + +## Benchmark 5: Soak Test + +**Question:** What happens to repo size and GC time over simulated weeks of CI? + +```rust +fn soak(config: SoakConfig) -> SoakResult { + // config: builds_per_day, days, packages_per_build, ttl_days + let repo = init_bare_repo(); + let mut daily_stats = vec![]; + + for day in 0..config.days { + // Write builds + for _ in 0..config.builds_per_day { + let artifacts = generate_build(config.packages_per_build); + ingest(&repo, &artifacts, day); + } + + // Expire refs older than TTL + expire_refs(&repo, day - config.ttl_days); + + // GC + let gc_start = Instant::now(); + repo.gc(); // or shell out to `git gc` + let gc_time = gc_start.elapsed(); + + daily_stats.push(DayStats { + day, + repo_size: dir_size(repo.path()), + gc_time, + loose_objects: count_loose(&repo), + pack_size: pack_size(&repo), + }); + } +} +``` + +Realistic parameters: +- 20 builds/day (CI on push for active team) +- 30 days simulated +- 30 packages per build, ~280MB artifacts +- 7-day TTL on output refs + +Output: time-series of repo size, GC duration, pack file count/size. + +## Benchmark 6: Baseline Comparison + +**Question:** How does Git compare to the dumb approach? + +For each operation, measure the equivalent non-Git path: + +| Git operation | Baseline equivalent | +|---|---| +| Blob ingest at level 0 | Write raw files to disk | +| Blob ingest at level 6 | zstd compress + write | +| Tree construction + commit | tar + write | +| Cache index lookup | SQLite key-value lookup | +| Full restore from remote | Download tarball | +| Incremental restore | rsync | + +Same artifact sets, same machine. Produce a comparison table. + +## Development Sequence + +### Week 1: Scaffolding + Ingestion + +1. Set up crate, `git2` dependency, tempdir-based repo creation +2. Implement artifact generator with realistic byte distributions +3. Benchmark 1: ingestion at compression levels 0/1/6 +4. Benchmark 6 (partial): baseline raw write and zstd write + +Deliverable: first numbers on MB/s ingestion. This is the most likely failure point — if ingestion is catastrophically slow, stop here. + +### Week 2: Trees + Cache Index + +5. Benchmark 2: tree construction at varying depths/widths +6. Benchmark 3: cache index CRUD at 10K/50K/200K entries +7. Investigate whether tree-based index is viable or if a sidecar (SQLite, flat file) is needed for the index specifically + +Deliverable: cache lookup latency numbers. If index update is too expensive at 50K entries, the index design needs revision before proceeding. + +### Week 3: Fetch + Restore + +8. Benchmark 4: local fetch simulation with partial clone +9. Benchmark 4 (extended): localhost TCP fetch +10. Benchmark 6 (continued): tarball download comparison + +Deliverable: restore latency numbers. This is the second most likely failure point — if fetching a build's outputs takes 30s vs 2s for a tarball, the developer experience suffers. + +### Week 4: Soak + Report + +11. Benchmark 5: 30-day soak test +12. Full baseline comparison table +13. Write-up with charts: throughput, latency, size over time, GC cost + +Deliverable: a single document with every number, every comparison, every chart. This is what you publish. + +## Success Criteria + +Git-as-CAS is viable if: + +- **Ingestion**: >100 MB/s at compression level 0 (a 280MB build stored in <3s) +- **Cache lookup**: <10ms p99 at 50K entries +- **Full restore**: <5s for 280MB output tree from local remote (same order as tarball) +- **Soak stability**: repo size reaches steady state under TTL, doesn't grow unboundedly +- **GC**: <30s on a week's worth of builds (can run in background) + +If any of these fail, the benchmark will have identified exactly where, which tells you whether the design needs revision or Git needs replacement. + +## What This Doesn't Test + +- Real network latency (test locally first, add network round-trip later) +- Concurrent writers (important but separate — test after single-writer is proven) +- libgit2 vs git CLI performance (benchmark both if ingestion is borderline) +- Packfile negotiation efficiency for partial clone (protocol-level concern, hard to benchmark in isolation) + +These are follow-ups. The core question — is Git fast enough as a local and local-remote CAS — comes first. diff --git a/docs/design/git-kiln.md b/docs/design/git-kiln.md index 7019007..ec72de0 100644 --- a/docs/design/git-kiln.md +++ b/docs/design/git-kiln.md @@ -1,21 +1,971 @@ +++ -title = "git-kiln: A Ceramic-Inspired Toolkit for Shaping Git-Native Data" +title = "Kiln: A Content-Addressed Build Engine Backed by Git" subtitle = "Design Specification" version = "0.1.0" date = 2026-03-11 status = "Draft" +summary = """ +Kiln is a build and execution engine defined as an open spec with a reference \ +implementation backed by Git's content-addressed object store. It treats build \ +actions as pure functions of content-addressed inputs, providing hermetic \ +caching, signed outputs, and graduated isolation — from no enforcement to full \ +sandboxing.""" +++ -# git-kiln: A Ceramic-Inspired Toolkit for Shaping Git-Native Data +# Kiln: A Content-Addressed Build Engine Backed by Git -## About -More documentation is en route! +## Foundation + +### Content-Addressed Storage as Build Primitive + +Containers achieve hermetic builds by isolating the environment. Content-addressed storage achieves hermeticity by making all inputs explicit and immutable by content hash. These are fundamentally different strategies for the same goal: ensuring builds are a pure function of their inputs. + +Every build input — source files, toolchains, libraries, even the compiler binary itself — is stored and referenced by its content hash. A build rule becomes a function: `hash(output) = f(hash(input₁), hash(input₂), ..., hash(toolchain))`. If all input hashes match a prior invocation, the executor skips execution entirely and retrieves the cached output from the store. + +Where containers provide filesystem isolation, content-addressed storage replaces it with explicitly declared inputs identified by hash. There is nothing ambient to leak because the build system only provides content-addressed artifacts to the build action. Where containers provide reproducible base images, content-addressed storage replaces them with content-addressed toolchains. The compiler isn't "whatever's in the container" — it is a specific hash. Network isolation is still needed and is typically provided by lightweight kernel-level mechanisms rather than full containers. + +This approach is strictly better than containers in several dimensions. Caching is granular at the individual artifact level rather than the coarse layer level of container rebuilds. Artifacts are shared across unrelated builds automatically when they share the same toolchain hash. And determinism is structural rather than conventional — a Dockerfile can still contain `apt-get update` nondeterminism, while a content-addressed system forbids undeclared inputs by design. + +### Git as the Store + +Git is already a content-addressed store. Its object database stores blobs, trees, and commits identified by SHA hashes. A tree object is a content-addressed filesystem snapshot. A commit is an immutable record pointing at a tree. The transport protocol supports fetching individual objects by hash. + +With partial clone and promisor remotes, Git becomes a content-addressed blob store with a well-defined transport protocol. A tool can fetch blobs on demand by hash, requesting only the specific trees and blobs needed. The concern that Git "downloads everything" disappears under this model. And for binary artifacts, the compression concern is overstated — most package managers store compressed tarballs anyway and are not performing cross-version delta compression either. + +Git's Merkle tree structure provides deduplication and change detection for free. Changing one file changes the root tree hash, which changes the action identity, which invalidates exactly the right caches. Shared source files across builds are stored once. The object database is already the storage layer. + +### Git Object Store Tuning + +Binary build artifacts — `.so`, `.a`, `.rlib`, `.dylib`, `.dll` — do not compress well with zlib and produce negligible savings from cross-version delta compression. Two knobs control Git's behavior for these files. + +Loose object compression (`core.looseCompression`) controls zlib level during `git add` and `git_odb_write`. The default level 6 is the bottleneck when ingesting large `target/` directories. Level 0 produces a valid zlib stream (stored blocks, no deflation) and cuts write time dramatically. Level 1 provides a modest compression ratio at minimal CPU cost. For a build system that ingests artifacts transiently before the next repack, level 0 or 1 is the right tradeoff. + +Delta compression (`.gitattributes -delta`) controls whether `git gc` and `git repack` attempt binary diffs between versions of an object. For compiled artifacts, delta compression is nearly always a waste of CPU for negligible size savings: + +```gitattributes +*.so -delta +*.a -delta +*.rlib -delta +*.dylib -delta +*.dll -delta +*.rmeta -delta +``` + +With libgit2 (`git-kiln`'s likely implementation path), these knobs are per-blob. `git_odb_backend_loose()` accepts a `compression_level` parameter. The adapter can write source blobs at default compression and binary artifact blobs at level 0, in the same repository, without a repo-wide config change. `git_odb_write()` dispatches to the backend — the choice of compression level is a storage-layer concern, invisible to refs, trees, gc, fsck, and every other layer. The SHA is computed from logical content, not compressed representation. + +### Actions as Pure Functions + +The core abstraction is that build actions are pure functions. An action takes a set of content-addressed inputs and produces a content-addressed output. Same inputs always produce the same output. There are no side effects — no network access, no reading undeclared files, no timestamps. All inputs are enumerated and content-addressed. + +The content-addressed store is the mechanism that enforces and exploits this property. Content-addressing inputs makes the function's domain explicit. Content-addressing outputs makes results memoizable. The entire system follows from taking "builds are pure functions" seriously and then asking what infrastructure is needed to guarantee and leverage that property. + ## Architecture -More documentation is en route! +Kiln is defined as an open spec and a Git-backed reference implementation called `git-kiln`. Language adapter authors interact with kiln directly. The spec defines content-addressed trees, actions, and cache lookups. The reference implementation backs the spec with Git's object store. Another implementation could use an OCI registry or a custom CAS. Language adapters target the kiln spec, not `git-kiln`, keeping the adapter ecosystem portable. + +The architecture has three layers. Layer zero is Git itself — refs, objects, trees, blobs, and signing. Layer one is Kiln, handling plans, actions, caching, and sandboxing, which is language-aware. Layer two contains language adapters like `cargo-kiln` and `uv-kiln`, which are language-specific. + +Tools built on top of Kiln — CI systems, collaboration platforms, release automation — consume its outputs (signed build refs, cached artifacts, action results) without Kiln knowing or caring about them. Kiln is a build engine, not a workflow system. + + +## The Plan Tree + +A language tool receives a worktree and emits a plan tree — a set of nodes describing build steps. The plan tree is stored in Git's object database, not in the working tree. No kiln-specific files clutter the source. + +The plan has one node type: actions. Actions are pure. They read declared inputs and produce outputs without modifying the source tree. The worktree is always immutable from Kiln's perspective. A build compilation is an action. A test suite is an action. Codegen is an action — it reads source like a protobuf definition and produces an output tree containing generated files like a Rust module. Actions are cached by hashing their inputs, and their outputs are stored and reused across machines. + +Actions can depend on other actions. Codegen before build — the build action declares the codegen action's output as an input. The planner emits the full DAG. Kiln executes it in dependency order. + +Operations that mutate the source tree — version bumping, changelog generation, formatting, license header insertion — are not build steps. They are repository maintenance workflows handled outside Kiln. These workflows make commits. Kiln then sees those commits as ordinary source. + +Each node in the plan has a small fixed schema: + +```toml +name # cache namespace (language-native package name by default) +toolchain # content-addressed tree ref (optional, defaults to planner's) +inputs # list of paths, hashed as trees +deps # dependency map (see below) +env # map of environment variables, part of cache key +run # list of command + arguments (exec-style, no shell interpretation) +outputs # named output map: key → path (artifact outputs, signed and shared) +expects # expected output hashes: key → hash (optional, see Verified Fetch) +state # named state map: key → path (convenience cache, not signed) +profiles # list of profiles this node participates in (optional, default: all) +``` + +The format is intentionally minimal. There are no conditionals, no templating, no variable substitution, no inheritance. Platform variants are separate nodes. If two nodes share configuration, the language adapter deduplicates when generating them. The plan format is a serialization boundary, not a programming language. All intelligence lives in the language adapter that generates plans and in kiln's DAG executor that evaluates them. + +### Profile Overrides + +Profile-specific overrides for `env` and `run` are declared inline on the node: + +```toml +[[action]] +name = "mylib" +inputs = ["crates/mylib/"] +run = ["cargo", "build", "-p", "mylib"] + +[env] +RUSTFLAGS = "" + +[env.profile.release] +RUSTFLAGS = "-C opt-level=3" + +[run.profile.release] +run = ["cargo", "build", "-p", "mylib", "--release"] +``` + +One plan, one planning action, no re-planning on profile switch. Kiln resolves overrides at execution time. The cache key for each action includes the resolved profile-specific fields, so debug and release outputs cache independently. The planner only re-runs when the worktree or planner binary changes — not when the developer switches between `--profile=debug` and `--profile=release`. This avoids the Bazel analysis-cache problem, where changing a configuration flag discards the entire analysis graph and forces expensive re-planning. + +For rare cases where a profile requires a structurally different DAG — PGO workflows (instrumented build → benchmark → rebuild with profile data), cross-compilation with proc macros — the planner can emit a profile-specific plan keyed as `plan/-`. Kiln checks for a profile-specific plan first, falls back to the shared plan with profile selection. Common profiles pay no re-planning cost. Exotic profiles pay it only when necessary. + +The `profiles` field is a whitelist. Omit it and the node runs in all profiles. Include it and kiln skips the node for unlisted profiles: + +```toml +[[action]] +name = "mylib-test" +inputs = ["crates/mylib/"] +profiles = ["debug", "ci"] +run = ["cargo", "test", "-p", "mylib"] +``` + +This covers the common cases: tests run in debug and CI but are skipped in release. LTO runs in release but is skipped in debug. Benchmarks run only in a `bench` profile. The DAG is the same object; the profile selects a subgraph. Skipped nodes aren't cache-invalidated because they never ran — switching from debug to release doesn't touch test cache entries, and they're still warm when you switch back. + +### Toolchain Field + +The `toolchain` field is a content-addressed tree ref, not a container image. It declares what tools the action needs, not how to run them. When Docker is the executor, kiln loads the tree into a container. When using `unshare`, kiln mounts the tree directly. The executor decides the mechanism. Most nodes omit this field and inherit the toolchain from the planner's environment. + +### Run Field + +The `run` field is an argv list — `["cargo", "build", "--release"]`, not a shell string. No shell interpretation, no quoting ambiguity, no implicit `/bin/sh -c`. The executor execs the command directly. If a node genuinely needs shell features (pipes, redirects, globbing), the command is `["sh", "-c", "..."]` and the shell dependency is explicit. This matters at higher isolation levels where the shell itself may not be in the sandbox unless declared. + +### Outputs and State + +The `outputs` field is a named map of artifact outputs — hermetic build products that are signed, shared, and cached across machines. Each key names a public output; the value is the path within the action's capture directory: + +```toml +[outputs] +lib = "target/release/libfoo.so" +bin = "target/release/foo" +``` + +The `expects` field is a map of expected content hashes for artifact outputs. It enables verified fetch actions — see the Verified Fetch section. + +The `state` field is a named map of state outputs — convenience caches for language tooling that are materialized to the worktree but never signed or shared. These are the `target/` and `.venv/` directories that LSPs and editors need: + +```toml +[state] +target = "target/" +``` + +The distinction is structural. Artifact outputs go into the object store, get signed by CI, and are fetchable by any consumer. State outputs are local-only convenience — restored to the worktree after a build so that `rust-analyzer` sees `target/` and `pyright` sees `.venv/`, but never shared across machines. If state is stale or missing, the language tool rebuilds incrementally. No correctness depends on state. + +### Dependencies Between Actions + +Consuming another node's outputs uses the dependency map: + +```toml +[deps.foo] +lib = "myservice/native.so" +``` + +The left side of a dep entry is an output key from the producing node. The right side is where it lands in the consuming node's input tree. Output keys are the interface contract — internal build artifacts that shouldn't leak across boundaries simply aren't named. To consume the entire output tree: + +```toml +[deps.foo] +tree = "vendor/foo/" +``` + +Node names default to the language's native package names. A Rust crate named `foo` produces a node named `foo`. A Python package named `myservice` produces a node named `myservice`. Users already know these names. This makes nodes referenceable without requiring knowledge of planner internals. + +### Per-Key Dependency Hashing + +When a node depends on another node's output, its action hash includes the content hash of the *specific output key* it references — not the root tree hash of the producing node's commit. Git's Merkle tree structure provides this naturally: every subtree and blob within a tree object has its own hash. + +This means a node's cache is only invalidated when the specific outputs it consumes change. If node A produces outputs `lib` and `metadata`, and node B depends only on `A.lib`, a rebuild of A that changes `metadata` but produces a bit-identical `lib` does not invalidate B's cache. + +This property is essential for cross-language composition. A Python service that depends on a Rust crate's `.so` file should not be invalidated when the Rust crate's `.rmeta` metadata changes. It is also essential for OCI image assembly, where a deployment image may consume specific facets of multiple build nodes — a binary from one, a shared library from another, a config tree from a third. Each facet's hash enters the image node's action hash independently. + +The commit parent links on the build graph still point at the producing node's commit (the whole action), preserving coarse-grained provenance. Per-key hashing is a cache-key concern, not a provenance concern. Provenance answers "what action produced the inputs to this build." Cache validity answers "did the specific bytes I depend on change." These are different questions answered by different mechanisms. + + +## Keeping the Working Tree Clean + +All kiln state lives in the Git object database under a custom ref namespace. The working tree stays pure source. `git status` is clean. `git branch` and `git log` show nothing kiln-related. + +### Build DAG and Cache Index + +The ref namespace separates two concerns: the build DAG (the record of truth) and the cache index (a derived acceleration structure). + +**Build DAG — per-target refs (ledger):** + +``` +refs/kiln/plans/ → plan tree +refs/kiln/outputs/// → signed commit (tree = build output) +``` + +Each target gets its own ref. Commits on that ref are builds of that target — the commit history is the build history. A commit's tree is the build output. A commit's additional parents link to the output commits of its dependencies, encoding the provenance DAG in Git's native structure. `git log refs/kiln/outputs/mylib/release/x86_64-linux` shows every build of mylib. `git log` with parent traversal shows the full dependency graph. + +Two CI jobs building different targets write to different refs — no contention. Two CI jobs building the same target race on the same ref, but both produce the same output tree hash, so last-write-wins is correct. + +Plans are keyed by worktree hash only (not per-profile) since plans are profile-agnostic — profile selection happens at execution time. + +**Cache index — metadata ref:** + +``` +refs/kiln/cache → commit → tree + / + / + commit-oid # blob: ":" +``` + +The cache index maps action hashes to commit OIDs using two-level fanout (the git-metadata pattern for hash keys). Cache lookup is: compute action hash, look up in the index, fetch the commit. Multiple CI jobs writing disjoint action hashes touch disjoint tree paths and auto-merge via three-way tree merge. + +The cache index is derived, not authoritative. If it is lost or stale, it can be rebuilt by walking all `refs/kiln/outputs/` refs and extracting action hashes from commit messages. Correctness never depends on the index — it exists for fast lookup. This is the same pattern as the sequential counter optimization in git-ledger: a performance structure, not a correctness requirement. + +**Why separate them:** The build DAG is the thing you `git log`, `git verify-commit`, and `git diff` against. The cache index is the thing the executor queries during bottom-up DAG traversal. One is a record of what was built, by whom, from what. The other is an optimization for "does this build exist." Conflating them (as a flat `refs/kiln/outputs/` scheme would) makes cache lookup O(1) but loses build history, target identity, and DAG structure. Separating them preserves all three while keeping cache lookup fast via the metadata index. + +Alternative cache indexes — a local-only one for unsigned dev builds, a CI-signed one for shared builds — can point into the same set of target refs. The DAG doesn't care how you found it. + +`git clone` doesn't fetch build refs because the default refspec only includes `refs/heads/*` and `refs/tags/*`. Build objects share deduplication with source blobs. And `git gc` handles eviction naturally — dropping a build ref makes its objects unreachable and they get pruned. + +Build tools that need artifacts explicitly fetch them: + +```sh +git fetch origin refs/kiln/outputs/mylib/release/x86_64-linux +``` + + +## Output Modes + +An action's output has two parts: artifact outputs (signed, shared) and state outputs (local convenience). Kiln provides three modes for consuming these: + +**Default materialization.** After a successful build, kiln restores each final target's state outputs to the worktree at the language's conventional paths — `target/` for Rust, `.venv/` for Python. This is what `cargo build` and `uv sync` do today. LSPs, editors, and language tooling see a normal project. Hermeticity is enforced during the build inside the sandbox. State restoration happens after. These are completely independent. This is a major advantage over Bazel, which breaks LSP integration by imposing its own output layout. + +**`kiln materialize [--output=key]`.** Explicitly copy an action's artifact outputs into the worktree. This is for codegen and similar cases where generated files should be visible to editors, grep, and other tools on disk. The files appear as untracked (the user decides whether to `.gitignore` or commit them). Kiln doesn't care — the write is a user-initiated side effect after the pure action completes. Without `--output`, all artifact outputs are materialized. With it, only the named output is materialized. + +**`kiln enter [--isolation=N]`.** Present a transient merged view of the worktree and the action's output tree. On exit, the worktree is untouched. The implementation mechanism is platform-dependent (see Platform Support). Isolation levels slot in naturally since transient contexts already operate in a namespace on Linux. This is the primary interface for environments: `kiln enter env` overlays a toolchain tree onto the worktree, giving the developer a shell with the right tools. + + +## State Restoration for Language Tooling + +State restoration is the mechanism that makes LSPs fast after a kiln build. The restored files must be exactly what the language's tooling expects. + +For Rust, `rust-analyzer` runs `cargo check` internally, which invokes `rustc --emit=metadata` on every crate. Without restored state, this recompiles every dependency from scratch — minutes on a large workspace. With restored state, `cargo check` skips all cached deps and only checks the crate the developer is editing. + +The minimal state set for Rust is: + +- `target//deps/*.rmeta` — type metadata for all crates (this is what `cargo check` produces and what rust-analyzer consumes) +- `target//deps/*.rlib` — compiled crate archives (needed if the developer later runs `cargo build` outside kiln) +- `target//build/*/out/` — build script outputs (generated code, cfg flags) +- `target//.fingerprint/` — cargo's staleness records (so cargo agrees on what's current) + +Rust-analyzer expects a debug profile build. `cargo-kiln` should always emit a debug build action so that LSP integration works out of the box. If the developer only requested a release build, the debug `.rmeta` files are still needed for the editor experience. + +The `.d` files (dependency tracking) and top-level binaries/rlibs are not needed for state restoration. Cargo regenerates `.d` files from fingerprints, and top-level binaries are just the final link step — seconds to reproduce from cached `.rlib` files. + + +## Isolation Levels + +Hermeticity is a dial, not a switch. Kiln defines five isolation levels, each a superset of the previous. + +Level zero provides declared inputs with no enforcement. The build runs on the host. Kiln hashes declared inputs for cache keys. If the build reads undeclared files, kiln doesn't know. Caching works but is only as correct as the declarations. + +Level one provides read-only inputs. The build runs on the host but the input tree is mounted read-only. Writes go to a capture directory. The build can still see the host filesystem and network, but it cannot mutate inputs. + +Level two provides filesystem isolation. The build runs in a mount namespace. Only declared inputs are visible. The host filesystem is gone. Environment variables are controlled. The build can still access the network and see the real clock. + +Level three adds network isolation. No network access at all. If the build needs to download something, it fails. All dependencies must be in the input tree. This is where builds become truly reproducible. + +Level four provides a full sandbox. Fixed timestamps, PID namespace, no access to host information through `/proc`, deterministic file ordering. The build sees a completely synthetic environment. + +The plan declares the minimum supported level. Kiln can enforce higher than declared but never lower. CI can require level three as a minimum while local development defaults to level zero. The same plan, the same rules, different enforcement. This eliminates the need for separate development and release build modes. + +When a developer tightens isolation, the system tells them exactly what breaks: + +``` +$ kiln build --isolation=2 crates/cli/ +ERROR: action reads /etc/ssl/certs (undeclared input) +``` + +Each node can also specify its own isolation level as a field, allowing fine-grained control within a single build. + + +## Verified Fetch + +Build actions at isolation level three and above cannot access the network. But dependency fetching is inherently a network operation. This creates a tension: how does a project with a level-three policy acquire external dependencies? + +The answer is that fetch actions run at isolation level zero but declare expected output hashes. Verification replaces isolation as the trust mechanism. + +Lock files already contain content hashes for every dependency. `Cargo.lock` has them. `uv.lock` has them. `go.sum` has them. `package-lock.json` has them. The planner reads these hashes and declares them on each fetch action's `expects` field. + +The enforcement rule becomes: `enforce(max(declared, policy))` unless the action has expected output hashes, in which case the declared level is used and verification replaces isolation as the trust mechanism. This is not an exception to the rule — it is a stronger guarantee. A hash-verified output from level zero is more trustworthy than an unverified output from level three, because verification is a proof and isolation is a precaution. + +The expected hashes are not authored or managed by the developer. They are derived mechanically from the lock file by the planner. CI runs the same planner, reads the same lock file, derives the same hashes, and verifies independently. The developer's workflow is unchanged: `cargo update` modifies `Cargo.lock`, the next `kiln build` re-plans (because the lock file is a planner input), emits new expected hashes, and the vendor action runs with a new cache key. + +Build actions downstream of verified fetch nodes run at the policy-required isolation level. The fetched output is their input. Only the fetch itself is exempted from network isolation, and only because verification provides a stronger correctness guarantee than isolation alone. + +Cache hits are also verified. If a cached fetch output exists but its hash doesn't match the expected value, kiln treats it as a miss and re-fetches. Cache poisoning is caught. + +### Per-Crate Vendor Nodes + +Fetch actions are emitted at per-crate (or per-package) granularity, not as a single monolithic vendor action per lock file. The planner reads the lock file and emits one fetch node per dependency: + +```toml +[[action]] +name = "vendor-serde" +isolation = 0 +inputs = ["Cargo.lock"] +run = ["cargo-kiln-fetch", "--crate=serde", "--version=1.0.210"] + +[outputs] +src = "vendor/serde/" + +[expects] +src = "sha256:ab34ef..." +``` + +This granularity is critical for cache invalidation. If fetch actions are coarse-grained — one action producing the entire vendor tree — then bumping a single dependency changes the vendor output tree hash, which enters every downstream build node's action hash via per-key dep hashing, invalidating the entire build cache. Per-crate vendor nodes confine invalidation: bumping serde changes only `vendor-serde`'s output, and only crates that depend on serde recompute their action hashes. + +The implementation can batch the actual network fetch (one `cargo vendor --locked` invocation) and split the output directory into per-crate trees. The per-node granularity is a cache-key concern, not necessarily a network-operation concern. Git deduplicates unchanged crate source blobs across vendor runs at the blob level. + +Downstream build nodes depend on specific vendor crates by name: + +```toml +[[action]] +name = "mylib" +isolation = 3 +inputs = ["crates/mylib/"] + +[deps.vendor-serde] +src = "vendor/serde/" + +[deps.vendor-tokio] +src = "vendor/tokio/" + +run = ["cargo", "build", "-p", "mylib", "--frozen"] + +[outputs] +lib = "target/release/libmylib.rlib" +``` + +The `--frozen` flag tells Cargo not to touch the network or update the lock file. All crates are already local in the vendored trees. At isolation level three, the network is blocked anyway — but the flag makes the intent explicit even at lower levels. + +The same pattern applies to every language with a lock file: `uv-kiln` emits per-package fetch nodes reading `uv.lock`, `go-kiln` emits per-module fetch nodes reading `go.sum`. + + +## Platform Support + +Isolation levels zero and one work everywhere — they require only filesystem permissions and directory structure, no kernel features. Level zero is pure convention. Level one uses read-only mounts on Linux and read-only directory permissions on macOS (weaker enforcement but catches accidental writes). + +Levels two through four require Linux kernel features: mount namespaces (`unshare`), network namespaces, PID namespaces, `seccomp`. These are not available on macOS or Windows. On non-Linux platforms, kiln enforces up to level one natively. For higher levels, kiln delegates to a Linux VM — Docker Desktop on macOS, WSL2 on Windows. The build runs inside the VM with full namespace isolation. The overhead is the VM boundary, but correctness is preserved. + +This is an honest tradeoff. Most developers work at level zero locally. CI runs on Linux at level two or three. The rare developer who wants local level-two enforcement on macOS pays the VM cost. The common case is fast; the strict case is correct. + +`kiln enter` on macOS at level zero uses a tmpdir copy or symlink farm — heavier than overlayfs but functional. At higher isolation levels on macOS, it enters a Linux VM context. + + +## The Executor + +The executor evaluates a plan through a bootstrap sequence that bottoms out at a single hardcoded action. The bootstrap has two phases: setup and execution. + +**Setup** (outside the action system): + +1. Read `env.toml` (file read, not an action). +2. Fetch planner and toolchain trees from the package store (git fetch, not an action). + +These two steps are necessarily outside the action system. You cannot execute an action to fetch the tool that creates actions. `env.toml` is a fixed-format file that kiln reads directly. Git fetch is a transport operation. Neither benefits from being modeled as actions. + +**Execution** (everything is an action from here): + +3. Require a clean worktree (the worktree hash is HEAD's tree, making it deterministic). +4. Hash the worktree tree. +5. Construct the planning action — the single hardcoded action kiln knows how to build without a planner. Its inputs are the worktree and the planner binary from `env.toml`. Its command is the planner invocation. Its output is the plan tree. The profile is not part of the planning action's cache key — plans are profile-agnostic. +6. Check the cache for the planning action. On a hit, retrieve the cached plan. On a miss, execute the planner and store the plan. The cache key is `hash(worktree, planner_binary, env.toml)`. +7. Read `kiln.toml` files (root-level and subdirectory). Graft extra actions and edges onto the plan. +8. Select the active profile. Resolve per-node profile overrides for `env` and `run`. Skip nodes whose `profiles` whitelist excludes the active profile. +9. Traverse the full plan DAG bottom-up. For each active node, compute the action hash. The action hash includes: the node's resolved `env` and `run` fields, the hashes of its `inputs`, and — critically — the content hashes of the *specific output keys* it references from its dependencies, not the root tree hashes of those dependencies' commits. This per-key hashing means a dependency that rebuilds but produces bit-identical outputs at the referenced keys does not invalidate its dependents. Look up the action hash in the cache index (`refs/kiln/cache`). On a hit, retrieve the cached commit and its output tree. On a miss, build. For nodes with `expects`, verify output hashes after execution. +10. Store artifact outputs as commits on per-target refs (`refs/kiln/outputs///`). The commit's tree is the build output. The first parent is the previous build of that target. Additional parents are the output commits of the node's dependencies. Update the cache index with the new action hash → commit OID mapping. +11. Materialize state outputs to conventional paths for final build targets. + +The planning action is cacheable like any other. Same worktree and same planner version produce the same plan. The planner doesn't even run on a cache hit. Switching profiles reuses the cached plan and only recomputes action-level cache keys with resolved overrides. + +For dirty worktrees, kiln rejects by default. A `--allow-dirty` flag hashes the actual working tree instead of HEAD's tree. The cache key includes uncommitted changes, so results are correct but only locally useful — they cannot be signed or shared because they don't correspond to a commit. + + +## Mtime Normalization Contract + +Cargo's fingerprinting is mtime-based. It records the modification time and absolute path of every source file and artifact. When fingerprints are restored from a remote cache onto a different machine, mtimes and paths will not match unless the executor enforces determinism. This is the single hardest integration problem for `cargo-kiln`. + +Kiln's executor must normalize mtimes and paths before invoking cargo. This is a contract, not an optimization — without it, cargo recompiles everything on every cache restore, defeating the purpose of the system. + +**Path determinism.** The executor must ensure that cargo always runs from a canonical working directory. At isolation level two and above, this is trivial — the sandbox mounts the worktree at a fixed path (e.g., `/kiln/workspace/`). Fingerprints are recorded against that path on every machine. At levels zero and one, the executor creates a symlink from a fixed canonical path to the actual checkout and invokes cargo from there: + +```sh +ln -sfn /actual/checkout /kiln/workspace +cd /kiln/workspace +cargo build -p foo +``` + +This only works if kiln is the sole entry point for builds. If the developer runs `cargo build` directly from their checkout path, cargo records fingerprints against the real path, clobbering the kiln-compatible ones. Kiln should detect non-canonical fingerprint paths and warn. + +**Mtime determinism.** Before every cargo invocation, kiln sets all source file mtimes and all restored artifact mtimes to a deterministic value — the git commit timestamp of HEAD. The value itself does not matter as long as it is reproducible. Both CI and the local machine perform the same normalization: + +```sh +git ls-files | xargs touch -t +``` + +For restored artifacts, kiln sets mtimes during the restore step, before cargo ever sees them. Fingerprints were recorded on CI against the same normalized mtimes. The check passes. + +**Full cache hit path.** When every action in the DAG is a cache hit, kiln never invokes cargo at all. Kiln checks action hashes, all hit, restores outputs. There are no fingerprints to consult. The mtime normalization contract only matters for partial rebuilds — when some crates are cache hits and one is modified, and kiln needs cargo to rebuild just the modified crate without recompiling its cached dependencies. + +**Limitation: fingerprint internal format.** Cargo serializes its own structs into `.fingerprint/` with embedded mtime data and path hashes. The format is not stable across cargo versions. Kiln does not patch fingerprint internals — it ensures the conditions under which fingerprints were created are reproduced exactly. If cargo changes its fingerprint format, the existing cache entries become misses, which is correct behavior — a toolchain change invalidates the cache. + +**Future direction: direct rustc invocation.** If cargo's fingerprinting proves too fragile, `cargo-kiln` can bypass cargo for execution entirely. The planner already calls `cargo metadata` for the dependency graph and can capture exact `rustc` invocations from `cargo build -v`. Each action's `run` field becomes a direct `rustc` call with `--extern` flags pointing at dependency `.rlib` inputs. This is what Buck2 and Bazel's `rules_rust` do. It eliminates the fingerprint problem but requires maintaining compatibility with rustc's unstable CLI surface. This is deferred unless mtime normalization proves insufficient. + + +## Action Failure + +When an action exits with a non-zero status, kiln does not cache the output. The action is considered failed. Partial outputs are discarded. Downstream dependents do not run. + +This means flaky tests are never cached as failures. Re-running the build re-executes the failed action. If it passes, the output is cached normally. + +However, the build work preceding a failed test is still cached. If action A (compilation) succeeds and action B (tests) fails, A's output is in the cache. Re-running only re-executes B. + +For reporting, failed actions produce a structured error ref containing the exit code, stderr capture, and the action's identity hash. The error ref is not a cached output — it is a record of what happened, not a reusable result. + +A `--cache-failures` flag exists for specific use cases like expensive test suites where a known failure should not be re-executed on every invocation. When enabled, the cached failure is returned immediately with its original error output. This is opt-in and never the default. + + +## Bootstrapping With Docker + +The executor needs a sandbox. Docker is a reasonable starting point. The bootstrapping ladder has clear stages. + +At stage zero, Docker is the executor. The toolchain is a Docker container. The executor accepts the impurity of Docker's runtime but gets the caching model working. At stage one, the toolchain becomes a tree in the Git object store. Docker provides only the namespace and sandbox — a dumb isolation shell around a content-addressed filesystem. At stage two, Docker is replaced with lighter isolation using `unshare`, `pivot_root`, and `seccomp`. There is no container runtime dependency. Stage three, which is optional and Nix-like, builds the toolchain itself through the system, turtles all the way down to a bootstrap binary. + +For most use cases, stage one is where the cost-benefit ratio peaks. Content-addressed caching and explicit inputs are achieved while Docker handles the boring isolation work. + + +## Caching + +The simplest useful form of kiln is a cache. Existing lock files are already content-addressed dependency specifications. `Cargo.lock`, `uv.lock`, `go.sum`, and `package-lock.json` contain content hashes of every dependency. Hash the lock file plus the source tree plus the toolchain, and you have a cache key. Hit means return the stored output. Miss means build and store the result. + +The Git remote is the cache. CI runs `kiln build`, populates the cache, pushes build refs. A developer clones and runs `kiln build`. Every external crate, every pinned dependency, every unchanged module is an instant cache hit fetched from the remote. No sccache, no S3 bucket, no cache key heuristics. + +### Cargo-Specific Cache Boundaries + +For Rust projects, the minimal set of artifacts needed to make `cargo build` a no-op for a cached crate is: + +- `target//deps/` — compiled `.rlib`, `.rmeta`, `.so` files for every crate, including transitive dependencies +- `target//.fingerprint/` — cargo's staleness records +- `target//build/` — build script outputs + +Everything else is regenerable. Top-level binaries and rlibs (`target/debug/my-binary`, `target/debug/libmy_crate.rlib`) are just the final link step from `.rlib` files in `deps/` — seconds to reproduce. `.d` files (dependency tracking) are regenerated from fingerprints. The `incremental/` directory is the incremental compilation cache — large, fragile, and inherently non-hermetic. The `examples/` directory is built on demand. `.cargo-lock`, `.rustc_info.json`, and `CACHEDIR.TAG` at the target root are metadata files cargo recreates. + +`cargo-kiln` stores `deps/` + `.fingerprint/` + `build/` per action. It does not store `incremental/` — incremental compilation is a local convenience that conflicts with hermetic caching. The `state` output restores these three directories after a build so that subsequent cargo invocations (including rust-analyzer's `cargo check`) see a warm cache. + +### Cache Key Refinement + +Rustc tracks every file it opens during compilation and writes the results to `.d` files when `--emit=dep-info` is passed. These files list the actual source files read, providing finer-grained dependency information than directory-level hashing. + +On a debug build, `.d` files are produced as part of the output. A release build can use that information for refined cache keys. Since both builds are happening anyway, there is no extra pass and no performance cost. The refinement can also be triggered explicitly with an optimization flag for CI environments that want tighter cache keys. + +Refinement data is stored per-node in the index, decoupled from the worktree hash. When a new commit arrives, kiln checks whether a refinement exists for a given node. If so, it hashes only the files that matter instead of the whole directory. If a refined cache lookup produces a false hit — because a new import added a file the refinement didn't know about — the build proceeds normally, captures a new `.d` file, and updates the refinement. The refinement is optimistic with a correctness backstop. + + +## Profiles + +Builds support profiles. The profile is passed to kiln at invocation time: + +```sh +kiln build --profile=debug +kiln build --profile=release +``` + +Kiln resolves the profile by selecting the subgraph of nodes whose `profiles` whitelist includes the active profile (or all nodes if `profiles` is omitted), then applying per-node overrides from `env.profile.` and `run.profile.`. The plan itself is not regenerated — the same cached plan serves all profiles. + +Different profiles produce different resolved `env` and `run` fields, and therefore different action hashes and cache keys. Debug and release outputs cache independently. Switching profiles is free at the planning level; only actions whose resolved fields change produce new cache keys. + + +## Container Environments as Actions + +A container environment is itself a build artifact. A Dockerfile is an input; the output is a filesystem tree: + +```toml +[[action]] +name = "runtime" +toolchain = "git://kiln-packages/docker@27" +inputs = ["Dockerfile"] +run = ["docker", "build", "-t", "scratch", "."] + +[action.outputs] +rootfs = "rootfs/" +``` + +The output tree is a content-addressed filesystem. Other actions use it as their `toolchain`. `kiln enter runtime` drops the developer into it. The container image isn't special infrastructure — it's a cached, content-addressed build artifact like everything else. Change the Dockerfile, the action hash changes, the environment rebuilds. Don't change it, cache hit. + +This unifies the development path. `kiln enter runtime` is the same operation whether the tree came from a Dockerfile, an import, or a from-scratch kiln build. The entry mechanism changes (Docker → unshare → chroot), the abstraction doesn't. Teams get reproducible Docker-based environments through kiln's caching before any native isolation exists. + +The one honest impurity: the action that builds the container environment needs a host Docker daemon. That's the bootstrap dependency acknowledged at stage zero — Docker is the one ambient input accepted until the system can self-host the sandbox. + + +## Deployment Images + +Deployment images are compositions of build output trees assembled into a root filesystem: + +``` +deployment_tree = merge( + toolchain_output_tree, + app_output_tree, + config_tree +) +``` + +The result is a Git tree representing a complete filesystem. It can be materialized and run directly with `unshare` and `chroot` — no Docker daemon, no image layers, no registry pull. + +Deduplication is structural. If two deployment images share OpenSSL, they share the same output tree hash because they were built from the same inputs. Deduplication happens at the file level, not the layer level. There is no Dockerfile — the image is a composition of verified build outputs. Incremental image updates swap a single entry in the tree when one component changes. + +For compatibility with existing infrastructure, a `kiln export --oci` command serializes the tree as a standard OCI image pushable to any registry. The internal representation is Git trees. The export format is whatever the deployment target needs. Docker, Kubernetes, ECS, and Cloud Run all accept the result without knowing it came from a Git tree. + + +## Ephemeral Inputs + +Not all action inputs are content-addressed. Secrets — API keys, deploy tokens, signing credentials — must be available during execution but cannot be stored in the object database, included in cache keys, or appear in signed outputs. Git repos get cloned, forked, mirrored. A secret in a ref is a secret on every machine that fetches. + +Kiln models these as ephemeral inputs. They are declared in the action node but handled differently from regular inputs: + +```toml +[[action]] +name = "deploy" +inputs = ["target/release/myapp"] +run = ["./scripts/deploy.sh"] + +[ephemeral] +AWS_ACCESS_KEY = { type = "file", mount = "/run/secrets/aws" } +DEPLOY_TOKEN = { type = "file", mount = "/run/secrets/deploy" } +``` + +Properties of ephemeral inputs: + +- **Declared but not hashed.** The action node names which ephemeral inputs it requires. The names are part of the action definition (and therefore visible in review), but their values are excluded from the action hash and cache key. Two runs with different secret values but identical content-addressed inputs produce the same cache key. This is correct — the secret enables a side effect (deployment, signing), not a deterministic build output. +- **Never cached.** Ephemeral inputs are not stored in the object database. They exist only for the duration of the action's execution. +- **Injected by the executor, not the runner.** The executor (or whatever host system manages the action's lifecycle) provides ephemeral values. A compromised action cannot request secrets it didn't declare. The host verifies that the declared ephemeral names are authorized for the runner's identity before injection. +- **Mounted, not exported.** At isolation level two and above, ephemeral inputs are written to a tmpfs volume mounted read-only into the sandbox (e.g., `/run/secrets/`). tmpfs is memory-backed — never hits disk. No environment variable exposure, no `/proc//environ` leakage, no child process inheritance, no accidental logging. At isolation levels zero and one, the executor writes to a temporary directory and deletes it after execution. Weaker guarantee, but the common accidental-leak vectors are still covered. +- **Destroyed on exit.** The tmpfs mount is torn down when the action exits. The temporary directory is deleted. No remnant. + +Ephemeral inputs are an honest exception to the "everything is content-addressed" principle. The alternative — storing secrets in Git — is worse in every dimension. The design contains the exception: ephemeral inputs are declared (reviewable), excluded from the cache (no leakage into outputs), and scoped to execution (no persistence). + +### Authorization + +The executor must decide which ephemeral inputs a given runner is allowed to receive. This is outside Kiln's scope — Kiln defines the declaration format and the injection contract, but the secret store and ACL are the host system's responsibility. + +A typical integration: the host maintains an encrypted secret store keyed by name. An ACL maps secret names to authorized runner identities (signing key fingerprints). When Kiln's executor prepares to run an action with ephemeral inputs, it authenticates the runner, checks the ACL for each declared name, retrieves the values, and mounts them. If any name is unauthorized, the action is rejected before execution. + +### Audit + +Every ephemeral input access — which secret, which runner, which action, when — should be logged by the host. Kiln does not define the audit format, but it provides the action identity hash and runner identity to the host at injection time, giving the host everything it needs for a complete audit trail. + + +## Language Adapters + +### The Adapter Contract + +A language adapter is a command that reads a project and emits a plan tree. That is its entire responsibility. It does not fetch dependencies, orchestrate builds, or interact with the cache. It produces plan nodes. Kiln does the rest. + +The adapter contract requires the adapter to answer two questions: what are my inputs, and what is the build command? The kiln substrate handles storage, caching, verification, and distribution. The adapter implements: + +``` + kiln plan → emits action nodes with inputs and dependencies +``` + +The adapter calls into the language's own tooling for introspection. `cargo-kiln` calls `cargo metadata`. `uv-kiln` reads `uv.lock`. `go-kiln` calls `go list -deps`. The adapter is a translation layer that serializes the language tool's own understanding of the project into kiln's plan format. + +This means cache key correctness is the language tool's responsibility. It knows about feature flags, build profiles, platform-specific dependencies, and conditional compilation. Kiln never guesses at language-specific semantics. The adapter hashes the inputs it knows matter. This resolves the fundamental tension in build caching: the less a build system knows about a language's internals, the coarser the caching. The more it knows, the more it becomes language-specific. By delegating to adapters, kiln gets fine-grained caching without centralizing language knowledge. + +### Dependency Fetch Actions + +The adapter emits per-crate (or per-package) fetch actions alongside build actions. For Rust, `cargo-kiln` reads `Cargo.lock` and emits one fetch node per dependency with expected output hashes derived from the lock file's per-crate content hashes: + +```toml +[[action]] +name = "vendor-serde" +isolation = 0 +inputs = ["Cargo.lock"] +run = ["cargo-kiln-fetch", "--crate=serde", "--version=1.0.210"] + +[outputs] +src = "vendor/serde/" + +[expects] +src = "sha256:ab34ef..." +``` + +Build actions depend on specific vendor crate nodes: + +```toml +[[action]] +name = "mylib" +isolation = 3 +inputs = ["crates/mylib/"] + +[deps.vendor-serde] +src = "vendor/serde/" + +[deps.vendor-tokio] +src = "vendor/tokio/" + +run = ["cargo", "build", "-p", "mylib", "--frozen"] + +[outputs] +lib = "target/release/libmylib.rlib" +``` + +The `--frozen` flag tells Cargo not to touch the network or update the lock file. All crates are already local in the vendored trees. At isolation level three, the network is blocked anyway — but the flag makes the intent explicit even at lower levels. + +The adapter doesn't fetch anything. It declares nodes. Kiln executes the vendor actions (or retrieves cache hits), verifies each output against expected hashes, and stages the vendored crates as inputs to downstream build actions. The same pattern applies to every language with a lock file: `uv-kiln` emits per-package fetch nodes reading `uv.lock`, `go-kiln` emits per-module fetch nodes reading `go.sum`. + +### Planner Detection and Resolution + +Planners are resolved from the environment, not discovered from PATH. The `env.toml` file pins the planner binary as a content-addressed tree in the package store. This avoids the ambient dependency problem — "whatever `cargo-kiln` is on PATH" is exactly the kind of undeclared input the system is designed to eliminate. + +Detection is by convention. Kiln scans the worktree for language marker files — `Cargo.toml`, `pyproject.toml`, `go.mod`, `package.json` — and invokes the corresponding planner from the environment tree. For a project with both `Cargo.toml` and `pyproject.toml`, kiln invokes both `cargo-kiln` and `uv-kiln`, each emitting its own subgraph. + +The planner version is part of the cache key for the planning action. A planner upgrade that emits different node granularity invalidates the plan cache. This is correct — the plan is a function of the worktree and the planner binary. + +### Planner Isolation + +The planning action needs an isolation level, but the planner hasn't run yet — it can't declare its isolation level in a plan node. Instead, isolation is declared in the planner's package metadata: + +```toml +# in the cargo-kiln package tree metadata +isolation = 0 +``` + +The planner author decides the minimum. `env.toml` can raise it: + +```toml +[planners] +cargo-kiln = { ref = "git://kiln-packages/cargo-kiln@0.3.0", isolation = 1 } +``` + +Kiln enforces the higher of the two. Same rule as action nodes: kiln can enforce higher than declared but never lower. + +### Granularity Tiers + +Language adapters can emit nodes at different granularities. At the workspace level, the entire project is one node. The planner is trivial and caching is coarse. At the crate or module level, one node per logical package uses moderate planning effort and achieves good caching. At the compilation unit level, one node per compiler invocation achieves maximum caching but effectively replaces the language's own build orchestration. + +For most languages, the module level is the practical sweet spot. The language tool already thinks in these units. The planner uses standard introspection APIs. And the language's native build system still runs inside the sandbox, handling the fine-grained details. + +### Codegen and Planner Limitations + +Planners infer project structure from language tooling introspection. This works well for standard dependency graphs but breaks down for codegen. `cargo metadata` reports that a crate has a `build.rs`, but build scripts are opaque — Cargo doesn't know what files they read or generate until runtime. `cargo:rerun-if-changed=` directives are emitted via stdout during execution, not available for static introspection. Python has no equivalent concept at all. + +This is the primary use case for `kiln.toml`. When the planner can't infer a dependency edge — protobuf codegen, FFI bindings, generated parsers — the developer declares it explicitly. `kiln.toml` is a sparse BUILD file: you only write the parts the planner couldn't figure out. Users coming from Bazel will recognize this immediately — it's the same explicit declaration, but only for the edges that can't be inferred. + +A sufficiently motivated planner could handle common codegen patterns. `cargo-kiln` could detect `prost` or `tonic` in dependencies, scan for `.proto` files, and emit codegen nodes. But this is an optimization, not a requirement. The escape hatch always works. + +### Version Metadata + +Adapters also emit version metadata alongside the build graph. Because the adapter already introspects the project, it knows where versions live. `cargo-kiln` knows that `Cargo.toml` has a version field and understands workspace inheritance. `uv-kiln` knows `pyproject.toml`. This metadata enables release automation by upstream tools without separate configuration: + +```toml +version: + file: "crates/core/Cargo.toml" + field: "version" + current: "1.2.0" + updater: "cargo set-version {version} -p core" +``` + +Adding a new crate to a Cargo workspace automatically makes it discoverable because the planner finds it. No configuration to update. + +For projects without a planner, upstream tools can fall back to scanning for known files — `Cargo.toml`, `pyproject.toml`, `package.json`, `go.mod` — and apply default patterns. + +### LSP as a Future Planning Source + +Language servers already maintain real-time dependency graphs for type checking and autocompletion. An LSP knows which files import which modules, which symbols come from where, and which packages are used. This data could supplement or replace language-specific introspection tools for planning. + +However, LSP dependency graphs serve type checking, not compilation. They don't capture build-time dependencies like build scripts, proc macros, or link-time inputs. LSP startup is slow — rust-analyzer takes 30 to 60 seconds on a large workspace — making it unsuitable for CI where no editor is running. And expecting LSP maintainers to implement custom extensions for a new build tool is unrealistic in the short term. + +The practical approach is to use language-specific introspection tools as the primary planning path and let LSP integration evolve as an optimization. The daemon can subscribe to LSP updates for instant plan invalidation when the developer has an editor open, making the LSP a real-time notification source rather than a query target. + +### Cross-Language Composition + +Each language adapter emits its own subgraph. Cross-language edges are declared in `kiln.toml` files: + +```toml +# python/service/kiln.toml +[deps.native-ext] +lib = "myservice/native.so" +``` + +Kiln reads `kiln.toml` files after all planners run, fetches each project's plan from the index, and stitches the DAGs together at these edges. Within a language, the planner handles all internal dependencies. The developer only specifies what no single language tool can know: the relationships between languages. + +Every node in every graph has the same shape — input tree, action, output tree with named keys. A node in one graph can reference named outputs of a node in another graph. The Python side doesn't care that the native extension was built with Cargo. It references `native-ext`'s `lib` key and gets a `.so` file at the path it declared. The Rust side doesn't know Python will consume it. The output keys are the interface contract — the producing node declares what's public, the consuming node declares where it goes. + +Per-key dependency hashing makes cross-language composition cache-efficient. The Python node's action hash includes only the hash of the `.so` blob it references, not the full Rust output tree. A Rust rebuild that changes `.rmeta` metadata but produces a bit-identical `.so` does not invalidate the Python node's cache. + +For external dependencies, each project publishes its output under a kiln ref. Depending on an external project is referencing a tree hash from another Git remote: + +```toml +serde = "git://github.com/serde-rs/serde@refs/kiln/manifest" +``` + +The consumer can trust the pre-built output or fetch the source and rebuild for verification. + + +## Configuration + +### `env.toml`: The Environment + +`env.toml` defines the content-addressed environment for the project — what toolchains are available and what planners generate the build graph. It replaces language-native version management files (`rust-toolchain.toml`, `.python-version`, `.nvmrc`) with a single source of truth. + +```toml +[toolchains] +rust = "git://kiln-packages/rust@1.82.0" +python = "git://kiln-packages/cpython@3.12.0" +postgres-client = "git://kiln-packages/postgres@16" + +[planners] +cargo-kiln = "git://kiln-packages/cargo-kiln@0.3.0" +uv-kiln = "git://kiln-packages/uv-kiln@0.1.0" +``` + +`env.toml` is the single file that governs `kiln enter env`. The environment assembles trees for all listed toolchains and planners, overlays them onto the worktree, and gives the developer a shell with the right tools. The planner runs inside this environment — it doesn't resolve versions, it just calls `cargo metadata` or `uv lock` with whatever toolchain is provided. + +Toolchains and planners are in the same file because the relationship between them is the most important thing to understand about the build configuration. `cargo-kiln@0.3.0` next to `rust@1.82.0` — you see immediately what's planning what and whether versions are compatible. Adding a language is one logical change (toolchain + planner) that should be one commit to one file. + +Architecture is encoded in the ref path. `refs/kiln/cpython/3.12.0/x86_64-linux` and `refs/kiln/cpython/3.12.0/aarch64-darwin` point to different trees. The tool detects the local platform and fetches the matching ref. Cross-platform environments are explicit. + +For projects that need reproducible environments without the build system, `env.toml` with only `[toolchains]` provides immediate value. The moment they adopt kiln planners, they add `[planners]` and everything is already in the right place. + +### `kiln.toml`: The Escape Hatch + +`kiln.toml` is optional. It declares actions and edges that planners can't infer. The root-level `kiln.toml` can live alongside `env.toml` or at the repository root. Subdirectory `kiln.toml` files live alongside source at the subtree boundary — `python/service/kiln.toml`. The root file is project-wide configuration. Subdirectory files are owned by the team that owns that subtree. + +```toml +# root kiln.toml +[[action]] +name = "protogen" +toolchain = "git://kiln-packages/protoc@27.0" +inputs = ["proto/"] +run = ["protoc", "--rust_out=gen/", "proto/api.proto"] + +[action.outputs] +rust_mod = "gen/" + +[deps.foo] +protogen.rust_mod = "src/generated/" +``` + +Kiln reads the root `kiln.toml` first, then walks the worktree for subdirectory `kiln.toml` files. Each file's actions and edges are scoped to its subtree's nodes. Node names in `[deps]` reference language-native package names — `foo` is the crate named `foo`, not a planner-internal identifier. Dep entries map producing node output keys to paths in the consuming node's input tree. + +`kiln.toml` is a sparse BUILD file. The spectrum from "planner does everything" to "fully manual" is how much you write in `kiln.toml`. If the overrides replace everything a planner emitted, that's effectively a new plan. No special case needed — it's the escape hatch turned to full. + + +## The Daemon + +### Purpose + +The daemon is an optional local process that keeps the object store hot and watches for changes. If it is running, builds are faster. If it is not, everything works the same, just cold. + +The daemon watches the worktree. On every file save, it re-hashes affected subtrees, recomputes action hashes, and checks the cache. By the time the developer runs a build command, the daemon already knows which nodes are cache hits and which need building. Planning is done and the build starts executing immediately. + +The daemon also watches `env.toml`. When toolchain or planner refs change, it fetches the new trees in the background. By the time the developer runs `kiln build` after updating a toolchain, the trees are already local. + +### Speculative Builds + +The daemon can observe editing patterns and start building proactively. When it sees changes to `crates/core/src/lib.rs`, it knows core's dependents will need rebuilding. It can start building core as soon as the file is saved, before the developer asks. By the time a full build is requested, core is already done. + +### Background Prefetch + +When the developer pulls, the daemon sees new commits. It fetches signed output refs from the remote for those commits' worktree hashes in the background. By the time a build runs, the cache is already populated locally. + +### Plan Caching Across Branches + +Switching branches produces a different worktree hash. The daemon diffs the two trees, identifies which nodes are affected, and only re-plans those. The rest of the DAG carries over from the previous branch's plan. + + +## Signed Builds and Verification + +### The Trust Model + +Only CI runners can sign commits on the build graph. Local builds populate the local cache. The shared remote only accepts build refs signed by CI keys. Consumers verify the signature before trusting a cached output. + +The developer pushes source code. CI pulls, builds at whatever isolation level the project requires, and signs the output refs. Anyone can verify that the output was built by CI from the declared inputs. Anyone can also rebuild from source and check that their output matches CI's. + +Unsigned local cache is a convenience for the developer's own machine. Signed CI cache is the source of truth for everyone else. The signature attests not just that CI built the artifact but that CI built it at a specific isolation level from a specific input tree. + +### Build Graph as Git History + +Each target has its own ref at `refs/kiln/outputs///`. Each build of that target produces a signed commit on the ref. The commit's tree is the build output. The commit's first parent is the previous build of that same target (version history). Additional parents are the output commits of the target's dependencies (provenance DAG). + +For example, building `mylib` which depends on `serde`: + +``` +refs/kiln/outputs/vendor-serde/release/x86_64-linux → commit VS1 (signed) + tree: vendor/serde/ + parents: (none, first build) + message: action=vendor-serde, source=, isolation=0, verified=sha256:ab34ef... + +refs/kiln/outputs/serde/release/x86_64-linux → commit S1 (signed) + tree: deps/libserde.rlib, deps/libserde.rmeta, .fingerprint/... + parents: VS1 (vendor-serde output) + message: action=serde, source=, toolchain=, isolation=3 + +refs/kiln/outputs/mylib/release/x86_64-linux → commit M1 (signed) + tree: deps/libmylib.rlib, deps/libmylib.rmeta, .fingerprint/... + parents: M0 (previous mylib build), S1 (serde output) + message: action=mylib, source=, toolchain=, isolation=3 +``` + +`git log M1` walks the full DAG: M1 ← S1 ← VS1. `git verify-commit` on any node proves CI built it. `git diff S1_old S1_new` shows what changed when serde was bumped. `git log refs/kiln/outputs/mylib/release/x86_64-linux` shows every build of mylib over time. + +The final deliverable for a release is a merge commit whose parents are the individual build commits. The resulting tree is the complete build output. Provenance is preserved through the commit structure. The source repository has its history. The build graph has its own parallel history. They are linked by the input hashes in commit messages. + +The cache index (`refs/kiln/cache`) is a separate metadata ref that maps action hashes to commit OIDs for fast lookup. It is derived, not authoritative — if lost, it can be rebuilt by walking the target refs. See the Build DAG and Cache Index section. + +### Architecture Encoding + +Architecture and OS are part of the cache key. The plan's target field is part of the action hash. Same source compiled for different targets produces different cache keys. There are no accidental cross-architecture cache hits. + +CI builds per target in a matrix. Each target produces a separate set of signed output refs. A consumer machine detects its local target, computes action hashes with that target, and looks for signed output matching those hashes. If CI didn't build for that target, there is no cache entry and the consumer builds locally. The toolchain tree is also part of the cache key — a build against one base environment versus another produces a different action hash. + +### Nondeterminism + +Some compilers do not produce bit-identical output across runs. When verification fails — the local rebuild produces a different hash than CI's signed output — the outputs can be diffed with Git to trace the difference. Depending on the index structure, a bisect can help locate the divergence. + +At isolation levels below two, the system defaults to trusting the last binary produced rather than demanding hash equality. Strict verification applies only at higher isolation levels where the build environment is controlled enough to expect determinism. This is an honest tradeoff: verification works for the common case, and the cases where it fails are diagnosable rather than silent. + + +## Environments + +### Git Trees as Environments + +The `env.toml` file defines the project environment as content-addressed Git trees. `kiln enter env` assembles the trees for all listed toolchains and planners, overlays them onto the worktree, and gives the developer a shell with the right tools. This is `kiln enter` applied to an environment action — the environment is just an action whose output is a usable toolchain tree. + +`env.toml` replaces language-native version management. Instead of `rust-toolchain.toml` telling rustup which tarball to download, `env.toml` pins a content-addressed tree. Instead of `.python-version` telling pyenv which Python to activate, `env.toml` pins a content-addressed tree. The role is the same — "when you enter this project, use this toolchain" — the mechanism is different. + +For Rust, this is a clean substitution. `cargo` doesn't read `rust-toolchain.toml` — rustup does. Cargo just calls whatever `rustc` is on PATH. Kiln provides the `rustc`, no rustup needed. + +For Python, `uv` reads `.python-version` and uses the Python version as a dependency resolution input via `requires-python` in `pyproject.toml`. But these serve different purposes: `requires-python` is a constraint (minimum bound), `env.toml` is the concrete pin. They don't conflict. If `env.toml` changes Python from 3.12 to 3.13, `uv.lock` may need regeneration — but that's correct behavior, because changing a toolchain should invalidate dependency resolution. + +### Bootstrapping Packages + +Initial toolchain packages are bootstrapped from official release binaries imported into the Git object store. A `kiln import` command fetches from official release URLs and writes trees locally. The binaries never pass through a kiln-hosted server, avoiding redistribution concerns. + +As the build system matures, bootstrap packages are replaced with kiln-built ones. The same ref gets a new tree hash backed by a signed build graph. Consumers don't change anything. Trust properties improve transparently. + +For a healthy ecosystem, community-maintained package repositories follow the Arch Linux model: comply with licensing per package, make source available for copyleft-licensed tools, and exclude proprietary software from public repos. Proprietary toolchains live in private stores. + +### Path to Replacing Docker + +The progression moves through clear stages. Initially, Kiln builds artifacts and Docker assembles images from those artifacts. Next, Kiln assembles the root filesystem tree and Docker just runs it via `docker import`. Eventually, Kiln assembles and runs via `unshare` and `chroot` with no Docker involvement. + +At every stage, export to OCI format provides compatibility with the existing container ecosystem. + + +## Remote Build Execution + +### Architecture + +Kiln already has the primitives for remote execution. An action is a self-contained unit of work: input tree hash, action blob, expected output tree hash. It can be sent anywhere. + +On a cache miss that isn't satisfied locally or from the remote cache, kiln submits the action to an RBE worker pool. A worker fetches the input tree from the Git remote, materializes it into a sandbox, executes the action, and pushes the output tree back to the remote. Trusted workers sign the output. + +Workers are dumb. They receive an action hash and an input tree hash. They have no knowledge of the project, the language, or the DAG. They fetch blobs, run a command in a container, and push the result. Workers are stateless and share nothing except the Git remote. + +### Simplicity Over Bazel RBE + +Bazel's Remote Execution API defines a complex protobuf protocol with dedicated servers. Kiln's version uses Git's transport protocol directly. The worker is a Git client. It fetches with `git fetch` and pushes with `git push`. No custom API, no dedicated build farm servers. + +Scaling is straightforward: add more workers. Two workers building the same action is harmless — both produce the same output hash, last push wins. + + +## Licensing + +### Cache as Distribution + +Build cache entries stored on a Git remote are accessible to anyone who can fetch from that remote. For private repositories, cache access is scoped to the organization — sharing compiled artifacts internally is not "distribution" under copyright law. This is the same legal posture as a company's existing CI cache, Artifactory, or S3 artifact bucket. + +For public repositories, build refs under `refs/kiln/outputs/` are fetchable by anyone, even though the default refspec doesn't include them. This constitutes distribution. The licensing implications depend on what the cache contains and what licenses govern the dependencies. + +### Compiled Artifacts and Dependency Licenses + +Most open-source licenses (MIT, Apache-2.0, BSD) permit redistribution of compiled derivatives with attribution. A `NOTICE` file in the output tree satisfies this. The planner already has license metadata from dependency manifests — generating the notice file as part of the build output is trivially automated. + +Copyleft licenses (GPL, LGPL, AGPL) require that recipients of compiled artifacts can obtain the complete corresponding source. For a public open-source project, the source is in the same public repository. The vendored source tree (from the vendor actions) provides an even stronger guarantee — the exact source used to produce the binary is in the same Git object store, fetchable by hash. + +### Vendored Source in the Cache + +The vendor actions' output trees contain copies of each dependency's source. Redistributing these trees is redistributing those dependencies' source code. For permissive licenses, this is fine with attribution. For copyleft licenses, this is actually desirable — GPL requires source availability, and the vendor trees provide it. + +The only problematic case is proprietary dependencies with redistribution restrictions. These should not appear in a public cache. A license enforcement policy can prevent this by denying builds that include disallowed licenses. + + +## CLI + +``` +kiln +├── build [target] [--profile=P] [--isolation=N] +├── plan [target] +├── materialize [--output=key] +├── enter [--isolation=N] +├── import +├── export --oci +├── cache +│ ├── status +│ ├── prune +│ └── fetch +└── status +``` + +All list commands support `--json` for scripting. Every subcommand maps to a ref operation. There is no hidden state outside Git. + + +## Development Roadmap + +### Phase 1: Single-Node Actions + +Implement the kiln executor with single-node plans and Docker-based execution. A project's entire build is one action — the worktree is the input, Docker provides the toolchain, the build script is the command. This is the universal plan that works for every language with zero configuration. No planners, no language awareness, no adapters. But the caching model, the ref format, the signed output infrastructure, and action failure semantics are real from day one. + +### Phase 2: Isolation Level One + +Implement read-only input enforcement. Builds run on the host but the input tree is mounted read-only and writes go to a capture directory. This works on both Linux and macOS (with weaker enforcement on macOS via filesystem permissions). This is the minimum isolation needed for planner development — without at least this level, a planner can emit incomplete inputs and nobody notices. + +### Phase 3: First Planner + +Build the first language adapter, either `cargo-kiln` or `uv-kiln`. The planner calls into the language's own introspection tools and emits multi-node plans at the crate or package level with profile-agnostic DAGs and per-node profile overrides. The planner also emits per-crate vendor fetch actions with expected output hashes derived from the lock file. This phase validates whether language tooling exposes enough information for correct cache keys and whether the node format is sufficient without language-specific extensions. + +### Phase 4: Output Modes + +Implement `kiln materialize` and `kiln enter`. This phase validates the three output modes (default state materialization, explicit artifact materialization, transient context) and forces solutions to tree composition and platform-specific mounting mechanisms. On macOS, `kiln enter` at level zero uses tmpdir copy or symlink farms. + +### Phase 5: Isolation Level Two + +Implement filesystem isolation via mount namespaces on Linux. Only declared inputs are visible to the build. On macOS, this level delegates to a Linux VM. This is where builds become genuinely reproducible and where the planner's input declarations are strictly enforced. + +### Phase 6: Second Planner + +Build the second language adapter targeting a language with a very different build model from the first. If the plan format needs language-specific fields to accommodate the second language, the abstraction is leaking. If it works cleanly, the spec is sound. + +### Phase 7: Cross-Language Composition + +Wire `kiln.toml` dependency edges between projects using different languages. A project with a Rust native extension consumed by a Python service, built end-to-end by `kiln build`, is the reference test case. + +### Phase 8: Isolation Level Three + +Add network isolation. Verified fetch actions enable projects to acquire external dependencies while maintaining a level-three policy. Combined with CI signing, this level enables full build verification. + +### Phase 9: Environments + +Implement `kiln enter env` with bootstrapped toolchain packages imported from official release binaries via `kiln import`. Developers get reproducible environments as Git trees defined by `env.toml`. + +### Phase 10: Remote Build Execution + +Implement remote workers that fetch input trees, execute actions in sandboxes, and push signed output trees. Workers are stateless Git clients. -## Crates +### Deferred Indefinitely -More documentation is en route! +Isolation level four — fixed timestamps, PID namespaces, deterministic file ordering — provides diminishing returns for enormous effort. Deferred until clear demand materializes.