diff --git a/.cargo/config.toml b/.cargo/config.toml index c853041..bff29e6 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,2 @@ [build] -rustflags = ["-C", "link-arg=-fuse-ld=lld"] - -[target.x86_64-unknown-linux-gnu] -linker = "clang" -rustflags = ["-C", "link-arg=-fuse-ld=lld"] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fff5e52 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,32 @@ +# Git +.git/ +.github/ + +# Nix +.direnv/ +.envrc +flake.lock +flake.nix +nix/ +result* + +# Build artifacts +target/ +*.tar.gz + +# Documentation +*.md +docs/ + +# CI/CD +.gitlab-ci.yml + +# Charts (not needed in Docker build context except for E2E) +charts/ + +# Tests +hack/ + +# IDE +.vscode/ +.idea/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c48983..ef9f63d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,160 +5,185 @@ on: branches: [main] pull_request: -env: - CARGO_TERM_COLOR: always - jobs: - # ── Layer 1 (all parallel) ───────────────────── + # ── L1: fast checks (all parallel) ─────────────── fmt: - name: "L1 · Format" + name: "L1 · fmt" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix develop -c cargo fmt --check + - uses: dtolnay/rust-toolchain@1.88 + with: + components: rustfmt + - run: cargo fmt --check clippy: - name: "L1 · Clippy" + name: "L1 · clippy" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix develop -c cargo clippy --workspace --all-targets -- -D warnings + - uses: dtolnay/rust-toolchain@1.88 + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + - run: cargo clippy --workspace --all-targets -- --deny warnings deny: - name: "L1 · Deny" + name: "L1 · deny" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix develop -c cargo deny check + - uses: dtolnay/rust-toolchain@1.88 + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@cargo-deny + - run: cargo deny check - unit: - name: "L1 · Unit Tests" + test: + name: "L1 · test" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix develop -c cargo nextest run --workspace --lib --bins + - uses: dtolnay/rust-toolchain@1.88 + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + - run: cargo nextest run --workspace --exclude niphas-e2e - # ── Layer 2 (all parallel, gates on layer 1) ─── + # ── L2: integration tests (parallel, gate on L1) ─ eval-http: name: "L2 · Eval HTTP" - needs: [fmt, clippy, deny, unit] + needs: [fmt, clippy, deny, test] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix develop -c cargo nextest run -p niphas-eval --test '*' + - uses: dtolnay/rust-toolchain@1.88 + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + - run: cargo nextest run -p niphas-eval --test '*' csi-grpc: name: "L2 · CSI gRPC" - needs: [fmt, clippy, deny, unit] + needs: [fmt, clippy, deny, test] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix develop -c cargo nextest run -p niphas-csi --test '*' + - uses: dtolnay/rust-toolchain@1.88 + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + - run: cargo nextest run -p niphas-csi --test '*' - # ── Layer 3 (gates on layer 2) ───────────────── + # ── L3: E2E (gates on L2) ─────────────────────── e2e-kind: name: "L3 · E2E Kind" needs: [eval-http, csi-grpc] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: dtolnay/rust-toolchain@1.88 + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest - # Build all OCI images in a single nix build (shared cargoArtifacts) - - name: Build images - run: nix build .#all-images -o result-images + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + + - name: Build Docker images + run: | + docker build -t ghcr.io/fullzer4/niphas-operator:dev -f Dockerfile.operator . + docker build -t ghcr.io/fullzer4/niphas-eval:dev -f Dockerfile.eval . + docker build -t ghcr.io/fullzer4/niphas-csi:dev -f Dockerfile.csi . + docker build -t ghcr.io/fullzer4/niphas-runner:dev -f Dockerfile.runner . - # Create kind cluster - uses: helm/kind-action@v1 with: cluster_name: niphas-test config: hack/kind-config.yaml - # Load images into kind - name: Load images into kind run: | - kind load image-archive result-images/operator.tar.gz --name niphas-test - kind load image-archive result-images/eval.tar.gz --name niphas-test - kind load image-archive result-images/csi.tar.gz --name niphas-test - kind load image-archive result-images/runner.tar.gz --name niphas-test - kind load image-archive result-images/mock-eval.tar.gz --name niphas-test + kind load docker-image ghcr.io/fullzer4/niphas-operator:dev --name niphas-test + kind load docker-image ghcr.io/fullzer4/niphas-eval:dev --name niphas-test + kind load docker-image ghcr.io/fullzer4/niphas-csi:dev --name niphas-test + kind load docker-image ghcr.io/fullzer4/niphas-runner:dev --name niphas-test - # Label nodes for CSI - name: Label nodes run: kubectl label nodes --all niphas.io/store=true - # Verify CRD freshness - name: Verify CRD freshness run: | - nix develop -c cargo run --bin niphas-crd-gen 2>/dev/null | grep -v '^niphas devShell ready$' | diff charts/niphas/crds/niphasworkloads.yaml - + cargo run --bin niphas-crd-gen 2>/dev/null | diff charts/niphas/crds/niphasworkloads.yaml - - # Install Helm chart - name: Install niphas run: | - nix develop -c helm install niphas charts/niphas \ + helm install niphas charts/niphas \ --namespace niphas-system \ --create-namespace \ --set operator.image.pullPolicy=Never \ --set operator.image.tag=dev \ - --set eval.image.repository=ghcr.io/fullzer4/niphas-mock-eval \ --set eval.image.pullPolicy=Never \ --set eval.image.tag=dev \ + --set eval.resources.limits.memory=4Gi \ --set csi.image.pullPolicy=Never \ --set csi.image.tag=dev \ - --set runner.image.tag=dev + --set runner.image.tag=dev \ + --set docs.enabled=true \ + --set docs.flakeRef=github:NixOS/nixpkgs \ + --set docs.attribute=legacyPackages.x86_64-linux.hello - # Wait for rollout - - name: Wait for deployments + - name: Wait for control plane run: | kubectl -n niphas-system rollout status deployment/niphas-operator --timeout=120s kubectl -n niphas-system rollout status deployment/niphas-eval --timeout=120s kubectl -n niphas-system rollout status daemonset/niphas-csi --timeout=120s - # Debug on failure - - name: Debug pods on failure + - name: Wait for docs workload + run: | + for i in $(seq 1 150); do + PHASE=$(kubectl -n niphas-system get niphasworkload niphas-docs -o jsonpath='{.status.phase}' 2>/dev/null || echo "") + echo "attempt $i: phase=$PHASE" + [ "$PHASE" = "Running" ] && exit 0 + sleep 2 + done + echo "FAIL: did not reach Running after 5 minutes" + kubectl -n niphas-system get niphasworkload niphas-docs -o yaml || true + exit 1 + + - name: Debug on failure if: failure() run: | - echo "=== Pod status ===" kubectl -n niphas-system get pods -o wide - echo "=== Pod descriptions ===" kubectl -n niphas-system describe pods - echo "=== CSI logs ===" - kubectl -n niphas-system logs -l app.kubernetes.io/name=niphas-csi --all-containers --tail=50 || true - echo "=== Operator logs ===" - kubectl -n niphas-system logs -l app.kubernetes.io/name=niphas-operator --tail=50 || true - echo "=== Eval logs ===" - kubectl -n niphas-system logs -l app.kubernetes.io/name=niphas-eval --tail=50 || true - echo "=== Events ===" + for l in niphas-csi niphas-operator niphas-eval; do + echo "=== $l ===" && kubectl -n niphas-system logs -l app.kubernetes.io/name=$l --all-containers --tail=50 || true + done + kubectl -n niphas-system get niphasworkload niphas-docs -o yaml || true + kubectl -n niphas-system get deployment -l niphas.io/workload=niphas-docs -o yaml || true kubectl -n niphas-system get events --sort-by=.lastTimestamp - # Create e2e namespace - - name: Create e2e namespace - run: kubectl create namespace niphas-e2e - - # Port-forward - - name: Port-forward services + - name: Run E2E tests run: | + kubectl create namespace niphas-e2e kubectl -n niphas-system port-forward svc/niphas-eval 8443:8443 & kubectl -n niphas-system port-forward deployment/niphas-operator 8080:8080 & sleep 5 - - # Run e2e tests - - name: Run E2E tests + cargo nextest run -p niphas-e2e --test '*' env: OPERATOR_HEALTH_URL: http://localhost:8080/healthz EVAL_HEALTH_URL: http://localhost:8443/healthz E2E_NAMESPACE: niphas-e2e - run: nix develop -c cargo nextest run -p niphas-e2e --test '*' diff --git a/Cargo.lock b/Cargo.lock index 5280fc3..61bc236 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,7 +147,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", @@ -596,7 +596,6 @@ dependencies = [ "pear", "serde", "serde_yaml", - "toml", "uncased", "version_check", ] @@ -788,12 +787,6 @@ dependencies = [ "wasip3", ] -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - [[package]] name = "gloo-timers" version = "0.3.0" @@ -818,19 +811,13 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.14.0", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.15.5" @@ -1110,16 +1097,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.14.0" @@ -1296,7 +1273,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", - "tower 0.5.3", + "tower", "tower-http", "tracing", ] @@ -1518,10 +1495,6 @@ dependencies = [ "futures", "k8s-openapi", "kube", - "opentelemetry", - "opentelemetry-appender-tracing", - "opentelemetry-otlp", - "opentelemetry_sdk", "reqwest", "schemars", "serde", @@ -1531,7 +1504,6 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", - "tracing-opentelemetry", "tracing-subscriber", ] @@ -1546,13 +1518,12 @@ dependencies = [ "libc", "niphas-core", "niphas-csi", - "prost 0.14.4", - "prost-types", + "prost", "serde", "tikv-jemallocator", "tokio", "tokio-stream", - "tonic 0.14.6", + "tonic", "tonic-build", "tonic-prost", "tonic-prost-build", @@ -1582,7 +1553,6 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", - "futures", "http-body-util", "niphas-core", "reqwest", @@ -1591,10 +1561,9 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "tokio", - "tower 0.5.3", + "tower", "tower-http", "tracing", - "tracing-subscriber", ] [[package]] @@ -1629,7 +1598,7 @@ dependencies = [ "time", "tokio", "tokio-util", - "tower 0.5.3", + "tower", "tower-http", "tracing", "tracing-subscriber", @@ -1671,98 +1640,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "opentelemetry" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror 2.0.18", - "tracing", -] - -[[package]] -name = "opentelemetry-appender-tracing" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e716f864eb23007bdd9dc4aec381e188a1cee28eecf22066772b5fd822b9727d" -dependencies = [ - "opentelemetry", - "tracing", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "opentelemetry-http" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" -dependencies = [ - "async-trait", - "bytes", - "http", - "opentelemetry", - "reqwest", - "tracing", -] - -[[package]] -name = "opentelemetry-otlp" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" -dependencies = [ - "futures-core", - "http", - "opentelemetry", - "opentelemetry-http", - "opentelemetry-proto", - "opentelemetry_sdk", - "prost 0.13.5", - "reqwest", - "thiserror 2.0.18", - "tokio", - "tonic 0.12.3", - "tracing", -] - -[[package]] -name = "opentelemetry-proto" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" -dependencies = [ - "opentelemetry", - "opentelemetry_sdk", - "prost 0.13.5", - "tonic 0.12.3", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" -dependencies = [ - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry", - "percent-encoding", - "rand 0.9.4", - "serde_json", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tracing", -] - [[package]] name = "ordered-float" version = "2.10.1" @@ -1891,7 +1768,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap", ] [[package]] @@ -2007,16 +1884,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive 0.13.5", -] - [[package]] name = "prost" version = "0.14.4" @@ -2024,7 +1891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", - "prost-derive 0.14.4", + "prost-derive", ] [[package]] @@ -2039,7 +1906,7 @@ dependencies = [ "multimap", "petgraph", "prettyplease", - "prost 0.14.4", + "prost", "prost-types", "pulldown-cmark", "pulldown-cmark-to-cmark", @@ -2048,19 +1915,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "prost-derive" version = "0.14.4" @@ -2080,7 +1934,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "prost 0.14.4", + "prost", ] [[package]] @@ -2132,7 +1986,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.4", + "rand", "ring", "rustc-hash", "rustls", @@ -2179,37 +2033,16 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -2304,9 +2137,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "futures-channel", "futures-core", - "futures-util", "http", "http-body", "http-body-util", @@ -2326,7 +2157,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -2591,15 +2422,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2618,7 +2440,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.0", + "indexmap", "itoa", "ryu", "serde", @@ -2957,73 +2779,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", -] - -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-trait", - "base64", - "bytes", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost 0.13.5", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tonic" version = "0.14.6" @@ -3047,7 +2802,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-stream", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", @@ -3072,8 +2827,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", - "prost 0.14.4", - "tonic 0.14.6", + "prost", + "tonic", ] [[package]] @@ -3092,26 +2847,6 @@ dependencies = [ "tonic-build", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.6", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.3" @@ -3120,7 +2855,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.0", + "indexmap", "pin-project-lite", "slab", "sync_wrapper", @@ -3150,7 +2885,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", @@ -3213,24 +2948,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd8e764bd6f5813fd8bebc3117875190c5b0415be8f7f8059bffb6ecd979c444" -dependencies = [ - "js-sys", - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time", -] - [[package]] name = "tracing-serde" version = "0.2.0" @@ -3454,7 +3171,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.14.0", + "indexmap", "wasm-encoder", "wasmparser", ] @@ -3467,7 +3184,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap", "semver", ] @@ -3662,15 +3379,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -3705,7 +3413,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.14.0", + "indexmap", "prettyplease", "syn", "wasm-metadata", @@ -3736,7 +3444,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap 2.14.0", + "indexmap", "log", "serde", "serde_derive", @@ -3755,7 +3463,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.14.0", + "indexmap", "log", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index ab53383..80c0347 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,105 +14,65 @@ version = "0.1.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/fullzer4/niphas" -rust-version = "1.85" +rust-version = "1.88" [workspace.dependencies] -# Kubernetes kube = { version = "3.1", features = ["runtime", "derive", "client"] } k8s-openapi = { version = "0.27", features = ["latest", "schemars"] } kube-runtime = "3.1" kube-leader-election = "0.43" - -# Async tokio = { version = "1.52", features = ["rt-multi-thread", "net", "time", "signal", "macros", "process"] } tokio-util = "0.7" - -# Web axum = "0.8" -tower = "0.5" -tower-http = { version = "0.6", features = ["trace"] } - -# gRPC (CSI) +tower = { version = "0.5", features = ["limit"] } +tower-http = { version = "0.6", features = ["trace", "timeout", "catch-panic"] } tonic = "0.14" tonic-build = "0.14" tonic-prost-build = "0.14" +tonic-prost = "0.14" prost = "0.14" -prost-types = "0.14" - -# P2P -libp2p = { version = "0.56", features = [ - "tcp", "quic", "noise", "yamux", - "dns", "identify", "gossipsub", "request-response", - "tokio", "macros", -] } - -# Serialization serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" - -# Zero-copy (P2P protocol) -rkyv = "0.8" - -# Error handling thiserror = "2" anyhow = "1" - -# Observability tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "registry"] } -tracing-opentelemetry = "0.30" -opentelemetry = "0.29" -opentelemetry_sdk = { version = "0.29", features = ["rt-tokio"] } -opentelemetry-otlp = { version = "0.29", features = ["grpc-tonic", "trace", "logs"] } -opentelemetry-appender-tracing = "0.29" - -# Schema / CRD schemars = "1" -garde = { version = "0.23", features = ["derive", "serde"] } - -# Config -figment = { version = "0.10", features = ["env", "toml", "yaml"] } - -# Allocator +figment = { version = "0.10", features = ["env", "yaml"] } tikv-jemallocator = { version = "0.7", features = ["profiling", "stats"] } - -# Performance primitives -smallvec = "1" -compact_str = "0.9" -bumpalo = { version = "3", features = ["collections"] } -memmap2 = "0.9" bytes = "1" - -# Object pooling -lockfree-object-pool = "0.1" - -# Async utilities futures = "0.3" async-compression = { version = "0.4", features = ["tokio", "zstd", "xz", "bzip2"] } -tonic-prost = "0.14" - -# Cryptography / hashing sha2 = "0.10" ed25519-dalek = { version = "2", features = ["pkcs8"] } base64 = "0.22" - -# Time time = { version = "0.3", features = ["formatting"] } - -# HTTP client reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "gzip"] } - -# Internal niphas-core = { path = "crates/niphas-core" } +[profile.dev] +debug = 1 +incremental = true + +[profile.dev.build-override] +opt-level = 3 + [profile.release] opt-level = 3 -lto = "fat" +lto = "thin" codegen-units = 1 panic = "abort" strip = "symbols" debug = 0 +[profile.release.build-override] +opt-level = 3 + [profile.release.package."*"] opt-level = 3 + +[profile.ci] +inherits = "release" +lto = false +codegen-units = 16 diff --git a/Dockerfile.csi b/Dockerfile.csi new file mode 100644 index 0000000..d7d7440 --- /dev/null +++ b/Dockerfile.csi @@ -0,0 +1,35 @@ +# Multi-stage build for niphas-csi +FROM rust:1.88-slim-bookworm AS builder + +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + +# Copy workspace files +COPY Cargo.toml Cargo.lock ./ +COPY crates/ crates/ + +# Build release binary +RUN cargo build --release --bin niphas-csi + +# Runtime image +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + tzdata \ + util-linux \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /build/target/release/niphas-csi /bin/niphas-csi + +# CSI needs to run as root for mount operations +USER 0 +ENTRYPOINT ["/bin/niphas-csi"] diff --git a/Dockerfile.eval b/Dockerfile.eval new file mode 100644 index 0000000..33c1cb2 --- /dev/null +++ b/Dockerfile.eval @@ -0,0 +1,41 @@ +# Multi-stage build for niphas-eval +FROM rust:1.88-slim-bookworm AS builder + +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + +# Copy workspace files +COPY Cargo.toml Cargo.lock ./ +COPY crates/ crates/ + +# Build release binary +RUN cargo build --release --bin niphas-eval + +# Runtime image - use NixOS/nix Docker image as base +FROM nixos/nix:2.24.11 + +# Install runtime dependencies +RUN nix-channel --update && \ + nix-env -iA nixpkgs.cacert nixpkgs.git && \ + nix-collect-garbage -d + +# Create Nix config +RUN mkdir -p /etc/nix && echo 'sandbox = false\n\ +experimental-features = nix-command flakes\n\ +accept-flake-config = false\n\ +filter-syscalls = false' > /etc/nix/nix.conf + +# Create nobody user if it doesn't exist +RUN id -u 65534 &>/dev/null || adduser -u 65534 -D -s /bin/false nobody + +COPY --from=builder /build/target/release/niphas-eval /bin/niphas-eval + +USER 65534 +ENV HOME=/tmp +ENTRYPOINT ["/bin/niphas-eval"] diff --git a/Dockerfile.operator b/Dockerfile.operator new file mode 100644 index 0000000..83f9d36 --- /dev/null +++ b/Dockerfile.operator @@ -0,0 +1,36 @@ +# Multi-stage build for niphas-operator +FROM rust:1.88-slim-bookworm AS builder + +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + +# Copy workspace files +COPY Cargo.toml Cargo.lock ./ +COPY crates/ crates/ + +# Build release binary +RUN cargo build --release --bin niphas-operator + +# Runtime image +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +# Create nobody user if it doesn't exist +RUN id -u 65534 &>/dev/null || useradd -u 65534 -r -s /bin/false nobody + +COPY --from=builder /build/target/release/niphas-operator /bin/niphas-operator + +USER 65534 +ENTRYPOINT ["/bin/niphas-operator"] diff --git a/Dockerfile.runner b/Dockerfile.runner new file mode 100644 index 0000000..9da7074 --- /dev/null +++ b/Dockerfile.runner @@ -0,0 +1,8 @@ +# Simple runner image with busybox +FROM busybox:1.36-glibc + +# Copy CA certificates from debian +COPY --from=debian:bookworm-slim /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +ENTRYPOINT ["/bin/sh"] diff --git a/README.md b/README.md index 54d8353..225fd33 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,15 @@ No Nix-only tool replaces this. The OCI image is the unnecessary middleman. Remove it: ``` -git push → nix eval → build (in-cluster) → binary cache → pod mounts the closure +git push → CI builds → binary cache → nix eval (in-cluster) → pod mounts the closure ``` +CI builds and pushes to a binary cache. The cluster only evaluates +(resolves the store path) and fetches pre-built closures. No builds +happen inside the cluster. + Pods reference store paths directly. Nodes share a deduplicated `/nix/store`. -Manifests are rendered from Nix and delivered by your existing GitOps +NiphasWorkload CRDs are delivered by your existing GitOps (Fleet, Argo CD – untouched). No registry, no Dockerfile, no image tags. Kubernetes keeps doing what it does best – scheduling, networking, healing, scaling – on top of a package layer that actually deserves the word @@ -63,10 +67,52 @@ scaling – on top of a package layer that actually deserves the word | Crate | Role | |---|---| -| `niphas-eval` | Webhook → evaluates flakes, schedules builds as Jobs | +| `niphas-eval` | Webhook that evaluates flakes via Nix C API (in-process FFI), resolves closures | | `niphas-operator` | Reconciles the `NiphasWorkload` CRD | | `niphas-csi` | CSI driver that mounts closures into pods | | `niphas-mesh` | P2P substitution of store paths between nodes | -100% Rust + Nix. Built on [kube-rs](https://kube.rs), +100% Rust + Nix. Built on [kube-rs](https://kube.rs). + +## Failure handling + +Node dies? Pods reschedule in ~70s. The CSI driver on the new node +fetches the closure through a fallback chain: + +``` +1. local NAR cache (instant, if workload ran here before) +2. mesh peers via libp2p (LAN speed, if mesh enabled) +3. binary cache HTTP (WAN, always available) +``` + +Each node only caches what its pods actually used -- no proactive +replication, no wasted bandwidth. The mesh is optional: it accelerates +fetches by letting nodes pull NARs from peers instead of the binary cache. + +CSI and mesh DaemonSets only run on nodes labeled `niphas.io/store=true`. +Unlabeled nodes have zero Niphas overhead. + +The operator creates PDBs, topology spread constraints, and status +conditions following K8s conventions. Infrastructure pods +(CSI, operator) run as `system-cluster-critical` so they survive +resource pressure. + +Details: [`docs/RESILIENCE.md`](docs/RESILIENCE.md) + +## Security + +Every component runs with least privilege. Separate ServiceAccount +and RBAC per component. The CSI driver gets **zero RBAC** (talks to +kubelet via Unix socket only). All other components run under +Restricted Pod Security Standards. + +NAR signature verification is mandatory on every fetch (binary cache +or mesh peer). Unverified NARs never reach a pod. Nix eval runs +in-process via Nix C API FFI with sandbox mode, IFD disabled, flake +allowlisting, and dedicated NetworkPolicy. + +The mesh is a closed network: libp2p Noise XX provides mutual +authentication with PFS. Only known cluster PeerIds can connect. + +Details: [`docs/SECURITY_DESIGN.md`](docs/SECURITY_DESIGN.md) diff --git a/charts/niphas/templates/configmap.yaml b/charts/niphas/templates/configmap.yaml index 160aef6..ca2fbbb 100644 --- a/charts/niphas/templates/configmap.yaml +++ b/charts/niphas/templates/configmap.yaml @@ -7,20 +7,23 @@ metadata: {{- include "niphas.labels" . | nindent 4 }} data: config.yaml: | - log_level: {{ .Values.config.logLevel | quote }} - binary_caches: + logLevel: {{ .Values.config.logLevel | quote }} + binaryCaches: {{- range .Values.config.binaryCaches }} - - {{ . | quote }} + - url: {{ .url | quote }} + {{- if .publicKey }} + publicKey: {{ .publicKey | quote }} + {{- end }} {{- end }} - allowed_flake_origins: + allowedFlakeOrigins: {{- range .Values.config.allowedFlakeOrigins }} - {{ . | quote }} {{- end }} - eval_webhook_url: {{ .Values.config.evalWebhookUrl | quote }} - health_port: {{ .Values.config.healthPort }} - eval_timeout: {{ .Values.config.evalTimeout | quote }} + evalWebhookUrl: {{ .Values.config.evalWebhookUrl | quote }} + healthPort: {{ .Values.config.healthPort }} + evalTimeout: {{ .Values.config.evalTimeout | quote }} cache: path: {{ .Values.config.cache.path | quote }} - high_watermark: {{ .Values.config.cache.highWatermark }} - low_watermark: {{ .Values.config.cache.lowWatermark }} - gc_interval: {{ .Values.config.cache.gcInterval | quote }} + highWatermarkBytes: {{ .Values.config.cache.highWatermark | int64 }} + lowWatermarkBytes: {{ .Values.config.cache.lowWatermark | int64 }} + gcInterval: {{ .Values.config.cache.gcInterval | quote }} diff --git a/charts/niphas/templates/docs/workload.yaml b/charts/niphas/templates/docs/workload.yaml new file mode 100644 index 0000000..fae709a --- /dev/null +++ b/charts/niphas/templates/docs/workload.yaml @@ -0,0 +1,17 @@ +{{- if .Values.docs.enabled }} +apiVersion: niphas.io/v1alpha1 +kind: NiphasWorkload +metadata: + name: niphas-docs + namespace: {{ .Values.namespace }} + labels: + app.kubernetes.io/name: niphas-docs + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + flakeRef: {{ .Values.docs.flakeRef | quote }} + attribute: {{ .Values.docs.attribute | quote }} + replicas: {{ .Values.docs.replicas }} + command: + - /bin/sleep + - "3600" +{{- end }} diff --git a/charts/niphas/templates/eval/deployment.yaml b/charts/niphas/templates/eval/deployment.yaml index 7115302..258f3bf 100644 --- a/charts/niphas/templates/eval/deployment.yaml +++ b/charts/niphas/templates/eval/deployment.yaml @@ -38,10 +38,17 @@ spec: periodSeconds: 10 readinessProbe: httpGet: - path: /healthz + path: /readyz port: http initialDelaySeconds: 5 periodSeconds: 5 + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL resources: {{- toYaml .Values.eval.resources | nindent 12 }} volumeMounts: diff --git a/charts/niphas/templates/operator/clusterrole.yaml b/charts/niphas/templates/operator/clusterrole.yaml index b0103ad..42eabd5 100644 --- a/charts/niphas/templates/operator/clusterrole.yaml +++ b/charts/niphas/templates/operator/clusterrole.yaml @@ -29,8 +29,8 @@ rules: - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - # Events - - apiGroups: [""] + # Events (core + events.k8s.io) + - apiGroups: ["", "events.k8s.io"] resources: ["events"] verbs: ["create", "patch"] # Pod status (for ready replica counting) diff --git a/charts/niphas/templates/operator/deployment.yaml b/charts/niphas/templates/operator/deployment.yaml index dbf1259..f652fb0 100644 --- a/charts/niphas/templates/operator/deployment.yaml +++ b/charts/niphas/templates/operator/deployment.yaml @@ -51,6 +51,13 @@ spec: port: health initialDelaySeconds: 5 periodSeconds: 10 + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL resources: {{- toYaml .Values.operator.resources | nindent 12 }} volumeMounts: diff --git a/charts/niphas/values.yaml b/charts/niphas/values.yaml index 47835f6..8a8e1e3 100644 --- a/charts/niphas/values.yaml +++ b/charts/niphas/values.yaml @@ -58,10 +58,18 @@ runner: repository: ghcr.io/fullzer4/niphas-runner tag: latest +docs: + enabled: false + flakeRef: "github:NixOS/nixpkgs" + attribute: "legacyPackages.x86_64-linux.hello" + replicas: 1 + service: + port: 80 + config: logLevel: info binaryCaches: - - https://cache.nixos.org + - url: https://cache.nixos.org allowedFlakeOrigins: - "github:*" evalWebhookUrl: http://niphas-eval.niphas-system.svc:8443 diff --git a/crates/niphas-core/Cargo.toml b/crates/niphas-core/Cargo.toml index 53c3ce5..b415208 100644 --- a/crates/niphas-core/Cargo.toml +++ b/crates/niphas-core/Cargo.toml @@ -23,11 +23,6 @@ thiserror.workspace = true # Observability tracing.workspace = true tracing-subscriber.workspace = true -tracing-opentelemetry.workspace = true -opentelemetry.workspace = true -opentelemetry_sdk.workspace = true -opentelemetry-otlp.workspace = true -opentelemetry-appender-tracing.workspace = true # Config figment.workspace = true @@ -51,6 +46,8 @@ base64.workspace = true name = "niphas-crd-gen" path = "src/bin/crd_gen.rs" +[dev-dependencies] + [features] default = [] runtime = ["kube/runtime"] diff --git a/crates/niphas-core/src/config.rs b/crates/niphas-core/src/config.rs index 702286c..1514a32 100644 --- a/crates/niphas-core/src/config.rs +++ b/crates/niphas-core/src/config.rs @@ -74,6 +74,14 @@ pub struct ClosureResolutionConfig { /// Timeout for resolving the full closure. #[serde(default = "default_closure_timeout", with = "humantime_serde")] pub timeout: Duration, + + /// Maximum number of store paths in a closure. + #[serde(default = "default_max_paths")] + pub max_paths: usize, + + /// Maximum total NAR bytes (uncompressed) for a closure. + #[serde(default = "default_max_nar_bytes")] + pub max_nar_bytes: u64, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -124,6 +132,15 @@ fn default_closure_timeout() -> Duration { Duration::from_secs(60) } +fn default_max_paths() -> usize { + 50_000 +} + +fn default_max_nar_bytes() -> u64 { + // 50 GB + 50 * 1024 * 1024 * 1024 +} + fn default_cache_path() -> PathBuf { PathBuf::from("/var/lib/niphas/cache") } @@ -163,6 +180,8 @@ impl Default for ClosureResolutionConfig { Self { concurrency: default_closure_concurrency(), timeout: default_closure_timeout(), + max_paths: default_max_paths(), + max_nar_bytes: default_max_nar_bytes(), } } } @@ -201,6 +220,42 @@ impl NiphasConfig { figment.extract() } + /// Validate configuration invariants. Returns a list of problems found. + pub fn validate(&self) -> Result<(), Vec> { + let mut errors = Vec::new(); + + if self.cache.high_watermark_bytes <= self.cache.low_watermark_bytes { + errors.push(format!( + "cache.highWatermarkBytes ({}) must be greater than cache.lowWatermarkBytes ({})", + self.cache.high_watermark_bytes, self.cache.low_watermark_bytes + )); + } + + if self.eval_timeout.is_zero() { + errors.push("evalTimeout must be greater than 0".into()); + } + + if self.closure_resolution.concurrency == 0 { + errors.push("closureResolution.concurrency must be greater than 0".into()); + } + + if self.closure_resolution.timeout.is_zero() { + errors.push("closureResolution.timeout must be greater than 0".into()); + } + + if self.binary_caches.is_empty() { + errors.push( + "binaryCaches must not be empty (at least one binary cache is required)".into(), + ); + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + /// Load configuration, falling back to defaults with a warning on error. pub fn load_or_default(path: Option<&str>) -> Self { match Self::load(path) { @@ -242,6 +297,53 @@ mod tests { assert_eq!(cs.gc_interval, Duration::from_secs(300)); } + fn valid_config() -> NiphasConfig { + NiphasConfig { + binary_caches: vec![CacheConfig { + url: "https://cache.nixos.org".into(), + public_key: None, + priority: 40, + }], + ..Default::default() + } + } + + #[test] + fn test_validate_high_watermark_less_than_low() { + let config = NiphasConfig { + cache: CacheStorageConfig { + high_watermark_bytes: 100, + low_watermark_bytes: 200, + ..Default::default() + }, + ..valid_config() + }; + let errors = config.validate().unwrap_err(); + assert!(errors.iter().any(|e| e.contains("highWatermarkBytes"))); + } + + #[test] + fn test_validate_zero_eval_timeout() { + let config = NiphasConfig { + eval_timeout: Duration::from_secs(0), + ..valid_config() + }; + let errors = config.validate().unwrap_err(); + assert!(errors.iter().any(|e| e.contains("evalTimeout"))); + } + + #[test] + fn test_validate_empty_binary_caches() { + let config = NiphasConfig::default(); + let errors = config.validate().unwrap_err(); + assert!(errors.iter().any(|e| e.contains("binaryCaches"))); + } + + #[test] + fn test_validate_valid_config() { + assert!(valid_config().validate().is_ok()); + } + #[test] fn test_humantime_parse_seconds() { assert_eq!( diff --git a/crates/niphas-core/src/crd.rs b/crates/niphas-core/src/crd.rs index 406b407..47156b4 100644 --- a/crates/niphas-core/src/crd.rs +++ b/crates/niphas-core/src/crd.rs @@ -285,7 +285,7 @@ pub struct ArchStatus { pub ready_replicas: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct NiphasCondition { /// Condition type: Evaluated, ClosureCached, Available, Progressing, Degraded. diff --git a/crates/niphas-core/src/eval.rs b/crates/niphas-core/src/eval.rs index 48ae8c9..374e09f 100644 --- a/crates/niphas-core/src/eval.rs +++ b/crates/niphas-core/src/eval.rs @@ -18,7 +18,7 @@ pub struct EvalRequest { /// Successful response from the eval webhook. /// /// Shared between niphas-eval (serializer) and niphas-operator (deserializer). -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EvalResponse { pub store_path: String, diff --git a/crates/niphas-core/src/nix/cache_client.rs b/crates/niphas-core/src/nix/cache_client.rs index a3c7cee..23d2f48 100644 --- a/crates/niphas-core/src/nix/cache_client.rs +++ b/crates/niphas-core/src/nix/cache_client.rs @@ -101,33 +101,79 @@ impl CacheClient { /// /// Returns the raw compressed bytes. Caller is responsible for /// decompression and hash verification. + /// + /// Retries transient errors (timeouts, connection resets, 5xx) up to 3 times + /// with exponential backoff (100ms, 500ms, 2s). Uses a dynamic per-request + /// timeout scaled by file size (30s + 1s per MB). async fn fetch_nar_impl(&self, narinfo: &NarInfo) -> Result { + const MAX_RETRIES: u32 = 3; + const BACKOFF_MS: [u64; 3] = [100, 500, 2000]; + + // Dynamic timeout: 30s base + 1s per MB + let timeout_secs = 30 + (narinfo.file_size / 1_000_000); + let req_timeout = Duration::from_secs(timeout_secs); + // The URL in narinfo is relative to the cache base URL. // Try each cache in case one has the NAR. let mut last_err = None; for cache in &self.caches { let url = format!("{}/{}", cache.url.trim_end_matches('/'), narinfo.url); - debug!(url = %url, "fetching NAR"); - - match self.http.get(&url).send().await { - Ok(resp) => { - if resp.status().is_success() { - let data = resp.bytes().await.map_err(|e| { - NiphasError::Cache(format!("failed to read NAR body: {e}")) - })?; - return Ok(data); + debug!(url = %url, timeout_secs = timeout_secs, "fetching NAR"); + + let mut attempt = 0; + loop { + match self.http.get(&url).timeout(req_timeout).send().await { + Ok(resp) => { + if resp.status().is_success() { + let data = resp.bytes().await.map_err(|e| { + NiphasError::Cache(format!("failed to read NAR body: {e}")) + })?; + if (data.len() as u64) < narinfo.file_size { + // Truncated download — fewer bytes than expected. + // More bytes is tolerable (CDN/cache inconsistency); + // the SHA256 hash check downstream catches corruption. + return Err(NiphasError::Cache(format!( + "incomplete NAR download: expected {} bytes, got {}", + narinfo.file_size, + data.len() + ))); + } + return Ok(data); + } + + let status = resp.status(); + let is_transient = status.is_server_error(); + + if is_transient && attempt < MAX_RETRIES { + warn!(url = %url, %status, attempt, "transient HTTP error, retrying"); + tokio::time::sleep(Duration::from_millis(BACKOFF_MS[attempt as usize])) + .await; + attempt += 1; + continue; + } + + warn!(url = %url, %status, "failed to fetch NAR"); + last_err = Some(NiphasError::Cache(format!( + "HTTP {status} fetching NAR from {url}" + ))); + break; + } + Err(e) => { + let is_transient = e.is_timeout() || e.is_connect() || e.is_request(); + + if is_transient && attempt < MAX_RETRIES { + warn!(url = %url, error = %e, attempt, "transient error, retrying"); + tokio::time::sleep(Duration::from_millis(BACKOFF_MS[attempt as usize])) + .await; + attempt += 1; + continue; + } + + warn!(url = %url, error = %e, "failed to fetch NAR"); + last_err = Some(NiphasError::Cache(format!("NAR request failed: {e}"))); + break; } - - let status = resp.status(); - warn!(url = %url, %status, "failed to fetch NAR"); - last_err = Some(NiphasError::Cache(format!( - "HTTP {status} fetching NAR from {url}" - ))); - } - Err(e) => { - warn!(url = %url, error = %e, "failed to fetch NAR"); - last_err = Some(NiphasError::Cache(format!("NAR request failed: {e}"))); } } } diff --git a/crates/niphas-core/src/nix/closure.rs b/crates/niphas-core/src/nix/closure.rs index bfdb659..1cfdba8 100644 --- a/crates/niphas-core/src/nix/closure.rs +++ b/crates/niphas-core/src/nix/closure.rs @@ -1,12 +1,13 @@ use crate::error::NiphasError; use crate::nix::cache_client::BinaryCacheClient; use crate::nix::narinfo::NarInfo; +use crate::nix::signature::{self, TrustedKey}; use crate::nix::store_path::StorePath; use futures::StreamExt; use futures::stream::FuturesUnordered; use std::collections::{HashMap, HashSet}; use std::time::Duration; -use tracing::{debug, warn}; +use tracing::debug; /// Result of resolving a full closure. #[derive(Debug)] @@ -39,17 +40,25 @@ impl ResolvedClosure { /// Performs BFS over `.narinfo` References using parallel HTTP fetches /// (bounded by `concurrency`). Returns all narinfos needed to materialize /// the store path. +/// +/// If `trusted_keys` is non-empty, each narinfo's signature is verified +/// against the trusted keys. If empty, no signature verification is performed +/// (backwards compatible). pub async fn resolve_closure( client: &C, root: &StorePath, concurrency: usize, timeout: Duration, + trusted_keys: &[TrustedKey], + max_paths: usize, + max_nar_bytes: u64, ) -> Result { let deadline = tokio::time::Instant::now() + timeout; let mut visited: HashSet = HashSet::new(); let mut narinfos: HashMap = HashMap::new(); let mut order: Vec = Vec::new(); + let mut total_nar_bytes: u64 = 0; // BFS queue: store path strings to resolve let mut queue: Vec = vec![root.clone()]; @@ -88,6 +97,12 @@ pub async fn resolve_closure( Ok(narinfo) => { debug!(store_path = %sp_str, refs = narinfo.references.len(), "resolved narinfo"); + // Verify signature if trusted keys are configured + if !trusted_keys.is_empty() { + let fp = narinfo.fingerprint(); + signature::verify_narinfo(&fp, &narinfo.signatures, trusted_keys)?; + } + // Enqueue unvisited references for ref_basename in &narinfo.references { let ref_path_str = ref_basename.to_store_path_string(); @@ -95,18 +110,32 @@ pub async fn resolve_closure( match StorePath::parse(&ref_path_str) { Ok(ref_sp) => queue.push(ref_sp), Err(e) => { - warn!( - reference = %ref_path_str, - error = %e, - "skipping invalid reference" - ); + return Err(NiphasError::ClosureResolution(format!( + "invalid reference '{}' in {}: {}", + ref_path_str, sp_str, e + ))); } } } } order.push(sp_str.clone()); + total_nar_bytes += narinfo.nar_size; narinfos.insert(sp_str, narinfo); + + // Check closure size limits + if narinfos.len() > max_paths { + return Err(NiphasError::ClosureResolution(format!( + "closure exceeds max path count ({})", + max_paths + ))); + } + if total_nar_bytes > max_nar_bytes { + return Err(NiphasError::ClosureResolution(format!( + "closure exceeds max NAR size ({} bytes)", + max_nar_bytes + ))); + } } Err(e) => { return Err(NiphasError::ClosureResolution(format!( @@ -132,6 +161,7 @@ pub async fn resolve_closure( #[cfg(test)] mod tests { use super::*; + use crate::nix::signature::{NarSignature, TrustedKey}; use crate::testutils::fake_cache::FakeCacheClient; use crate::testutils::narinfo_builder::NarInfoBuilder; @@ -156,9 +186,17 @@ mod tests { client.add_narinfo(NarInfoBuilder::new(&path).nar_size(5000).build()); let root = StorePath::parse(&path).unwrap(); - let result = resolve_closure(&client, &root, 4, Duration::from_secs(10)) - .await - .unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await + .unwrap(); assert_eq!(result.paths.len(), 1); assert_eq!(result.paths[0], path); @@ -187,9 +225,17 @@ mod tests { client.add_narinfo(NarInfoBuilder::new(&path_c).build()); let root = StorePath::parse(&path_a).unwrap(); - let result = resolve_closure(&client, &root, 4, Duration::from_secs(10)) - .await - .unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await + .unwrap(); assert_eq!(result.paths.len(), 3); assert!(result.paths.contains(&path_a)); @@ -228,9 +274,17 @@ mod tests { client.add_narinfo(NarInfoBuilder::new(&path_d).build()); let root = StorePath::parse(&path_a).unwrap(); - let result = resolve_closure(&client, &root, 4, Duration::from_secs(10)) - .await - .unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await + .unwrap(); // D should appear exactly once despite being referenced by both B and C. assert_eq!(result.paths.len(), 4); @@ -251,7 +305,16 @@ mod tests { ); let root = StorePath::parse(&path_a).unwrap(); - let result = resolve_closure(&client, &root, 4, Duration::from_secs(10)).await; + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await; assert!(result.is_err()); let err = result.unwrap_err().to_string(); @@ -280,9 +343,17 @@ mod tests { ); let root = StorePath::parse(&path_a).unwrap(); - let result = resolve_closure(&client, &root, 4, Duration::from_secs(10)) - .await - .unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await + .unwrap(); assert_eq!(result.paths().len(), 2); assert_eq!(result.nar_size(), 4000); @@ -302,10 +373,161 @@ mod tests { ); let root = StorePath::parse(&path_a).unwrap(); - let result = resolve_closure(&client, &root, 4, Duration::from_secs(10)) - .await - .unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await + .unwrap(); assert_eq!(result.paths.len(), 1); } + + /// Helper: generate a valid ed25519 keypair and sign a narinfo fingerprint. + fn sign_narinfo(narinfo: &NarInfo, key_name: &str) -> (TrustedKey, NarSignature) { + use base64::prelude::*; + use ed25519_dalek::{Signer, SigningKey}; + + let signing_key = SigningKey::from_bytes(&[42u8; 32]); + let verifying_key = signing_key.verifying_key(); + + let fp = narinfo.fingerprint(); + let sig = signing_key.sign(fp.as_bytes()); + + let sig_b64 = BASE64_STANDARD.encode(sig.to_bytes()); + let sig_str = format!("{key_name}:{sig_b64}"); + let nar_sig = NarSignature::parse(&sig_str).unwrap(); + + let key_b64 = BASE64_STANDARD.encode(verifying_key.to_bytes()); + let key_str = format!("{key_name}:{key_b64}"); + let trusted_key = TrustedKey::parse(&key_str).unwrap(); + + (trusted_key, nar_sig) + } + + #[tokio::test] + async fn test_signature_verification_valid() { + let mut client = FakeCacheClient::new(); + let path = sp(HASH_A, "hello-2.12.1"); + + let narinfo_unsigned = NarInfoBuilder::new(&path).nar_size(5000).build(); + let (trusted_key, nar_sig) = sign_narinfo(&narinfo_unsigned, "test-key-1"); + + client.add_narinfo( + NarInfoBuilder::new(&path) + .nar_size(5000) + .signatures(vec![nar_sig]) + .build(), + ); + + let root = StorePath::parse(&path).unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[trusted_key], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await; + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_signature_verification_no_valid_sig() { + let mut client = FakeCacheClient::new(); + let path = sp(HASH_A, "hello-2.12.1"); + + // Narinfo with no signatures, but trusted keys are configured + client.add_narinfo(NarInfoBuilder::new(&path).nar_size(5000).build()); + + let key_str = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + let trusted_key = TrustedKey::parse(key_str).unwrap(); + + let root = StorePath::parse(&path).unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[trusted_key], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("signature"), "unexpected error: {err}"); + } + + #[tokio::test] + async fn test_signature_verification_empty_keys_skips() { + let mut client = FakeCacheClient::new(); + let path = sp(HASH_A, "hello-2.12.1"); + + // Narinfo with no signatures, but no trusted keys configured → should pass + client.add_narinfo(NarInfoBuilder::new(&path).nar_size(5000).build()); + + let root = StorePath::parse(&path).unwrap(); + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 50_000, + 50 * 1024 * 1024 * 1024, + ) + .await; + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_closure_exceeds_max_paths() { + let mut client = FakeCacheClient::new(); + let path = sp(HASH_A, "hello-2.12.1"); + client.add_narinfo(NarInfoBuilder::new(&path).nar_size(5000).build()); + + let root = StorePath::parse(&path).unwrap(); + // max_paths = 0 means any closure with at least 1 path should fail + let result = resolve_closure( + &client, + &root, + 4, + Duration::from_secs(10), + &[], + 0, + 50 * 1024 * 1024 * 1024, + ) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("max path count"), "unexpected error: {err}"); + } + + #[tokio::test] + async fn test_closure_exceeds_max_nar_bytes() { + let mut client = FakeCacheClient::new(); + let path = sp(HASH_A, "hello-2.12.1"); + client.add_narinfo(NarInfoBuilder::new(&path).nar_size(5000).build()); + + let root = StorePath::parse(&path).unwrap(); + // max_nar_bytes = 100 is less than the 5000 nar_size + let result = + resolve_closure(&client, &root, 4, Duration::from_secs(10), &[], 50_000, 100).await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("max NAR size"), "unexpected error: {err}"); + } } diff --git a/crates/niphas-core/src/nix/nar.rs b/crates/niphas-core/src/nix/nar.rs index cf27975..397c6e4 100644 --- a/crates/niphas-core/src/nix/nar.rs +++ b/crates/niphas-core/src/nix/nar.rs @@ -130,25 +130,36 @@ impl NarReader { Ok((node, nar_hash)) } - /// Parse a single node: `"(" node_body ")"`. + /// Parse a single node: `"(" type_tag node_body ")"`. + /// + /// Per the NAR spec, a node is `str("(") nar-obj-inner str(")")`. + /// For directory nodes, `parse_directory()` already consumes the + /// closing `")"` (it reads `"entry"` or `")"` in a loop to detect + /// end-of-directory), so we must not read it again. async fn parse_node(&mut self) -> Result { self.expect_str("(").await?; self.expect_str("type").await?; let type_str = self.read_str().await?; - let node = match type_str.as_str() { - "regular" => self.parse_regular().await?, - "symlink" => self.parse_symlink().await?, - "directory" => self.parse_directory().await?, - _ => { - return Err(NiphasError::NarParse(format!( - "unknown node type: '{type_str}'" - ))); + match type_str.as_str() { + "regular" => { + let node = self.parse_regular().await?; + self.expect_str(")").await?; + Ok(node) } - }; - - self.expect_str(")").await?; - Ok(node) + "symlink" => { + let node = self.parse_symlink().await?; + self.expect_str(")").await?; + Ok(node) + } + "directory" => { + // parse_directory already consumes the closing ")" + self.parse_directory().await + } + _ => Err(NiphasError::NarParse(format!( + "unknown node type: '{type_str}'" + ))), + } } /// Parse a regular file node. @@ -218,12 +229,12 @@ impl NarReader { validate_entry_name(&name)?; // Check ordering - if let Some(ref prev) = last_name { - if name <= *prev { - return Err(NiphasError::NarParse(format!( - "directory entries not sorted: '{prev}' followed by '{name}'" - ))); - } + if let Some(ref prev) = last_name + && name <= *prev + { + return Err(NiphasError::NarParse(format!( + "directory entries not sorted: '{prev}' followed by '{name}'" + ))); } last_name = Some(name.clone()); @@ -390,6 +401,122 @@ mod tests { } } + /// Build a NAR for a directory with the given entries. + /// Each entry is (name, nar_node_bytes) where nar_node_bytes is + /// the inner serialization of a node (including "(" ... ")"). + fn build_directory_nar(entries: &[(&str, Vec)]) -> Vec { + let mut nar = Vec::new(); + nar.extend(nar_str("nix-archive-1")); + nar.extend(nar_str("(")); + nar.extend(nar_str("type")); + nar.extend(nar_str("directory")); + for (name, node_bytes) in entries { + nar.extend(nar_str("entry")); + nar.extend(nar_str("(")); + nar.extend(nar_str("name")); + nar.extend(nar_str(name)); + nar.extend(nar_str("node")); + nar.extend(node_bytes); + nar.extend(nar_str(")")); + } + nar.extend(nar_str(")")); + nar + } + + /// Build the inner NAR bytes for a regular file node (including parens). + fn build_regular_node(contents: &[u8]) -> Vec { + let mut node = Vec::new(); + node.extend(nar_str("(")); + node.extend(nar_str("type")); + node.extend(nar_str("regular")); + node.extend(nar_str("contents")); + node.extend(&(contents.len() as u64).to_le_bytes()); + node.extend(contents); + let pad = (8 - (contents.len() % 8)) % 8; + node.extend(std::iter::repeat_n(0u8, pad)); + node.extend(nar_str(")")); + node + } + + #[tokio::test] + async fn test_parse_empty_directory() { + let nar = build_directory_nar(&[]); + let reader = NarReader::new(&nar[..]); + let (node, hash) = reader.parse().await.unwrap(); + + match node { + NarNode::Directory { entries } => { + assert!(entries.is_empty()); + } + _ => panic!("expected directory"), + } + assert_eq!(hash.algo, crate::nix::hash::HashAlgo::Sha256); + assert_eq!(hash.digest.len(), 32); + } + + #[tokio::test] + async fn test_parse_directory_with_files() { + let nar = build_directory_nar(&[ + ("hello.txt", build_regular_node(b"hello")), + ("world.txt", build_regular_node(b"world")), + ]); + let reader = NarReader::new(&nar[..]); + let (node, _) = reader.parse().await.unwrap(); + + match node { + NarNode::Directory { entries } => { + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].name, "hello.txt"); + assert_eq!(entries[1].name, "world.txt"); + match &entries[0].node { + NarNode::Regular { contents, .. } => assert_eq!(contents, b"hello"), + _ => panic!("expected regular file"), + } + } + _ => panic!("expected directory"), + } + } + + #[tokio::test] + async fn test_parse_nested_directory() { + // Build: root/subdir/file.txt + let inner_dir = { + let file_node = build_regular_node(b"nested content"); + let mut d = Vec::new(); + d.extend(nar_str("(")); + d.extend(nar_str("type")); + d.extend(nar_str("directory")); + d.extend(nar_str("entry")); + d.extend(nar_str("(")); + d.extend(nar_str("name")); + d.extend(nar_str("file.txt")); + d.extend(nar_str("node")); + d.extend(&file_node); + d.extend(nar_str(")")); + d.extend(nar_str(")")); + d + }; + + let nar = build_directory_nar(&[("subdir", inner_dir)]); + let reader = NarReader::new(&nar[..]); + let (node, _) = reader.parse().await.unwrap(); + + match node { + NarNode::Directory { entries } => { + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].name, "subdir"); + match &entries[0].node { + NarNode::Directory { entries: sub } => { + assert_eq!(sub.len(), 1); + assert_eq!(sub[0].name, "file.txt"); + } + _ => panic!("expected nested directory"), + } + } + _ => panic!("expected directory"), + } + } + #[test] fn test_validate_entry_name() { assert!(validate_entry_name("hello").is_ok()); diff --git a/crates/niphas-core/src/telemetry.rs b/crates/niphas-core/src/telemetry.rs index 10de4ae..82cc50d 100644 --- a/crates/niphas-core/src/telemetry.rs +++ b/crates/niphas-core/src/telemetry.rs @@ -1,103 +1,14 @@ -use opentelemetry::trace::TracerProvider as _; -use opentelemetry_otlp::LogExporter; -use opentelemetry_otlp::SpanExporter; -use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::Resource; -use opentelemetry_sdk::logs::SdkLoggerProvider; -use opentelemetry_sdk::trace::SdkTracerProvider; -use tracing_opentelemetry::OpenTelemetryLayer; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, fmt}; -/// Guard that flushes pending spans/logs on drop. -/// The caller must keep it alive until shutdown. -pub struct TelemetryGuard { - tracer_provider: Option, - logger_provider: Option, -} - -impl Drop for TelemetryGuard { - fn drop(&mut self) { - if let Some(tp) = self.tracer_provider.take() { - let _ = tp.shutdown(); - } - if let Some(lp) = self.logger_provider.take() { - let _ = lp.shutdown(); - } - } -} - -/// Initializes tracing with optional OTEL support. -/// -/// If `OTEL_EXPORTER_OTLP_ENDPOINT` is set, adds: -/// - OTLP trace exporter (spans) -/// - OTLP log exporter (logs bridge) -/// -/// Without that env var, behavior is identical to before (JSON logs on stdout). -/// -/// Returns a guard that MUST be kept alive in main. -pub fn init_tracing(service_name: &'static str) -> TelemetryGuard { +pub fn init_tracing(service_name: &'static str) { let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let fmt_layer = fmt::layer().json(); - - let otel_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); - - if let Some(endpoint) = otel_endpoint { - let resource = Resource::builder().with_service_name(service_name).build(); - - // Trace exporter - let span_exporter = SpanExporter::builder() - .with_tonic() - .with_endpoint(&endpoint) - .build() - .expect("failed to create OTLP span exporter"); - - let tracer_provider = SdkTracerProvider::builder() - .with_batch_exporter(span_exporter) - .with_resource(resource.clone()) - .build(); - - let tracer = tracer_provider.tracer(service_name); - let otel_layer = OpenTelemetryLayer::new(tracer); - - // Log exporter - let log_exporter = LogExporter::builder() - .with_tonic() - .with_endpoint(&endpoint) - .build() - .expect("failed to create OTLP log exporter"); - - let logger_provider = SdkLoggerProvider::builder() - .with_batch_exporter(log_exporter) - .with_resource(resource) - .build(); - - let otel_log_layer = opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new( - &logger_provider, - ); - - tracing_subscriber::registry() - .with(filter) - .with(fmt_layer) - .with(otel_layer) - .with(otel_log_layer) - .init(); - - TelemetryGuard { - tracer_provider: Some(tracer_provider), - logger_provider: Some(logger_provider), - } - } else { - tracing_subscriber::registry() - .with(filter) - .with(fmt_layer) - .init(); + tracing_subscriber::registry() + .with(filter) + .with(fmt::layer().json().with_target(true)) + .init(); - TelemetryGuard { - tracer_provider: None, - logger_provider: None, - } - } + tracing::info!(service = service_name, "tracing initialized"); } diff --git a/crates/niphas-core/src/testutils/narinfo_builder.rs b/crates/niphas-core/src/testutils/narinfo_builder.rs index a7c2a8e..2ccb415 100644 --- a/crates/niphas-core/src/testutils/narinfo_builder.rs +++ b/crates/niphas-core/src/testutils/narinfo_builder.rs @@ -2,6 +2,7 @@ use crate::nix::hash::{HashAlgo, NixHash}; use crate::nix::narinfo::{Compression, NarInfo}; +use crate::nix::signature::NarSignature; use crate::nix::store_path::{StorePath, StorePathRef}; /// Builder for creating NarInfo fixtures in tests. @@ -21,6 +22,7 @@ pub struct NarInfoBuilder { nar_hash: NixHash, nar_size: u64, references: Vec, + signatures: Vec, } impl NarInfoBuilder { @@ -46,6 +48,7 @@ impl NarInfoBuilder { }, nar_size: 2000, references: Vec::new(), + signatures: Vec::new(), } } @@ -76,6 +79,12 @@ impl NarInfoBuilder { self } + /// Set signatures on the narinfo. + pub fn signatures(mut self, sigs: Vec) -> Self { + self.signatures = sigs; + self + } + pub fn build(self) -> NarInfo { NarInfo { store_path: self.store_path, @@ -87,7 +96,7 @@ impl NarInfoBuilder { nar_size: self.nar_size, references: self.references, deriver: None, - signatures: Vec::new(), + signatures: self.signatures, ca: None, } } diff --git a/crates/niphas-csi/Cargo.toml b/crates/niphas-csi/Cargo.toml index 925a47e..b36449d 100644 --- a/crates/niphas-csi/Cargo.toml +++ b/crates/niphas-csi/Cargo.toml @@ -16,7 +16,6 @@ niphas-core.workspace = true tonic.workspace = true tonic-prost.workspace = true prost.workspace = true -prost-types.workspace = true tokio.workspace = true tokio-stream = { version = "0.1", features = ["net"] } tracing.workspace = true diff --git a/crates/niphas-csi/src/cache.rs b/crates/niphas-csi/src/cache.rs index 04d4902..848b832 100644 --- a/crates/niphas-csi/src/cache.rs +++ b/crates/niphas-csi/src/cache.rs @@ -1,16 +1,16 @@ use niphas_core::config::NiphasConfig; use niphas_core::error::NiphasError; use niphas_core::nix::cache_client::{BinaryCacheClient, CacheClient}; -use niphas_core::nix::hash::sha256_hash; use niphas_core::nix::nar::NarReader; use niphas_core::nix::narinfo::NarInfo; +use niphas_core::nix::signature::{self, TrustedKey}; use niphas_core::nix::store_path::StorePath; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::io::AsyncReadExt; use tokio::sync::Mutex; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; /// Local NAR cache on each node. /// @@ -23,6 +23,7 @@ use tracing::{debug, info}; pub struct NarCache { cache_dir: PathBuf, cache_client: C, + trusted_keys: Vec, /// Per-path locks to prevent duplicate downloads. locks: Mutex>>>, } @@ -34,9 +35,17 @@ impl NarCache { let cache_client = CacheClient::new(config.binary_caches.clone())?; + let trusted_keys: Vec = config + .binary_caches + .iter() + .filter_map(|c| c.public_key.as_deref()) + .map(TrustedKey::parse) + .collect::, _>>()?; + Ok(NarCache { cache_dir, cache_client, + trusted_keys, locks: Mutex::new(HashMap::new()), }) } @@ -48,10 +57,16 @@ impl NarCache { NarCache { cache_dir, cache_client, + trusted_keys: Vec::new(), locks: Mutex::new(HashMap::new()), } } + /// Return the cache directory path (used as the bind mount source for /nix/store). + pub fn store_dir(&self) -> &Path { + &self.cache_dir + } + /// Return the filesystem path for a given store path basename. pub fn path_for(&self, store_path: &str) -> String { // Strip /nix/store/ prefix if present. @@ -95,7 +110,13 @@ impl NarCache { continue; } - fetch_and_extract(&self.cache_client, basename, &self.cache_dir).await?; + fetch_and_extract( + &self.cache_client, + basename, + &self.cache_dir, + &self.trusted_keys, + ) + .await?; } Ok(()) @@ -118,26 +139,76 @@ impl NarCache { } /// Fetch a NAR from binary cache and extract it to the cache directory. +/// +/// Verification strategy (matches Nix behaviour): +/// - `narHash` (uncompressed NAR) is the authoritative integrity check. +/// - `fileHash` (compressed NAR) is NOT verified — Nix never checks it on +/// download, and CDN staleness can cause it to diverge from the actual file. +/// - On `narHash` mismatch the narinfo is likely stale (CDN served a cached +/// narinfo while S3 has a newer NAR). We re-fetch narinfo once and retry, +/// mirroring the approach proposed in NixOS/nix PR #3969. async fn fetch_and_extract( client: &impl BinaryCacheClient, basename: &str, cache_dir: &Path, + trusted_keys: &[TrustedKey], ) -> Result<(), NiphasError> { info!(basename, "fetching from binary cache"); let store_path = StorePath::parse_basename(basename)?; - let narinfo = client.fetch_narinfo_by_hash(&store_path.hash_str()).await?; - let nar_data = client.fetch_nar(&narinfo).await?; + let hash = store_path.hash_str(); + + let narinfo = client.fetch_narinfo_by_hash(&hash).await?; + verify_narinfo_sig(&narinfo, trusted_keys)?; + + match try_fetch_and_verify(client, &narinfo, basename, cache_dir).await { + Ok(()) => Ok(()), + Err(NiphasError::HashMismatch { expected, actual }) => { + // Stale narinfo — re-fetch and retry once (PR #3969 pattern). + warn!( + basename, + expected = %expected, + actual = %actual, + "narHash mismatch, re-fetching narinfo (possible CDN staleness)" + ); + + let fresh_narinfo = client.fetch_narinfo_by_hash(&hash).await?; + verify_narinfo_sig(&fresh_narinfo, trusted_keys)?; + + // If the narinfo hasn't changed, no point retrying the same NAR. + if fresh_narinfo.nar_hash == narinfo.nar_hash && fresh_narinfo.url == narinfo.url { + return Err(NiphasError::HashMismatch { expected, actual }); + } - let file_hash = sha256_hash(&nar_data); - if file_hash != narinfo.file_hash { - return Err(NiphasError::HashMismatch { - expected: narinfo.file_hash.to_string(), - actual: file_hash.to_string(), - }); + info!( + basename, + "narinfo changed after re-fetch, retrying download" + ); + try_fetch_and_verify(client, &fresh_narinfo, basename, cache_dir).await + } + Err(e) => Err(e), } +} - let decompressed = decompress_nar(&narinfo, &nar_data).await?; +/// Verify narinfo signature against trusted keys (if any are configured). +fn verify_narinfo_sig(narinfo: &NarInfo, trusted_keys: &[TrustedKey]) -> Result<(), NiphasError> { + if !trusted_keys.is_empty() { + let fp = narinfo.fingerprint(); + signature::verify_narinfo(&fp, &narinfo.signatures, trusted_keys)?; + } + Ok(()) +} + +/// Download a NAR, decompress, parse, verify narHash, and extract to cache dir. +async fn try_fetch_and_verify( + client: &impl BinaryCacheClient, + narinfo: &NarInfo, + basename: &str, + cache_dir: &Path, +) -> Result<(), NiphasError> { + let nar_data = client.fetch_nar(narinfo).await?; + + let decompressed = decompress_nar(narinfo, &nar_data).await?; let tmp_name = format!(".tmp-{}-{}", basename, std::process::id()); let tmp_dir = cache_dir.join(&tmp_name); @@ -176,22 +247,32 @@ async fn decompress_nar(narinfo: &NarInfo, data: &[u8]) -> Result, Nipha macro_rules! decode { ($decoder:ty, $data:expr) => {{ let decoder = <$decoder>::new(tokio::io::BufReader::new($data)); - let mut output = Vec::new(); + let mut buf = Vec::new(); tokio::pin!(decoder); - decoder.read_to_end(&mut output).await?; - Ok(output) + decoder.read_to_end(&mut buf).await?; + buf }}; } - match narinfo.compression { - Compression::None => Ok(data.to_vec()), + let output = match narinfo.compression { + Compression::None => data.to_vec(), Compression::Zstd => decode!(async_compression::tokio::bufread::ZstdDecoder<_>, data), Compression::Xz => decode!(async_compression::tokio::bufread::XzDecoder<_>, data), Compression::Bzip2 => decode!(async_compression::tokio::bufread::BzDecoder<_>, data), - other => Err(NiphasError::Cache(format!( - "unsupported compression: {other}" - ))), + other => { + return Err(NiphasError::Cache(format!( + "unsupported compression: {other}" + ))); + } + }; + if output.len() as u64 != narinfo.nar_size { + return Err(NiphasError::Cache(format!( + "decompressed NAR size mismatch: expected {} bytes, got {}", + narinfo.nar_size, + output.len() + ))); } + Ok(output) } /// Extract a NAR node tree to a filesystem directory. diff --git a/crates/niphas-csi/src/main.rs b/crates/niphas-csi/src/main.rs index aa7440f..227325e 100644 --- a/crates/niphas-csi/src/main.rs +++ b/crates/niphas-csi/src/main.rs @@ -17,7 +17,7 @@ const CSI_SOCKET_PATH: &str = "/csi/csi.sock"; #[tokio::main] async fn main() -> anyhow::Result<()> { - let _telemetry = niphas_core::telemetry::init_tracing("niphas-csi"); + niphas_core::telemetry::init_tracing("niphas-csi"); info!("starting niphas-csi"); let config = NiphasConfig::load_or_default(None); @@ -48,8 +48,20 @@ async fn main() -> anyhow::Result<()> { .add_service(csi::node_server::NodeServer::new(NodeService::new( node_id, nar_cache, ))) - .serve_with_incoming(stream) + .serve_with_incoming_shutdown(stream, shutdown_signal()) .await?; + info!("shutdown complete"); Ok(()) } + +async fn shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).expect("failed to install SIGTERM handler"); + + tokio::select! { + _ = tokio::signal::ctrl_c() => info!("received SIGINT"), + _ = sigterm.recv() => info!("received SIGTERM"), + } +} diff --git a/crates/niphas-csi/src/node.rs b/crates/niphas-csi/src/node.rs index fa1a3a3..0c944f3 100644 --- a/crates/niphas-csi/src/node.rs +++ b/crates/niphas-csi/src/node.rs @@ -116,8 +116,9 @@ impl csi::node_server::No ))); } - // Bind mount the primary store path (read-only). - let source = self.cache.path_for(store_path); + // Bind mount the entire cache directory so all closure paths are + // accessible at their correct /nix/store/- paths. + let source = self.cache.store_dir().to_string_lossy().to_string(); if let Err(e) = self.mount.bind_mount_readonly(&source, target_path) { // Clean up target dir on failure. warn!(target_path, err = %e, "bind mount failed, cleaning up"); diff --git a/crates/niphas-e2e/tests/e2e_platform.rs b/crates/niphas-e2e/tests/e2e_platform.rs index 1369f99..82864a9 100644 --- a/crates/niphas-e2e/tests/e2e_platform.rs +++ b/crates/niphas-e2e/tests/e2e_platform.rs @@ -1,12 +1,15 @@ use anyhow::{Context, Result, bail}; +use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition; use kube::{ Api, Client, - api::{DeleteParams, PostParams}, + api::{DeleteParams, ListParams, Patch, PatchParams, PostParams}, }; -use niphas_core::crd::{NiphasWorkload, WorkloadPhase}; +use niphas_core::crd::{ConditionStatus, NiphasWorkload, WorkloadPhase}; use std::time::Duration; +const FINALIZER: &str = "niphas.io/workload-cleanup"; + /// Poll the NiphasWorkload status until it reaches `expected` phase or timeout. async fn wait_for_phase( api: &Api, @@ -17,10 +20,17 @@ async fn wait_for_phase( let start = std::time::Instant::now(); loop { if start.elapsed() > timeout { + let current = api.get_opt(name).await?.map(|w| { + w.status + .as_ref() + .map(|s| format!("phase={:?}", s.phase)) + .unwrap_or_else(|| "no status".to_string()) + }); bail!( - "timed out waiting for workload '{}' to reach phase {:?}", + "timed out waiting for workload '{}' to reach phase {:?} (current: {:?})", name, - expected + expected, + current ); } if let Some(wl) = api.get_opt(name).await? { @@ -34,6 +44,42 @@ async fn wait_for_phase( } } +/// Poll the NiphasWorkload status until a condition with `type_` has `status`, +/// or timeout. This is more reliable than waiting for a phase, since phases +/// can flip-flop during reconciliation retries. +async fn wait_for_condition( + api: &Api, + name: &str, + condition_type: &str, + expected_status: ConditionStatus, + timeout: Duration, +) -> Result { + let start = std::time::Instant::now(); + loop { + if start.elapsed() > timeout { + bail!( + "timed out waiting for workload '{}' condition '{}' = {:?}", + name, + condition_type, + expected_status + ); + } + if let Some(wl) = api.get_opt(name).await? { + if let Some(ref status) = wl.status { + if let Some(conditions) = &status.conditions { + if conditions + .iter() + .any(|c| c.type_ == condition_type && c.status == expected_status) + { + return Ok(wl); + } + } + } + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + /// Wait until the workload no longer exists. async fn wait_for_deletion(api: &Api, name: &str, timeout: Duration) -> Result<()> { let start = std::time::Instant::now(); @@ -48,7 +94,42 @@ async fn wait_for_deletion(api: &Api, name: &str, timeout: Durat } } -fn test_workload(name: &str, flake_ref: &str) -> NiphasWorkload { +/// Wait until the workload has the niphas finalizer. +async fn wait_for_finalizer( + api: &Api, + name: &str, + timeout: Duration, +) -> Result { + let start = std::time::Instant::now(); + loop { + if start.elapsed() > timeout { + bail!("timed out waiting for workload '{}' to get finalizer", name); + } + if let Some(wl) = api.get_opt(name).await? { + let has_finalizer = wl + .metadata + .finalizers + .as_ref() + .is_some_and(|f| f.iter().any(|s| s == FINALIZER)); + if has_finalizer { + return Ok(wl); + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +/// Clean up a workload, ignoring errors if it doesn't exist. +async fn cleanup_workload(api: &Api, name: &str) { + let _ = api.delete(name, &DeleteParams::default()).await; + let _ = wait_for_deletion(api, name, Duration::from_secs(30)).await; +} + +fn e2e_namespace() -> String { + std::env::var("E2E_NAMESPACE").unwrap_or_else(|_| "niphas-e2e".to_string()) +} + +fn test_workload(name: &str, flake_ref: &str, attribute: &str) -> NiphasWorkload { serde_json::from_value(serde_json::json!({ "apiVersion": "niphas.io/v1alpha1", "kind": "NiphasWorkload", @@ -57,7 +138,8 @@ fn test_workload(name: &str, flake_ref: &str) -> NiphasWorkload { }, "spec": { "flakeRef": flake_ref, - "attribute": "packages.x86_64-linux.default", + "attribute": attribute, + "command": ["/bin/sleep", "3600"], } })) .expect("valid test workload") @@ -111,33 +193,140 @@ async fn test_eval_healthy() -> Result<()> { #[tokio::test] async fn test_workload_eval_failure_sets_failed() -> Result<()> { let client = Client::try_default().await?; - let ns = std::env::var("E2E_NAMESPACE").unwrap_or_else(|_| "niphas-e2e".to_string()); + let ns = e2e_namespace(); let api: Api = Api::namespaced(client, &ns); let name = "e2e-fail-test"; - let wl = test_workload(name, "github:nonexistent/does-not-exist-12345"); + let wl = test_workload( + name, + "github:nonexistent/does-not-exist-12345", + "packages.x86_64-linux.default", + ); - // Clean up if leftover from previous run - let _ = api.delete(name, &DeleteParams::default()).await; - tokio::time::sleep(Duration::from_secs(2)).await; + cleanup_workload(&api, name).await; api.create(&PostParams::default(), &wl) .await .context("failed to create test workload")?; - let result = wait_for_phase(&api, name, WorkloadPhase::Failed, Duration::from_secs(120)).await; + // Wait for the Evaluated condition to become False (eval failure). + // We check the condition rather than the phase because the reconciler + // retries eval failures, flip-flopping the phase between Evaluating and Failed. + let result = wait_for_condition( + &api, + name, + "Evaluated", + ConditionStatus::False, + Duration::from_secs(120), + ) + .await; - // Cleanup - let _ = api.delete(name, &DeleteParams::default()).await; + cleanup_workload(&api, name).await; - let wl = result.context("workload did not reach Failed phase")?; + let wl = result.context("Evaluated condition did not become False")?; let status = wl.status.expect("status should be set"); - // Check Evaluated condition is False - if let Some(conditions) = &status.conditions { - let eval_cond = conditions.iter().find(|c| c.type_ == "Evaluated"); - assert!(eval_cond.is_some(), "expected Evaluated condition to exist"); - } + let conditions = status + .conditions + .as_ref() + .expect("workload should have conditions after eval failure"); + let eval_cond = conditions + .iter() + .find(|c| c.type_ == "Evaluated") + .expect("Evaluated condition should exist"); + assert_eq!( + eval_cond.status, + ConditionStatus::False, + "Evaluated condition should be False on failure" + ); + assert_eq!(eval_cond.reason, "EvalFailed"); + + Ok(()) +} + +#[tokio::test] +async fn test_workload_full_pipeline() -> Result<()> { + let client = Client::try_default().await?; + let ns = e2e_namespace(); + let api: Api = Api::namespaced(client.clone(), &ns); + + let name = "e2e-pipeline-test"; + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); + + cleanup_workload(&api, name).await; + + api.create(&PostParams::default(), &wl) + .await + .context("failed to create test workload")?; + + // Wait for Provisioning phase (eval succeeded, child resources applied). + // Reaching Provisioning proves the full eval pipeline: + // Pending → Evaluating → [nix eval + closure resolution] → Provisioning + let wl = wait_for_phase( + &api, + name, + WorkloadPhase::Provisioning, + Duration::from_secs(180), + ) + .await + .context("workload did not reach Provisioning phase")?; + + let status = wl.status.expect("status should be set"); + + // Verify store_path starts with /nix/store/ + let store_path = status + .store_path + .as_ref() + .expect("store_path should be set after successful eval"); + assert!( + store_path.starts_with("/nix/store/"), + "store_path should start with /nix/store/, got: {store_path}" + ); + + // Verify closure_paths has more than 1 path (hello + glibc + etc) + let closure_paths = status + .closure_paths + .as_ref() + .expect("closure_paths should be set"); + assert!( + closure_paths.len() > 1, + "closure should contain multiple paths (hello + deps), got: {closure_paths:?}" + ); + + // Verify Evaluated condition is True + let conditions = status + .conditions + .as_ref() + .expect("should have conditions after eval"); + let eval_cond = conditions + .iter() + .find(|c| c.type_ == "Evaluated") + .expect("Evaluated condition should exist"); + assert_eq!( + eval_cond.status, + ConditionStatus::True, + "Evaluated condition should be True" + ); + assert_eq!(eval_cond.reason, "EvalSucceeded"); + + // Verify operator created a child Deployment + let deployments: Api = Api::namespaced(client, &ns); + let lp = ListParams::default().labels(&format!("niphas.io/workload={name}")); + let dep_list = deployments + .list(&lp) + .await + .context("failed to list child deployments")?; + assert_eq!( + dep_list.items.len(), + 1, + "operator should have created exactly one child Deployment" + ); + + cleanup_workload(&api, name).await; Ok(()) } @@ -145,22 +334,26 @@ async fn test_workload_eval_failure_sets_failed() -> Result<()> { #[tokio::test] async fn test_workload_deletion_cleanup() -> Result<()> { let client = Client::try_default().await?; - let ns = std::env::var("E2E_NAMESPACE").unwrap_or_else(|_| "niphas-e2e".to_string()); + let ns = e2e_namespace(); let api: Api = Api::namespaced(client, &ns); let name = "e2e-delete-test"; - let wl = test_workload(name, "github:nixos/nixpkgs"); + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); - // Clean up if leftover - let _ = api.delete(name, &DeleteParams::default()).await; - tokio::time::sleep(Duration::from_secs(2)).await; + cleanup_workload(&api, name).await; api.create(&PostParams::default(), &wl) .await .context("failed to create test workload")?; - // Wait for it to get a finalizer (operator picks it up) - tokio::time::sleep(Duration::from_secs(10)).await; + // Wait for the operator to add the finalizer + wait_for_finalizer(&api, name, Duration::from_secs(30)) + .await + .context("operator did not add finalizer")?; // Delete it api.delete(name, &DeleteParams::default()) @@ -173,3 +366,293 @@ async fn test_workload_deletion_cleanup() -> Result<()> { Ok(()) } + +// ───────────────────────────────────────────────────────────── +// New e2e tests: reconciler behavior +// ───────────────────────────────────────────────────────────── + +/// Verify that the operator adds the finalizer to a new workload. +#[tokio::test] +async fn test_workload_gets_finalizer() -> Result<()> { + let client = Client::try_default().await?; + let ns = e2e_namespace(); + let api: Api = Api::namespaced(client, &ns); + + let name = "e2e-finalizer-test"; + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); + + cleanup_workload(&api, name).await; + + api.create(&PostParams::default(), &wl) + .await + .context("failed to create test workload")?; + + let wl = wait_for_finalizer(&api, name, Duration::from_secs(30)) + .await + .context("operator did not add finalizer")?; + + let finalizers = wl.metadata.finalizers.as_ref().unwrap(); + assert!( + finalizers.contains(&FINALIZER.to_string()), + "expected finalizer '{}', got: {:?}", + FINALIZER, + finalizers + ); + + cleanup_workload(&api, name).await; + Ok(()) +} + +/// Verify that after eval succeeds, observedGeneration matches generation +/// and the workload has a valid store_path. +#[tokio::test] +async fn test_workload_eval_produces_valid_status() -> Result<()> { + let client = Client::try_default().await?; + let ns = e2e_namespace(); + let api: Api = Api::namespaced(client, &ns); + + let name = "e2e-stable-test"; + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); + + cleanup_workload(&api, name).await; + + api.create(&PostParams::default(), &wl) + .await + .context("failed to create test workload")?; + + // Wait for Evaluated=True condition (proves eval succeeded) + let wl = wait_for_condition( + &api, + name, + "Evaluated", + ConditionStatus::True, + Duration::from_secs(180), + ) + .await + .context("Evaluated condition did not become True")?; + + let status = wl.status.as_ref().unwrap(); + let generation = wl.metadata.generation.unwrap_or(0); + let observed = status.observed_generation.unwrap_or(-1); + assert_eq!( + observed, generation, + "observedGeneration ({observed}) should match generation ({generation})" + ); + + // Verify resolved_command is set (mainProgram resolved) + assert!( + status.resolved_command.is_some(), + "resolved_command should be set after eval" + ); + + cleanup_workload(&api, name).await; + Ok(()) +} + +/// Verify Evaluated and Progressing conditions are set correctly +/// after eval succeeds and provisioning starts. +#[tokio::test] +async fn test_workload_conditions_after_eval() -> Result<()> { + let client = Client::try_default().await?; + let ns = e2e_namespace(); + let api: Api = Api::namespaced(client, &ns); + + let name = "e2e-conditions-test"; + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); + + cleanup_workload(&api, name).await; + + api.create(&PostParams::default(), &wl) + .await + .context("failed to create test workload")?; + + // Wait for Provisioning (eval succeeded, child resources created) + let wl = wait_for_phase( + &api, + name, + WorkloadPhase::Provisioning, + Duration::from_secs(180), + ) + .await + .context("workload did not reach Provisioning")?; + + let conditions = wl + .status + .as_ref() + .and_then(|s| s.conditions.as_ref()) + .expect("Provisioning workload should have conditions"); + + // Evaluated = True + let evaluated = conditions + .iter() + .find(|c| c.type_ == "Evaluated") + .expect("Evaluated condition should exist"); + assert_eq!( + evaluated.status, + ConditionStatus::True, + "Evaluated should be True" + ); + assert_eq!(evaluated.reason, "EvalSucceeded"); + + // Progressing = True (still provisioning replicas) + let progressing = conditions + .iter() + .find(|c| c.type_ == "Progressing") + .expect("Progressing condition should exist"); + assert_eq!( + progressing.status, + ConditionStatus::True, + "Progressing should be True during Provisioning" + ); + + cleanup_workload(&api, name).await; + Ok(()) +} + +/// Verify that changing the spec triggers re-evaluation. +#[tokio::test] +async fn test_workload_spec_update_triggers_reeval() -> Result<()> { + let client = Client::try_default().await?; + let ns = e2e_namespace(); + let api: Api = Api::namespaced(client, &ns); + + let name = "e2e-reeval-test"; + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); + + cleanup_workload(&api, name).await; + + api.create(&PostParams::default(), &wl) + .await + .context("failed to create test workload")?; + + // Wait for initial eval to succeed + wait_for_condition( + &api, + name, + "Evaluated", + ConditionStatus::True, + Duration::from_secs(180), + ) + .await + .context("initial eval did not succeed")?; + + let generation_before = api.get(name).await?.metadata.generation.unwrap_or(0); + + // Update the spec — this bumps generation and should trigger re-eval + let patch = serde_json::json!({ + "spec": { + "args": ["--version"] + } + }); + api.patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .context("failed to patch workload spec")?; + + // Generation should have incremented + let wl_after_patch = api.get(name).await?; + let generation_after = wl_after_patch.metadata.generation.unwrap_or(0); + assert!( + generation_after > generation_before, + "generation should increment after spec change ({generation_before} -> {generation_after})" + ); + + // Wait for re-eval to complete — the Evaluated condition should update + // with the new observedGeneration matching the new generation. + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(180); + loop { + if start.elapsed() > timeout { + bail!("timed out waiting for re-evaluation after spec update"); + } + if let Some(wl) = api.get_opt(name).await? { + if let Some(ref status) = wl.status { + if status.observed_generation == Some(generation_after) { + // Re-eval happened for the new generation + break; + } + } + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + + let settled = api.get(name).await?; + let status = settled.status.as_ref().unwrap(); + assert_eq!( + status.observed_generation.unwrap_or(0), + generation_after, + "observedGeneration should match new generation after re-eval" + ); + + cleanup_workload(&api, name).await; + Ok(()) +} + +/// Verify that the workload passes through Evaluating before Provisioning. +#[tokio::test] +async fn test_workload_passes_through_evaluating() -> Result<()> { + let client = Client::try_default().await?; + let ns = e2e_namespace(); + let api: Api = Api::namespaced(client, &ns); + + let name = "e2e-provisioning-test"; + let wl = test_workload( + name, + "github:NixOS/nixpkgs", + "legacyPackages.x86_64-linux.hello", + ); + + cleanup_workload(&api, name).await; + + api.create(&PostParams::default(), &wl) + .await + .context("failed to create test workload")?; + + // Poll rapidly to observe the Evaluating → Provisioning transition + let mut saw_evaluating = false; + let mut saw_provisioning = false; + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(180); + + loop { + if start.elapsed() > timeout { + bail!( + "timed out (saw_evaluating={saw_evaluating}, saw_provisioning={saw_provisioning})" + ); + } + if let Some(wl) = api.get_opt(name).await? { + if let Some(ref status) = wl.status { + match status.phase { + WorkloadPhase::Evaluating => saw_evaluating = true, + WorkloadPhase::Provisioning => { + saw_provisioning = true; + break; + } + _ => {} + } + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + + assert!(saw_evaluating, "workload should pass through Evaluating"); + assert!(saw_provisioning, "workload should reach Provisioning"); + + cleanup_workload(&api, name).await; + Ok(()) +} diff --git a/crates/niphas-eval/Cargo.toml b/crates/niphas-eval/Cargo.toml index 782ea63..005a6aa 100644 --- a/crates/niphas-eval/Cargo.toml +++ b/crates/niphas-eval/Cargo.toml @@ -19,12 +19,9 @@ tokio.workspace = true serde.workspace = true serde_json.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true anyhow.workspace = true tikv-jemallocator = { workspace = true, optional = true } -reqwest.workspace = true thiserror.workspace = true -futures.workspace = true [dev-dependencies] http-body-util = "0.1" diff --git a/crates/niphas-eval/src/allowlist.rs b/crates/niphas-eval/src/allowlist.rs index 14c5833..8fb804a 100644 --- a/crates/niphas-eval/src/allowlist.rs +++ b/crates/niphas-eval/src/allowlist.rs @@ -3,13 +3,18 @@ use crate::error::AppError; /// Validate a flake reference against the allowlist. /// /// The allowlist uses simple glob patterns: -/// - `*` matches any sequence of non-`/` characters -/// - `**` is not supported (flake refs don't have deep paths) +/// - `*` matches any sequence of characters (including `/` and empty string) +/// +/// Matching is case-sensitive (Nix flake refs are case-sensitive). +/// +/// Note: `github:*` would match `github:` (empty path after scheme), but this +/// is harmless because `validate_eval_request()` in `eval.rs` already rejects +/// refs without a valid `scheme:path` format with non-empty path. /// /// Examples: -/// - `github:myorg/*` matches `github:myorg/myapp` +/// - `github:myorg/*` matches `github:myorg/myapp` and `github:myorg/myapp?dir=sub` /// - `github:nixos/nixpkgs` matches exactly -/// - `github:*/*` matches any github flake +/// - `github:*/*` matches any two-segment github flake pub fn validate_flake_ref(flake_ref: &str, allowlist: &[String]) -> Result<(), AppError> { if allowlist.is_empty() { // Empty allowlist = deny all @@ -29,41 +34,34 @@ pub fn validate_flake_ref(flake_ref: &str, allowlist: &[String]) -> Result<(), A ))) } -/// Simple glob matching for flake refs. +/// Simple glob matching for flake refs (zero allocation). fn matches_glob(input: &str, pattern: &str) -> bool { - let mut input_chars = input.chars().peekable(); - let mut pattern_chars = pattern.chars().peekable(); - - loop { - match (pattern_chars.peek(), input_chars.peek()) { - (Some(&'*'), _) => { - pattern_chars.next(); - // '*' matches everything to end if no more pattern - if pattern_chars.peek().is_none() { - return true; - } - // Try matching rest of pattern at each position - let remaining_pattern: String = pattern_chars.collect(); - let remaining_input: String = input_chars.collect(); - for i in 0..=remaining_input.len() { - if matches_glob(&remaining_input[i..], &remaining_pattern) { - return true; - } - } - return false; - } - (Some(&pc), Some(&ic)) => { - if pc == ic { - pattern_chars.next(); - input_chars.next(); - } else { - return false; - } - } - (None, None) => return true, - _ => return false, + let (ib, pb) = (input.as_bytes(), pattern.as_bytes()); + let (mut i, mut p) = (0, 0); + let (mut star_p, mut star_i) = (usize::MAX, 0); + + while i < ib.len() { + if p < pb.len() && pb[p] == b'*' { + star_p = p; + star_i = i; + p += 1; + } else if p < pb.len() && pb[p] == ib[i] { + i += 1; + p += 1; + } else if star_p != usize::MAX { + star_i += 1; + i = star_i; + p = star_p + 1; + } else { + return false; } } + + while p < pb.len() && pb[p] == b'*' { + p += 1; + } + + p == pb.len() } #[cfg(test)] @@ -110,4 +108,30 @@ mod tests { fn test_empty_allowlist_denies_all() { assert!(validate_flake_ref("github:myorg/app", &[]).is_err()); } + + #[test] + fn test_wildcard_matches_empty_suffix() { + // `*` matches empty string, so `github:*` matches `github:` + // This is harmless — validate_eval_request() rejects bare `github:`. + assert!(matches_glob("github:", "github:*")); + } + + #[test] + fn test_case_sensitivity() { + // Nix flake refs are case-sensitive + assert!(!matches_glob("GitHub:myorg/app", "github:myorg/*")); + assert!(!matches_glob("github:MyOrg/app", "github:myorg/*")); + assert!(matches_glob("github:myorg/app", "github:myorg/*")); + } + + #[test] + fn test_wildcard_with_query_params() { + assert!(matches_glob("github:myorg/myapp?dir=sub", "github:myorg/*")); + } + + #[test] + fn test_path_scheme() { + assert!(matches_glob("path:/local/flake", "path:*")); + assert!(!matches_glob("path:/local/flake", "github:*")); + } } diff --git a/crates/niphas-eval/src/error.rs b/crates/niphas-eval/src/error.rs index fafaf71..10a9e57 100644 --- a/crates/niphas-eval/src/error.rs +++ b/crates/niphas-eval/src/error.rs @@ -1,18 +1,61 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; +use niphas_core::error::NiphasError; use serde::Serialize; +use thiserror::Error; -#[derive(Debug)] +#[derive(Debug, Error)] pub enum AppError { + #[error("invalid input: {0}")] InvalidInput(String), + + #[error("flake not allowed: {0}")] FlakeNotAllowed(String), + + #[error("evaluation failed: {0}")] EvalFailed(String), + + #[error("evaluation exceeded timeout")] EvalTimeout, + + #[error("store path not cached: {0}")] StorePathNotCached(String), + + #[error("closure resolution failed: {0}")] ClosureResolutionFailed(String), + + #[error("internal error: {0}")] Internal(String), } +impl AppError { + /// HTTP status code for this error. + fn status_code(&self) -> StatusCode { + match self { + Self::InvalidInput(_) => StatusCode::BAD_REQUEST, + Self::FlakeNotAllowed(_) => StatusCode::FORBIDDEN, + Self::EvalFailed(_) => StatusCode::UNPROCESSABLE_ENTITY, + Self::EvalTimeout => StatusCode::REQUEST_TIMEOUT, + Self::StorePathNotCached(_) => StatusCode::NOT_FOUND, + Self::ClosureResolutionFailed(_) => StatusCode::BAD_GATEWAY, + Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + /// Machine-readable error code for the JSON response. + fn error_code(&self) -> &'static str { + match self { + Self::InvalidInput(_) => "InvalidInput", + Self::FlakeNotAllowed(_) => "FlakeNotAllowed", + Self::EvalFailed(_) => "EvalFailed", + Self::EvalTimeout => "EvalTimeout", + Self::StorePathNotCached(_) => "StorePathNotCached", + Self::ClosureResolutionFailed(_) => "ClosureResolutionFailed", + Self::Internal(_) => "InternalError", + } + } +} + #[derive(Serialize)] struct ErrorResponse { error: String, @@ -21,44 +64,24 @@ struct ErrorResponse { impl IntoResponse for AppError { fn into_response(self) -> Response { - let (status, code, message) = match self { - AppError::InvalidInput(msg) => (StatusCode::BAD_REQUEST, "InvalidInput", msg), - AppError::FlakeNotAllowed(msg) => (StatusCode::FORBIDDEN, "FlakeNotAllowed", msg), - AppError::EvalFailed(msg) => (StatusCode::UNPROCESSABLE_ENTITY, "EvalFailed", msg), - AppError::EvalTimeout => ( - StatusCode::REQUEST_TIMEOUT, - "EvalTimeout", - "Evaluation exceeded timeout".into(), - ), - AppError::StorePathNotCached(msg) => (StatusCode::NOT_FOUND, "StorePathNotCached", msg), - AppError::ClosureResolutionFailed(msg) => { - (StatusCode::BAD_GATEWAY, "ClosureResolutionFailed", msg) - } - AppError::Internal(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "InternalError", msg), - }; - + let status = self.status_code(); let body = ErrorResponse { - error: code.into(), - message, + error: self.error_code().into(), + message: self.to_string(), }; - (status, axum::Json(body)).into_response() } } -impl From for AppError { - fn from(e: niphas_core::error::NiphasError) -> Self { +impl From for AppError { + fn from(e: NiphasError) -> Self { match e { - niphas_core::error::NiphasError::InvalidInput(msg) => AppError::InvalidInput(msg), - niphas_core::error::NiphasError::FlakeNotAllowed(msg) => AppError::FlakeNotAllowed(msg), - niphas_core::error::NiphasError::EvalTimeout(_) => AppError::EvalTimeout, - niphas_core::error::NiphasError::StorePathNotCached(msg) => { - AppError::StorePathNotCached(msg) - } - niphas_core::error::NiphasError::ClosureResolution(msg) => { - AppError::ClosureResolutionFailed(msg) - } - other => AppError::Internal(other.to_string()), + NiphasError::InvalidInput(msg) => Self::InvalidInput(msg), + NiphasError::FlakeNotAllowed(msg) => Self::FlakeNotAllowed(msg), + NiphasError::EvalTimeout(_) => Self::EvalTimeout, + NiphasError::StorePathNotCached(msg) => Self::StorePathNotCached(msg), + NiphasError::ClosureResolution(msg) => Self::ClosureResolutionFailed(msg), + other => Self::Internal(other.to_string()), } } } @@ -68,62 +91,60 @@ mod tests { use super::*; use http_body_util::BodyExt; - async fn status_of(err: AppError) -> StatusCode { - let resp = err.into_response(); - resp.status() + fn status_of(err: AppError) -> StatusCode { + err.into_response().status() } async fn body_of(err: AppError) -> serde_json::Value { let resp = err.into_response(); - let body = resp.into_body(); - let bytes = body.collect().await.unwrap().to_bytes(); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); serde_json::from_slice(&bytes).unwrap() } - #[tokio::test] - async fn test_invalid_input_400() { + #[test] + fn test_invalid_input_400() { assert_eq!( - status_of(AppError::InvalidInput("bad flake_ref".into())).await, + status_of(AppError::InvalidInput("bad flake_ref".into())), StatusCode::BAD_REQUEST ); } - #[tokio::test] - async fn test_flake_not_allowed_403() { + #[test] + fn test_flake_not_allowed_403() { assert_eq!( - status_of(AppError::FlakeNotAllowed("bad".into())).await, + status_of(AppError::FlakeNotAllowed("bad".into())), StatusCode::FORBIDDEN ); } - #[tokio::test] - async fn test_eval_failed_422() { + #[test] + fn test_eval_failed_422() { assert_eq!( - status_of(AppError::EvalFailed("nix failed".into())).await, + status_of(AppError::EvalFailed("nix failed".into())), StatusCode::UNPROCESSABLE_ENTITY ); } - #[tokio::test] - async fn test_eval_timeout_408() { + #[test] + fn test_eval_timeout_408() { assert_eq!( - status_of(AppError::EvalTimeout).await, + status_of(AppError::EvalTimeout), StatusCode::REQUEST_TIMEOUT ); } - #[tokio::test] - async fn test_closure_resolution_502() { + #[test] + fn test_closure_resolution_502() { assert_eq!( - status_of(AppError::ClosureResolutionFailed("fail".into())).await, + status_of(AppError::ClosureResolutionFailed("fail".into())), StatusCode::BAD_GATEWAY ); } - #[tokio::test] - async fn test_internal_500() { + #[test] + fn test_internal_500() { assert_eq!( - status_of(AppError::Internal("oops".into())).await, + status_of(AppError::Internal("oops".into())), StatusCode::INTERNAL_SERVER_ERROR ); } @@ -132,20 +153,18 @@ mod tests { async fn test_error_body_shape() { let body = body_of(AppError::FlakeNotAllowed("github:evil/repo".into())).await; assert_eq!(body["error"], "FlakeNotAllowed"); - assert_eq!(body["message"], "github:evil/repo"); + assert_eq!(body["message"], "flake not allowed: github:evil/repo"); } #[tokio::test] async fn test_invalid_input_body_shape() { let body = body_of(AppError::InvalidInput("bad field".into())).await; assert_eq!(body["error"], "InvalidInput"); - assert_eq!(body["message"], "bad field"); + assert_eq!(body["message"], "invalid input: bad field"); } - #[tokio::test] - async fn test_from_niphas_error() { - use niphas_core::error::NiphasError; - + #[test] + fn test_from_niphas_error() { let err: AppError = NiphasError::InvalidInput("bad".into()).into(); assert!(matches!(err, AppError::InvalidInput(_))); diff --git a/crates/niphas-eval/src/evaluator.rs b/crates/niphas-eval/src/evaluator.rs index f8e40be..29b9c47 100644 --- a/crates/niphas-eval/src/evaluator.rs +++ b/crates/niphas-eval/src/evaluator.rs @@ -4,6 +4,7 @@ use niphas_core::config::NiphasConfig; use niphas_core::eval::{EvalRequest, EvalResponse}; use niphas_core::nix::cache_client::CacheClient; use niphas_core::nix::closure; +use niphas_core::nix::signature::TrustedKey; use niphas_core::nix::store_path::StorePath; use serde::Deserialize; use std::sync::atomic::{AtomicBool, Ordering}; @@ -13,6 +14,7 @@ use tracing::{debug, info}; pub struct Evaluator { config: NiphasConfig, cache_client: CacheClient, + trusted_keys: Vec, /// Set to true after at least one successful evaluation. warm: AtomicBool, } @@ -21,14 +23,25 @@ impl Evaluator { pub fn new(config: NiphasConfig) -> Result { let cache_client = CacheClient::new(config.binary_caches.clone())?; + let trusted_keys: Vec = config + .binary_caches + .iter() + .filter_map(|c| c.public_key.as_deref()) + .map(TrustedKey::parse) + .collect::, _>>()?; + Ok(Evaluator { config, cache_client, + trusted_keys, warm: AtomicBool::new(false), }) } - /// Whether at least one eval has succeeded (for readiness probe). + pub fn config(&self) -> &NiphasConfig { + &self.config + } + pub fn is_warm(&self) -> bool { self.warm.load(Ordering::Relaxed) } @@ -48,13 +61,10 @@ impl Evaluator { }; // 4. Nix evaluation via subprocess - // - // TODO: Phase 3B -- integrate nix-bindings-rust for in-process eval. - // For now, we use `nix eval` as a subprocess fallback. - let eval_result = self.nix_eval(&pinned_ref, &req.attribute).await?; + let nix_output = self.nix_eval(&pinned_ref, &req.attribute).await?; // 5. Resolve closure via binary cache - let root = StorePath::parse(&eval_result.store_path) + let root = StorePath::parse(&nix_output.store_path) .map_err(|e| AppError::Internal(format!("invalid store path from eval: {e}")))?; let resolved = closure::resolve_closure( @@ -62,6 +72,9 @@ impl Evaluator { &root, self.config.closure_resolution.concurrency, self.config.closure_resolution.timeout, + &self.trusted_keys, + self.config.closure_resolution.max_paths, + self.config.closure_resolution.max_nar_bytes, ) .await .map_err(|e| AppError::ClosureResolutionFailed(e.to_string()))?; @@ -69,15 +82,15 @@ impl Evaluator { self.warm.store(true, Ordering::Relaxed); info!( - store_path = %eval_result.store_path, + store_path = %nix_output.store_path, closure_size = resolved.paths.len(), "evaluation complete" ); Ok(EvalResponse { - store_path: eval_result.store_path, - name: eval_result.name, - main_program: eval_result.main_program, + store_path: nix_output.store_path, + name: nix_output.name, + main_program: nix_output.main_program, closure_paths: resolved.paths, }) } @@ -86,10 +99,10 @@ impl Evaluator { /// /// This is the Phase 3A fallback. Phase 3B will replace this with /// in-process Nix C FFI via nix-bindings-rust. - async fn nix_eval(&self, pinned_ref: &str, attribute: &str) -> Result { + async fn nix_eval(&self, pinned_ref: &str, attribute: &str) -> Result { let expr = format!( r#"let drv = (builtins.getFlake "{pinned_ref}").{attribute}; in builtins.toJSON {{ - outPath = drv.outPath; + storePath = drv.outPath; name = drv.name; mainProgram = drv.meta.mainProgram or null; }}"# @@ -98,17 +111,26 @@ impl Evaluator { debug!(expr = %expr, "running nix eval"); let timeout = self.config.eval_timeout; - let output = tokio::time::timeout(timeout, async { - tokio::process::Command::new("nix") - .args(["eval", "--raw", "--expr", &expr]) - .arg("--extra-experimental-features") - .arg("nix-command flakes") - .output() - .await - }) - .await - .map_err(|_| AppError::EvalTimeout)? - .map_err(|e| AppError::EvalFailed(format!("failed to run nix eval: {e}")))?; + let child = tokio::process::Command::new("nix") + .args(["eval", "--raw", "--impure", "--expr", &expr]) + .arg("--extra-experimental-features") + .arg("nix-command flakes") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|e| AppError::EvalFailed(format!("failed to spawn nix eval: {e}")))?; + + let output = match tokio::time::timeout(timeout, child.wait_with_output()).await { + Ok(result) => { + result.map_err(|e| AppError::EvalFailed(format!("failed to run nix eval: {e}")))? + } + Err(_) => { + // child is moved into wait_with_output, but on timeout the future + // is dropped — kill_on_drop(true) ensures the process is killed. + return Err(AppError::EvalTimeout); + } + }; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -116,28 +138,15 @@ impl Evaluator { } let stdout = String::from_utf8_lossy(&output.stdout); - let parsed: NixEvalOutput = serde_json::from_str(&stdout) - .map_err(|e| AppError::EvalFailed(format!("failed to parse nix eval output: {e}")))?; - - Ok(NixEvalResult { - store_path: parsed.out_path, - name: parsed.name, - main_program: parsed.main_program, - }) + serde_json::from_str(&stdout) + .map_err(|e| AppError::EvalFailed(format!("failed to parse nix eval output: {e}"))) } } -/// Raw output from nix eval. +/// Intermediate eval result from `nix eval` (before closure resolution). #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct NixEvalOutput { - out_path: String, - name: String, - main_program: Option, -} - -/// Intermediate eval result (before closure resolution). -struct NixEvalResult { store_path: String, name: String, main_program: Option, diff --git a/crates/niphas-eval/src/handlers.rs b/crates/niphas-eval/src/handlers.rs index f59ac45..245e189 100644 --- a/crates/niphas-eval/src/handlers.rs +++ b/crates/niphas-eval/src/handlers.rs @@ -28,13 +28,8 @@ pub async fn healthz() -> StatusCode { } /// GET /readyz -- readiness probe. -/// Returns 200 if at least one eval has succeeded (warm cache). -pub async fn readyz(State(evaluator): State>) -> StatusCode { - if evaluator.is_warm() { - StatusCode::OK - } else { - StatusCode::SERVICE_UNAVAILABLE - } +pub async fn readyz() -> StatusCode { + StatusCode::OK } #[cfg(test)] diff --git a/crates/niphas-eval/src/lib.rs b/crates/niphas-eval/src/lib.rs index e9a1c59..56eea56 100644 --- a/crates/niphas-eval/src/lib.rs +++ b/crates/niphas-eval/src/lib.rs @@ -4,15 +4,35 @@ pub mod evaluator; pub mod handlers; use axum::Router; +use axum::extract::DefaultBodyLimit; +use axum::http::StatusCode; use axum::routing::{get, post}; use evaluator::Evaluator; use std::sync::Arc; +use std::time::Duration; +use tower::ServiceBuilder; +use tower_http::catch_panic::CatchPanicLayer; +use tower_http::timeout::TimeoutLayer; +use tower_http::trace::TraceLayer; + +const MAX_REQUEST_BODY: usize = 1024 * 1024; // 1MB -/// Build the Axum router for the eval service. pub fn app(evaluator: Arc) -> Router { + let timeout = evaluator.config().eval_timeout + Duration::from_secs(5); + + let middleware = ServiceBuilder::new() + .layer(CatchPanicLayer::new()) + .layer(TraceLayer::new_for_http()) + .layer(TimeoutLayer::with_status_code( + StatusCode::GATEWAY_TIMEOUT, + timeout, + )); + Router::new() .route("/evaluate", post(handlers::evaluate)) .route("/healthz", get(handlers::healthz)) .route("/readyz", get(handlers::readyz)) + .layer(DefaultBodyLimit::max(MAX_REQUEST_BODY)) + .layer(middleware) .with_state(evaluator) } diff --git a/crates/niphas-eval/src/main.rs b/crates/niphas-eval/src/main.rs index 6ca3e80..9b9c7da 100644 --- a/crates/niphas-eval/src/main.rs +++ b/crates/niphas-eval/src/main.rs @@ -10,7 +10,7 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[tokio::main] async fn main() -> anyhow::Result<()> { - let _telemetry = niphas_core::telemetry::init_tracing("niphas-eval"); + niphas_core::telemetry::init_tracing("niphas-eval"); info!("starting niphas-eval"); let config = NiphasConfig::load_or_default(None); @@ -25,7 +25,21 @@ async fn main() -> anyhow::Result<()> { let listener = TcpListener::bind(&addr).await?; info!(addr = %addr, "listening"); - axum::serve(listener, app).await?; + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await?; + info!("shutdown complete"); Ok(()) } + +async fn shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).expect("failed to install SIGTERM handler"); + + tokio::select! { + _ = tokio::signal::ctrl_c() => info!("received SIGINT"), + _ = sigterm.recv() => info!("received SIGTERM"), + } +} diff --git a/crates/niphas-eval/tests/integration.rs b/crates/niphas-eval/tests/integration.rs index 82c5a28..e3294b9 100644 --- a/crates/niphas-eval/tests/integration.rs +++ b/crates/niphas-eval/tests/integration.rs @@ -32,10 +32,10 @@ async fn healthz_returns_200() { } #[tokio::test] -async fn readyz_returns_503_when_cold() { +async fn readyz_returns_200() { let base = spawn_app().await; let resp = reqwest::get(format!("{base}/readyz")).await.unwrap(); - assert_eq!(resp.status(), 503); + assert_eq!(resp.status(), 200); } #[tokio::test] diff --git a/crates/niphas-mesh/src/main.rs b/crates/niphas-mesh/src/main.rs index fa47145..7cce7ab 100644 --- a/crates/niphas-mesh/src/main.rs +++ b/crates/niphas-mesh/src/main.rs @@ -4,7 +4,7 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[tokio::main] async fn main() -> anyhow::Result<()> { - let _telemetry = niphas_core::telemetry::init_tracing("niphas-mesh"); + niphas_core::telemetry::init_tracing("niphas-mesh"); tracing::info!("starting niphas-mesh"); Ok(()) } diff --git a/crates/niphas-operator/src/context.rs b/crates/niphas-operator/src/context.rs index d25b54d..a8fa7ac 100644 --- a/crates/niphas-operator/src/context.rs +++ b/crates/niphas-operator/src/context.rs @@ -2,9 +2,8 @@ use kube::Client; use kube_runtime::events::{Recorder, Reporter}; use niphas_core::config::NiphasConfig; use reqwest::Client as HttpClient; -use std::sync::Arc; -/// Shared context for the reconciler and health server. +/// Shared context for the reconciler. pub struct Context { /// Kubernetes API client. pub client: Client, @@ -14,8 +13,6 @@ pub struct Context { pub recorder: Recorder, /// Operator configuration. pub config: NiphasConfig, - /// Whether the controller is ready (leader elected + watching). - pub ready: Arc, } impl Context { @@ -33,7 +30,6 @@ impl Context { http: HttpClient::new(), recorder, config, - ready: Arc::new(std::sync::atomic::AtomicBool::new(false)), } } } diff --git a/crates/niphas-operator/src/error.rs b/crates/niphas-operator/src/error.rs index 5a87aa8..0cd3dfc 100644 --- a/crates/niphas-operator/src/error.rs +++ b/crates/niphas-operator/src/error.rs @@ -24,7 +24,19 @@ impl OperatorError { pub fn is_transient(&self) -> bool { matches!( self, - OperatorError::Kube(_) | OperatorError::EvalWebhook(_) | OperatorError::EvalTimeout(_) + Self::Kube(_) | Self::EvalWebhook(_) | Self::EvalTimeout(_) ) } + + /// Extract a machine-readable reason and human-readable message for K8s conditions/events. + pub fn reason_and_message(&self) -> (String, String) { + match self { + Self::EvalTimeout(secs) => ( + "EvalTimeout".into(), + format!("Evaluation timed out after {secs}s"), + ), + Self::EvalFailed { code, message } => (code.clone(), message.clone()), + other => ("EvalFailed".into(), other.to_string()), + } + } } diff --git a/crates/niphas-operator/src/eval.rs b/crates/niphas-operator/src/eval.rs index e17595c..f25ed31 100644 --- a/crates/niphas-operator/src/eval.rs +++ b/crates/niphas-operator/src/eval.rs @@ -1,61 +1,45 @@ use crate::error::OperatorError; -use niphas_core::crd::NiphasWorkload; +use niphas_core::crd::{NiphasWorkload, NiphasWorkloadStatus}; use niphas_core::eval::{EvalRequest, EvalResponse}; +use niphas_core::nix::store_path::StorePath; use reqwest::Client; use serde::Deserialize; use std::time::Duration; use tracing::{debug, warn}; -/// Re-export as EvalResult for backward compatibility within the operator. -pub type EvalResult = EvalResponse; - -/// Error response from the eval webhook. #[derive(Debug, Deserialize)] pub struct EvalErrorResponse { pub error: String, pub message: String, } -/// Extension methods for EvalResult within the operator context. -pub trait EvalResultExt { - fn from_status(status: &niphas_core::crd::NiphasWorkloadStatus) -> Option; -} - -impl EvalResultExt for EvalResult { - /// Reconstruct an EvalResult from the workload status (for skipping re-eval). - fn from_status(status: &niphas_core::crd::NiphasWorkloadStatus) -> Option { - use niphas_core::nix::store_path::StorePath; +pub fn eval_result_from_status(status: &NiphasWorkloadStatus) -> Option { + let store_path = status.store_path.as_ref()?; + let closure_paths = status.closure_paths.clone().unwrap_or_default(); + let resolved_command = status.resolved_command.clone(); - let store_path = status.store_path.as_ref()?; - let closure_paths = status.closure_paths.clone().unwrap_or_default(); - let resolved_command = status.resolved_command.clone(); + let name = StorePath::parse(store_path) + .map(|sp| sp.name.clone()) + .unwrap_or_else(|_| "unknown".to_string()); - // Extract name from store path using the validated parser. - let name = StorePath::parse(store_path) - .map(|sp| sp.name.clone()) - .unwrap_or_else(|_| "unknown".to_string()); + let main_program = resolved_command + .as_ref() + .and_then(|cmd| cmd.rsplit('/').next().map(|s| s.to_string())); - // Extract mainProgram from resolved command. - let main_program = resolved_command - .as_ref() - .and_then(|cmd| cmd.rsplit('/').next().map(|s| s.to_string())); - - Some(EvalResult { - store_path: store_path.clone(), - name, - main_program, - closure_paths, - }) - } + Some(EvalResponse { + store_path: store_path.clone(), + name, + main_program, + closure_paths, + }) } -/// Call the eval webhook to evaluate a flake. pub async fn call_eval_webhook( http: &Client, eval_url: &str, workload: &NiphasWorkload, timeout: Duration, -) -> Result { +) -> Result { let req = EvalRequest { flake_ref: workload.spec.flake_ref.clone(), attribute: workload.spec.attribute.clone(), @@ -86,7 +70,7 @@ pub async fn call_eval_webhook( })?; if resp.status().is_success() { - let result: EvalResult = resp + let result: EvalResponse = resp .json() .await .map_err(|e| OperatorError::EvalWebhook(format!("invalid response body: {e}")))?; @@ -94,7 +78,6 @@ pub async fn call_eval_webhook( return Ok(result); } - // Try to parse error response let status = resp.status(); let body = resp.text().await.unwrap_or_default(); diff --git a/crates/niphas-operator/src/health.rs b/crates/niphas-operator/src/health.rs index 96a708c..6bc231b 100644 --- a/crates/niphas-operator/src/health.rs +++ b/crates/niphas-operator/src/health.rs @@ -1,18 +1,24 @@ use axum::{Router, extract::State, http::StatusCode, routing::get}; use std::sync::Arc; use std::sync::atomic::Ordering; +use tower::ServiceBuilder; +use tower_http::catch_panic::CatchPanicLayer; +use tower_http::trace::TraceLayer; -/// Shared health state. #[derive(Clone)] pub struct HealthState { pub ready: Arc, } -/// Build the health/metrics HTTP server router. pub fn router(state: HealthState) -> Router { + let middleware = ServiceBuilder::new() + .layer(CatchPanicLayer::new()) + .layer(TraceLayer::new_for_http()); + Router::new() .route("/healthz", get(healthz)) .route("/readyz", get(readyz)) + .layer(middleware) .with_state(state) } diff --git a/crates/niphas-operator/src/lib.rs b/crates/niphas-operator/src/lib.rs new file mode 100644 index 0000000..f3e53f6 --- /dev/null +++ b/crates/niphas-operator/src/lib.rs @@ -0,0 +1,6 @@ +pub mod context; +pub mod error; +pub mod eval; +pub mod health; +pub mod reconciler; +pub mod resources; diff --git a/crates/niphas-operator/src/main.rs b/crates/niphas-operator/src/main.rs index a7702e9..67f6465 100644 --- a/crates/niphas-operator/src/main.rs +++ b/crates/niphas-operator/src/main.rs @@ -16,7 +16,7 @@ use kube_leader_election::{LeaseLock, LeaseLockParams, LeaseLockResult}; use niphas_core::config::NiphasConfig; use niphas_core::crd::NiphasWorkload; use std::sync::Arc; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; @@ -29,12 +29,11 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; const LEASE_NAME: &str = "niphas-operator-leader"; const LEASE_TTL: Duration = Duration::from_secs(15); const LEASE_RENEW_INTERVAL: Duration = Duration::from_secs(5); -/// Max consecutive renewal failures before giving up leadership. const MAX_RENEW_FAILURES: u32 = 3; #[tokio::main] async fn main() -> anyhow::Result<()> { - let _telemetry = niphas_core::telemetry::init_tracing("niphas-operator"); + niphas_core::telemetry::init_tracing("niphas-operator"); info!("starting niphas-operator"); let config = NiphasConfig::load_or_default(None); @@ -43,11 +42,10 @@ async fn main() -> anyhow::Result<()> { let client = Client::try_default().await?; info!("connected to kubernetes"); + let ready = Arc::new(AtomicBool::new(false)); let ctx = Arc::new(Context::new(client.clone(), config)); - let ready = ctx.ready.clone(); let shutdown = CancellationToken::new(); - // Start health server (runs regardless of leadership) let health_state = health::HealthState { ready: ready.clone(), }; @@ -57,18 +55,14 @@ async fn main() -> anyhow::Result<()> { let health_shutdown = shutdown.clone(); tokio::spawn(async move { - let serve = axum::serve(health_listener, health_router); - tokio::select! { - result = serve => { - if let Err(e) = result { - error!(error = %e, "health server failed"); - } - } - _ = health_shutdown.cancelled() => {} + if let Err(e) = axum::serve(health_listener, health_router) + .with_graceful_shutdown(async move { health_shutdown.cancelled().await }) + .await + { + error!(error = %e, "health server failed"); } }); - // Leader election: only the leader runs the controller let holder_id = std::env::var("POD_NAME") .unwrap_or_else(|_| format!("niphas-operator-{}", std::process::id())); @@ -87,7 +81,6 @@ async fn main() -> anyhow::Result<()> { info!(holder_id = %holder_id, "starting leader election"); - // Acquire leadership before starting the controller loop { match leadership.try_acquire_or_renew().await? { LeaseLockResult::Acquired(_) => { @@ -106,9 +99,8 @@ async fn main() -> anyhow::Result<()> { } } - // Spawn lease renewal in the background let renew_shutdown = shutdown.clone(); - let ready_for_renewer = ready.clone(); + let renew_ready = ready.clone(); tokio::spawn(async move { let mut consecutive_failures: u32 = 0; @@ -125,7 +117,7 @@ async fn main() -> anyhow::Result<()> { } Ok(LeaseLockResult::NotAcquired(_)) => { warn!("lost leadership, initiating shutdown"); - ready_for_renewer.store(false, Ordering::Relaxed); + renew_ready.store(false, Ordering::Relaxed); renew_shutdown.cancel(); break; } @@ -139,7 +131,7 @@ async fn main() -> anyhow::Result<()> { ); if consecutive_failures >= MAX_RENEW_FAILURES { error!("max renewal failures reached, initiating shutdown"); - ready_for_renewer.store(false, Ordering::Relaxed); + renew_ready.store(false, Ordering::Relaxed); renew_shutdown.cancel(); break; } @@ -148,7 +140,6 @@ async fn main() -> anyhow::Result<()> { } }); - // Start controller (only runs as leader) let workloads: Api = Api::all(client.clone()); let deployments: Api = Api::all(client.clone()); let pods: Api = Api::all(client.clone()); @@ -181,10 +172,8 @@ async fn main() -> anyhow::Result<()> { } } - // Graceful shutdown: ready=false lets K8s drain traffic before restart ready.store(false, Ordering::Relaxed); info!("shutdown complete"); - // _telemetry guard drops here, flushing pending spans/logs Ok(()) } diff --git a/crates/niphas-operator/src/reconciler.rs b/crates/niphas-operator/src/reconciler.rs index cd7daa7..38f0a78 100644 --- a/crates/niphas-operator/src/reconciler.rs +++ b/crates/niphas-operator/src/reconciler.rs @@ -1,13 +1,16 @@ use crate::context::Context; use crate::error::OperatorError; -use crate::eval::{self, EvalResult, EvalResultExt}; +use crate::eval; use crate::resources; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::ObjectReference; use kube::api::{Api, Patch, PatchParams, ResourceExt}; use kube::runtime::controller::Action; use kube_runtime::events::{Event, EventType}; -use niphas_core::crd::{ConditionStatus, NiphasWorkload, NiphasWorkloadStatus, WorkloadPhase}; +use niphas_core::crd::{ + ConditionStatus, NiphasCondition, NiphasWorkload, NiphasWorkloadStatus, WorkloadPhase, +}; +use niphas_core::eval::EvalResponse; use serde_json::json; use std::sync::Arc; use std::time::Duration; @@ -15,165 +18,446 @@ use tracing::{debug, error, info, warn}; const FINALIZER: &str = "niphas.io/workload-cleanup"; -/// Main reconciliation function called by the kube-rs Controller. -pub async fn reconcile( - workload: Arc, - ctx: Arc, -) -> Result { - let name = workload.name_any(); - let ns = workload.namespace().unwrap_or_default(); - let generation = workload.metadata.generation.unwrap_or(0); +pub struct WorkloadState<'a> { + pub name: String, + pub namespace: String, + pub generation: i64, + pub deletion_pending: bool, + pub has_finalizer: bool, + pub existing_finalizers: Vec, + pub observed_generation: i64, + pub phase: Option, + pub status: Option<&'a NiphasWorkloadStatus>, + pub desired_replicas: i32, +} + +impl<'a> WorkloadState<'a> { + pub fn from_workload(workload: &'a NiphasWorkload) -> Self { + let status = workload.status.as_ref(); + let finalizers = workload.metadata.finalizers.clone().unwrap_or_default(); + let has_finalizer = finalizers.iter().any(|s| s == FINALIZER); + + WorkloadState { + name: workload.name_any(), + namespace: workload.namespace().unwrap_or_default(), + generation: workload.metadata.generation.unwrap_or(0), + deletion_pending: workload.metadata.deletion_timestamp.is_some(), + has_finalizer, + existing_finalizers: finalizers, + observed_generation: status.and_then(|s| s.observed_generation).unwrap_or(0), + phase: status.map(|s| s.phase), + status, + desired_replicas: workload.spec.replicas.unwrap_or(1), + } + } +} - debug!(%name, %ns, generation, "reconciling"); +#[derive(Debug, Clone, PartialEq)] +pub enum ReconcileDecision { + HandleDeletion, + AddFinalizer { + existing_finalizers: Vec, + }, + UpToDate, + RunEval, + Provision { + eval_result: EvalResponse, + }, + UpdateStatus { + eval_result: EvalResponse, + ready_replicas: i32, + desired_replicas: i32, + }, +} - if workload.metadata.deletion_timestamp.is_some() { - return handle_deletion(&workload, &ctx, &name, &ns).await; +enum ExecuteOutcome { + Done(Action), + EvalCompleted(EvalResponse), + Provisioned { + eval_result: EvalResponse, + ready_replicas: i32, + }, +} + +pub fn decide(state: &WorkloadState) -> ReconcileDecision { + if state.deletion_pending { + return ReconcileDecision::HandleDeletion; } - let has_finalizer = workload - .metadata - .finalizers - .as_ref() - .map(|f| f.iter().any(|s| s == FINALIZER)) - .unwrap_or(false); - if !has_finalizer { - add_finalizer(&ctx, &workload, &ns).await?; - return Ok(Action::requeue(Duration::from_secs(0))); + if !state.has_finalizer { + return ReconcileDecision::AddFinalizer { + existing_finalizers: state.existing_finalizers.clone(), + }; } - let status = workload.status.as_ref(); - let observed = status.and_then(|s| s.observed_generation).unwrap_or(0); + if state.observed_generation >= state.generation && state.phase == Some(WorkloadPhase::Running) + { + return ReconcileDecision::UpToDate; + } - if observed >= generation && status.map(|s| s.phase) == Some(WorkloadPhase::Running) { - return Ok(Action::requeue(Duration::from_secs(300))); + // Generation changed — must re-evaluate regardless of cache + if state.observed_generation < state.generation { + return ReconcileDecision::RunEval; } - let eval_result = if needs_eval(&workload, observed, generation) { - set_phase(&ctx, &workload, &ns, WorkloadPhase::Evaluating).await?; - publish_event( - &ctx, - &workload, - EventType::Normal, - "EvalStarted", - "Starting flake evaluation", - ) - .await; - - let timeout = ctx.config.eval_timeout; - let eval_url = &ctx.config.eval_webhook_url; - - match eval::call_eval_webhook(&ctx.http, eval_url, &workload, timeout).await { - Ok(result) => { - publish_event( - &ctx, - &workload, - EventType::Normal, - "EvalSucceeded", - &format!("Resolved store path: {}", result.store_path), - ) - .await; - result - } - Err(e) => { - let (reason, msg) = match &e { - OperatorError::EvalTimeout(secs) => ( - "EvalTimeout".to_string(), - format!("Evaluation timed out after {secs}s"), - ), - OperatorError::EvalFailed { code, message } => (code.clone(), message.clone()), - other => ("EvalFailed".to_string(), other.to_string()), - }; + // Same generation, not Running — use cached eval or re-evaluate + match state.status.and_then(eval::eval_result_from_status) { + Some(cached) => ReconcileDecision::Provision { + eval_result: cached, + }, + None => ReconcileDecision::RunEval, + } +} - publish_event(&ctx, &workload, EventType::Warning, &reason, &msg).await; - set_failed(&ctx, &workload, &ns, &reason, &msg, generation).await?; - return Ok(Action::requeue(Duration::from_secs(30))); - } +fn make_condition( + type_: &str, + status: ConditionStatus, + reason: &str, + message: &str, + generation: i64, +) -> NiphasCondition { + NiphasCondition { + type_: type_.to_string(), + status, + reason: reason.to_string(), + message: message.to_string(), + last_transition_time: iso_now(), + observed_generation: Some(generation), + } +} + +fn upsert_conditions( + existing: Option<&[NiphasCondition]>, + new: Vec, +) -> Vec { + let mut result: Vec = existing.map(|s| s.to_vec()).unwrap_or_default(); + + for cond in new { + if let Some(pos) = result.iter().position(|c| c.type_ == cond.type_) { + result[pos] = cond; + } else { + result.push(cond); } + } + + result +} + +fn build_success_conditions( + eval: &EvalResponse, + phase: WorkloadPhase, + ready: i32, + desired: i32, + generation: i64, +) -> Vec { + let (progressing_status, progressing_reason) = if phase == WorkloadPhase::Running { + (ConditionStatus::False, "NewReplicaSetAvailable") } else { - // Use cached eval from status - match EvalResult::from_status(status.unwrap_or(&NiphasWorkloadStatus::default())) { - Some(cached) => cached, - None => { - // No cached result, need full eval - set_phase(&ctx, &workload, &ns, WorkloadPhase::Evaluating).await?; - let timeout = ctx.config.eval_timeout; - let eval_url = &ctx.config.eval_webhook_url; - - match eval::call_eval_webhook(&ctx.http, eval_url, &workload, timeout).await { - Ok(result) => result, - Err(e) => { - set_failed( - &ctx, - &workload, - &ns, - "EvalFailed", - &e.to_string(), - generation, - ) - .await?; - return Ok(Action::requeue(Duration::from_secs(30))); - } - } - } - } + (ConditionStatus::True, "ReplicaSetUpdated") }; - set_phase(&ctx, &workload, &ns, WorkloadPhase::Provisioning).await?; + let mut conditions = vec![ + make_condition( + "Evaluated", + ConditionStatus::True, + "EvalSucceeded", + &format!("Resolved {}", eval.store_path), + generation, + ), + make_condition( + "Progressing", + progressing_status, + progressing_reason, + &format!("{ready} of {desired} replicas ready"), + generation, + ), + ]; - if let Err(e) = resources::apply_child_resources(&ctx, &workload, &eval_result).await { - error!(error = %e, "failed to apply child resources"); - set_failed( - &ctx, - &workload, - &ns, - "ProvisionFailed", - &e.to_string(), + if phase == WorkloadPhase::Running { + conditions.push(make_condition( + "Available", + ConditionStatus::True, + "ReplicasReady", + &format!("All {ready} replicas are ready"), generation, - ) - .await?; - return Ok(Action::requeue(Duration::from_secs(10))); + )); } - publish_event( - &ctx, - &workload, - EventType::Normal, - "Provisioned", - "Child resources applied", - ) - .await; + conditions +} - let ready = count_ready_replicas(&ctx, &workload, &ns).await?; - let desired = workload.spec.replicas.unwrap_or(1); +fn build_failed_conditions(reason: &str, message: &str, generation: i64) -> Vec { + vec![make_condition( + "Evaluated", + ConditionStatus::False, + reason, + message, + generation, + )] +} - let phase = if ready >= desired { - WorkloadPhase::Running - } else { - WorkloadPhase::Provisioning - }; +struct StatusPatch<'a> { + phase: WorkloadPhase, + generation: i64, + eval_result: Option<&'a EvalResponse>, + ready_replicas: Option, + conditions: Vec, +} - if phase == WorkloadPhase::Running { - publish_event( - &ctx, - &workload, - EventType::Normal, - "Available", - "All replicas ready", - ) - .await; +async fn patch_status( + ctx: &Context, + name: &str, + ns: &str, + patch: StatusPatch<'_>, +) -> Result<(), OperatorError> { + let api: Api = Api::namespaced(ctx.client.clone(), ns); + + let mut status = serde_json::Map::new(); + status.insert("phase".into(), json!(patch.phase)); + status.insert("observedGeneration".into(), json!(patch.generation)); + status.insert( + "conditions".into(), + serde_json::to_value(&patch.conditions)?, + ); + + if let Some(eval) = patch.eval_result { + status.insert("storePath".into(), json!(eval.store_path)); + status.insert("resolvedCommand".into(), json!(eval.resolved_command())); + status.insert("closurePaths".into(), json!(eval.closure_paths)); } - update_status(&ctx, &workload, &ns, phase, &eval_result, ready, generation).await?; + if let Some(ready) = patch.ready_replicas { + status.insert("readyReplicas".into(), json!(ready)); + } - let interval = if ready < desired { - Duration::from_secs(5) - } else { - Duration::from_secs(300) - }; - Ok(Action::requeue(interval)) + let body = json!({ "status": status }); + api.patch_status(name, &PatchParams::default(), &Patch::Merge(&body)) + .await?; + Ok(()) +} + +async fn execute( + decision: ReconcileDecision, + workload: &NiphasWorkload, + ctx: &Context, + state: &WorkloadState<'_>, +) -> Result { + match decision { + ReconcileDecision::HandleDeletion => { + info!(name = %state.name, ns = %state.namespace, "handling deletion"); + publish_event( + ctx, + workload, + EventType::Normal, + "Deleting", + &format!("Cleaning up workload {}", state.name), + ) + .await; + remove_finalizer(ctx, workload, &state.namespace).await?; + Ok(ExecuteOutcome::Done(Action::await_change())) + } + + ReconcileDecision::AddFinalizer { + mut existing_finalizers, + } => { + existing_finalizers.push(FINALIZER.to_string()); + let api: Api = Api::namespaced(ctx.client.clone(), &state.namespace); + let patch = json!({ "metadata": { "finalizers": existing_finalizers } }); + api.patch(&state.name, &PatchParams::default(), &Patch::Merge(&patch)) + .await?; + Ok(ExecuteOutcome::Done(Action::requeue(Duration::from_secs( + 0, + )))) + } + + ReconcileDecision::UpToDate => Ok(ExecuteOutcome::Done(Action::requeue( + Duration::from_secs(300), + ))), + + ReconcileDecision::RunEval => { + set_phase(ctx, workload, &state.namespace, WorkloadPhase::Evaluating).await?; + publish_event( + ctx, + workload, + EventType::Normal, + "EvalStarted", + "Starting flake evaluation", + ) + .await; + + let timeout = ctx.config.eval_timeout; + let eval_url = &ctx.config.eval_webhook_url; + + match eval::call_eval_webhook(&ctx.http, eval_url, workload, timeout).await { + Ok(result) => { + publish_event( + ctx, + workload, + EventType::Normal, + "EvalSucceeded", + &format!("Resolved store path: {}", result.store_path), + ) + .await; + Ok(ExecuteOutcome::EvalCompleted(result)) + } + Err(e) => { + let (reason, msg) = e.reason_and_message(); + publish_event(ctx, workload, EventType::Warning, &reason, &msg).await; + + let conditions = upsert_conditions( + state.status.and_then(|s| s.conditions.as_deref()), + build_failed_conditions(&reason, &msg, state.generation), + ); + patch_status( + ctx, + &state.name, + &state.namespace, + StatusPatch { + phase: WorkloadPhase::Failed, + generation: state.generation, + eval_result: None, + ready_replicas: None, + conditions, + }, + ) + .await?; + Ok(ExecuteOutcome::Done(Action::requeue(Duration::from_secs( + 30, + )))) + } + } + } + + ReconcileDecision::Provision { eval_result } => { + set_phase(ctx, workload, &state.namespace, WorkloadPhase::Provisioning).await?; + + if let Err(e) = resources::apply_child_resources(ctx, workload, &eval_result).await { + error!(error = %e, "failed to apply child resources"); + let conditions = upsert_conditions( + state.status.and_then(|s| s.conditions.as_deref()), + build_failed_conditions("ProvisionFailed", &e.to_string(), state.generation), + ); + patch_status( + ctx, + &state.name, + &state.namespace, + StatusPatch { + phase: WorkloadPhase::Failed, + generation: state.generation, + eval_result: None, + ready_replicas: None, + conditions, + }, + ) + .await?; + return Ok(ExecuteOutcome::Done(Action::requeue(Duration::from_secs( + 10, + )))); + } + + publish_event( + ctx, + workload, + EventType::Normal, + "Provisioned", + "Child resources applied", + ) + .await; + + let ready = count_ready_replicas(ctx, workload, &state.namespace).await?; + Ok(ExecuteOutcome::Provisioned { + eval_result, + ready_replicas: ready, + }) + } + + ReconcileDecision::UpdateStatus { + eval_result, + ready_replicas, + desired_replicas, + } => { + let phase = if ready_replicas >= desired_replicas { + WorkloadPhase::Running + } else { + WorkloadPhase::Provisioning + }; + + if phase == WorkloadPhase::Running { + publish_event( + ctx, + workload, + EventType::Normal, + "Available", + "All replicas ready", + ) + .await; + } + + let conditions = upsert_conditions( + state.status.and_then(|s| s.conditions.as_deref()), + build_success_conditions( + &eval_result, + phase, + ready_replicas, + desired_replicas, + state.generation, + ), + ); + + patch_status( + ctx, + &state.name, + &state.namespace, + StatusPatch { + phase, + generation: state.generation, + eval_result: Some(&eval_result), + ready_replicas: Some(ready_replicas), + conditions, + }, + ) + .await?; + + let interval = if ready_replicas < desired_replicas { + Duration::from_secs(5) + } else { + Duration::from_secs(300) + }; + Ok(ExecuteOutcome::Done(Action::requeue(interval))) + } + } +} + +pub async fn reconcile( + workload: Arc, + ctx: Arc, +) -> Result { + let state = WorkloadState::from_workload(&workload); + debug!(name = %state.name, ns = %state.namespace, generation = state.generation, "reconciling"); + + let mut decision = decide(&state); + loop { + match execute(decision, &workload, &ctx, &state).await? { + ExecuteOutcome::Done(action) => return Ok(action), + ExecuteOutcome::EvalCompleted(result) => { + decision = ReconcileDecision::Provision { + eval_result: result, + }; + } + ExecuteOutcome::Provisioned { + eval_result, + ready_replicas, + } => { + decision = ReconcileDecision::UpdateStatus { + eval_result, + ready_replicas, + desired_replicas: state.desired_replicas, + }; + } + } + } } -/// Error policy: determine retry behavior on reconcile error. pub fn error_policy( _workload: Arc, error: &OperatorError, @@ -187,50 +471,12 @@ pub fn error_policy( } } -fn needs_eval(workload: &NiphasWorkload, observed: i64, generation: i64) -> bool { - if observed < generation { - return true; - } - // No store path yet (first reconciliation after crash) - workload - .status - .as_ref() - .and_then(|s| s.store_path.as_ref()) - .is_none() -} - -async fn add_finalizer( - ctx: &Context, - workload: &NiphasWorkload, - ns: &str, -) -> Result<(), OperatorError> { - let api: Api = Api::namespaced(ctx.client.clone(), ns); - // Append our finalizer without clobbering existing ones from other controllers. - let mut finalizers = workload.metadata.finalizers.clone().unwrap_or_default(); - if !finalizers.iter().any(|f| f == FINALIZER) { - finalizers.push(FINALIZER.to_string()); - } - let patch = json!({ - "metadata": { - "finalizers": finalizers - } - }); - api.patch( - &workload.name_any(), - &PatchParams::default(), - &Patch::Merge(&patch), - ) - .await?; - Ok(()) -} - async fn remove_finalizer( ctx: &Context, workload: &NiphasWorkload, ns: &str, ) -> Result<(), OperatorError> { let api: Api = Api::namespaced(ctx.client.clone(), ns); - // Remove only our finalizer, preserving others. let finalizers: Vec = workload .metadata .finalizers @@ -254,28 +500,6 @@ async fn remove_finalizer( Ok(()) } -async fn handle_deletion( - workload: &NiphasWorkload, - ctx: &Context, - name: &str, - ns: &str, -) -> Result { - info!(%name, %ns, "handling deletion"); - publish_event( - ctx, - workload, - EventType::Normal, - "Deleting", - &format!("Cleaning up workload {name}"), - ) - .await; - - // Child resources are cleaned by ownerReferences. - // Remove finalizer to allow K8s to complete deletion. - remove_finalizer(ctx, workload, ns).await?; - Ok(Action::await_change()) -} - async fn set_phase( ctx: &Context, workload: &NiphasWorkload, @@ -283,118 +507,7 @@ async fn set_phase( phase: WorkloadPhase, ) -> Result<(), OperatorError> { let api: Api = Api::namespaced(ctx.client.clone(), ns); - let patch = json!({ - "status": { - "phase": phase - } - }); - api.patch_status( - &workload.name_any(), - &PatchParams::default(), - &Patch::Merge(&patch), - ) - .await?; - Ok(()) -} - -async fn set_failed( - ctx: &Context, - workload: &NiphasWorkload, - ns: &str, - reason: &str, - message: &str, - generation: i64, -) -> Result<(), OperatorError> { - let api: Api = Api::namespaced(ctx.client.clone(), ns); - let now = iso_now(); - let patch = json!({ - "status": { - "phase": WorkloadPhase::Failed, - "observedGeneration": generation, - "conditions": [{ - "type": "Evaluated", - "status": ConditionStatus::False, - "reason": reason, - "message": message, - "lastTransitionTime": now, - "observedGeneration": generation, - }] - } - }); - api.patch_status( - &workload.name_any(), - &PatchParams::default(), - &Patch::Merge(&patch), - ) - .await?; - Ok(()) -} - -async fn update_status( - ctx: &Context, - workload: &NiphasWorkload, - ns: &str, - phase: WorkloadPhase, - eval_result: &EvalResult, - ready_replicas: i32, - generation: i64, -) -> Result<(), OperatorError> { - let api: Api = Api::namespaced(ctx.client.clone(), ns); - let now = iso_now(); - - let progressing_status = if phase == WorkloadPhase::Provisioning { - ConditionStatus::True - } else { - ConditionStatus::False - }; - let progressing_reason = if phase == WorkloadPhase::Running { - "NewReplicaSetAvailable" - } else { - "ReplicaSetUpdated" - }; - - let mut conditions = vec![ - json!({ - "type": "Evaluated", - "status": ConditionStatus::True, - "reason": "EvalSucceeded", - "message": format!("Resolved {}", eval_result.store_path), - "lastTransitionTime": now, - "observedGeneration": generation, - }), - json!({ - "type": "Progressing", - "status": progressing_status, - "reason": progressing_reason, - "message": format!("{ready_replicas} of {} replicas ready", workload.spec.replicas.unwrap_or(1)), - "lastTransitionTime": now, - "observedGeneration": generation, - }), - ]; - - if phase == WorkloadPhase::Running { - conditions.push(json!({ - "type": "Available", - "status": ConditionStatus::True, - "reason": "ReplicasReady", - "message": format!("All {ready_replicas} replicas are ready"), - "lastTransitionTime": now, - "observedGeneration": generation, - })); - } - - let patch = json!({ - "status": { - "phase": phase, - "observedGeneration": generation, - "storePath": eval_result.store_path, - "resolvedCommand": eval_result.resolved_command(), - "closurePaths": eval_result.closure_paths, - "readyReplicas": ready_replicas, - "conditions": conditions, - } - }); - + let patch = json!({ "status": { "phase": phase } }); api.patch_status( &workload.name_any(), &PatchParams::default(), @@ -413,14 +526,11 @@ async fn count_ready_replicas( let name = workload.name_any(); match deploys.get_opt(&name).await? { - Some(deploy) => { - let ready = deploy - .status - .as_ref() - .and_then(|s| s.ready_replicas) - .unwrap_or(0); - Ok(ready) - } + Some(deploy) => Ok(deploy + .status + .as_ref() + .and_then(|s| s.ready_replicas) + .unwrap_or(0)), None => Ok(0), } } @@ -458,3 +568,223 @@ fn iso_now() -> String { now.format(&time::format_description::well_known::Rfc3339) .expect("RFC 3339 formatting cannot fail") } + +#[cfg(test)] +mod tests { + use super::*; + use niphas_core::crd::{NiphasWorkloadSpec, NiphasWorkloadStatus, WorkloadPhase}; + + fn base_workload() -> NiphasWorkload { + NiphasWorkload { + metadata: kube::api::ObjectMeta { + name: Some("test-app".to_string()), + namespace: Some("default".to_string()), + generation: Some(1), + uid: Some("uid-1234".to_string()), + ..Default::default() + }, + spec: NiphasWorkloadSpec { + flake_ref: "github:test/app".into(), + attribute: "packages.x86_64-linux.default".into(), + revision: None, + replicas: Some(2), + binary_cache: None, + command: None, + args: None, + env: None, + ports: None, + resources: None, + liveness_probe: None, + readiness_probe: None, + startup_probe: None, + node_selector: None, + tolerations: None, + affinity: None, + service: None, + ingress: None, + extra_volumes: None, + extra_volume_mounts: None, + architectures: None, + }, + status: None, + } + } + + fn with_finalizer(workload: &mut NiphasWorkload) { + workload.metadata.finalizers = Some(vec![FINALIZER.to_string()]); + } + + fn running_status(generation: i64) -> NiphasWorkloadStatus { + NiphasWorkloadStatus { + observed_generation: Some(generation), + phase: WorkloadPhase::Running, + store_path: Some("/nix/store/abc123-myapp-1.0".to_string()), + resolved_command: Some("/nix/store/abc123-myapp-1.0/bin/myapp".to_string()), + closure_paths: Some(vec!["/nix/store/abc123-myapp-1.0".to_string()]), + ready_replicas: Some(2), + conditions: None, + last_eval: None, + architectures: None, + } + } + + #[test] + fn decide_deletion_takes_priority() { + let state = WorkloadState { + name: "test-app".into(), + namespace: "default".into(), + generation: 1, + deletion_pending: true, + has_finalizer: true, + existing_finalizers: vec![FINALIZER.to_string()], + observed_generation: 1, + phase: Some(WorkloadPhase::Running), + status: None, + desired_replicas: 1, + }; + assert_eq!(decide(&state), ReconcileDecision::HandleDeletion); + } + + #[test] + fn decide_missing_finalizer() { + let workload = base_workload(); + let state = WorkloadState::from_workload(&workload); + assert_eq!( + decide(&state), + ReconcileDecision::AddFinalizer { + existing_finalizers: vec![], + } + ); + } + + #[test] + fn decide_up_to_date() { + let mut workload = base_workload(); + with_finalizer(&mut workload); + workload.status = Some(running_status(1)); + let state = WorkloadState::from_workload(&workload); + assert_eq!(decide(&state), ReconcileDecision::UpToDate); + } + + #[test] + fn decide_needs_eval_new_generation() { + let mut workload = base_workload(); + with_finalizer(&mut workload); + workload.metadata.generation = Some(2); + workload.status = Some(running_status(1)); + let state = WorkloadState::from_workload(&workload); + assert_eq!(decide(&state), ReconcileDecision::RunEval); + } + + #[test] + fn decide_uses_cached_eval() { + let mut workload = base_workload(); + with_finalizer(&mut workload); + let mut status = running_status(1); + status.phase = WorkloadPhase::Provisioning; + workload.status = Some(status); + let state = WorkloadState::from_workload(&workload); + match decide(&state) { + ReconcileDecision::Provision { eval_result } => { + assert_eq!(eval_result.store_path, "/nix/store/abc123-myapp-1.0"); + } + other => panic!("expected Provision, got {:?}", other), + } + } + + #[test] + fn conditions_upsert_replaces_by_type() { + let existing = vec![ + make_condition("Evaluated", ConditionStatus::True, "Ok", "ok", 1), + make_condition( + "Progressing", + ConditionStatus::True, + "Updating", + "updating", + 1, + ), + ]; + let new = vec![make_condition( + "Evaluated", + ConditionStatus::False, + "EvalFailed", + "timeout", + 2, + )]; + let result = upsert_conditions(Some(&existing), new); + assert_eq!(result.len(), 2); + assert_eq!(result[0].type_, "Evaluated"); + assert_eq!(result[0].status, ConditionStatus::False); + assert_eq!(result[0].reason, "EvalFailed"); + assert_eq!(result[1].type_, "Progressing"); + assert_eq!(result[1].status, ConditionStatus::True); + } + + #[test] + fn conditions_upsert_preserves_others() { + let existing = vec![make_condition( + "Custom", + ConditionStatus::True, + "Something", + "custom condition", + 1, + )]; + let new = vec![make_condition( + "Evaluated", + ConditionStatus::True, + "EvalSucceeded", + "ok", + 1, + )]; + let result = upsert_conditions(Some(&existing), new); + assert_eq!(result.len(), 2); + assert_eq!(result[0].type_, "Custom"); + assert_eq!(result[1].type_, "Evaluated"); + } + + #[test] + fn build_success_conditions_running_has_available() { + let eval = EvalResponse { + store_path: "/nix/store/abc-app".into(), + name: "app".into(), + main_program: Some("app".into()), + closure_paths: vec![], + }; + let conditions = build_success_conditions(&eval, WorkloadPhase::Running, 3, 3, 5); + assert_eq!(conditions.len(), 3); + assert_eq!(conditions[0].type_, "Evaluated"); + assert_eq!(conditions[0].status, ConditionStatus::True); + assert_eq!(conditions[1].type_, "Progressing"); + assert_eq!(conditions[1].status, ConditionStatus::False); + assert_eq!(conditions[1].reason, "NewReplicaSetAvailable"); + assert_eq!(conditions[2].type_, "Available"); + assert_eq!(conditions[2].status, ConditionStatus::True); + } + + #[test] + fn build_success_conditions_provisioning_no_available() { + let eval = EvalResponse { + store_path: "/nix/store/abc-app".into(), + name: "app".into(), + main_program: Some("app".into()), + closure_paths: vec![], + }; + let conditions = build_success_conditions(&eval, WorkloadPhase::Provisioning, 1, 3, 5); + assert_eq!(conditions.len(), 2); + assert_eq!(conditions[0].type_, "Evaluated"); + assert_eq!(conditions[1].type_, "Progressing"); + assert_eq!(conditions[1].status, ConditionStatus::True); + assert_eq!(conditions[1].reason, "ReplicaSetUpdated"); + assert!(conditions.iter().all(|c| c.type_ != "Available")); + } + + #[test] + fn build_failed_conditions_sets_evaluated_false() { + let conditions = build_failed_conditions("EvalTimeout", "timed out after 60s", 3); + assert_eq!(conditions.len(), 1); + assert_eq!(conditions[0].type_, "Evaluated"); + assert_eq!(conditions[0].status, ConditionStatus::False); + assert_eq!(conditions[0].reason, "EvalTimeout"); + assert_eq!(conditions[0].observed_generation, Some(3)); + } +} diff --git a/crates/niphas-operator/src/resources.rs b/crates/niphas-operator/src/resources.rs index 3f53b3e..e975051 100644 --- a/crates/niphas-operator/src/resources.rs +++ b/crates/niphas-operator/src/resources.rs @@ -1,23 +1,33 @@ use crate::context::Context; use crate::error::OperatorError; -use crate::eval::EvalResult; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::Service; use k8s_openapi::api::networking::v1::Ingress; use k8s_openapi::api::policy::v1::PodDisruptionBudget; use kube::api::{Api, Patch, PatchParams, ResourceExt}; use niphas_core::crd::NiphasWorkload; +use niphas_core::eval::EvalResponse; use serde_json::json; use tracing::debug; const FIELD_MANAGER: &str = "niphas-operator"; const RUNNER_IMAGE: &str = "ghcr.io/fullzer4/niphas-runner:latest"; -/// Apply all child resources for a NiphasWorkload using server-side apply. +macro_rules! insert_opt { + ($obj:expr, $key:expr, $val:expr) => { + if let Some(ref v) = $val { + $obj.insert( + $key.into(), + serde_json::to_value(v).expect("k8s-openapi types are serializable"), + ); + } + }; +} + pub async fn apply_child_resources( ctx: &Context, workload: &NiphasWorkload, - eval_result: &EvalResult, + eval_result: &EvalResponse, ) -> Result<(), OperatorError> { let name = workload.name_any(); let ns = workload.namespace().unwrap_or_default(); @@ -47,7 +57,6 @@ pub async fn apply_child_resources( Ok(()) } -/// Generic SSA apply for any K8s resource type. async fn apply_resource( ctx: &Context, name: &str, @@ -74,7 +83,7 @@ where pub(crate) fn build_deployment( workload: &NiphasWorkload, - eval_result: &EvalResult, + eval_result: &EvalResponse, name: &str, ns: &str, ) -> serde_json::Value { @@ -108,7 +117,7 @@ pub(crate) fn build_deployment( ]; if let Some(ref user_tols) = workload.spec.tolerations { for t in user_tols { - tolerations.push(serde_json::to_value(t).unwrap_or_default()); + tolerations.push(serde_json::to_value(t).expect("k8s-openapi types are serializable")); } } @@ -129,51 +138,19 @@ pub(crate) fn build_deployment( }] }); - let container_obj = container.as_object_mut().unwrap(); - - if let Some(ref args) = workload.spec.args { - container_obj.insert("args".into(), json!(args)); - } - if let Some(ref env) = workload.spec.env { - container_obj.insert("env".into(), serde_json::to_value(env).unwrap_or_default()); - } - if let Some(ref ports) = workload.spec.ports { - container_obj.insert( - "ports".into(), - serde_json::to_value(ports).unwrap_or_default(), - ); - } - if let Some(ref resources) = workload.spec.resources { - container_obj.insert( - "resources".into(), - serde_json::to_value(resources).unwrap_or_default(), - ); - } - if let Some(ref probe) = workload.spec.liveness_probe { - container_obj.insert( - "livenessProbe".into(), - serde_json::to_value(probe).unwrap_or_default(), - ); - } - if let Some(ref probe) = workload.spec.readiness_probe { - container_obj.insert( - "readinessProbe".into(), - serde_json::to_value(probe).unwrap_or_default(), - ); - } - if let Some(ref probe) = workload.spec.startup_probe { - container_obj.insert( - "startupProbe".into(), - serde_json::to_value(probe).unwrap_or_default(), - ); - } + let co = container.as_object_mut().unwrap(); + insert_opt!(co, "args", workload.spec.args); + insert_opt!(co, "env", workload.spec.env); + insert_opt!(co, "ports", workload.spec.ports); + insert_opt!(co, "resources", workload.spec.resources); + insert_opt!(co, "livenessProbe", workload.spec.liveness_probe); + insert_opt!(co, "readinessProbe", workload.spec.readiness_probe); + insert_opt!(co, "startupProbe", workload.spec.startup_probe); if let Some(ref extra_mounts) = workload.spec.extra_volume_mounts { - if let Some(mounts) = container_obj.get_mut("volumeMounts") { - if let Some(arr) = mounts.as_array_mut() { - for m in extra_mounts { - arr.push(serde_json::to_value(m).unwrap_or_default()); - } + if let Some(arr) = co.get_mut("volumeMounts").and_then(|v| v.as_array_mut()) { + for m in extra_mounts { + arr.push(serde_json::to_value(m).expect("k8s-openapi types are serializable")); } } } @@ -183,43 +160,17 @@ pub(crate) fn build_deployment( "csi": { "driver": "niphas.io.csi", "volumeAttributes": { + "storePath": eval_result.store_path, "closurePaths": closure_csv } } })]; if let Some(ref extra_vols) = workload.spec.extra_volumes { for v in extra_vols { - volumes.push(serde_json::to_value(v).unwrap_or_default()); + volumes.push(serde_json::to_value(v).expect("k8s-openapi types are serializable")); } } - let topology_spread = if replicas >= 2 { - json!([ - { - "maxSkew": 1, - "topologyKey": "kubernetes.io/hostname", - "whenUnsatisfiable": "DoNotSchedule", - "labelSelector": { - "matchLabels": { - "niphas.io/workload": name - } - } - }, - { - "maxSkew": 1, - "topologyKey": "topology.kubernetes.io/zone", - "whenUnsatisfiable": "ScheduleAnyway", - "labelSelector": { - "matchLabels": { - "niphas.io/workload": name - } - } - } - ]) - } else { - json!(null) - }; - let mut pod_spec = json!({ "nodeSelector": node_selector, "tolerations": tolerations, @@ -227,20 +178,30 @@ pub(crate) fn build_deployment( "volumes": volumes, }); - if topology_spread != json!(null) { - pod_spec - .as_object_mut() - .unwrap() - .insert("topologySpreadConstraints".into(), topology_spread); - } + let ps = pod_spec.as_object_mut().unwrap(); - if let Some(ref affinity) = workload.spec.affinity { - pod_spec.as_object_mut().unwrap().insert( - "affinity".into(), - serde_json::to_value(affinity).unwrap_or_default(), + if replicas >= 2 { + ps.insert( + "topologySpreadConstraints".into(), + json!([ + { + "maxSkew": 1, + "topologyKey": "kubernetes.io/hostname", + "whenUnsatisfiable": "DoNotSchedule", + "labelSelector": { "matchLabels": { "niphas.io/workload": name } } + }, + { + "maxSkew": 1, + "topologyKey": "topology.kubernetes.io/zone", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": { "matchLabels": { "niphas.io/workload": name } } + } + ]), ); } + insert_opt!(ps, "affinity", workload.spec.affinity); + json!({ "apiVersion": "apps/v1", "kind": "Deployment", @@ -481,8 +442,8 @@ mod tests { } } - fn test_eval_result() -> EvalResult { - EvalResult { + fn test_eval_result() -> EvalResponse { + EvalResponse { store_path: "/nix/store/abc123-myapp-1.0.0".into(), name: "myapp-1.0.0".into(), main_program: Some("myapp".into()), @@ -505,17 +466,22 @@ mod tests { assert_eq!(dep["metadata"]["name"], "myapp"); assert_eq!(dep["metadata"]["namespace"], "default"); - // Check CSI volume let volumes = dep["spec"]["template"]["spec"]["volumes"] .as_array() .unwrap(); assert_eq!(volumes[0]["csi"]["driver"], "niphas.io.csi"); + assert_eq!( + volumes[0]["csi"]["volumeAttributes"]["storePath"], "/nix/store/abc123-myapp-1.0.0", + "storePath must be passed in volumeAttributes" + ); + assert_eq!( + volumes[0]["csi"]["volumeAttributes"]["closurePaths"], + "/nix/store/abc123-myapp-1.0.0,/nix/store/def456-glibc-2.37" + ); - // Check nodeSelector includes niphas.io/store=true let ns = &dep["spec"]["template"]["spec"]["nodeSelector"]; assert_eq!(ns["niphas.io/store"], "true"); - // Check command resolves to main_program let containers = dep["spec"]["template"]["spec"]["containers"] .as_array() .unwrap(); @@ -659,11 +625,9 @@ mod tests { assert_eq!(oref["blockOwnerDeletion"], true); } - // -- EvalResult tests -- - #[test] fn test_resolved_command_with_main_program() { - let eval = EvalResult { + let eval = EvalResponse { store_path: "/nix/store/abc-hello-2.12.1".into(), name: "hello-2.12.1".into(), main_program: Some("hello".into()), @@ -677,7 +641,7 @@ mod tests { #[test] fn test_resolved_command_without_main_program() { - let eval = EvalResult { + let eval = EvalResponse { store_path: "/nix/store/abc-hello-2.12.1".into(), name: "hello-2.12.1".into(), main_program: None, diff --git a/deny.toml.bak b/deny.toml.bak new file mode 100644 index 0000000..466adab --- /dev/null +++ b/deny.toml.bak @@ -0,0 +1,33 @@ +[advisories] +vulnerability = "deny" +unmaintained = "workspace" +yanked = "workspace" +notice = "workspace" + +[licenses] +unlicensed = "deny" +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Unicode-3.0", + "Unicode-DFS-2016", + "Zlib", + "OpenSSL", + "MPL-2.0", +] +copyleft = "deny" + +[bans] +multiple-versions = "warn" +wildcards = "deny" +highlight = "all" + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +allow-git = [] diff --git a/nix/devShell.nix b/nix/devShell.nix index 66cefc6..8ab4fa1 100644 --- a/nix/devShell.nix +++ b/nix/devShell.nix @@ -13,7 +13,7 @@ pkgs.pkg-config pkgs.protobuf pkgs.clang - pkgs.lld + pkgs.mold pkgs.git pkgs.cargo-deny pkgs.cargo-nextest @@ -21,28 +21,13 @@ pkgs.kubectl pkgs.kubernetes-helm pkgs.kind - pkgs.mdbook ]; - buildInputs = [ - pkgs.openssl - pkgs.zstd - pkgs.xz - pkgs.bzip2 - ]; + buildInputs = [ pkgs.openssl pkgs.zstd pkgs.xz pkgs.bzip2 ]; PROTOC = "${pkgs.protobuf}/bin/protoc"; - - LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ - pkgs.openssl - pkgs.zstd - pkgs.xz - pkgs.bzip2 - ]; - - shellHook = '' - echo "niphas devShell ready" - ''; + CARGO_BUILD_RUSTFLAGS = "-C linker=clang -C link-arg=-fuse-ld=mold --cfg tokio_unstable"; + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ pkgs.openssl pkgs.zstd pkgs.xz pkgs.bzip2 ]; }; }; } diff --git a/nix/docs.nix b/nix/docs.nix index 68dd3e8..58938b8 100644 --- a/nix/docs.nix +++ b/nix/docs.nix @@ -7,10 +7,7 @@ src = self; nativeBuildInputs = [ pkgs.mdbook ]; buildPhase = "mdbook build docs"; - installPhase = '' - mkdir -p $out - cp -r docs/book/* $out/ - ''; + installPhase = "mkdir -p $out && cp -r docs/book/* $out/"; }; in { @@ -24,18 +21,7 @@ exec darkhttpd ${docs-site} --port "$PORT" ''; }; - niphas-docs-site = docs-site; - - image-niphas-docs = pkgs.dockerTools.buildLayeredImage { - name = "ghcr.io/fullzer4/niphas-docs"; - tag = "dev"; - contents = [ pkgs.darkhttpd docs-site pkgs.cacert ]; - config = { - Entrypoint = [ "${pkgs.darkhttpd}/bin/darkhttpd" "${docs-site}" "--port" "8080" ]; - ExposedPorts = { "8080/tcp" = {}; }; - }; - }; }; }; } diff --git a/nix/images.nix b/nix/images.nix index 0b3183d..3e17472 100644 --- a/nix/images.nix +++ b/nix/images.nix @@ -2,17 +2,9 @@ { perSystem = { pkgs, self', system, ... }: let - # Runtime shared libraries needed by all Rust binaries - runtimeLibs = [ - pkgs.openssl.out - pkgs.zstd.out - pkgs.xz.out - pkgs.bzip2.out - ]; - + runtimeLibs = [ pkgs.openssl.out pkgs.zstd.out pkgs.xz.out pkgs.bzip2.out ]; runtimeLibPath = pkgs.lib.makeLibraryPath runtimeLibs; - # Nix configuration for eval container (single-user, no sandbox) nixConf = pkgs.writeTextDir "etc/nix/nix.conf" '' sandbox = false experimental-features = nix-command flakes @@ -20,23 +12,31 @@ filter-syscalls = false ''; - mkImage = { name, pkg, extraPaths ? [], entrypoint }: + + mkImage = { name, pkg, extraPaths ? [], extraEnv ? [], entrypoint, runAsRoot ? true }: pkgs.dockerTools.buildLayeredImage { inherit name; tag = "dev"; - contents = [ - pkg - pkgs.cacert - pkgs.tzdata - ] ++ runtimeLibs ++ extraPaths; + contents = [ pkg pkgs.cacert pkgs.tzdata ] ++ runtimeLibs ++ extraPaths; config = { Entrypoint = [ entrypoint ]; + User = if runAsRoot then "0" else "65534"; Env = [ "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" "TZDIR=${pkgs.tzdata}/share/zoneinfo" "LD_LIBRARY_PATH=${runtimeLibPath}" - ]; + ] ++ extraEnv; }; + # Create real /etc/passwd and /etc/group for non-root images. + # writeTextDir creates symlinks into /nix/store which containerd + # rejects as "path escapes from parent", so we use fakeRootCommands + # to write real files directly into the image layer. + fakeRootCommands = pkgs.lib.optionalString (!runAsRoot) '' + mkdir -p ./etc + echo 'nobody:x:65534:65534:Nobody:/tmp:/bin/sh' > ./etc/passwd + echo 'nogroup:x:65534:' > ./etc/group + ''; + enableFakechroot = !runAsRoot; }; in { @@ -45,13 +45,16 @@ name = "ghcr.io/fullzer4/niphas-operator"; pkg = self'.packages.niphas-operator; entrypoint = "/bin/niphas-operator"; + runAsRoot = false; }; image-niphas-eval = mkImage { name = "ghcr.io/fullzer4/niphas-eval"; pkg = self'.packages.niphas-eval; extraPaths = [ pkgs.nix pkgs.git nixConf ]; + extraEnv = [ "HOME=/tmp" ]; entrypoint = "/bin/niphas-eval"; + runAsRoot = false; }; image-niphas-csi = mkImage { @@ -64,30 +67,18 @@ image-niphas-runner = pkgs.dockerTools.buildLayeredImage { name = "ghcr.io/fullzer4/niphas-runner"; tag = "dev"; - contents = [ - pkgs.busybox - pkgs.cacert - ]; + contents = [ pkgs.busybox pkgs.cacert ]; config = { Entrypoint = [ "/bin/sh" ]; - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" - ]; + Env = [ "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" ]; }; }; - image-niphas-mock-eval = mkImage { - name = "ghcr.io/fullzer4/niphas-mock-eval"; - pkg = self'.packages.niphas-mock-eval; - entrypoint = "/bin/niphas-mock-eval"; - }; - all-images = pkgs.linkFarm "niphas-all-images" [ { name = "operator.tar.gz"; path = self'.packages.image-niphas-operator; } { name = "eval.tar.gz"; path = self'.packages.image-niphas-eval; } { name = "csi.tar.gz"; path = self'.packages.image-niphas-csi; } { name = "runner.tar.gz"; path = self'.packages.image-niphas-runner; } - { name = "mock-eval.tar.gz"; path = self'.packages.image-niphas-mock-eval; } ]; }; }; diff --git a/nix/packages.nix b/nix/packages.nix index a9620f5..9e394ea 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -18,43 +18,53 @@ inherit src; pname = "niphas"; strictDeps = true; - - nativeBuildInputs = [ - pkgs.pkg-config - pkgs.protobuf - pkgs.clang - pkgs.lld - ]; - - buildInputs = [ - pkgs.openssl - pkgs.zstd - pkgs.xz - pkgs.bzip2 - ]; - + nativeBuildInputs = [ pkgs.pkg-config pkgs.protobuf pkgs.clang pkgs.mold ]; + buildInputs = [ pkgs.openssl pkgs.zstd pkgs.xz pkgs.bzip2 ]; PROTOC = "${pkgs.protobuf}/bin/protoc"; + CARGO_BUILD_RUSTFLAGS = "-C linker=clang -C link-arg=-fuse-ld=mold --cfg tokio_unstable"; }; - cargoArtifacts = craneLib.buildDepsOnly commonArgs; + cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { CARGO_PROFILE = "ci"; }); mkBin = name: craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; cargoExtraArgs = "--bin ${name}"; + CARGO_PROFILE = "ci"; doCheck = false; }); in { packages = { - default = craneLib.buildPackage (commonArgs // { - inherit cargoArtifacts; - }); - + default = craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; }); niphas-operator = mkBin "niphas-operator"; niphas-eval = mkBin "niphas-eval"; niphas-csi = mkBin "niphas-csi"; niphas-crd-gen = mkBin "niphas-crd-gen"; - niphas-mock-eval = mkBin "niphas-mock-eval"; + }; + + checks = { + fmt = craneLib.cargoFmt { inherit src; }; + clippy = craneLib.cargoClippy (commonArgs // { + inherit cargoArtifacts; + cargoClippyExtraArgs = "--workspace --all-targets -- --deny warnings"; + }); + deny = craneLib.cargoDeny { inherit src; }; + nextest = craneLib.cargoNextest (commonArgs // { + inherit cargoArtifacts; + partitions = 1; + partitionType = "count"; + cargoNextestExtraArgs = "--workspace --exclude niphas-e2e"; + }); + eval-http = craneLib.cargoNextest (commonArgs // { + inherit cargoArtifacts; + pnameSuffix = "-eval-http"; + cargoNextestExtraArgs = "-p niphas-eval --test '*'"; + }); + csi-grpc = craneLib.cargoNextest (commonArgs // { + inherit cargoArtifacts; + pnameSuffix = "-csi-grpc"; + cargoNextestExtraArgs = "-p niphas-csi --test '*'"; + }); }; }; }