From bedd21ad5995511af8c4e9a924714162bc8c5253 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Tue, 3 Mar 2026 19:27:36 -0800 Subject: [PATCH 1/2] nix: extract K8s manifests, remove BCC, improve build isolation Standalone K8s manifest generation (nix build .#pcp-k8s-manifests) so users can inspect or deploy PCP's DaemonSet without minikube. Manifests are validated at build time with kube-linter; all 9 default findings are suppressed with documented rationale (PCP is a privileged node monitoring agent that needs host PID, /proc, /sys, and BPF access by design). The nix/k8s-test module now imports from nix/k8s-manifests with a namespace override ("pcp-test") for test isolation, replacing the old nix/k8s-test/manifests.nix. Completely remove BCC (pmdabcc was removed from PCP upstream in commit 9a576b8076). This drops the unused bcc build dependency from nix/package.nix, invalid --with-pmdabcc configure flags, the dead nix/bcc.nix NixOS module (304 lines), and all enableBcc plumbing from nix/microvm.nix and flake.nix. Pin minikube's --kubernetes-version to v1.33.0 so the test suite doesn't break when nixpkgs updates minikube. The start script now auto-deletes stale clusters with incompatible k8s versions instead of failing. Exclude nix/, flake.nix, flake.lock, and CLAUDE.md from the PCP source filter so that Nix infrastructure changes don't trigger a full PCP rebuild. Co-Authored-By: Claude Opus 4.6 --- docs/HowTos/nix/index.rst | 37 +++ flake.nix | 14 +- nix/bcc.nix | 304 ------------------ nix/bpf.nix | 21 +- nix/constants.nix | 12 - nix/k8s-manifests/constants.nix | 45 +++ .../daemonset.nix} | 70 ++-- nix/k8s-manifests/default.nix | 116 +++++++ nix/k8s-manifests/kube-linter-config.yaml | 50 +++ nix/k8s-manifests/kube-linter.nix | 74 +++++ nix/k8s-manifests/namespace.nix | 17 + nix/k8s-test/constants.nix | 14 +- nix/k8s-test/default.nix | 70 ++-- nix/microvm.nix | 68 +--- nix/package.nix | 16 +- 15 files changed, 465 insertions(+), 463 deletions(-) delete mode 100644 nix/bcc.nix create mode 100644 nix/k8s-manifests/constants.nix rename nix/{k8s-test/manifests.nix => k8s-manifests/daemonset.nix} (51%) create mode 100644 nix/k8s-manifests/default.nix create mode 100644 nix/k8s-manifests/kube-linter-config.yaml create mode 100644 nix/k8s-manifests/kube-linter.nix create mode 100644 nix/k8s-manifests/namespace.nix diff --git a/docs/HowTos/nix/index.rst b/docs/HowTos/nix/index.rst index a3c5b10ba3..d135e52058 100644 --- a/docs/HowTos/nix/index.rst +++ b/docs/HowTos/nix/index.rst @@ -84,6 +84,13 @@ File Structure ├── pmie-test.nix # pmie testing module (stress-ng workload) │ ├── container.nix # OCI container image + │ + ├── k8s-manifests/ # Standalone K8s manifest generator + │ ├── default.nix # Entry point, produces dir + combined YAML + │ ├── constants.nix # Deployment constants (namespace, resources) + │ ├── namespace.nix # Namespace resource YAML + │ └── daemonset.nix # DaemonSet resource YAML + │ ├── network-setup.nix # TAP/bridge network scripts ├── shell.nix # Development shell ├── vm-test.nix # NixOS VM integration test @@ -311,6 +318,14 @@ Flake Outputs Reference - Quick test (skip build, assume image loaded) * - ``pcp-minikube-start`` - Start minikube with optimal settings for PCP testing + * - **K8s Manifests** (no minikube needed) + - + * - ``pcp-k8s-manifests`` + - Directory with all K8s manifests + README + * - ``pcp-k8s-manifest-namespace`` + - Namespace YAML only + * - ``pcp-k8s-manifest-daemonset`` + - DaemonSet YAML only * - **Checks** - * - ``vm-test`` @@ -1801,6 +1816,28 @@ for PCP testing (4 CPUs, 8GB RAM, docker driver):: nix run .#pcp-minikube-start +Standalone K8s Manifests +"""""""""""""""""""""""" + +You can generate Kubernetes manifests without minikube or the full test suite. +This is useful for inspecting the PCP DaemonSet deployment configuration or +applying it to an existing cluster:: + + # Build all manifests into a directory + nix build .#pcp-k8s-manifests + ls result/ # README.md combined.yaml daemonset.yaml namespace.yaml + + # Apply to a cluster + cat result/combined.yaml | kubectl apply -f - + + # Or build individual manifests + nix build .#pcp-k8s-manifest-daemonset + cat result + +The standalone manifests use the ``pcp`` namespace by default (the test suite +overrides this to ``pcp-test`` for isolation). Resource limits, image settings, +and host mount paths are all defined in ``nix/k8s-manifests/constants.nix``. + Development Shell ----------------- diff --git a/flake.nix b/flake.nix index b44de14ef2..53901003e7 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,10 @@ # nix run .#pcp-k8s-test # Kubernetes DaemonSet (needs minikube) # nix run .#pcp-test-all-microvms # All MicroVM variants # +# K8s Manifests (no minikube needed): +# nix build .#pcp-k8s-manifests # All manifests + README +# nix build .#pcp-k8s-manifest-daemonset # Individual DaemonSet YAML +# # MicroVM with TAP networking (for Grafana dashboards): # nix run .#pcp-check-host # Verify host environment # sudo nix run .#pcp-network-setup # Create TAP bridge (requires sudo) @@ -69,14 +73,13 @@ enablePmieTest ? false, enableGrafana ? false, enableBpf ? false, - enableBcc ? false, portOffset ? 0, variant ? "base", }: import ./nix/microvm.nix { inherit pkgs lib pcp microvm nixosModule nixpkgs system; inherit networking debugMode enablePmlogger enableEvalTools - enablePmieTest enableGrafana enableBpf enableBcc + enablePmieTest enableGrafana enableBpf portOffset variant; }; @@ -137,6 +140,11 @@ } ); + # Import standalone K8s manifest generator (Linux only) + k8sManifests = lib.optionalAttrs pkgs.stdenv.isLinux ( + import ./nix/k8s-manifests { inherit pkgs lib; } + ); + # Import test-all runner (Linux only) testAll = lib.optionalAttrs pkgs.stdenv.isLinux ( import ./nix/test-all { @@ -162,6 +170,8 @@ // containerTest.packages # Kubernetes testing packages // k8sTest.packages + # Standalone K8s manifests + // k8sManifests.packages # Test-all runner // testAll.packages ); diff --git a/nix/bcc.nix b/nix/bcc.nix deleted file mode 100644 index c129a33cde..0000000000 --- a/nix/bcc.nix +++ /dev/null @@ -1,304 +0,0 @@ -# nix/bcc.nix -# -# NixOS module for BCC PMDA (eBPF metrics). -# -# ══════════════════════════════════════════════════════════════════════════════ -# DEPRECATED: pmdabcc is deprecated upstream and will be removed in a future -# PCP release. Use pmdabpf instead (see nix/bpf.nix). -# -# From pmdabcc(1): -# "This PMDA is now deprecated and will be removed in a future release, -# transition to using its replacement pmdabpf(1) instead." -# -# This module is provided for reference but is NOT functional on NixOS. -# No further development will be done on this module. -# ══════════════════════════════════════════════════════════════════════════════ -# -# ─── Known Issues (will not be fixed) ───────────────────────────────────────── -# -# BCC compilation works but metrics don't register with pmcd: -# - pminfo bcc returns "Unknown metric name" -# - Modules compile but don't register metrics via pmcd pipe protocol -# - Log file path doesn't work correctly -# -# ─── Use pmdabpf Instead ────────────────────────────────────────────────────── -# -# pmdabpf is the supported replacement: -# - Pre-compiled CO-RE eBPF (no runtime compilation) -# - Works reliably on NixOS -# - Provides runqlat, biolatency, oomkill, execsnoop, etc. -# - Fast startup, ~512MB memory vs ~2GB for BCC -# - Actively maintained -# -# ══════════════════════════════════════════════════════════════════════════════ -# -# Architecture (for reference): -# pmcd reads pmcd.conf which includes bcc PMDA entry -# pmcd spawns pmdabcc as a pipe PMDA (binary protocol) -# pmdabcc loads eBPF modules via BCC library -# Grafana PCP Vector datasource queries pmproxy → pmcd → pmdabcc -# -{ config, lib, pkgs, ... }: -with lib; -let - constants = import ./constants.nix; - cfg = config.services.pcp; - bccCfg = cfg.bcc; - - # Use the kernel from pkgs.linuxPackages (NixOS default). - # Note: We can't use config.boot.kernelPackages.kernel here because it creates - # an infinite recursion with fileSystems evaluation. - # - # This works for MicroVMs because they use the default NixOS kernel (pkgs.linuxPackages). - # If you customize boot.kernelPackages, ensure it matches pkgs.linuxPackages or - # the bind mounts will be at the wrong path. - # - # BCC uses uname -r to find headers at /lib/modules/$(uname -r)/build - kernel = pkgs.linuxPackages.kernel; - - # All modules (defaults + extras) - allModules = bccCfg.modules ++ bccCfg.extraModules; - - # Generate bcc.conf from enabled modules - moduleConfigs = concatStringsSep "\n" (map (mod: '' - [${mod.name}] - module = ${if mod.module != null then mod.module else mod.name} - cluster = ${toString mod.cluster} - ${optionalString (mod.extraConfig != "") mod.extraConfig} - '') allModules); - - bccConf = pkgs.writeText "bcc.conf" '' - # - # PCP BCC PMDA config - generated by NixOS - # See pmdabcc(1) for configuration options - # - - [pmda] - modules = ${concatMapStringsSep "," (m: m.name) allModules} - prefix = bcc. - process_refresh = ${toString bccCfg.processRefresh} - module_failure_fatal = ${if bccCfg.moduleFailureFatal then "True" else "False"} - - ${moduleConfigs} - ''; - - # Python path for BCC - use the same Python as PCP - python = pkgs.python3; - pythonVersion = python.pythonVersion; - -in { - # ═══════════════════════════════════════════════════════════════════════ - # Options interface - # ═══════════════════════════════════════════════════════════════════════ - - options.services.pcp.bcc = { - enable = mkEnableOption "BCC PMDA for eBPF metrics (requires kernel eBPF support)"; - - modules = mkOption { - type = types.listOf (types.submodule { - options = { - name = mkOption { - type = types.str; - description = "Module name (used in config section header)"; - }; - module = mkOption { - type = types.nullOr types.str; - default = null; - description = "Module to load (defaults to name if not specified)"; - }; - cluster = mkOption { - type = types.int; - description = "Unique cluster ID (0-99 for BCC tools, 100-199 for PCP-specific)"; - }; - extraConfig = mkOption { - type = types.str; - default = ""; - description = "Additional config lines for this module"; - }; - }; - }); - default = [ - # Modules required for eBPF/BCC Overview dashboard - { name = "runqlat"; cluster = 4; } - { name = "biolatency"; cluster = 0; } - { name = "tcptop"; cluster = 12; } - { name = "tcplife"; cluster = 3; } - ]; - description = '' - BCC modules to enable. Default includes modules required for - the Grafana eBPF/BCC Overview dashboard. - ''; - }; - - processRefresh = mkOption { - type = types.int; - default = 60; - description = "Interval in seconds to refresh monitored processes (0 to disable)"; - }; - - moduleFailureFatal = mkOption { - type = types.bool; - default = false; - description = '' - Whether BCC PMDA should exit when a module fails to compile. - Set to false for development (partial functionality) or true for production. - ''; - }; - - extraModules = mkOption { - type = types.listOf (types.submodule { - options = { - name = mkOption { type = types.str; }; - module = mkOption { type = types.nullOr types.str; default = null; }; - cluster = mkOption { type = types.int; }; - extraConfig = mkOption { type = types.str; default = ""; }; - }; - }); - default = []; - description = "Additional BCC modules beyond the dashboard defaults"; - example = literalExpression '' - [ - { name = "netproc"; cluster = 40; } - { name = "bioperpid"; cluster = 1; } - ] - ''; - }; - }; - - # ═══════════════════════════════════════════════════════════════════════ - # Implementation - # ═══════════════════════════════════════════════════════════════════════ - - config = mkIf (cfg.enable && bccCfg.enable) { - # Warning about deprecation - warnings = [ - "pmdabcc is DEPRECATED and will be removed in a future PCP release. Use pmdabpf instead (services.pcp.bpf.enable = true)." - "BCC PMDA is NOT functional on NixOS - metrics will not register with pmcd." - ]; - - # Add BCC to system packages for debugging - environment.systemPackages = [ pkgs.bcc ]; - - # Register BCC PMDA with pmcd via the pmdas option - services.pcp.pmdas.bcc = { - domain = 149; # BCC domain ID - type = "pipe"; - flags = "binary notready"; - command = let - # Create wrapper script that sets up BCC environment - # Note: BCC requires clang for runtime eBPF compilation - # We use clang-unwrapped to avoid NixOS clang-wrapper validation issues - # that cause objdump to fail during pmdabcc initialization - - # Create an ld wrapper that prepends glibc library path - # Problem: cffi runs `ld -t -L -o /dev/null -lc` to verify libraries - # This fails because glibc isn't in . Our wrapper injects -L/lib - ldWrapper = pkgs.writeShellScript "ld" '' - exec ${pkgs.llvmPackages.bintools-unwrapped}/bin/ld -L${pkgs.glibc}/lib "$@" - ''; - - # Create a clang wrapper that adds kernel include paths for BPF compilation - # BCC calls clang directly without respecting environment variables for includes - # This wrapper prepends the NixOS kernel dev paths - kernelDev = "${kernel.dev}/lib/modules/${kernel.version}"; - clangWrapper = pkgs.writeShellScript "clang" '' - exec ${pkgs.llvmPackages.clang-unwrapped}/bin/clang \ - -I${kernelDev}/build/include/generated \ - -I${kernelDev}/build/include \ - -I${kernelDev}/source/include \ - -I${kernelDev}/source/arch/x86/include \ - -I${kernelDev}/build/arch/x86/include/generated \ - -I${kernelDev}/source/include/uapi \ - -I${kernelDev}/source/arch/x86/include/uapi \ - "$@" - ''; - - # Directory containing our wrappers (must be first in PATH) - wrapperDir = pkgs.runCommand "bcc-wrapper-dir" {} '' - mkdir -p $out/bin - ln -s ${ldWrapper} $out/bin/ld - ln -s ${clangWrapper} $out/bin/clang - ''; - - bccWrapper = pkgs.writeShellScript "pmdabcc-wrapper" '' - # BCC Python bindings - export PYTHONPATH="${pkgs.bcc}/lib/python${pythonVersion}/site-packages:${cfg.package}/lib/python${pythonVersion}/site-packages" - - # Put our wrappers FIRST in PATH: - # - clang wrapper: adds kernel include paths for BPF compilation - # - ld wrapper: adds -L/lib for cffi library checks - # Also add kmod for modprobe (needed by BCC for loading kernel modules) - # See: nix/tests/BCC_PMDA_DEFECT.md for details - export PATH="${wrapperDir}/bin:${pkgs.llvmPackages.llvm}/bin:${pkgs.llvmPackages.bintools-unwrapped}/bin:${pkgs.bcc}/bin:${pkgs.kmod}/bin:$PATH" - - # Library paths for dynamic linking (required for pcp.pmapi to find libpcp.so) - export LD_LIBRARY_PATH="${cfg.package}/lib:${pkgs.glibc}/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - - # Provide library paths that would normally come from the wrapper - export NIX_CFLAGS_COMPILE="-isystem ${pkgs.glibc.dev}/include -isystem ${kernel.dev}/include" - export NIX_LDFLAGS="-L${pkgs.glibc}/lib" - - # Library search paths (for tools that respect these variables) - export LIBRARY_PATH="${pkgs.glibc}/lib:${cfg.package}/lib" - export LDFLAGS="-L${pkgs.glibc}/lib" - - # Kernel headers for eBPF compilation - # DO NOT set BCC_KERNEL_SOURCE - this breaks split-layout detection! - # Instead, we create symlinks at the NixOS kernel modules path to our bind mounts - # See systemd.tmpfiles.rules for the symlink creation - - # Disable wrapper purity enforcement - export NIX_ENFORCE_PURITY=0 - - # Override PCP_PMDAS_DIR so pmdabcc.python finds our custom config - # pmdabcc.python uses: pmGetConfig('PCP_PMDAS_DIR') + '/bcc/bcc.conf' - # Our config is at: /var/lib/pcp/config/bcc/bcc.conf - # So we set PCP_PMDAS_DIR=/var/lib/pcp/config - export PCP_PMDAS_DIR=/var/lib/pcp/config - - # Log file for debugging (pmdabcc.python uses the -l flag) - exec ${cfg.package}/bin/pmpython \ - ${cfg.package}/var/lib/pcp/pmdas/bcc/pmdabcc.python \ - -l /var/log/pcp/pmcd/bcc.log \ - "$@" - ''; - in "${bccWrapper} -d 149"; - }; - - # Create BCC config directory and files - # Note: /var/lib/pcp/config is owned by pcp:pcp, so subdirs must also be pcp-owned - # to avoid "unsafe path transition" errors from systemd-tmpfiles - systemd.tmpfiles.rules = [ - # Writable config directory for BCC (must be pcp:pcp to match parent) - "d /var/lib/pcp/config/bcc 0755 pcp pcp -" - "d /var/log/pcp/pmcd 0755 root root -" - - # Create kernel headers mount points for BCC at /lib/modules/VERSION - "d /lib/modules 0755 root root -" - "d /lib/modules/${kernel.version} 0755 root root -" - "d /lib/modules/${kernel.version}/build 0755 root root -" - "d /lib/modules/${kernel.version}/source 0755 root root -" - - # Copy BCC config (using L+ to create symlink, since C doesn't work well here) - # The config is read-only which is fine for the PMDA - "L+ /var/lib/pcp/config/bcc/bcc.conf - - - - ${bccConf}" - - # Symlink BCC module Python files from package - "L+ /var/lib/pcp/config/bcc/modules - - - - ${cfg.package}/var/lib/pcp/pmdas/bcc/modules" - ]; - - # Bind mount kernel dev headers to /lib/modules/VERSION/{build,source} - # NixOS BCC uses KERNEL_MODULES_DIR=/run/booted-system/... but that's a symlink we can't mount into - # So we mount at /lib/modules/ and set BCC_KERNEL_SOURCE to point there - fileSystems."/lib/modules/${kernel.version}/build" = { - device = "${kernel.dev}/lib/modules/${kernel.version}/build"; - fsType = "none"; - options = [ "bind" "ro" ]; - }; - fileSystems."/lib/modules/${kernel.version}/source" = { - device = "${kernel.dev}/lib/modules/${kernel.version}/source"; - fsType = "none"; - options = [ "bind" "ro" ]; - }; - }; -} diff --git a/nix/bpf.nix b/nix/bpf.nix index f7b9cd9822..0c520f4c5a 100644 --- a/nix/bpf.nix +++ b/nix/bpf.nix @@ -3,23 +3,16 @@ # NixOS module for BPF PMDA (pre-compiled eBPF metrics). # # This module uses pmdabpf which loads pre-compiled CO-RE (Compile Once, Run Everywhere) -# eBPF programs. Unlike pmdabcc, no runtime compilation is needed - the eBPF bytecode -# is compiled at Nix build time and loaded directly by the kernel via libbpf. +# eBPF programs. No runtime compilation is needed — the eBPF bytecode is compiled at +# Nix build time and loaded directly by the kernel via libbpf. # -# Advantages over pmdabcc: +# Characteristics: # - Fast startup: No eBPF compilation at runtime -# - Low memory: No clang/LLVM needed in the VM (~512MB vs ~2GB) -# - Simpler: Just needs libbpf + BTF-enabled kernel +# - Low memory: No clang/LLVM needed in the VM (~512MB) +# - Simple: Just needs libbpf + BTF-enabled kernel # -# Trade-offs: -# - Fewer modules available compared to pmdabcc -# - Missing: tcptop, tcplife (required for some Grafana BCC dashboards) -# -# If you need tcptop/tcplife metrics, see bcc.nix which uses pmdabcc with runtime -# eBPF compilation. This requires: -# - 2GB+ VM memory (clang/LLVM are memory-hungry) -# - BTF-enabled kernel (CONFIG_DEBUG_INFO_BTF=y) -# - Extended pmcd timeout (-q 60) for module compilation +# Note: pmdabcc (runtime BCC compilation) was removed from PCP upstream. +# pmdabpf is the only supported eBPF PMDA. # # Available pmdabpf modules: # - biolatency: Block device I/O latency histogram diff --git a/nix/constants.nix b/nix/constants.nix index 6cda9c0d04..3b35372ba1 100644 --- a/nix/constants.nix +++ b/nix/constants.nix @@ -102,8 +102,6 @@ rec { pmcd = 2; # src/pmdas/pmcd/domain.h linux = 60; # src/pmdas/linux/domain.h bpf = 157; # src/pmdas/bpf/domain.h (pre-compiled CO-RE eBPF) - # NOTE: BCC is deprecated - use BPF PMDA instead (CO-RE eBPF) - # bcc = 149; # src/pmdas/bcc/domain.h (runtime-compiled eBPF) }; # ─── Test Configuration ──────────────────────────────────────────────── @@ -131,8 +129,6 @@ rec { eval = 100; # pcp-microvm-eval, pcp-microvm-eval-tap grafana = 200; # pcp-microvm-grafana, pcp-microvm-grafana-tap bpf = 300; # pcp-microvm-bpf - # NOTE: BCC is deprecated - use BPF PMDA instead - # bcc = 400; # pcp-microvm-bcc }; # ─── Serial Console Configuration ──────────────────────────────────── @@ -166,8 +162,6 @@ rec { eval = 10; # 24510-24519 grafana = 20; # 24520-24529 bpf = 30; # 24530-24539 - # NOTE: BCC is deprecated - use BPF PMDA instead - # bcc = 40; # 24540-24549 }; }; @@ -205,12 +199,6 @@ rec { eval = {}; grafana = { serviceReady = 90; }; # Grafana takes longer to start bpf = {}; - # NOTE: BCC is deprecated - use BPF PMDA instead (CO-RE eBPF) - # BCC used runtime eBPF compilation which required longer timeouts - # bcc = { - # serviceReady = 180; # BCC modules compile at pmcd startup - # metricsReady = 120; # BCC metrics appear after compilation - # }; }; }; diff --git a/nix/k8s-manifests/constants.nix b/nix/k8s-manifests/constants.nix new file mode 100644 index 0000000000..5f5e3854db --- /dev/null +++ b/nix/k8s-manifests/constants.nix @@ -0,0 +1,45 @@ +# nix/k8s-manifests/constants.nix +# +# Kubernetes deployment constants for PCP DaemonSet manifests. +# These are the production defaults; test suites can override +# (e.g. namespace -> "pcp-test") via the namespaceOverride parameter +# in default.nix. +# +{ }: +let + mainConstants = import ../constants.nix; +in +{ + # Re-export ports from main constants + inherit (mainConstants) ports; + + # ─── Kubernetes Deployment Settings ────────────────────────────────── + k8s = { + namespace = "pcp"; + daemonSetName = "pcp"; + + image = { + name = "pcp"; + tag = "latest"; + pullPolicy = "Never"; + }; + + resources = { + limits = { + memory = "512Mi"; + cpu = "500m"; + }; + requests = { + memory = "256Mi"; + cpu = "100m"; + }; + }; + + hostMounts = { + root = "/host"; + proc = "/host/proc"; + sys = "/host/sys"; + kernelDebug = "/sys/kernel/debug"; + }; + }; +} diff --git a/nix/k8s-test/manifests.nix b/nix/k8s-manifests/daemonset.nix similarity index 51% rename from nix/k8s-test/manifests.nix rename to nix/k8s-manifests/daemonset.nix index 56b3c8e001..11d692e651 100644 --- a/nix/k8s-test/manifests.nix +++ b/nix/k8s-manifests/daemonset.nix @@ -1,44 +1,37 @@ -# nix/k8s-test/manifests.nix +# nix/k8s-manifests/daemonset.nix # -# Generates Kubernetes manifests for PCP DaemonSet deployment. -# Creates privileged DaemonSet with full node monitoring capabilities. +# Generates the Kubernetes DaemonSet resource YAML for PCP deployment. +# Creates a privileged DaemonSet with full node monitoring capabilities +# including BPF metrics. # -{ pkgs, lib }: +{ pkgs, constants }: let - constants = import ./constants.nix { }; - mainConstants = import ../constants.nix; + k = constants.k8s; - # ─── DaemonSet Manifest ──────────────────────────────────────────────── - # Privileged DaemonSet for full node monitoring including BPF metrics - daemonSetManifest = '' - apiVersion: v1 - kind: Namespace - metadata: - name: ${constants.k8s.namespace} - --- + yaml = '' apiVersion: apps/v1 kind: DaemonSet metadata: - name: ${constants.k8s.daemonSetName} - namespace: ${constants.k8s.namespace} + name: ${k.daemonSetName} + namespace: ${k.namespace} labels: - app: ${constants.k8s.daemonSetName} + app: ${k.daemonSetName} spec: selector: matchLabels: - app: ${constants.k8s.daemonSetName} + app: ${k.daemonSetName} template: metadata: labels: - app: ${constants.k8s.daemonSetName} + app: ${k.daemonSetName} spec: # Required for seeing all node processes hostPID: true containers: - name: pcp - image: ${constants.k8s.imageName}:${constants.k8s.imageTag} - imagePullPolicy: Never + image: ${k.image.name}:${k.image.tag} + imagePullPolicy: ${k.image.pullPolicy} # Privileged for BPF and full /proc access # Run as root to override container's default pcp user @@ -47,33 +40,33 @@ let runAsUser: 0 ports: - - containerPort: ${toString mainConstants.ports.pmcd} + - containerPort: ${toString constants.ports.pmcd} name: pmcd - - containerPort: ${toString mainConstants.ports.pmproxy} + - containerPort: ${toString constants.ports.pmproxy} name: pmproxy env: # Tell PCP where host filesystem is mounted - name: HOST_MOUNT - value: "/host" + value: "${k.hostMounts.root}" - name: PCP_SYSFS_DIR - value: "/host/sys" + value: "${k.hostMounts.sys}" volumeMounts: # Host root filesystem (read-only) - name: host-root - mountPath: /host + mountPath: ${k.hostMounts.root} readOnly: true # Required for BPF - name: sys-kernel-debug - mountPath: /sys/kernel/debug + mountPath: ${k.hostMounts.kernelDebug} # Host /proc for process metrics - name: host-proc - mountPath: /host/proc + mountPath: ${k.hostMounts.proc} readOnly: true # Host /sys for system metrics - name: host-sys - mountPath: /host/sys + mountPath: ${k.hostMounts.sys} readOnly: true readinessProbe: @@ -84,11 +77,11 @@ let resources: limits: - memory: "512Mi" - cpu: "500m" + memory: "${k.resources.limits.memory}" + cpu: "${k.resources.limits.cpu}" requests: - memory: "256Mi" - cpu: "100m" + memory: "${k.resources.requests.memory}" + cpu: "${k.resources.requests.cpu}" volumes: - name: host-root @@ -108,15 +101,8 @@ let tolerations: - operator: Exists ''; - in { - # The full DaemonSet manifest as a string - manifest = daemonSetManifest; - - # Write manifest to a file for kubectl apply - manifestFile = pkgs.writeText "pcp-daemonset.yaml" daemonSetManifest; - - # Helper to get the manifest path - getManifestPath = "${pkgs.writeText "pcp-daemonset.yaml" daemonSetManifest}"; + inherit yaml; + file = pkgs.writeText "pcp-daemonset.yaml" yaml; } diff --git a/nix/k8s-manifests/default.nix b/nix/k8s-manifests/default.nix new file mode 100644 index 0000000000..2a4938fdad --- /dev/null +++ b/nix/k8s-manifests/default.nix @@ -0,0 +1,116 @@ +# nix/k8s-manifests/default.nix +# +# Standalone Kubernetes manifest generator for PCP DaemonSet deployment. +# Produces namespace and daemonset YAML files that can be applied directly +# with kubectl, without needing minikube or the full test suite. +# +# Manifests are validated with kube-linter at build time. +# See kube-linter.nix for the full rationale on suppressed checks. +# +# Usage in flake.nix: +# k8sManifests = import ./nix/k8s-manifests { inherit pkgs lib; }; +# +# With namespace override (for test suites): +# k8sManifests = import ./nix/k8s-manifests { +# inherit pkgs lib; +# namespaceOverride = "pcp-test"; +# }; +# +# Generated outputs: +# k8sManifests.manifestFile - Combined YAML file +# k8sManifests.dir - Directory with individual files +# k8sManifests.packages.pcp-k8s-manifests - Directory derivation +# k8sManifests.packages.pcp-k8s-manifest-namespace - Namespace YAML +# k8sManifests.packages.pcp-k8s-manifest-daemonset - DaemonSet YAML +# +{ pkgs, lib, namespaceOverride ? null }: +let + baseConstants = import ./constants.nix { }; + + # Apply namespace override if provided (used by test suites) + constants = baseConstants // { + k8s = baseConstants.k8s // lib.optionalAttrs (namespaceOverride != null) { + namespace = namespaceOverride; + }; + }; + + namespace = import ./namespace.nix { inherit pkgs constants; }; + daemonset = import ./daemonset.nix { inherit pkgs constants; }; + + # Combined manifest (namespace + daemonset, separated by ---) + combinedYaml = namespace.yaml + "---\n" + daemonset.yaml; + manifestFile = pkgs.writeText "pcp-daemonset.yaml" combinedYaml; + + # Lint validation — fails the build if manifests have unexpected errors + # (see kube-linter.nix for rationale on suppressed checks) + lintResult = import ./kube-linter.nix { + inherit pkgs; + manifestFiles = [ namespace.file daemonset.file ]; + }; + + readme = '' + PCP Kubernetes Manifests + ======================== + + These manifests deploy PCP as a privileged DaemonSet for full node + monitoring including BPF metrics. + + Namespace: ${constants.k8s.namespace} + + Quick start: + + # Apply all manifests at once + kubectl apply -f combined.yaml + + # Or apply individually + kubectl apply -f namespace.yaml + kubectl apply -f daemonset.yaml + + # Verify deployment + kubectl get pods -n ${constants.k8s.namespace} + + # Query metrics from a pod + kubectl exec -n ${constants.k8s.namespace} -- pminfo -f kernel.all.load + + # Cleanup + kubectl delete namespace ${constants.k8s.namespace} + + Generated by: nix build .#pcp-k8s-manifests + ''; + + # Directory derivation with all manifests (depends on lint passing) + dir = pkgs.runCommand "pcp-k8s-manifests" { + inherit lintResult; + } '' + mkdir -p $out + cp ${namespace.file} $out/namespace.yaml + cp ${daemonset.file} $out/daemonset.yaml + cp ${manifestFile} $out/combined.yaml + cp ${pkgs.writeText "README.md" readme} $out/README.md + ''; + +in +{ + # Backwards-compatible: combined manifest as a single file + inherit manifestFile; + + # Individual manifest components + inherit namespace daemonset; + + # Directory with all files + inherit dir; + + # Lint validation result + inherit lintResult; + + # Constants (for consumers that need deployment settings) + inherit constants; + + # Packages for flake integration + packages = { + pcp-k8s-manifests = dir; + pcp-k8s-manifest-namespace = namespace.file; + pcp-k8s-manifest-daemonset = daemonset.file; + pcp-k8s-manifests-lint = lintResult; + }; +} diff --git a/nix/k8s-manifests/kube-linter-config.yaml b/nix/k8s-manifests/kube-linter-config.yaml new file mode 100644 index 0000000000..1322bf3d49 --- /dev/null +++ b/nix/k8s-manifests/kube-linter-config.yaml @@ -0,0 +1,50 @@ +# nix/k8s-manifests/kube-linter-config.yaml +# +# KubeLinter configuration for PCP Kubernetes manifests. +# +# PCP is a node-level monitoring agent deployed as a privileged DaemonSet. +# Many default security checks are intentionally suppressed because full +# host access is fundamental to what PCP does (system metrics collection, +# BPF tracing, process monitoring). +# +# Each suppression is documented with the rationale below. +# +checks: + exclude: + # ─── Host Access (required for node monitoring) ──────────────────── + # + # PCP must see all processes on the node to collect per-process metrics. + - "host-pid" + + # PCP mounts /, /proc, and /sys from the host to collect system metrics. + # This is the fundamental purpose of a node monitoring DaemonSet. + - "sensitive-host-mounts" + + # ─── Privileged Mode (required for BPF and /proc) ───────────────── + # + # The BPF PMDA needs privileged mode to load eBPF programs for kernel + # tracing (runqueue latency, disk I/O latency, etc.). Full /proc + # access also requires elevated privileges. + - "privileged-container" + - "privilege-escalation-container" + + # PCP services (pmcd, pmlogger) run as root inside the container to + # access BPF syscalls and host /proc. The container itself provides + # the isolation boundary. + - "run-as-non-root" + + # ─── Filesystem (PCP needs writable state directories) ───────────── + # + # PCP writes to /var/lib/pcp (PMNS cache, PMDA state), + # /var/log/pcp (pmlogger archives), and /run/pcp (sockets). + # A read-only root filesystem would require extensive tmpfs mounts + # for all these paths with no meaningful security benefit given + # the container is already privileged. + - "no-read-only-root-fs" + + # ─── Image Tag (local build, not from a registry) ────────────────── + # + # The PCP container image is built by Nix (nix build .#pcp-container) + # and loaded directly into minikube's Docker daemon. There is no + # registry with versioned tags; "latest" is the only tag available. + - "latest-tag" diff --git a/nix/k8s-manifests/kube-linter.nix b/nix/k8s-manifests/kube-linter.nix new file mode 100644 index 0000000000..2b74e25aad --- /dev/null +++ b/nix/k8s-manifests/kube-linter.nix @@ -0,0 +1,74 @@ +# nix/k8s-manifests/kube-linter.nix +# +# KubeLinter validation for PCP Kubernetes manifests. +# +# Runs kube-linter (https://github.com/stackrox/kube-linter) at build time +# to catch manifest errors. If someone introduces a change that triggers +# an unexpected kube-linter finding, the Nix build fails. +# +# ─── Why PCP suppresses certain kube-linter checks ──────────────────── +# +# PCP is a *node-level monitoring agent*. Its job is to collect system +# metrics (CPU, memory, disk I/O, network, per-process stats) and kernel +# trace data (via eBPF). This is fundamentally different from a typical +# application workload — PCP needs deep access to the host it monitors. +# +# kube-linter's defaults assume a least-privilege application container. +# All 9 default findings are expected and intentional for PCP: +# +# Check Why PCP needs it +# ──────────────────────────── ────────────────────────────────────── +# host-pid Required to see all node processes +# for per-process metric collection +# (proc.* metric namespace). +# +# sensitive-host-mounts (x3) Mounts /, /proc, /sys from the host. +# This IS the monitoring — PCP reads +# /proc for process/kernel metrics and +# /sys for device/block metrics. +# +# privileged-container The BPF PMDA loads eBPF programs into +# privilege-escalation the kernel for tracing (runqueue +# latency, disk I/O latency, etc.). +# This requires CAP_SYS_ADMIN + BPF +# access, which effectively means +# privileged mode. +# +# run-as-non-root PCP services (pmcd, pmlogger) run as +# root inside the container to access +# BPF syscalls and host /proc. The +# container boundary provides isolation. +# +# no-read-only-root-fs PCP writes to /var/lib/pcp (PMNS +# cache, PMDA state), /var/log/pcp +# (pmlogger archives), and /run/pcp +# (sockets). Extensive tmpfs mounts +# would add complexity with no security +# benefit given privileged mode. +# +# latest-tag The container image is built locally +# by Nix (nix build .#pcp-container) +# and loaded directly into minikube. +# There is no registry; "latest" is +# the only tag. +# +# The suppression rules live in kube-linter-config.yaml. If PCP's +# deployment model changes (e.g. dropping BPF, using a registry), +# the corresponding suppression should be removed so kube-linter +# can enforce stricter defaults. +# +{ pkgs, manifestFiles }: +let + lintConfig = ./kube-linter-config.yaml; + + fileArgs = builtins.concatStringsSep " " (map (f: "${f}") manifestFiles); +in +pkgs.runCommand "pcp-k8s-manifests-lint" { + nativeBuildInputs = [ pkgs.kube-linter ]; +} '' + echo "Running kube-linter on PCP K8s manifests..." + kube-linter lint \ + --config ${lintConfig} \ + ${fileArgs} + echo "kube-linter: all checks passed" > $out +'' diff --git a/nix/k8s-manifests/namespace.nix b/nix/k8s-manifests/namespace.nix new file mode 100644 index 0000000000..ea80d936c5 --- /dev/null +++ b/nix/k8s-manifests/namespace.nix @@ -0,0 +1,17 @@ +# nix/k8s-manifests/namespace.nix +# +# Generates the Kubernetes Namespace resource YAML for PCP deployment. +# +{ pkgs, constants }: +let + yaml = '' + apiVersion: v1 + kind: Namespace + metadata: + name: ${constants.k8s.namespace} + ''; +in +{ + inherit yaml; + file = pkgs.writeText "pcp-namespace.yaml" yaml; +} diff --git a/nix/k8s-test/constants.nix b/nix/k8s-test/constants.nix index 271a45a380..fa55a61bbf 100644 --- a/nix/k8s-test/constants.nix +++ b/nix/k8s-test/constants.nix @@ -1,22 +1,25 @@ # nix/k8s-test/constants.nix # # Kubernetes DaemonSet testing configuration. -# Provides constants for Minikube-based PCP testing with BPF metrics. +# Imports shared deployment settings from k8s-manifests/constants.nix +# and overrides namespace for test isolation. # { }: let common = import ../test-common/constants.nix { }; + manifestConstants = import ../k8s-manifests/constants.nix { }; in rec { # Re-export from shared constants inherit (common) ports user colors; # ─── Kubernetes Settings ─────────────────────────────────────────────── - k8s = { + # Inherit deployment defaults, override namespace for test isolation + k8s = manifestConstants.k8s // { namespace = "pcp-test"; - daemonSetName = "pcp"; - imageName = "pcp"; - imageTag = "latest"; + # Flatten image attrs for backwards compatibility with test scripts + imageName = manifestConstants.k8s.image.name; + imageTag = manifestConstants.k8s.image.tag; }; # ─── Timeouts (seconds) ──────────────────────────────────────────────── @@ -38,6 +41,7 @@ rec { cpus = 4; # More CPUs for parallel workloads memory = 8192; # 8GB RAM for comfortable operation diskSize = "20g"; # 20GB disk + kubernetesVersion = "v1.33.0"; # Pin to avoid version drift with nixpkgs minikube }; # ─── Verification Checks ─────────────────────────────────────────────── diff --git a/nix/k8s-test/default.nix b/nix/k8s-test/default.nix index 5bcfcdd01b..201cc4f28a 100644 --- a/nix/k8s-test/default.nix +++ b/nix/k8s-test/default.nix @@ -18,7 +18,7 @@ let constants = import ./constants.nix { }; mainConstants = import ../constants.nix; helpers = import ./lib.nix { inherit pkgs lib; }; - manifests = import ./manifests.nix { inherit pkgs lib; }; + manifests = import ../k8s-manifests { inherit pkgs lib; namespaceOverride = constants.k8s.namespace; }; inherit (helpers) colorHelpers timingHelpers k8sHelpers metricHelpers @@ -576,25 +576,41 @@ let CPUS="${toString constants.minikube.cpus}" MEMORY="${toString constants.minikube.memory}" DISK="${constants.minikube.diskSize}" + K8S_VERSION="${constants.minikube.kubernetesVersion}" - # Check if minikube is already running + # Check if minikube is already running with the right version if minikube status --format='{{.Host}}' 2>/dev/null | grep -q "Running"; then - warn "Minikube is already running." - echo "" - info "Current configuration:" - minikube config view 2>/dev/null || true - echo "" - info "To recreate with optimal settings, run:" - echo " minikube delete" - echo " nix run .#pcp-minikube-start" - exit 0 + CURRENT_K8S=$(minikube kubectl -- version --client=false -o json 2>/dev/null \ + | jq -r '.serverVersion | "v\(.major).\(.minor).\(.patch // 0)"' 2>/dev/null || echo "unknown") + if [[ "$CURRENT_K8S" == "$K8S_VERSION"* ]]; then + success "Minikube is already running (Kubernetes $CURRENT_K8S)." + exit 0 + else + warn "Minikube is running Kubernetes $CURRENT_K8S but we need $K8S_VERSION." + info "Deleting existing cluster to recreate with correct version..." + minikube delete + echo "" + fi + fi + + # Check for a stopped cluster that may be incompatible + if minikube status &>/dev/null || minikube profile list 2>/dev/null | grep -q minikube; then + # A cluster exists but isn't running; delete to avoid version conflicts + EXISTING_K8S=$(minikube config view 2>/dev/null | grep -oP 'kubernetes-version:\s*\K.*' || echo "") + if [[ -n "$EXISTING_K8S" && "$EXISTING_K8S" != "$K8S_VERSION" ]]; then + warn "Existing stopped cluster uses Kubernetes $EXISTING_K8S (need $K8S_VERSION)." + info "Deleting stale cluster..." + minikube delete + echo "" + fi fi info "Starting minikube with settings for PCP testing:" - echo " Driver: $DRIVER" - echo " CPUs: $CPUS" - echo " Memory: $MEMORY MB" - echo " Disk: $DISK" + echo " Driver: $DRIVER" + echo " CPUs: $CPUS" + echo " Memory: $MEMORY MB" + echo " Disk: $DISK" + echo " Kubernetes: $K8S_VERSION" echo "" info "Tip: For better I/O performance, use KVM2 driver:" echo " minikube start --driver=kvm2 --cpus=$CPUS --memory=$MEMORY" @@ -606,15 +622,33 @@ let --driver="$DRIVER" \ --cpus="$CPUS" \ --memory="$MEMORY" \ - --disk-size="$DISK"; then + --disk-size="$DISK" \ + --kubernetes-version="$K8S_VERSION"; then echo "" success "Minikube started successfully!" echo "" info "Run the PCP Kubernetes test:" echo " nix run .#pcp-k8s-test" else - error "Failed to start minikube" - exit 1 + # If start failed, try deleting and retrying once + warn "First attempt failed. Deleting cluster and retrying..." + minikube delete 2>/dev/null || true + echo "" + if minikube start \ + --driver="$DRIVER" \ + --cpus="$CPUS" \ + --memory="$MEMORY" \ + --disk-size="$DISK" \ + --kubernetes-version="$K8S_VERSION"; then + echo "" + success "Minikube started successfully (after clean start)!" + echo "" + info "Run the PCP Kubernetes test:" + echo " nix run .#pcp-k8s-test" + else + error "Failed to start minikube" + exit 1 + fi fi ''; }; diff --git a/nix/microvm.nix b/nix/microvm.nix index abddb6af81..9b20198b00 100644 --- a/nix/microvm.nix +++ b/nix/microvm.nix @@ -11,7 +11,6 @@ # enablePmieTest - Enable stress-ng workload + pmie rules (default: false) # enableGrafana - Enable Grafana + Prometheus for visual monitoring (default: false) # enableBpf - Enable pre-compiled BPF PMDA (default: false) -# enableBcc - Enable runtime BCC PMDA, requires 2GB (default: false) # # Helper scripts (see nix/microvm-scripts.nix): # nix run .#pcp-vm-check - List running PCP MicroVMs and show count @@ -39,9 +38,8 @@ enablePmieTest ? false, # stress-ng + pmie rules enableGrafana ? false, # Grafana + Prometheus enableBpf ? false, # Pre-compiled CO-RE eBPF (lightweight) - enableBcc ? false, # Runtime eBPF compilation (heavyweight, 2GB) portOffset ? 0, # Port offset for all forwarded ports (see constants.variantPortOffsets) - variant ? "base", # Variant name for console port allocation (base, eval, grafana, bpf, bcc) + variant ? "base", # Variant name for console port allocation (base, eval, grafana, bpf) }: let constants = import ./constants.nix; @@ -50,35 +48,17 @@ let # Get serial console ports for this variant consolePorts = constants.getConsolePorts variant; - # BCC overlay: change KERNEL_MODULES_DIR from /run/booted-system/... to /lib/modules - # This is required because /run/booted-system is a symlink to read-only Nix store, - # but we need to bind mount kernel dev headers at this location. - # See bcc.nix for the bind mount configuration. - bccOverlay = final: prev: { - bcc = prev.bcc.overrideAttrs (old: { - cmakeFlags = builtins.map (flag: - if builtins.match ".*BCC_KERNEL_MODULES_DIR.*" flag != null - then "-DBCC_KERNEL_MODULES_DIR=/lib/modules" - else flag - ) old.cmakeFlags; - }); - }; - # Dynamic hostname based on enabled features - # Priority: bcc > grafana > bpf > eval > base + # Priority: grafana > bpf > eval > base # Note: grafana comes before bpf because grafana variant enables bpf for dashboards # but should still be identified as grafana-vm for lifecycle testing hostname = - if enableBcc then "pcp-bcc-vm" - else if enableGrafana then "pcp-grafana-vm" + if enableGrafana then "pcp-grafana-vm" else if enableBpf then "pcp-bpf-vm" else if enableEvalTools || enablePmieTest then "pcp-eval-vm" else "pcp-vm"; - # Dynamic memory: 2GB+ for BCC (clang/LLVM), 1GB otherwise - # Use 2049 instead of 2048 to avoid QEMU hang with exactly 2GB - # See: https://github.com/microvm-nix/microvm.nix/issues/171 - memoryMB = if enableBcc then 2049 else constants.vm.memoryMB; + memoryMB = constants.vm.memoryMB; # Build a NixOS system with MicroVM support vmConfig = nixpkgs.lib.nixosSystem { @@ -103,9 +83,6 @@ let # Import BPF module (provides services.pcp.bpf option) ./bpf.nix - # Import BCC module (provides services.pcp.bcc option) - ./bcc.nix - # PCP service configuration ({ pcp, ... }: { services.pcp = { @@ -130,10 +107,6 @@ let nixpkgs.hostPlatform = system; - # Apply BCC overlay when BCC is enabled - # This changes BCC's KERNEL_MODULES_DIR to /lib/modules (see bccOverlay above) - nixpkgs.overlays = lib.optionals enableBcc [ bccOverlay ]; - microvm = { hypervisor = "qemu"; mem = memoryMB; @@ -329,26 +302,6 @@ let ]; }) - # ─── BCC PMDA (Runtime-compiled eBPF) ─────────────────────────── - # Runtime eBPF compilation: slow startup (~30-60s), 2GB memory. - # Required for: tcptop, tcplife metrics (Grafana eBPF/BCC Overview dashboard) - (lib.mkIf enableBcc { - services.pcp.bcc = { - enable = true; - moduleFailureFatal = false; # Continue if some modules fail to compile - }; - - boot.kernelPatches = [ - { - name = "btf-for-bcc"; - patch = null; - structuredExtraConfig = with lib.kernel; { - DEBUG_INFO_BTF = yes; - }; - } - ]; - }) - # ─── Debug Mode Module ──────────────────────────────────────────── # Enables password auth for quick local testing. (lib.mkIf debugMode { @@ -365,14 +318,7 @@ let users.users.root.password = "pcp"; environment.etc."motd".text = '' - ${lib.optionalString enableBcc '' - ╔═══════════════════════════════════════════════════════════════╗ - ║ PCP MicroVM with BCC PMDA (runtime eBPF compilation) ║ - ╠═══════════════════════════════════════════════════════════════╣ - ║ BCC modules take 30-60s to compile at pmcd startup. ║ - ║ Check status: journalctl -u pmcd -f ║ - ╚═══════════════════════════════════════════════════════════════╝ - ''}${lib.optionalString enableBpf '' + ${lib.optionalString enableBpf '' ╔═══════════════════════════════════════════════════════════════╗ ║ PCP MicroVM with BPF PMDA (pre-compiled eBPF) ║ ╠═══════════════════════════════════════════════════════════════╣ @@ -385,7 +331,7 @@ let ║ Grafana: http://localhost:${toString constants.ports.grafanaForward} (admin/pcp) ║ ║ Prometheus: http://localhost:${toString constants.ports.prometheusForward} ║ ╚═══════════════════════════════════════════════════════════════╝ - ''}${lib.optionalString (enableEvalTools && !enableGrafana && !enableBpf && !enableBcc) '' + ''}${lib.optionalString (enableEvalTools && !enableGrafana && !enableBpf ) '' ╔═══════════════════════════════════════════════════════════════╗ ║ PCP Evaluation MicroVM ║ ╠═══════════════════════════════════════════════════════════════╣ @@ -393,7 +339,7 @@ let ║ node_exporter: curl localhost:9100/metrics ║ ║ below: below live ║ ╚═══════════════════════════════════════════════════════════════╝ - ''}${lib.optionalString (!enableEvalTools && !enableGrafana && !enableBpf && !enableBcc) '' + ''}${lib.optionalString (!enableEvalTools && !enableGrafana && !enableBpf ) '' ╔═══════════════════════════════════════════════════════════════╗ ║ PCP Base MicroVM ║ ╠═══════════════════════════════════════════════════════════════╣ diff --git a/nix/package.nix b/nix/package.nix index ffc05bbf02..3b42cf71c9 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -12,7 +12,9 @@ # 4. Using `./..` directly would cause hash changes on every file modification # # The source filtering (cleanSourceWith) further improves cache hits by -# excluding build artifacts, editor files, and other non-essential content. +# excluding build artifacts, editor files, Nix packaging infrastructure +# (nix/, flake.nix, flake.lock), and other non-essential content. +# This ensures that changes to .nix files don't trigger PCP rebuilds. # { pkgs, src }: let @@ -69,7 +71,7 @@ let # withSystemd = pkgs.stdenv.isLinux; # systemd service management withPfm = pkgs.stdenv.isLinux; # hardware performance counters (libpfm) - withBpf = pkgs.stdenv.isLinux; # eBPF tracing (bcc, bpf, bpftrace PMDAs) + withBpf = pkgs.stdenv.isLinux; # eBPF tracing (bpf, bpftrace PMDAs) withSnmp = true; # SNMP network monitoring withPythonHttp = true; # Python HTTP client (requests) withPerlHttp = true; # Perl HTTP client (LWP) @@ -99,6 +101,13 @@ let # Build output symlinks (only at top level, and only exact matches or result-*) (isTopLevel && (baseName == "result" || lib.hasPrefix "result-" baseName)) || baseName == "test-results" || + # Nix packaging infrastructure (not needed by PCP build) + # Changes to these should not trigger a full PCP rebuild + (isTopLevel && baseName == "nix") || + (isTopLevel && baseName == "flake.nix") || + (isTopLevel && baseName == "flake.lock") || + (isTopLevel && baseName == "CLAUDE.md") || + (isTopLevel && baseName == ".claude") || # Editor/IDE artifacts baseName == ".vscode" || baseName == ".idea" || @@ -164,7 +173,6 @@ pkgs.stdenv.mkDerivation rec { libpfm ] ++ lib.optionals withBpf [ libbpf - bcc elfutils ] ++ lib.optionals withSnmp [ net-snmp @@ -225,13 +233,11 @@ pkgs.stdenv.mkDerivation rec { ( if withBpf then [ - "--with-pmdabcc=yes" "--with-pmdabpf=yes" "--with-pmdabpftrace=yes" ] else [ - "--with-pmdabcc=no" "--with-pmdabpf=no" "--with-pmdabpftrace=no" ] From e591e3331b2f0fa23f5c4ad1caa343cbfa02859d Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Wed, 4 Mar 2026 15:30:38 -0800 Subject: [PATCH 2/2] nix: move nix/ to build/nix/ for consistent project layout Move the Nix packaging directory from the repository root into the build/ directory, alongside the existing platform-specific packaging (build/mac/, build/rpm/, build/aix/, etc.). Centralize the path in flake.nix via a single nixDir variable so all 17 import statements reference it indirectly: nixDir = ./build/nix; pcp = import (nixDir + "/package.nix") { ... }; Internal cross-references between nix files use relative paths and require no changes. The source filter in package.nix is updated to exclude build/nix/ from PCP source builds (preventing unnecessary rebuilds when only .nix files change). Update all file-header comments, usage-example comments, CLAUDE.md, and docs/HowTos/nix/index.rst to reflect the new location. Verified: nix flake show, nix run .#pcp-test-all pass. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 2 +- {nix => build/nix}/bpf.nix | 2 +- {nix => build/nix}/constants.nix | 2 +- .../nix}/container-test/constants.nix | 2 +- {nix => build/nix}/container-test/default.nix | 4 +- {nix => build/nix}/container-test/lib.nix | 2 +- {nix => build/nix}/container.nix | 2 +- .../nix}/dashboards/pcp-bpf-overview.json | 0 {nix => build/nix}/grafana.nix | 2 +- .../nix}/k8s-manifests/constants.nix | 2 +- .../nix}/k8s-manifests/daemonset.nix | 2 +- {nix => build/nix}/k8s-manifests/default.nix | 6 +-- .../k8s-manifests/kube-linter-config.yaml | 2 +- .../nix}/k8s-manifests/kube-linter.nix | 2 +- .../nix}/k8s-manifests/namespace.nix | 2 +- {nix => build/nix}/k8s-test/constants.nix | 2 +- {nix => build/nix}/k8s-test/default.nix | 4 +- {nix => build/nix}/k8s-test/lib.nix | 2 +- {nix => build/nix}/lifecycle/constants.nix | 2 +- {nix => build/nix}/lifecycle/default.nix | 4 +- {nix => build/nix}/lifecycle/lib.nix | 2 +- {nix => build/nix}/lifecycle/pcp-checks.nix | 2 +- .../nix}/lifecycle/scripts/vm-debug.exp | 0 .../nix}/lifecycle/scripts/vm-expect.exp | 0 .../nix}/lifecycle/scripts/vm-lib.exp | 0 .../nix}/lifecycle/scripts/vm-verify-pcp.exp | 0 {nix => build/nix}/microvm-scripts.nix | 2 +- {nix => build/nix}/microvm.nix | 4 +- {nix => build/nix}/network-setup.nix | 2 +- {nix => build/nix}/nixos-module.nix | 2 +- {nix => build/nix}/package.nix | 6 +-- .../nix}/patches/gnumakefile-nix-fixes.patch | 0 .../nix}/patches/perl-install-path-fix.patch | 0 .../nix}/patches/python-libpcp-nix.patch | 0 .../patches/python-pmapi-no-reconnect.patch | 0 .../nix}/patches/shell-portable-pwd.patch | 0 {nix => build/nix}/pmie-test.nix | 4 +- {nix => build/nix}/shell.nix | 2 +- {nix => build/nix}/test-all/default.nix | 4 +- {nix => build/nix}/test-common/constants.nix | 2 +- {nix => build/nix}/test-common/inputs.nix | 2 +- .../nix}/test-common/shell-helpers.nix | 2 +- {nix => build/nix}/test-lib.nix | 2 +- {nix => build/nix}/tests/microvm-test.nix | 2 +- .../nix}/tests/test-all-microvms.nix | 2 +- {nix => build/nix}/variants.nix | 4 +- {nix => build/nix}/vm-test.nix | 2 +- docs/HowTos/nix/index.rst | 22 +++++------ flake.nix | 39 ++++++++++--------- 49 files changed, 79 insertions(+), 78 deletions(-) rename {nix => build/nix}/bpf.nix (99%) rename {nix => build/nix}/constants.nix (99%) rename {nix => build/nix}/container-test/constants.nix (98%) rename {nix => build/nix}/container-test/default.nix (99%) rename {nix => build/nix}/container-test/lib.nix (99%) rename {nix => build/nix}/container.nix (99%) rename {nix => build/nix}/dashboards/pcp-bpf-overview.json (100%) rename {nix => build/nix}/grafana.nix (99%) rename {nix => build/nix}/k8s-manifests/constants.nix (96%) rename {nix => build/nix}/k8s-manifests/daemonset.nix (98%) rename {nix => build/nix}/k8s-manifests/default.nix (95%) rename {nix => build/nix}/k8s-manifests/kube-linter-config.yaml (97%) rename {nix => build/nix}/k8s-manifests/kube-linter.nix (98%) rename {nix => build/nix}/k8s-manifests/namespace.nix (87%) rename {nix => build/nix}/k8s-test/constants.nix (99%) rename {nix => build/nix}/k8s-test/default.nix (99%) rename {nix => build/nix}/k8s-test/lib.nix (99%) rename {nix => build/nix}/lifecycle/constants.nix (99%) rename {nix => build/nix}/lifecycle/default.nix (99%) rename {nix => build/nix}/lifecycle/lib.nix (99%) rename {nix => build/nix}/lifecycle/pcp-checks.nix (99%) rename {nix => build/nix}/lifecycle/scripts/vm-debug.exp (100%) rename {nix => build/nix}/lifecycle/scripts/vm-expect.exp (100%) rename {nix => build/nix}/lifecycle/scripts/vm-lib.exp (100%) rename {nix => build/nix}/lifecycle/scripts/vm-verify-pcp.exp (100%) rename {nix => build/nix}/microvm-scripts.nix (99%) rename {nix => build/nix}/microvm.nix (99%) rename {nix => build/nix}/network-setup.nix (99%) rename {nix => build/nix}/nixos-module.nix (99%) rename {nix => build/nix}/package.nix (99%) rename {nix => build/nix}/patches/gnumakefile-nix-fixes.patch (100%) rename {nix => build/nix}/patches/perl-install-path-fix.patch (100%) rename {nix => build/nix}/patches/python-libpcp-nix.patch (100%) rename {nix => build/nix}/patches/python-pmapi-no-reconnect.patch (100%) rename {nix => build/nix}/patches/shell-portable-pwd.patch (100%) rename {nix => build/nix}/pmie-test.nix (98%) rename {nix => build/nix}/shell.nix (96%) rename {nix => build/nix}/test-all/default.nix (98%) rename {nix => build/nix}/test-common/constants.nix (98%) rename {nix => build/nix}/test-common/inputs.nix (95%) rename {nix => build/nix}/test-common/shell-helpers.nix (98%) rename {nix => build/nix}/test-lib.nix (99%) rename {nix => build/nix}/tests/microvm-test.nix (99%) rename {nix => build/nix}/tests/test-all-microvms.nix (99%) rename {nix => build/nix}/variants.nix (96%) rename {nix => build/nix}/vm-test.nix (98%) diff --git a/CLAUDE.md b/CLAUDE.md index a41eca2cb0..9b4c9c1da6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -184,7 +184,7 @@ cd build/mac/test && ./run-all-tests.sh # Daily: build + test (20-30s) ## Nix Development -Nix packaging is provided via `flake.nix` and the `nix/` directory. +Nix packaging is provided via `flake.nix` and the `build/nix/` directory. See `docs/HowTos/nix/index.rst` for full documentation. ```bash diff --git a/nix/bpf.nix b/build/nix/bpf.nix similarity index 99% rename from nix/bpf.nix rename to build/nix/bpf.nix index 0c520f4c5a..6c52964bd5 100644 --- a/nix/bpf.nix +++ b/build/nix/bpf.nix @@ -1,4 +1,4 @@ -# nix/bpf.nix +# build/nix/bpf.nix # # NixOS module for BPF PMDA (pre-compiled eBPF metrics). # diff --git a/nix/constants.nix b/build/nix/constants.nix similarity index 99% rename from nix/constants.nix rename to build/nix/constants.nix index 3b35372ba1..82dd723a7e 100644 --- a/nix/constants.nix +++ b/build/nix/constants.nix @@ -1,4 +1,4 @@ -# nix/constants.nix +# build/nix/constants.nix # # Shared constants for PCP MicroVM infrastructure. # Import this file in microvm.nix, network-setup.nix, test-lib.nix, etc. diff --git a/nix/container-test/constants.nix b/build/nix/container-test/constants.nix similarity index 98% rename from nix/container-test/constants.nix rename to build/nix/container-test/constants.nix index 0b7ef57a26..a6e1e1ba4e 100644 --- a/nix/container-test/constants.nix +++ b/build/nix/container-test/constants.nix @@ -1,4 +1,4 @@ -# nix/container-test/constants.nix +# build/nix/container-test/constants.nix # # Container lifecycle testing configuration. # Provides constants for OCI container testing phases. diff --git a/nix/container-test/default.nix b/build/nix/container-test/default.nix similarity index 99% rename from nix/container-test/default.nix rename to build/nix/container-test/default.nix index 31a36b83a5..4663f22988 100644 --- a/nix/container-test/default.nix +++ b/build/nix/container-test/default.nix @@ -1,10 +1,10 @@ -# nix/container-test/default.nix +# build/nix/container-test/default.nix # # Entry point for PCP OCI container lifecycle testing. # Generates lifecycle test scripts for the PCP container image. # # Usage in flake.nix: -# containerTest = import ./nix/container-test { inherit pkgs lib pcp; }; +# containerTest = import (nixDir + "/container-test") { inherit pkgs lib pcp; }; # # Generated outputs: # containerTest.packages.pcp-container-test - Full lifecycle test diff --git a/nix/container-test/lib.nix b/build/nix/container-test/lib.nix similarity index 99% rename from nix/container-test/lib.nix rename to build/nix/container-test/lib.nix index 0af7732be6..2e26993f01 100644 --- a/nix/container-test/lib.nix +++ b/build/nix/container-test/lib.nix @@ -1,4 +1,4 @@ -# nix/container-test/lib.nix +# build/nix/container-test/lib.nix # # Shell helper functions for PCP container lifecycle testing. # Provides container-specific operations on top of shared helpers. diff --git a/nix/container.nix b/build/nix/container.nix similarity index 99% rename from nix/container.nix rename to build/nix/container.nix index c27d29d157..9aa6cd8b5f 100644 --- a/nix/container.nix +++ b/build/nix/container.nix @@ -1,4 +1,4 @@ -# nix/container.nix +# build/nix/container.nix # # OCI container image with PCP. # Uses buildLayeredImage for better Docker layer caching. diff --git a/nix/dashboards/pcp-bpf-overview.json b/build/nix/dashboards/pcp-bpf-overview.json similarity index 100% rename from nix/dashboards/pcp-bpf-overview.json rename to build/nix/dashboards/pcp-bpf-overview.json diff --git a/nix/grafana.nix b/build/nix/grafana.nix similarity index 99% rename from nix/grafana.nix rename to build/nix/grafana.nix index dad0376254..05e47e804c 100644 --- a/nix/grafana.nix +++ b/build/nix/grafana.nix @@ -1,4 +1,4 @@ -# nix/grafana.nix +# build/nix/grafana.nix # # NixOS module for Grafana with PCP dashboards. # Provides visualization for PCP metrics via the grafana-pcp plugin. diff --git a/nix/k8s-manifests/constants.nix b/build/nix/k8s-manifests/constants.nix similarity index 96% rename from nix/k8s-manifests/constants.nix rename to build/nix/k8s-manifests/constants.nix index 5f5e3854db..d37d33e055 100644 --- a/nix/k8s-manifests/constants.nix +++ b/build/nix/k8s-manifests/constants.nix @@ -1,4 +1,4 @@ -# nix/k8s-manifests/constants.nix +# build/nix/k8s-manifests/constants.nix # # Kubernetes deployment constants for PCP DaemonSet manifests. # These are the production defaults; test suites can override diff --git a/nix/k8s-manifests/daemonset.nix b/build/nix/k8s-manifests/daemonset.nix similarity index 98% rename from nix/k8s-manifests/daemonset.nix rename to build/nix/k8s-manifests/daemonset.nix index 11d692e651..f6de311ebb 100644 --- a/nix/k8s-manifests/daemonset.nix +++ b/build/nix/k8s-manifests/daemonset.nix @@ -1,4 +1,4 @@ -# nix/k8s-manifests/daemonset.nix +# build/nix/k8s-manifests/daemonset.nix # # Generates the Kubernetes DaemonSet resource YAML for PCP deployment. # Creates a privileged DaemonSet with full node monitoring capabilities diff --git a/nix/k8s-manifests/default.nix b/build/nix/k8s-manifests/default.nix similarity index 95% rename from nix/k8s-manifests/default.nix rename to build/nix/k8s-manifests/default.nix index 2a4938fdad..9736fdbbf5 100644 --- a/nix/k8s-manifests/default.nix +++ b/build/nix/k8s-manifests/default.nix @@ -1,4 +1,4 @@ -# nix/k8s-manifests/default.nix +# build/nix/k8s-manifests/default.nix # # Standalone Kubernetes manifest generator for PCP DaemonSet deployment. # Produces namespace and daemonset YAML files that can be applied directly @@ -8,10 +8,10 @@ # See kube-linter.nix for the full rationale on suppressed checks. # # Usage in flake.nix: -# k8sManifests = import ./nix/k8s-manifests { inherit pkgs lib; }; +# k8sManifests = import (nixDir + "/k8s-manifests") { inherit pkgs lib; }; # # With namespace override (for test suites): -# k8sManifests = import ./nix/k8s-manifests { +# k8sManifests = import (nixDir + "/k8s-manifests") { # inherit pkgs lib; # namespaceOverride = "pcp-test"; # }; diff --git a/nix/k8s-manifests/kube-linter-config.yaml b/build/nix/k8s-manifests/kube-linter-config.yaml similarity index 97% rename from nix/k8s-manifests/kube-linter-config.yaml rename to build/nix/k8s-manifests/kube-linter-config.yaml index 1322bf3d49..e775816f96 100644 --- a/nix/k8s-manifests/kube-linter-config.yaml +++ b/build/nix/k8s-manifests/kube-linter-config.yaml @@ -1,4 +1,4 @@ -# nix/k8s-manifests/kube-linter-config.yaml +# build/nix/k8s-manifests/kube-linter-config.yaml # # KubeLinter configuration for PCP Kubernetes manifests. # diff --git a/nix/k8s-manifests/kube-linter.nix b/build/nix/k8s-manifests/kube-linter.nix similarity index 98% rename from nix/k8s-manifests/kube-linter.nix rename to build/nix/k8s-manifests/kube-linter.nix index 2b74e25aad..0d69abf5b4 100644 --- a/nix/k8s-manifests/kube-linter.nix +++ b/build/nix/k8s-manifests/kube-linter.nix @@ -1,4 +1,4 @@ -# nix/k8s-manifests/kube-linter.nix +# build/nix/k8s-manifests/kube-linter.nix # # KubeLinter validation for PCP Kubernetes manifests. # diff --git a/nix/k8s-manifests/namespace.nix b/build/nix/k8s-manifests/namespace.nix similarity index 87% rename from nix/k8s-manifests/namespace.nix rename to build/nix/k8s-manifests/namespace.nix index ea80d936c5..eea2c55bab 100644 --- a/nix/k8s-manifests/namespace.nix +++ b/build/nix/k8s-manifests/namespace.nix @@ -1,4 +1,4 @@ -# nix/k8s-manifests/namespace.nix +# build/nix/k8s-manifests/namespace.nix # # Generates the Kubernetes Namespace resource YAML for PCP deployment. # diff --git a/nix/k8s-test/constants.nix b/build/nix/k8s-test/constants.nix similarity index 99% rename from nix/k8s-test/constants.nix rename to build/nix/k8s-test/constants.nix index fa55a61bbf..2002355f77 100644 --- a/nix/k8s-test/constants.nix +++ b/build/nix/k8s-test/constants.nix @@ -1,4 +1,4 @@ -# nix/k8s-test/constants.nix +# build/nix/k8s-test/constants.nix # # Kubernetes DaemonSet testing configuration. # Imports shared deployment settings from k8s-manifests/constants.nix diff --git a/nix/k8s-test/default.nix b/build/nix/k8s-test/default.nix similarity index 99% rename from nix/k8s-test/default.nix rename to build/nix/k8s-test/default.nix index 201cc4f28a..099a663dba 100644 --- a/nix/k8s-test/default.nix +++ b/build/nix/k8s-test/default.nix @@ -1,11 +1,11 @@ -# nix/k8s-test/default.nix +# build/nix/k8s-test/default.nix # # Entry point for PCP Kubernetes DaemonSet lifecycle testing. # Generates test scripts for deploying PCP as a privileged DaemonSet # in minikube with full node monitoring including BPF metrics. # # Usage in flake.nix: -# k8sTest = import ./nix/k8s-test { inherit pkgs lib pcp; }; +# k8sTest = import (nixDir + "/k8s-test") { inherit pkgs lib pcp; }; # # Generated outputs: # k8sTest.packages.pcp-k8s-test - Full lifecycle test diff --git a/nix/k8s-test/lib.nix b/build/nix/k8s-test/lib.nix similarity index 99% rename from nix/k8s-test/lib.nix rename to build/nix/k8s-test/lib.nix index 6f6fe5d6bc..37664d6522 100644 --- a/nix/k8s-test/lib.nix +++ b/build/nix/k8s-test/lib.nix @@ -1,4 +1,4 @@ -# nix/k8s-test/lib.nix +# build/nix/k8s-test/lib.nix # # Shell helper functions for PCP Kubernetes DaemonSet testing. # Provides k8s-specific operations on top of shared helpers. diff --git a/nix/lifecycle/constants.nix b/build/nix/lifecycle/constants.nix similarity index 99% rename from nix/lifecycle/constants.nix rename to build/nix/lifecycle/constants.nix index d284906ef8..d5cfbcc1fd 100644 --- a/nix/lifecycle/constants.nix +++ b/build/nix/lifecycle/constants.nix @@ -1,4 +1,4 @@ -# nix/lifecycle/constants.nix +# build/nix/lifecycle/constants.nix # # Lifecycle testing configuration for PCP MicroVMs. # Extends the main constants.nix with lifecycle-specific values. diff --git a/nix/lifecycle/default.nix b/build/nix/lifecycle/default.nix similarity index 99% rename from nix/lifecycle/default.nix rename to build/nix/lifecycle/default.nix index 576e5a8d5c..20424575a2 100644 --- a/nix/lifecycle/default.nix +++ b/build/nix/lifecycle/default.nix @@ -1,10 +1,10 @@ -# nix/lifecycle/default.nix +# build/nix/lifecycle/default.nix # # Entry point for PCP MicroVM lifecycle testing. # Generates lifecycle test scripts for all MicroVM variants. # # Usage in flake.nix: -# lifecycle = import ./nix/lifecycle { inherit pkgs lib; }; +# lifecycle = import (nixDir + "/lifecycle") { inherit pkgs lib; }; # # Generated outputs: # lifecycle.scripts.. - Individual phase scripts diff --git a/nix/lifecycle/lib.nix b/build/nix/lifecycle/lib.nix similarity index 99% rename from nix/lifecycle/lib.nix rename to build/nix/lifecycle/lib.nix index 1040a89ddf..4568778b72 100644 --- a/nix/lifecycle/lib.nix +++ b/build/nix/lifecycle/lib.nix @@ -1,4 +1,4 @@ -# nix/lifecycle/lib.nix +# build/nix/lifecycle/lib.nix # # Script generators for PCP MicroVM lifecycle testing. # Provides functions to generate bash scripts for each lifecycle phase. diff --git a/nix/lifecycle/pcp-checks.nix b/build/nix/lifecycle/pcp-checks.nix similarity index 99% rename from nix/lifecycle/pcp-checks.nix rename to build/nix/lifecycle/pcp-checks.nix index 37ddef8090..42a1bfbd85 100644 --- a/nix/lifecycle/pcp-checks.nix +++ b/build/nix/lifecycle/pcp-checks.nix @@ -1,4 +1,4 @@ -# nix/lifecycle/pcp-checks.nix +# build/nix/lifecycle/pcp-checks.nix # # PCP-specific verification functions for MicroVM lifecycle testing. # Provides service checks, metric verification, and HTTP endpoint testing. diff --git a/nix/lifecycle/scripts/vm-debug.exp b/build/nix/lifecycle/scripts/vm-debug.exp similarity index 100% rename from nix/lifecycle/scripts/vm-debug.exp rename to build/nix/lifecycle/scripts/vm-debug.exp diff --git a/nix/lifecycle/scripts/vm-expect.exp b/build/nix/lifecycle/scripts/vm-expect.exp similarity index 100% rename from nix/lifecycle/scripts/vm-expect.exp rename to build/nix/lifecycle/scripts/vm-expect.exp diff --git a/nix/lifecycle/scripts/vm-lib.exp b/build/nix/lifecycle/scripts/vm-lib.exp similarity index 100% rename from nix/lifecycle/scripts/vm-lib.exp rename to build/nix/lifecycle/scripts/vm-lib.exp diff --git a/nix/lifecycle/scripts/vm-verify-pcp.exp b/build/nix/lifecycle/scripts/vm-verify-pcp.exp similarity index 100% rename from nix/lifecycle/scripts/vm-verify-pcp.exp rename to build/nix/lifecycle/scripts/vm-verify-pcp.exp diff --git a/nix/microvm-scripts.nix b/build/nix/microvm-scripts.nix similarity index 99% rename from nix/microvm-scripts.nix rename to build/nix/microvm-scripts.nix index e5cb9e020d..667db08dad 100644 --- a/nix/microvm-scripts.nix +++ b/build/nix/microvm-scripts.nix @@ -1,4 +1,4 @@ -# nix/microvm-scripts.nix +# build/nix/microvm-scripts.nix # # Helper scripts for managing PCP MicroVMs. # These provide simple Unix-idiomatic ways to check, stop, and connect to VMs. diff --git a/nix/microvm.nix b/build/nix/microvm.nix similarity index 99% rename from nix/microvm.nix rename to build/nix/microvm.nix index 9b20198b00..2a22e5f674 100644 --- a/nix/microvm.nix +++ b/build/nix/microvm.nix @@ -1,4 +1,4 @@ -# nix/microvm.nix +# build/nix/microvm.nix # # Parametric MicroVM generator for PCP. # This single module handles all MicroVM configurations via parameters. @@ -12,7 +12,7 @@ # enableGrafana - Enable Grafana + Prometheus for visual monitoring (default: false) # enableBpf - Enable pre-compiled BPF PMDA (default: false) # -# Helper scripts (see nix/microvm-scripts.nix): +# Helper scripts (see build/nix/microvm-scripts.nix): # nix run .#pcp-vm-check - List running PCP MicroVMs and show count # nix run .#pcp-vm-stop - Stop all running PCP MicroVMs # nix run .#pcp-vm-ssh - SSH into the VM as root (debug mode only) diff --git a/nix/network-setup.nix b/build/nix/network-setup.nix similarity index 99% rename from nix/network-setup.nix rename to build/nix/network-setup.nix index 01155fd1bc..9d623e68ee 100644 --- a/nix/network-setup.nix +++ b/build/nix/network-setup.nix @@ -1,4 +1,4 @@ -# nix/network-setup.nix +# build/nix/network-setup.nix # # TAP/bridge/vhost-net setup and teardown scripts. # All network parameters come from constants.nix. diff --git a/nix/nixos-module.nix b/build/nix/nixos-module.nix similarity index 99% rename from nix/nixos-module.nix rename to build/nix/nixos-module.nix index 88af2a63e5..935d015cff 100644 --- a/nix/nixos-module.nix +++ b/build/nix/nixos-module.nix @@ -1,4 +1,4 @@ -# nix/nixos-module.nix +# build/nix/nixos-module.nix # # NixOS module for Performance Co-Pilot. # Mirrors a standard RHEL/Debian standalone PCP deployment. diff --git a/nix/package.nix b/build/nix/package.nix similarity index 99% rename from nix/package.nix rename to build/nix/package.nix index 3b42cf71c9..3bc323c225 100644 --- a/nix/package.nix +++ b/build/nix/package.nix @@ -1,4 +1,4 @@ -# nix/package.nix +# build/nix/package.nix # # PCP package derivation. # Version is automatically derived from VERSION.pcp (the configure script's @@ -13,7 +13,7 @@ # # The source filtering (cleanSourceWith) further improves cache hits by # excluding build artifacts, editor files, Nix packaging infrastructure -# (nix/, flake.nix, flake.lock), and other non-essential content. +# (build/nix/, flake.nix, flake.lock), and other non-essential content. # This ensures that changes to .nix files don't trigger PCP rebuilds. # { pkgs, src }: @@ -103,7 +103,7 @@ let baseName == "test-results" || # Nix packaging infrastructure (not needed by PCP build) # Changes to these should not trigger a full PCP rebuild - (isTopLevel && baseName == "nix") || + (parentDir == srcPath + "/build" && baseName == "nix") || (isTopLevel && baseName == "flake.nix") || (isTopLevel && baseName == "flake.lock") || (isTopLevel && baseName == "CLAUDE.md") || diff --git a/nix/patches/gnumakefile-nix-fixes.patch b/build/nix/patches/gnumakefile-nix-fixes.patch similarity index 100% rename from nix/patches/gnumakefile-nix-fixes.patch rename to build/nix/patches/gnumakefile-nix-fixes.patch diff --git a/nix/patches/perl-install-path-fix.patch b/build/nix/patches/perl-install-path-fix.patch similarity index 100% rename from nix/patches/perl-install-path-fix.patch rename to build/nix/patches/perl-install-path-fix.patch diff --git a/nix/patches/python-libpcp-nix.patch b/build/nix/patches/python-libpcp-nix.patch similarity index 100% rename from nix/patches/python-libpcp-nix.patch rename to build/nix/patches/python-libpcp-nix.patch diff --git a/nix/patches/python-pmapi-no-reconnect.patch b/build/nix/patches/python-pmapi-no-reconnect.patch similarity index 100% rename from nix/patches/python-pmapi-no-reconnect.patch rename to build/nix/patches/python-pmapi-no-reconnect.patch diff --git a/nix/patches/shell-portable-pwd.patch b/build/nix/patches/shell-portable-pwd.patch similarity index 100% rename from nix/patches/shell-portable-pwd.patch rename to build/nix/patches/shell-portable-pwd.patch diff --git a/nix/pmie-test.nix b/build/nix/pmie-test.nix similarity index 98% rename from nix/pmie-test.nix rename to build/nix/pmie-test.nix index a5bdf1476b..112c3cd3f8 100644 --- a/nix/pmie-test.nix +++ b/build/nix/pmie-test.nix @@ -1,4 +1,4 @@ -# nix/pmie-test.nix +# build/nix/pmie-test.nix # # pmie testing module with synthetic workload. # Creates predictable CPU spikes that a dedicated pmie instance detects. @@ -54,7 +54,7 @@ let pmieRules = pkgs.writeText "pmie-test.rules" '' // // pmie test rules - detect stress-ng-test workload - // Deployed by nix/pmie-test.nix + // Deployed by build/nix/pmie-test.nix // delta = 5 sec; diff --git a/nix/shell.nix b/build/nix/shell.nix similarity index 96% rename from nix/shell.nix rename to build/nix/shell.nix index 066b74ea5e..95898ca7da 100644 --- a/nix/shell.nix +++ b/build/nix/shell.nix @@ -1,4 +1,4 @@ -# nix/shell.nix +# build/nix/shell.nix # # Development shell for PCP. # Provides build dependencies plus debugging tools. diff --git a/nix/test-all/default.nix b/build/nix/test-all/default.nix similarity index 98% rename from nix/test-all/default.nix rename to build/nix/test-all/default.nix index 55f29e22c1..b5dfa9f04d 100644 --- a/nix/test-all/default.nix +++ b/build/nix/test-all/default.nix @@ -1,10 +1,10 @@ -# nix/test-all/default.nix +# build/nix/test-all/default.nix # # Runs all PCP tests sequentially (container-test, k8s-test, microvm-tests). # Reports overall pass/fail status. # # Usage in flake.nix: -# testAll = import ./nix/test-all { inherit pkgs lib containerTest k8sTest; }; +# testAll = import (nixDir + "/test-all") { inherit pkgs lib containerTest k8sTest; }; # # Generated outputs: # testAll.packages.pcp-test-all - Sequential test runner diff --git a/nix/test-common/constants.nix b/build/nix/test-common/constants.nix similarity index 98% rename from nix/test-common/constants.nix rename to build/nix/test-common/constants.nix index d1db13cf7f..b62a29b895 100644 --- a/nix/test-common/constants.nix +++ b/build/nix/test-common/constants.nix @@ -1,4 +1,4 @@ -# nix/test-common/constants.nix +# build/nix/test-common/constants.nix # # Shared constants for PCP test scripts. # Provides common values used by container and k8s tests. diff --git a/nix/test-common/inputs.nix b/build/nix/test-common/inputs.nix similarity index 95% rename from nix/test-common/inputs.nix rename to build/nix/test-common/inputs.nix index 9e353ea3a4..35dff7ac30 100644 --- a/nix/test-common/inputs.nix +++ b/build/nix/test-common/inputs.nix @@ -1,4 +1,4 @@ -# nix/test-common/inputs.nix +# build/nix/test-common/inputs.nix # # Shared runtime inputs for PCP test scripts. # Provides common packages needed by container and k8s tests. diff --git a/nix/test-common/shell-helpers.nix b/build/nix/test-common/shell-helpers.nix similarity index 98% rename from nix/test-common/shell-helpers.nix rename to build/nix/test-common/shell-helpers.nix index 0f9ed9f4f6..49af1b8afc 100644 --- a/nix/test-common/shell-helpers.nix +++ b/build/nix/test-common/shell-helpers.nix @@ -1,4 +1,4 @@ -# nix/test-common/shell-helpers.nix +# build/nix/test-common/shell-helpers.nix # # Shared shell helper functions for PCP test scripts. # Provides color output, timing, and result formatting. diff --git a/nix/test-lib.nix b/build/nix/test-lib.nix similarity index 99% rename from nix/test-lib.nix rename to build/nix/test-lib.nix index 1b811dc0db..5b4abf1904 100644 --- a/nix/test-lib.nix +++ b/build/nix/test-lib.nix @@ -1,4 +1,4 @@ -# nix/test-lib.nix +# build/nix/test-lib.nix # # Shared test functions for MicroVM validation. # All timeouts, thresholds, and ports come from constants.nix. diff --git a/nix/tests/microvm-test.nix b/build/nix/tests/microvm-test.nix similarity index 99% rename from nix/tests/microvm-test.nix rename to build/nix/tests/microvm-test.nix index a48e86a82f..c3dcd8e5c0 100644 --- a/nix/tests/microvm-test.nix +++ b/build/nix/tests/microvm-test.nix @@ -1,4 +1,4 @@ -# nix/tests/microvm-test.nix +# build/nix/tests/microvm-test.nix # # Test script builder for MicroVM variants. # Uses constants.nix for all configuration. diff --git a/nix/tests/test-all-microvms.nix b/build/nix/tests/test-all-microvms.nix similarity index 99% rename from nix/tests/test-all-microvms.nix rename to build/nix/tests/test-all-microvms.nix index 32808f27f6..4d74fecd33 100644 --- a/nix/tests/test-all-microvms.nix +++ b/build/nix/tests/test-all-microvms.nix @@ -1,4 +1,4 @@ -# nix/tests/test-all-microvms.nix +# build/nix/tests/test-all-microvms.nix # # Comprehensive test runner for all PCP MicroVM variants. # diff --git a/nix/variants.nix b/build/nix/variants.nix similarity index 96% rename from nix/variants.nix rename to build/nix/variants.nix index 17f7f30425..ea91e3c667 100644 --- a/nix/variants.nix +++ b/build/nix/variants.nix @@ -1,4 +1,4 @@ -# nix/variants.nix +# build/nix/variants.nix # # Centralized MicroVM variant definitions for PCP. # @@ -7,7 +7,7 @@ # it supports TAP networking. # # USAGE: -# variants = import ./nix/variants.nix { inherit constants; }; +# variants = import (nixDir + "/variants.nix") { inherit constants; }; # variants.definitions.grafana.config # -> { enableGrafana = true; ... } # variants.mkPackageName "grafana" "tap" # -> "pcp-microvm-grafana-tap" # diff --git a/nix/vm-test.nix b/build/nix/vm-test.nix similarity index 98% rename from nix/vm-test.nix rename to build/nix/vm-test.nix index 33909d8718..facc87781b 100644 --- a/nix/vm-test.nix +++ b/build/nix/vm-test.nix @@ -1,4 +1,4 @@ -# nix/vm-test.nix +# build/nix/vm-test.nix # # NixOS VM integration test for PCP. # Uses the shared NixOS module for service configuration. diff --git a/docs/HowTos/nix/index.rst b/docs/HowTos/nix/index.rst index d135e52058..45f76d6dc6 100644 --- a/docs/HowTos/nix/index.rst +++ b/docs/HowTos/nix/index.rst @@ -70,7 +70,7 @@ File Structure :: - nix/ + build/nix/ ├── package.nix # PCP derivation (version from VERSION.pcp) ├── nixos-module.nix # NixOS module for services.pcp ├── constants.nix # Shared configuration constants @@ -601,7 +601,7 @@ hardcode ``/var/tmp`` for temporary files, causing errors:: - ``src/libpcp/doc/mk.cgraph`` - ``src/pmlogcompress/check-optimize`` -**Solution**: The patch file ``nix/patches/tmpdir-portability.patch`` replaces +**Solution**: The patch file ``build/nix/patches/tmpdir-portability.patch`` replaces ``/var/tmp`` with ``${TMPDIR:-/tmp}`` in all affected scripts. This is portable: - On Nix, ``$TMPDIR`` is set to a writable sandbox directory @@ -629,7 +629,7 @@ Additionally, PCP's ``GNUmakefile`` has a hardcoded fallback to SYSTEMD_TMPFILESDIR = "${placeholder "out"}/lib/tmpfiles.d"; SYSTEMD_SYSUSERSDIR = "${placeholder "out"}/lib/sysusers.d"; -2. The patch file ``nix/patches/gnumakefile-nix-fixes.patch`` redirects the +2. The patch file ``build/nix/patches/gnumakefile-nix-fixes.patch`` redirects the tmpfiles installation from ``/usr/lib/tmpfiles.d`` to ``$(PCP_SHARE_DIR)/tmpfiles.d``. @@ -653,7 +653,7 @@ reached. No patch is needed - the ``preConfigure`` export handles it completely. .. note:: - The patch file ``nix/patches/configure-ar-portable.patch`` exists but is not + The patch file ``build/nix/patches/configure-ar-portable.patch`` exists but is not applied in ``flake.nix``. It changes the fallback to ``ar`` (PATH lookup) and could be submitted upstream to improve portability for non-Nix builds. @@ -666,7 +666,7 @@ in the Nix sandbox:: install: cannot change ownership: Operation not permitted -**Solution**: The patch file ``nix/patches/gnumakefile-nix-fixes.patch`` removes +**Solution**: The patch file ``build/nix/patches/gnumakefile-nix-fixes.patch`` removes the ownership flags from install commands. The NixOS module (when created) will handle proper ownership at activation time. @@ -701,7 +701,7 @@ Patch Files ^^^^^^^^^^^ The Nix-specific fixes have been implemented as proper ``.patch`` files in -``nix/patches/`` rather than inline ``substituteInPlace`` calls: +``build/nix/patches/`` rather than inline ``substituteInPlace`` calls: **Applied in flake.nix:** @@ -759,7 +759,7 @@ entirely, avoiding slow shebang patching of test scripts. NixOS VM Test ^^^^^^^^^^^^^ -A NixOS VM integration test is provided in ``nix/vm-test.nix`` to verify the +A NixOS VM integration test is provided in ``build/nix/vm-test.nix`` to verify the package works correctly in a real NixOS environment. The test: 1. Boots a NixOS VM with PCP installed @@ -822,7 +822,7 @@ Use ``-L`` to see test output in real-time:: # >>> machine.succeed("pminfo --version") # >>> machine.shell_interact() # Get a shell in the VM -**Test configuration** (from ``nix/vm-test.nix``): +**Test configuration** (from ``build/nix/vm-test.nix``): The VM is configured with: @@ -850,7 +850,7 @@ journal health, and pmie functionality. # Clean up nix run .#pcp-vm-stop -**Test phases** (from ``nix/tests/microvm-test.nix``): +**Test phases** (from ``build/nix/tests/microvm-test.nix``): 1. **SSH connectivity** - Wait for VM to accept SSH connections 2. **Service status** - Verify pmcd, pmproxy, node_exporter are active @@ -1836,7 +1836,7 @@ applying it to an existing cluster:: The standalone manifests use the ``pcp`` namespace by default (the test suite overrides this to ``pcp-test`` for isolation). Resource limits, image settings, -and host mount paths are all defined in ``nix/k8s-manifests/constants.nix``. +and host mount paths are all defined in ``build/nix/k8s-manifests/constants.nix``. Development Shell ----------------- @@ -1899,7 +1899,7 @@ without rebuilding the package, via ``PCP_PMDA_PATH`` or similar mechanism. NixOS Module Enhancements ^^^^^^^^^^^^^^^^^^^^^^^^^ -The current NixOS module (``nix/nixos-module.nix``) provides basic functionality. +The current NixOS module (``build/nix/nixos-module.nix``) provides basic functionality. Future enhancements could include: - Declarative PMDA configuration (enable/disable specific PMDAs) diff --git a/flake.nix b/flake.nix index 53901003e7..55412f6e2d 100644 --- a/flake.nix +++ b/flake.nix @@ -50,21 +50,22 @@ flake-utils.lib.eachDefaultSystem ( system: let + nixDir = ./build/nix; pkgs = nixpkgs.legacyPackages.${system}; lib = pkgs.lib; # Import modular package definition - # Pass self for stable source hashing - see nix/package.nix for details - pcp = import ./nix/package.nix { inherit pkgs; src = self; }; + # Pass self for stable source hashing - see build/nix/package.nix for details + pcp = import (nixDir + "/package.nix") { inherit pkgs; src = self; }; # Import shared constants and variant definitions - constants = import ./nix/constants.nix; - variants = import ./nix/variants.nix { inherit constants; }; - nixosModule = import ./nix/nixos-module.nix; + constants = import (nixDir + "/constants.nix"); + variants = import (nixDir + "/variants.nix") { inherit constants; }; + nixosModule = import (nixDir + "/nixos-module.nix"); # ─── MicroVM Generator ─────────────────────────────────────────── # Creates a MicroVM runner with the specified configuration. - # See nix/microvm.nix for full parameter documentation. + # See build/nix/microvm.nix for full parameter documentation. mkMicroVM = { networking ? "user", debugMode ? true, @@ -76,7 +77,7 @@ portOffset ? 0, variant ? "base", }: - import ./nix/microvm.nix { + import (nixDir + "/microvm.nix") { inherit pkgs lib pcp microvm nixosModule nixpkgs system; inherit networking debugMode enablePmlogger enableEvalTools enablePmieTest enableGrafana enableBpf @@ -116,17 +117,17 @@ # Import lifecycle testing framework (Linux only) lifecycle = lib.optionalAttrs pkgs.stdenv.isLinux ( - import ./nix/lifecycle { inherit pkgs lib; } + import (nixDir + "/lifecycle") { inherit pkgs lib; } ); # Import container module (Linux only) - returns { image, inputsHash } containerModule = lib.optionalAttrs pkgs.stdenv.isLinux ( - import ./nix/container.nix { inherit pkgs pcp; } + import (nixDir + "/container.nix") { inherit pkgs pcp; } ); # Import container testing framework (Linux only) containerTest = lib.optionalAttrs pkgs.stdenv.isLinux ( - import ./nix/container-test { + import (nixDir + "/container-test") { inherit pkgs lib pcp; containerInputsHash = containerModule.inputsHash or ""; } @@ -134,7 +135,7 @@ # Import Kubernetes testing framework (Linux only) k8sTest = lib.optionalAttrs pkgs.stdenv.isLinux ( - import ./nix/k8s-test { + import (nixDir + "/k8s-test") { inherit pkgs lib pcp; containerInputsHash = containerModule.inputsHash or ""; } @@ -142,12 +143,12 @@ # Import standalone K8s manifest generator (Linux only) k8sManifests = lib.optionalAttrs pkgs.stdenv.isLinux ( - import ./nix/k8s-manifests { inherit pkgs lib; } + import (nixDir + "/k8s-manifests") { inherit pkgs lib; } ); # Import test-all runner (Linux only) testAll = lib.optionalAttrs pkgs.stdenv.isLinux ( - import ./nix/test-all { + import (nixDir + "/test-all") { inherit pkgs lib containerTest k8sTest; } ); @@ -177,19 +178,19 @@ ); checks = lib.optionalAttrs pkgs.stdenv.isLinux { - vm-test = import ./nix/vm-test.nix { + vm-test = import (nixDir + "/vm-test.nix") { inherit pkgs pcp; }; }; # Import modular development shell - devShells.default = import ./nix/shell.nix { inherit pkgs pcp; }; + devShells.default = import (nixDir + "/shell.nix") { inherit pkgs pcp; }; # ─── Apps (Linux only) ───────────────────────────────────────────── apps = lib.optionalAttrs pkgs.stdenv.isLinux ( let - networkScripts = import ./nix/network-setup.nix { inherit pkgs; }; - vmScripts = import ./nix/microvm-scripts.nix { inherit pkgs; }; + networkScripts = import (nixDir + "/network-setup.nix") { inherit pkgs; }; + vmScripts = import (nixDir + "/microvm-scripts.nix") { inherit pkgs; }; # ─── MicroVM Test Apps ──────────────────────────────────────────── # Generate test apps for each variant @@ -204,7 +205,7 @@ name = testName; value = { type = "app"; - program = "${import ./nix/tests/microvm-test.nix { + program = "${import (nixDir + "/tests/microvm-test.nix") { inherit pkgs lib; variant = "${variant}-${networkMode}"; inherit host sshPort; @@ -253,7 +254,7 @@ # Comprehensive test runner pcp-test-all-microvms = { type = "app"; - program = "${import ./nix/tests/test-all-microvms.nix { inherit pkgs lib; }}/bin/pcp-test-all-microvms"; + program = "${import (nixDir + "/tests/test-all-microvms.nix") { inherit pkgs lib; }}/bin/pcp-test-all-microvms"; }; } # Per-variant test apps