nix: extract K8s manifests, remove BCC, improve build isolation

CLAUDE.md

@@ -184,7 +184,7 @@ cd build/mac/test && ./run-all-tests.sh  # Daily: build + test (20-30s)
 
 ## Nix Development
 
-Nix packaging is provided via `flake.nix` and the `nix/` directory.
+Nix packaging is provided via `flake.nix` and the `build/nix/` directory.
 See `docs/HowTos/nix/index.rst` for full documentation.
 
 ```bash

nix/bpf.nix → build/nix/bpf.nix

@@ -1,25 +1,18 @@
-# nix/bpf.nix
+# build/nix/bpf.nix
 #
 # NixOS module for BPF PMDA (pre-compiled eBPF metrics).
 #
 # This module uses pmdabpf which loads pre-compiled CO-RE (Compile Once, Run Everywhere)
-# eBPF programs. Unlike pmdabcc, no runtime compilation is needed - the eBPF bytecode
-# is compiled at Nix build time and loaded directly by the kernel via libbpf.
+# eBPF programs.  No runtime compilation is needed — the eBPF bytecode is compiled at
+# Nix build time and loaded directly by the kernel via libbpf.
 #
-# Advantages over pmdabcc:
+# Characteristics:
 #   - Fast startup: No eBPF compilation at runtime
-#   - Low memory: No clang/LLVM needed in the VM (~512MB vs ~2GB)
-#   - Simpler: Just needs libbpf + BTF-enabled kernel
+#   - Low memory: No clang/LLVM needed in the VM (~512MB)
+#   - Simple: Just needs libbpf + BTF-enabled kernel
 #
-# Trade-offs:
-#   - Fewer modules available compared to pmdabcc
-#   - Missing: tcptop, tcplife (required for some Grafana BCC dashboards)
-#
-# If you need tcptop/tcplife metrics, see bcc.nix which uses pmdabcc with runtime
-# eBPF compilation. This requires:
-#   - 2GB+ VM memory (clang/LLVM are memory-hungry)
-#   - BTF-enabled kernel (CONFIG_DEBUG_INFO_BTF=y)
-#   - Extended pmcd timeout (-q 60) for module compilation
+# Note: pmdabcc (runtime BCC compilation) was removed from PCP upstream.
+# pmdabpf is the only supported eBPF PMDA.
 #
 # Available pmdabpf modules:
 #   - biolatency: Block device I/O latency histogram

nix/constants.nix → build/nix/constants.nix

@@ -1,4 +1,4 @@
-# nix/constants.nix
+# build/nix/constants.nix
 #
 # Shared constants for PCP MicroVM infrastructure.
 # Import this file in microvm.nix, network-setup.nix, test-lib.nix, etc.
@@ -102,8 +102,6 @@ rec {
     pmcd = 2;      # src/pmdas/pmcd/domain.h
     linux = 60;    # src/pmdas/linux/domain.h
     bpf = 157;     # src/pmdas/bpf/domain.h (pre-compiled CO-RE eBPF)
-    # NOTE: BCC is deprecated - use BPF PMDA instead (CO-RE eBPF)
-    # bcc = 149;   # src/pmdas/bcc/domain.h (runtime-compiled eBPF)
   };
 
   # ─── Test Configuration ────────────────────────────────────────────────
@@ -131,8 +129,6 @@ rec {
     eval = 100;     # pcp-microvm-eval, pcp-microvm-eval-tap
     grafana = 200;  # pcp-microvm-grafana, pcp-microvm-grafana-tap
     bpf = 300;      # pcp-microvm-bpf
-    # NOTE: BCC is deprecated - use BPF PMDA instead
-    # bcc = 400;    # pcp-microvm-bcc
   };
 
   # ─── Serial Console Configuration ────────────────────────────────────
@@ -166,8 +162,6 @@ rec {
       eval = 10;     # 24510-24519
       grafana = 20;  # 24520-24529
       bpf = 30;      # 24530-24539
-      # NOTE: BCC is deprecated - use BPF PMDA instead
-      # bcc = 40;    # 24540-24549
     };
   };
 
@@ -205,12 +199,6 @@ rec {
       eval = {};
       grafana = { serviceReady = 90; };  # Grafana takes longer to start
       bpf = {};
-      # NOTE: BCC is deprecated - use BPF PMDA instead (CO-RE eBPF)
-      # BCC used runtime eBPF compilation which required longer timeouts
-      # bcc = {
-      #   serviceReady = 180;    # BCC modules compile at pmcd startup
-      #   metricsReady = 120;    # BCC metrics appear after compilation
-      # };
     };
   };
 

nix/container-test/constants.nix → build/nix/container-test/constants.nix

@@ -1,4 +1,4 @@
-# nix/container-test/constants.nix
+# build/nix/container-test/constants.nix
 #
 # Container lifecycle testing configuration.
 # Provides constants for OCI container testing phases.

nix/container-test/default.nix → build/nix/container-test/default.nix

@@ -1,10 +1,10 @@
-# nix/container-test/default.nix
+# build/nix/container-test/default.nix
 #
 # Entry point for PCP OCI container lifecycle testing.
 # Generates lifecycle test scripts for the PCP container image.
 #
 # Usage in flake.nix:
-#   containerTest = import ./nix/container-test { inherit pkgs lib pcp; };
+#   containerTest = import (nixDir + "/container-test") { inherit pkgs lib pcp; };
 #
 # Generated outputs:
 #   containerTest.packages.pcp-container-test  - Full lifecycle test

nix/container-test/lib.nix → build/nix/container-test/lib.nix

@@ -1,4 +1,4 @@
-# nix/container-test/lib.nix
+# build/nix/container-test/lib.nix
 #
 # Shell helper functions for PCP container lifecycle testing.
 # Provides container-specific operations on top of shared helpers.

nix/container.nix → build/nix/container.nix

@@ -1,4 +1,4 @@
-# nix/container.nix
+# build/nix/container.nix
 #
 # OCI container image with PCP.
 # Uses buildLayeredImage for better Docker layer caching.

nix/grafana.nix → build/nix/grafana.nix

@@ -1,4 +1,4 @@
-# nix/grafana.nix
+# build/nix/grafana.nix
 #
 # NixOS module for Grafana with PCP dashboards.
 # Provides visualization for PCP metrics via the grafana-pcp plugin.

build/nix/k8s-manifests/constants.nix

@@ -0,0 +1,45 @@
+# build/nix/k8s-manifests/constants.nix
+#
+# Kubernetes deployment constants for PCP DaemonSet manifests.
+# These are the production defaults; test suites can override
+# (e.g. namespace -> "pcp-test") via the namespaceOverride parameter
+# in default.nix.
+#
+{ }:
+let
+  mainConstants = import ../constants.nix;
+in
+{
+  # Re-export ports from main constants
+  inherit (mainConstants) ports;
+
+  # ─── Kubernetes Deployment Settings ──────────────────────────────────
+  k8s = {
+    namespace = "pcp";
+    daemonSetName = "pcp";
+
+    image = {
+      name = "pcp";
+      tag = "latest";
+      pullPolicy = "Never";
+    };
+
+    resources = {
+      limits = {
+        memory = "512Mi";
+        cpu = "500m";
+      };
+      requests = {
+        memory = "256Mi";
+        cpu = "100m";
+      };
+    };
+
+    hostMounts = {
+      root = "/host";
+      proc = "/host/proc";
+      sys = "/host/sys";
+      kernelDebug = "/sys/kernel/debug";
+    };
+  };
+}

nix/k8s-test/manifests.nix → build/nix/k8s-manifests/daemonset.nix

@@ -1,44 +1,37 @@
-# nix/k8s-test/manifests.nix
+# build/nix/k8s-manifests/daemonset.nix
 #
-# Generates Kubernetes manifests for PCP DaemonSet deployment.
-# Creates privileged DaemonSet with full node monitoring capabilities.
+# Generates the Kubernetes DaemonSet resource YAML for PCP deployment.
+# Creates a privileged DaemonSet with full node monitoring capabilities
+# including BPF metrics.
 #
-{ pkgs, lib }:
+{ pkgs, constants }:
 let
-  constants = import ./constants.nix { };
-  mainConstants = import ../constants.nix;
+  k = constants.k8s;
 
-  # ─── DaemonSet Manifest ────────────────────────────────────────────────
-  # Privileged DaemonSet for full node monitoring including BPF metrics
-  daemonSetManifest = ''
-    apiVersion: v1
-    kind: Namespace
-    metadata:
-      name: ${constants.k8s.namespace}
-    ---
+  yaml = ''
     apiVersion: apps/v1
     kind: DaemonSet
     metadata:
-      name: ${constants.k8s.daemonSetName}
-      namespace: ${constants.k8s.namespace}
+      name: ${k.daemonSetName}
+      namespace: ${k.namespace}
       labels:
-        app: ${constants.k8s.daemonSetName}
+        app: ${k.daemonSetName}
     spec:
       selector:
         matchLabels:
-          app: ${constants.k8s.daemonSetName}
+          app: ${k.daemonSetName}
       template:
         metadata:
           labels:
-            app: ${constants.k8s.daemonSetName}
+            app: ${k.daemonSetName}
         spec:
           # Required for seeing all node processes
           hostPID: true
 
           containers:
           - name: pcp
-            image: ${constants.k8s.imageName}:${constants.k8s.imageTag}
-            imagePullPolicy: Never
+            image: ${k.image.name}:${k.image.tag}
+            imagePullPolicy: ${k.image.pullPolicy}
 
             # Privileged for BPF and full /proc access
             # Run as root to override container's default pcp user
@@ -47,33 +40,33 @@ let
               runAsUser: 0
 
             ports:
-            - containerPort: ${toString mainConstants.ports.pmcd}
+            - containerPort: ${toString constants.ports.pmcd}
               name: pmcd
-            - containerPort: ${toString mainConstants.ports.pmproxy}
+            - containerPort: ${toString constants.ports.pmproxy}
               name: pmproxy
 
             env:
             # Tell PCP where host filesystem is mounted
             - name: HOST_MOUNT
-              value: "/host"
+              value: "${k.hostMounts.root}"
             - name: PCP_SYSFS_DIR
-              value: "/host/sys"
+              value: "${k.hostMounts.sys}"
 
             volumeMounts:
             # Host root filesystem (read-only)
             - name: host-root
-              mountPath: /host
+              mountPath: ${k.hostMounts.root}
               readOnly: true
             # Required for BPF
             - name: sys-kernel-debug
-              mountPath: /sys/kernel/debug
+              mountPath: ${k.hostMounts.kernelDebug}
             # Host /proc for process metrics
             - name: host-proc
-              mountPath: /host/proc
+              mountPath: ${k.hostMounts.proc}
               readOnly: true
             # Host /sys for system metrics
             - name: host-sys
-              mountPath: /host/sys
+              mountPath: ${k.hostMounts.sys}
               readOnly: true
 
             readinessProbe:
@@ -84,11 +77,11 @@ let
 
             resources:
               limits:
-                memory: "512Mi"
-                cpu: "500m"
+                memory: "${k.resources.limits.memory}"
+                cpu: "${k.resources.limits.cpu}"
               requests:
-                memory: "256Mi"
-                cpu: "100m"
+                memory: "${k.resources.requests.memory}"
+                cpu: "${k.resources.requests.cpu}"
 
           volumes:
           - name: host-root
@@ -108,15 +101,8 @@ let
           tolerations:
           - operator: Exists
   '';
-
 in
 {
-  # The full DaemonSet manifest as a string
-  manifest = daemonSetManifest;
-
-  # Write manifest to a file for kubectl apply
-  manifestFile = pkgs.writeText "pcp-daemonset.yaml" daemonSetManifest;
-
-  # Helper to get the manifest path
-  getManifestPath = "${pkgs.writeText "pcp-daemonset.yaml" daemonSetManifest}";
+  inherit yaml;
+  file = pkgs.writeText "pcp-daemonset.yaml" yaml;
 }