From fd8cb71146ef20bf81770fbebbb16f3135f2a00a Mon Sep 17 00:00:00 2001 From: ZPascal Date: Fri, 12 Jun 2026 15:31:37 +0200 Subject: [PATCH] feat: Add Podman support as drop-in replacement for Docker Signed-off-by: ZPascal --- .github/workflows/kind-cats.yaml | 2 +- .github/workflows/kind-smoke.yaml | 125 ++----------- .github/workflows/smoke-run.yaml | 173 ++++++++++++++++++ .gitignore | 2 +- Makefile | 31 +++- README.md | 41 ++++- .../base-chart/templates/network-policy.yaml | 2 + assets/base-chart/values.yaml | 4 + assets/values/cilium-podman.yaml | 47 +++++ helmfile.yaml.gotmpl | 50 ++++- kind-podman.yaml | 91 +++++++++ .../helm/templates/nfsv3driver.yaml | 105 +++++++++++ releases/nfs-volume/helm/values.schema.json | 152 +++++++++++++++ scripts/create-kind.sh | 50 ++++- scripts/delete-kind.sh | 13 +- scripts/detect-runtime.sh | 43 +++++ scripts/init.sh | 7 +- scripts/setup-podman-vm.sh | 156 ++++++++++++++++ 18 files changed, 957 insertions(+), 137 deletions(-) create mode 100644 .github/workflows/smoke-run.yaml create mode 100644 assets/values/cilium-podman.yaml create mode 100644 kind-podman.yaml create mode 100644 releases/nfs-volume/helm/templates/nfsv3driver.yaml create mode 100644 releases/nfs-volume/helm/values.schema.json create mode 100755 scripts/detect-runtime.sh create mode 100755 scripts/setup-podman-vm.sh diff --git a/.github/workflows/kind-cats.yaml b/.github/workflows/kind-cats.yaml index 49c0d659..0c73d83a 100644 --- a/.github/workflows/kind-cats.yaml +++ b/.github/workflows/kind-cats.yaml @@ -37,7 +37,7 @@ jobs: --privileged \ --pid host \ -v /lib/modules:/lib/modules:ro \ - alpine sh -c "sysctl -w fs.inotify.max_user_instances=512 && modprobe nfs && modprobe nfsd" + alpine sh -c "sysctl -w fs.inotify.max_user_instances=512 && sysctl -w net.ipv4.ip_unprivileged_port_start=80 && modprobe nfs && modprobe nfsd" - name: Install dependencies if: steps.check_changes.outputs.skip != 'true' run: | diff --git a/.github/workflows/kind-smoke.yaml b/.github/workflows/kind-smoke.yaml index c48efc8b..e853471c 100644 --- a/.github/workflows/kind-smoke.yaml +++ b/.github/workflows/kind-smoke.yaml @@ -21,114 +21,23 @@ on: jobs: kind-smoke: - runs-on: ubuntu-latest + uses: ./.github/workflows/smoke-run.yaml permissions: id-token: write contents: read - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Check if only ignored paths changed - id: check_changes - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }}) - RELEVANT_FILES=$(echo "$CHANGED_FILES" | grep -v -E '.*\.Dockerfile|^releases/.*/files/|^\.github/|^docs/|\.md$|^renovate\.json$' || true) - if [ -z "$RELEVANT_FILES" ]; then - echo "Only ignored paths were changed. Skipping workflow." - echo "skip=true" >> $GITHUB_OUTPUT - else - echo "skip=false" >> $GITHUB_OUTPUT - fi - else - echo "skip=false" >> $GITHUB_OUTPUT - fi - - name: Set kernel settings - if: steps.check_changes.outputs.skip != 'true' - run: | - docker run --rm \ - --privileged \ - --pid host \ - alpine sh -c "sysctl -w fs.inotify.max_user_instances=512" - - name: Install dependencies - if: steps.check_changes.outputs.skip != 'true' - run: | - mkdir -p $HOME/.local/bin && echo "$HOME/.local/bin" >> "$GITHUB_PATH" - curl -L https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 -o $HOME/.local/bin/kind - chmod +x $HOME/.local/bin/kind - curl -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_amd64.tar.gz | tar -zx - mv helmfile $HOME/.local/bin/helmfile - curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=${CF_CLI_VERSION}&source=github" | tar -zx - mv cf8 $HOME/.local/bin/cf - env: - # renovate: dataSource=github-releases depName=cloudfoundry/cli - CF_CLI_VERSION: "8.18.3" - # renovate: dataSource=github-releases depName=kubernetes-sigs/kind - KIND_VERSION: "0.32.0" - # renovate: dataSource=github-releases depName=helmfile/helmfile - HELMFILE_VERSION: "1.5.3" - - name: Use develop versions of cf-deployment - id: pre_validation - if: steps.check_changes.outputs.skip != 'true' && github.event_name == 'workflow_dispatch' && github.event.inputs.fresh-validation == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip3 install -r scripts/requirements.txt - python3 scripts/sync-cf-deployment-versions.py --ref develop - - name: Run make up (default) - if: steps.check_changes.outputs.skip != 'true' && github.event_name == 'pull_request' - run: make up - - name: Run make up - if: steps.check_changes.outputs.skip != 'true' && github.event_name != 'pull_request' - run: make up - env: - INSTALL_OPTIONAL_COMPONENTS: ${{ inputs.minimal }} - - name: Login - if: steps.check_changes.outputs.skip != 'true' - run: make login - - name: Init - if: steps.check_changes.outputs.skip != 'true' - run: make bootstrap-complete - - name: setup CF tests - if: steps.check_changes.outputs.skip != 'true' - uses: ./.github/actions/setup-cf-tests - with: - test-repo: cf-smoke-tests - test-branch: main - config-template: ./.github/smoke-config.tpl - config-output: ./.github/smoke-config.json - - name: run smoke test - if: steps.check_changes.outputs.skip != 'true' - env: - CONFIG: "${{ github.workspace }}/.github/smoke-config.json" - GINKGO_NO_COLOR: "true" - run: | - ./cf-smoke-tests/bin/test --no-color --github-output --timeout=30m --procs=4 --json-report report.json - - name: debug events - if: failure() - run: kubectl get events -A --sort-by='.lastTimestamp' - - name: debug pods - if: failure() - run: | - echo "===== CLOUD_CONTROLLER =====" - kubectl get pod -n cf-system -l app.kubernetes.io/name=cloud-controller -o wide - kubectl get pod -n cf-system -l app.kubernetes.io/name=cloud-controller -o jsonpath='{.status.containerStatuses[*].state}' | jq - kubectl logs -n cf-system -l app.kubernetes.io/name=cloud-controller --all-containers=true - echo "===== CC_WORKER =====" - kubectl get pod -n cf-system -l app.kubernetes.io/name=cc-worker -o wide - kubectl logs -n cf-system -l app.kubernetes.io/name=cc-worker - echo "===== CC_UPLOADER =====" - kubectl get pod -n cf-system -l app=cc-uploader -o wide - kubectl logs -n cf-system -l app=cc-uploader - echo "===== CC_DEPLOYMENT_UPDATER =====" - kubectl get pod -n cf-system -l app.kubernetes.io/name=cc-deployment-updater -o wide - kubectl logs -n cf-system -l app.kubernetes.io/name=cc-deployment-updater - echo "===== CC_WORKER_CLOCK =====" - kubectl get pod -n cf-system -l app.kubernetes.io/name=cc-worker-clock -o wide - kubectl logs -n cf-system -l app.kubernetes.io/name=cc-worker-clock - - uses: actions/upload-artifact@v7 - if: always() - with: - name: report.json - path: ./cf-smoke-tests/report.json + with: + container-runtime: docker + install-optional-components: ${{ github.event_name == 'pull_request' && 'false' || inputs.minimal != true && 'true' || 'false' }} + fresh-validation: ${{ inputs.fresh-validation == true }} + artifact-name: report.json + + kind-smoke-podman: + uses: ./.github/workflows/smoke-run.yaml + permissions: + id-token: write + contents: read + with: + container-runtime: podman + install-optional-components: ${{ github.event_name == 'pull_request' && 'false' || inputs.minimal != true && 'true' || 'false' }} + fresh-validation: ${{ inputs.fresh-validation == true }} + artifact-name: report-podman.json diff --git a/.github/workflows/smoke-run.yaml b/.github/workflows/smoke-run.yaml new file mode 100644 index 00000000..5b74aaf4 --- /dev/null +++ b/.github/workflows/smoke-run.yaml @@ -0,0 +1,173 @@ +name: Smoke Run (reusable) + +on: + workflow_call: + inputs: + container-runtime: + description: 'Container runtime to use (docker or podman)' + type: string + required: true + install-optional-components: + description: 'Whether to install optional CF components' + type: string + required: true + artifact-name: + description: 'Name of the uploaded test report artifact' + type: string + required: true + fresh-validation: + description: 'Use versions of cf-deployment from the develop branch' + type: boolean + required: false + default: false + +jobs: + smoke-run: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Check if only ignored paths changed + id: check_changes + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }}) + RELEVANT_FILES=$(echo "$CHANGED_FILES" | grep -v -E '.*\.Dockerfile|^releases/.*/files/|^\.github/|^docs/|\.md$|^renovate\.json$' || true) + if [ -z "$RELEVANT_FILES" ]; then + echo "Only ignored paths were changed. Skipping workflow." + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + - name: Pin nip.io domains to localhost + if: steps.check_changes.outputs.skip != 'true' + run: | + # nip.io is an external DNS service; CI runners sometimes can't resolve it. + printf '127.0.0.1 api.127-0-0-1.nip.io\n127.0.0.1 uaa.127-0-0-1.nip.io\n127.0.0.1 login.127-0-0-1.nip.io\n127.0.0.1 apps.127-0-0-1.nip.io\n127.0.0.1 doppler.127-0-0-1.nip.io\n127.0.0.1 log-stream.127-0-0-1.nip.io\n' | sudo tee -a /etc/hosts + - name: Set kernel settings + if: steps.check_changes.outputs.skip != 'true' + run: | + sudo sysctl -w fs.inotify.max_user_instances=512 + sudo sysctl -w net.ipv4.ip_unprivileged_port_start=80 + - name: Set kernel settings (Podman extras) + if: steps.check_changes.outputs.skip != 'true' && inputs.container-runtime == 'podman' + run: | + # Mount BPF filesystem on the host (needed by kindnet on some kernels) + sudo mount bpffs /sys/fs/bpf -t bpf -o nosuid,nodev,noexec,relatime || true + - name: Install podman-compose + if: steps.check_changes.outputs.skip != 'true' && inputs.container-runtime == 'podman' + run: | + sudo apt-get update -qq + sudo apt-get install -y podman-compose + # Ensure podman-compose is used instead of the docker-compose plugin shim + sudo ln -sf /usr/bin/podman-compose /usr/local/bin/podman-compose + - name: Install dependencies + if: steps.check_changes.outputs.skip != 'true' + run: | + mkdir -p $HOME/.local/bin && echo "$HOME/.local/bin" >> "$GITHUB_PATH" + curl -L https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 -o $HOME/.local/bin/kind + chmod +x $HOME/.local/bin/kind + curl -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_amd64.tar.gz | tar -zx + mv helmfile $HOME/.local/bin/helmfile + curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=${CF_CLI_VERSION}&source=github" | tar -zx + mv cf8 $HOME/.local/bin/cf + env: + # renovate: dataSource=github-releases depName=cloudfoundry/cli + CF_CLI_VERSION: "8.18.3" + # renovate: dataSource=github-releases depName=kubernetes-sigs/kind + KIND_VERSION: "0.32.0" + # renovate: dataSource=github-releases depName=helmfile/helmfile + HELMFILE_VERSION: "1.5.3" + - name: Use develop versions of cf-deployment + id: pre_validation + if: steps.check_changes.outputs.skip != 'true' && inputs.fresh-validation == true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip3 install -r scripts/requirements.txt + python3 scripts/sync-cf-deployment-versions.py --ref develop + - name: Run make up + if: steps.check_changes.outputs.skip != 'true' + run: make up + env: + CONTAINER_RUNTIME: ${{ inputs.container-runtime }} + INSTALL_OPTIONAL_COMPONENTS: ${{ inputs.install-optional-components }} + - name: Login + if: steps.check_changes.outputs.skip != 'true' + run: make login + - name: Init + if: steps.check_changes.outputs.skip != 'true' + run: make bootstrap-complete + - name: setup CF tests + if: steps.check_changes.outputs.skip != 'true' + uses: ./.github/actions/setup-cf-tests + with: + test-repo: cf-smoke-tests + test-branch: main + config-template: ./.github/smoke-config.tpl + config-output: ./.github/smoke-config.json + - name: run smoke test + if: steps.check_changes.outputs.skip != 'true' + env: + CONFIG: "${{ github.workspace }}/.github/smoke-config.json" + GINKGO_NO_COLOR: "true" + run: | + ./cf-smoke-tests/bin/test --no-color --github-output --timeout=30m --procs=4 --json-report report.json + - name: debug events + if: failure() + run: kubectl get events -A --sort-by='.lastTimestamp' + - name: debug cilium + if: failure() && inputs.container-runtime == 'podman' + run: | + echo "===== CILIUM PODS =====" + kubectl get pod -n kube-system -l k8s-app=cilium -o wide + echo "===== CILIUM AGENT LOGS (all pods) =====" + for pod in $(kubectl get pod -n kube-system -l k8s-app=cilium -o jsonpath='{.items[*].metadata.name}'); do + echo "--- $pod ---" + kubectl logs -n kube-system "$pod" --all-containers=true --previous 2>/dev/null || kubectl logs -n kube-system "$pod" --all-containers=true 2>/dev/null || true + done + echo "===== CILIUM OPERATOR LOGS =====" + kubectl logs -n kube-system -l name=cilium-operator --all-containers=true 2>/dev/null || true + echo "===== CILIUM STATUS =====" + kubectl exec -n kube-system $(kubectl get pod -n kube-system -l k8s-app=cilium -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) -- cilium status 2>/dev/null || true + - name: debug pods + if: failure() + run: | + echo "===== CLOUD_CONTROLLER =====" + kubectl get pod -n cf-system -l app.kubernetes.io/name=cloud-controller -o wide + kubectl get pod -n cf-system -l app.kubernetes.io/name=cloud-controller -o jsonpath='{.status.containerStatuses[*].state}' | jq + kubectl logs -n cf-system -l app.kubernetes.io/name=cloud-controller --all-containers=true + echo "===== CC_WORKER =====" + kubectl get pod -n cf-system -l app.kubernetes.io/name=cc-worker -o wide + kubectl logs -n cf-system -l app.kubernetes.io/name=cc-worker + echo "===== CC_UPLOADER =====" + kubectl get pod -n cf-system -l app=cc-uploader -o wide + kubectl logs -n cf-system -l app=cc-uploader + echo "===== CC_DEPLOYMENT_UPDATER =====" + kubectl get pod -n cf-system -l app.kubernetes.io/name=cc-deployment-updater -o wide + kubectl logs -n cf-system -l app.kubernetes.io/name=cc-deployment-updater + echo "===== CC_WORKER_CLOCK =====" + kubectl get pod -n cf-system -l app.kubernetes.io/name=cc-worker-clock -o wide + kubectl logs -n cf-system -l app.kubernetes.io/name=cc-worker-clock + - name: debug pods (Podman extras) + if: failure() && inputs.container-runtime == 'podman' + run: | + echo "===== CF_TCP_ROUTER =====" + kubectl get pod -n cf-system -l app=cf-tcp-router -o wide + kubectl get pod -n cf-system -l app=cf-tcp-router -o jsonpath='{.items[*].status.containerStatuses[*].state}' | jq 2>/dev/null || true + kubectl logs -n cf-system -l app=cf-tcp-router --all-containers=true 2>/dev/null || true + echo "===== ROUTING_API =====" + kubectl get pod -n cf-system -l app=routing-api -o wide + kubectl logs -n cf-system -l app=routing-api --all-containers=true 2>/dev/null || true + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ${{ inputs.artifact-name }} + path: ./cf-smoke-tests/report.json diff --git a/.gitignore b/.gitignore index 0e9e5612..896ccd5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ temp/ - +.idea diff --git a/Makefile b/Makefile index 283b1809..c0da2ec5 100644 --- a/Makefile +++ b/Makefile @@ -3,19 +3,40 @@ TARGET_ARCH ?= $(if $(filter true,$(LOCAL)),$(shell go env GOARCH),amd64) # renovate: dataSource=github-releases depName=helmfile/helmfile HELMFILE_VERSION ?= "1.5.3" +build: + @ . ./scripts/detect-runtime.sh; \ + if [ "$$CONTAINER_RUNTIME" = "podman" ]; then \ + echo "Building with Podman is not yet supported via docker-bake.hcl."; \ + echo "Use 'podman build' manually with the Dockerfiles in releases/."; \ + exit 1; \ + fi; \ + docker buildx bake --file docker-bake.hcl --set "*.platform=linux/$(TARGET_ARCH)" $(BAKE_TARGETS) + init: temp/certs/ca.key temp/certs/ca.crt temp/certs/ssh_key temp/certs/ssh_key.pub temp/secrets.sh temp/secrets.env temp/certs/ca.key temp/certs/ca.crt temp/certs/ssh_key temp/certs/ssh_key.pub temp/secrets.sh temp/secrets.env: @ ./scripts/init.sh install: - kind get kubeconfig --name cfk8s > temp/kubeconfig - docker run --rm --net=host --env-file temp/secrets.env \ + @ . ./scripts/detect-runtime.sh; \ + if [ "$$IS_PODMAN" = "true" ]; then export SKIP_CILIUM="true"; fi; \ + kind get kubeconfig --name cfk8s > temp/kubeconfig; \ + $$CONTAINER_RUNTIME run --rm --net=host --env-file temp/secrets.env \ --env INSTALL_OPTIONAL_COMPONENTS \ + --env CILIUM_EXTRA_VALUES \ + --env SKIP_CILIUM \ -v "$$PWD/temp/certs:/certs" -v "$$PWD/temp/kubeconfig:/helm/.kube/config:ro" -v "$$PWD:/wd" --workdir /wd ghcr.io/helmfile/helmfile:v$(HELMFILE_VERSION) helmfile sync login: - @ . temp/secrets.sh; \ + @ echo "Waiting for CF API to become ready..."; \ + for i in $$(seq 1 60); do \ + status=$$(curl -sk -o /dev/null -w "%{http_code}" https://api.127-0-0-1.nip.io/v2/info); \ + if [ "$$status" = "200" ]; then echo "CF API is ready."; break; fi; \ + echo " attempt $$i/60: HTTP $$status – retrying in 10s..."; \ + sleep 10; \ + done; \ + if [ "$$status" != "200" ]; then echo "ERROR: CF API did not become ready after 10 minutes." >&2; exit 1; fi; \ + . temp/secrets.sh; \ cf login -a https://api.127-0-0-1.nip.io -u ccadmin -p "$$CC_ADMIN_PASSWORD" --skip-ssl-validation create-kind: @@ -33,7 +54,7 @@ create-org: bootstrap: create-org @ ./scripts/upload_buildpacks.sh -bootstrap-complete: create-org +bootstrap-complete: create-org @ ALL_BUILDPACKS=true ./scripts/upload_buildpacks.sh up: create-kind init install @@ -41,4 +62,4 @@ up: create-kind init install down: delete-kind @ rm -rf temp -PHONY: install login create-kind delete-kind up down create-org bootstrap bootstrap-complete +.PHONY: build install login create-kind delete-kind up down create-org bootstrap bootstrap-complete diff --git a/README.md b/README.md index ecb4fac6..b5829abd 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,16 @@ This repository provides a simple and fast way to run Cloud Foundry locally. It The following tools need to be installed: -- [`docker`](https://docs.docker.com/engine/install/) +- [`docker`](https://docs.docker.com/engine/install/) or [`podman`](https://podman.io/docs/installation) (v4.0 or higher) +- [`docker-compose`](https://docs.docker.com/compose/install) or [`podman-compose`](https://github.com/containers/podman-compose) (v1.0 or higher) - [`kind`](https://kind.sigs.k8s.io/docs/user/quick-start/#installing-from-release-binaries) (v0.31.0 or higher) - [`kubectl`](https://kubernetes.io/docs/tasks/tools/#kubectl) (v1.35.1 or higher) - `make`: - It should be already installed on MacOS and Linux. - For Windows installation see: +The container runtime is auto-detected. Set `CONTAINER_RUNTIME=docker` or `CONTAINER_RUNTIME=podman` to override. + ## Run the Installation ```bash @@ -48,6 +51,40 @@ You can configure the installation by setting the environment variable `INSTALL_ `bosh-dns`, `cf-tcp-router`, `credhub`, `loggregator`, `nfsbroker`, `policy-agent`, `policy-server`, `routing-api`, `service-discovery-controller` +## Podman Support + +Podman is supported as a drop-in replacement for Docker. The runtime is detected automatically; no aliasing is required. + +### Linux (rootless Podman) + +Rootless Podman is fully supported on Linux. The following kernel settings must be applied before running `make up` — either manually or via your system's sysctl configuration: + +```bash +sudo sysctl -w fs.inotify.max_user_instances=512 +sudo sysctl -w net.ipv4.ip_unprivileged_port_start=80 +``` + +The first setting prevents inotify exhaustion under heavy workloads. The second allows the kind node containers and pods to bind privileged ports (80, 443, 2222) without root. + +### macOS / Windows (Podman Desktop) + +A Podman machine is created and configured automatically by `make up`. The machine is created in rootful mode with 4 CPUs, 8 GB RAM, and 60 GB disk. No manual configuration is needed. + +### Limitations + +- **CNI:** Cilium is skipped under rootless Podman (Linux CI / rootless desktop) because Cilium 1.18.x requires `CAP_NET_ADMIN` in the host user namespace, which rootless containers cannot provide. [kindnet](https://github.com/aojea/kindnet) is used instead, providing full pod-to-pod connectivity without eBPF privileges. Cilium network policies are therefore not enforced in this mode. +- **Image builds:** `make build` (which uses `docker buildx bake`) is not supported with Podman. Use `podman build` directly with the Dockerfiles in `releases/` for local image development. + +## ARM / Apple Silicon Limitations + +The CF stack (`cflinuxfs4`) and all buildpacks are **x86-64 (amd64) only**. CF applications run inside `cflinuxfs4` containers, which are amd64 images and require x86 emulation on ARM hosts. + +- **Docker Desktop on Apple Silicon:** Enable Rosetta emulation (Settings → General → Use Rosetta for x86_64/amd64 emulation). This is the recommended and well-tested path. +- **Podman on Apple Silicon:** The Podman machine is created with `--rootful` and runs under QEMU/Rosetta. Functional, but noticeably slower than Docker Desktop with Rosetta. +- **Linux ARM64:** Not supported. The `cflinuxfs4` stack image and pre-compiled buildpack zip files are amd64-only. CF app staging and execution will fail on a native ARM64 host without kernel-level x86 emulation (`binfmt_misc` with QEMU). + +The CF platform components themselves (gorouter, diego, CAPI, etc.) are built for the local architecture (`make build` targets the native arch), so control-plane operations are native-speed on ARM. Only the **application workload layer** (buildpacks, cflinuxfs4) is restricted to amd64. + ## Unsupported Features - Routing isolation segments are not fully feature complete since this relies on more than one gateway which is not possible to realize in a local kind setup (see [FAQ](./docs/faq.md)) @@ -61,4 +98,4 @@ You can configure the installation by setting the environment variable `INSTALL_ Please check our [contributing guidelines](/CONTRIBUTING.md). -This project follows [Cloud Foundry Code of Conduct](https://www.cloudfoundry.org/code-of-conduct/). +This project follows [Cloud Foundry Code of Conduct](https://www.cloudfoundry.org/code-of-conduct/) diff --git a/assets/base-chart/templates/network-policy.yaml b/assets/base-chart/templates/network-policy.yaml index c5a6416c..68452e82 100644 --- a/assets/base-chart/templates/network-policy.yaml +++ b/assets/base-chart/templates/network-policy.yaml @@ -1,3 +1,4 @@ +{{- if .Values.ciliumNetworkPolicies.enabled }} --- apiVersion: cilium.io/v2 kind: CiliumClusterwideNetworkPolicy @@ -85,3 +86,4 @@ spec: - ports: - port: "8443" protocol: TCP +{{- end }} diff --git a/assets/base-chart/values.yaml b/assets/base-chart/values.yaml index 2bb606dd..0c8bf6b7 100644 --- a/assets/base-chart/values.yaml +++ b/assets/base-chart/values.yaml @@ -2,3 +2,7 @@ caCert: "" caKey: "" tlsCert: "" tlsKey: "" +# Set to false when Cilium is not installed (e.g. rootless Podman CI). +# CiliumClusterwideNetworkPolicy CRDs won't exist without Cilium. +ciliumNetworkPolicies: + enabled: true diff --git a/assets/values/cilium-podman.yaml b/assets/values/cilium-podman.yaml new file mode 100644 index 00000000..b991b762 --- /dev/null +++ b/assets/values/cilium-podman.yaml @@ -0,0 +1,47 @@ +ipam: + mode: kubernetes +envoy: + enabled: false +# kube-proxy replacement via eBPF requires privileges not available in +# rootless Podman containers (used in CI). Fall back to standard kube-proxy mode. +kubeProxyReplacement: false +k8sServiceHost: cfk8s-control-plane +k8sServicePort: 6443 +operator: + replicas: 1 +routingMode: tunnel +tunnelProtocol: vxlan +# Disable all eBPF features that require CAP_BPF / kernel privileges +# not available inside rootless Podman-spawned KinD nodes. +bpf: + masquerade: false + autoMount: + enabled: false + preallocateMaps: false +hostServices: + enabled: false +nodePort: + enabled: false +socketLB: + enabled: false +externalIPs: + enabled: false +hubble: + enabled: false +# Tell Cilium where the cgroup root is (since we can't auto-mount) +cgroup: + autoMount: + enabled: false + hostRoot: /sys/fs/cgroup +bandwidthManager: + enabled: false +localRedirectPolicy: false +enableIPv4Masquerade: true +loadBalancer: + mode: snat +sysctlfix: + enabled: false +hostFirewall: + enabled: false +sessionAffinity: false +enableEnvoyConfig: false diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index 7d48d446..09253af4 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -32,8 +32,12 @@ releases: namespace: kube-system chart: {{ .Values.charts.cilium.url }} version: {{ .Values.charts.cilium.version }} + installed: {{ ne (env "SKIP_CILIUM") "true" }} values: - ./assets/values/cilium.yaml + {{- if env "CILIUM_EXTRA_VALUES" }} + - {{ env "CILIUM_EXTRA_VALUES" }} + {{- end }} hooks: - events: ["postsync"] showlogs: true @@ -41,22 +45,17 @@ releases: args: - "-c" - | - echo "Waiting for nodes to become ready after CNI installation..." + echo "Cilium installed – waiting for nodes to become ready..." kubectl wait --for=condition=Ready nodes --all --timeout=300s - corefile=$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | sed '/kubernetes/i \ - rewrite name regex (.*)\\.127-0-0-1\\.nip\\.io istio-gateway-istio.{{ .Values.systemNamespace }}.svc.cluster.local answer auto\ - ') - kubectl -n kube-system patch configmap coredns --type=json \ - -p="$(jq -n --arg cf "$corefile" '[{"op":"replace","path":"/data/Corefile","value":$cf}]')" - kubectl -n kube-system rollout restart deployment/coredns - - name: istio-base namespace: istio-system chart: {{ .Values.charts.istioBase.url }} version: {{ .Values.charts.istioBase.version }} needs: + {{- if ne (env "SKIP_CILIUM") "true" }} - kube-system/cilium + {{- end }} values: - defaultRevision: minimal @@ -81,7 +80,9 @@ releases: namespace: {{ .Values.systemNamespace }} chart: ./assets/base-chart needs: + {{- if ne (env "SKIP_CILIUM") "true" }} - kube-system/cilium + {{- end }} - istio-system/istiod set: - name: caCert @@ -92,6 +93,8 @@ releases: file: {{ .Values.certsDir }}/all-in-one.crt - name: tlsKey file: {{ .Values.certsDir }}/all-in-one.key + - name: ciliumNetworkPolicies.enabled + value: {{ ne (env "SKIP_CILIUM") "true" }} hooks: - events: ["presync"] showlogs: true @@ -99,7 +102,21 @@ releases: args: - "-c" - | + echo "Waiting for nodes to become ready..." + kubectl wait --for=condition=Ready nodes --all --timeout=300s kubectl apply --server-side --filename assets/gateway-api.yaml + - events: ["postsync"] + showlogs: true + command: "/bin/sh" + args: + - "-c" + - | + corefile=$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | sed '/kubernetes/i \ + rewrite name regex (.*)\\.127-0-0-1\\.nip\\.io istio-gateway-istio.{{ .Values.systemNamespace }}.svc.cluster.local answer auto\ + ') + kubectl -n kube-system patch configmap coredns --type=json \ + -p="$(jq -n --arg cf "$corefile" '[{"op":"replace","path":"/data/Corefile","value":$cf}]')" + kubectl -n kube-system rollout restart deployment/coredns - name: nats namespace: {{ .Values.systemNamespace }} @@ -405,6 +422,7 @@ releases: - minio - diego - postgresql + - "{{ .Values.systemNamespace }}/nats" values: - cloudController: enabled: true @@ -427,6 +445,7 @@ releases: chart: {{ .Values.charts.nfsVolume.url }} version: {{ .Values.charts.nfsVolume.version }} installed: {{ .Values.installOptionalComponents }} + timeout: 600 needs: - capi values: @@ -478,6 +497,21 @@ releases: - name: pollInterval value: "5s" hooks: + - events: ["presync"] + showlogs: true + command: "/bin/sh" + args: + - "-c" + - | + # The policy-agent chart hardcodes imagePullSecrets: [{name: image-pull-secret}]. + # Kubernetes refuses to pull images when a referenced secret does not exist, + # even for public images. Create an empty placeholder if it's missing. + kubectl create secret docker-registry image-pull-secret \ + --namespace {{ .Values.systemNamespace }} \ + --docker-server=ghcr.io \ + --docker-username=x \ + --docker-password=x \ + --dry-run=client -o yaml | kubectl apply -f - - events: ["postsync"] showlogs: true command: "/bin/sh" diff --git a/kind-podman.yaml b/kind-podman.yaml new file mode 100644 index 00000000..027af106 --- /dev/null +++ b/kind-podman.yaml @@ -0,0 +1,91 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: false + +featureGates: + PodAndContainerStatsFromCRI: true +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +kubeadmConfigPatches: + - | + kind: ClusterConfiguration + metadata: + name: config + controllerManager: + extraArgs: + authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics + bind-address: 0.0.0.0 + scheduler: + extraArgs: + authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics + bind-address: 0.0.0.0 +nodes: +- role: control-plane + extraMounts: + - hostPath: /sys/fs/bpf + containerPath: /sys/fs/bpf + propagation: HostToContainer +- role: worker + labels: + extraMounts: + - hostPath: /sys/fs/bpf + containerPath: /sys/fs/bpf + propagation: HostToContainer + extraPortMappings: + - containerPort: 31080 + hostPort: 80 + - containerPort: 31443 + hostPort: 443 + - containerPort: 31222 + hostPort: 2222 + - containerPort: 32000 + hostPort: 32000 + - containerPort: 32001 + hostPort: 32001 + - containerPort: 32002 + hostPort: 32002 + - containerPort: 32003 + hostPort: 32003 + - containerPort: 32004 + hostPort: 32004 + - containerPort: 32005 + hostPort: 32005 + - containerPort: 32006 + hostPort: 32006 + - containerPort: 32007 + hostPort: 32007 + - containerPort: 32008 + hostPort: 32008 + - containerPort: 32009 + hostPort: 32009 + - containerPort: 32010 + hostPort: 32010 + - containerPort: 32011 + hostPort: 32011 + - containerPort: 32012 + hostPort: 32012 + - containerPort: 32013 + hostPort: 32013 + - containerPort: 32014 + hostPort: 32014 + - containerPort: 32015 + hostPort: 32015 + - containerPort: 32016 + hostPort: 32016 + - containerPort: 32017 + hostPort: 32017 + - containerPort: 32018 + hostPort: 32018 + - containerPort: 32019 + hostPort: 32019 +- role: worker + labels: + cloudfoundry.org/cell: "true" + cloudfoundry.org/zone: z1 + extraMounts: + - hostPath: /sys/fs/bpf + containerPath: /sys/fs/bpf + propagation: HostToContainer diff --git a/releases/nfs-volume/helm/templates/nfsv3driver.yaml b/releases/nfs-volume/helm/templates/nfsv3driver.yaml new file mode 100644 index 00000000..cc9f8ce7 --- /dev/null +++ b/releases/nfs-volume/helm/templates/nfsv3driver.yaml @@ -0,0 +1,105 @@ +{{- if .Values.nfsv3driver.enabled }} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: nfsv3driver + labels: + app: nfsv3driver +spec: + selector: + matchLabels: + app: nfsv3driver + template: + metadata: + labels: + app: nfsv3driver + spec: + initContainers: + - name: init-rpcbind-dir + image: {{ .Values.nfsv3driver.image.repository }}:{{ default .Chart.AppVersion .Values.nfsv3driver.image.tag }} + imagePullPolicy: {{ .Values.nfsv3driver.image.imagePullPolicy }} + command: + - /bin/bash + - -ec + - | + mkdir -p /run/rpcbind + # rpcbind init script checks 'stat -c "%U"' == "root" (by name). + chown root:root /run/rpcbind + chmod 0755 /run/rpcbind + securityContext: + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: rpcbind-run + mountPath: /run/rpcbind + containers: + - name: nfsv3driver + image: {{ .Values.nfsv3driver.image.repository }}:{{ default .Chart.AppVersion .Values.nfsv3driver.image.tag }} + imagePullPolicy: {{ .Values.nfsv3driver.image.imagePullPolicy }} + # nfsv3driver.sh (v7.53.0+) calls 'service rpcbind start' which fails + # in containers because the Ubuntu init script requires /run/rpcbind + # to be owned by root *before* it is created by the service wrapper. + # We bypass nfsv3driver.sh entirely: start rpcbind directly with -w + # (warm start / no daemon-ize check) and exec the driver binary. + command: + - /bin/bash + - -c + - | + mkdir -p /run/rpcbind + chown root:root /run/rpcbind + chmod 0755 /run/rpcbind + /sbin/rpcbind -w + exec /usr/local/bin/nfsv3driver "$@" + - -- + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: 7589 + args: + - --listenAddr=$(POD_IP):7589 + - --transport=tcp-json + - --debugAddr=127.0.0.1:7689 + - --adminAddr=127.0.0.1:7590 + - --driversPath=/var/lib/rep/voldrivers + - --mountDir=/var/lib/rep/volumes/nfs + - --logLevel=info + - --timeFormat=rfc3339 + - --mapfsPath=/usr/local/bin/mapfs + {{- if .Values.nfsv3driver.resources }} + resources: + {{- toYaml .Values.nfsv3driver.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: voldrivers + mountPath: /var/lib/rep/voldrivers + mountPropagation: Bidirectional + - name: nfs-mounts + mountPath: /var/lib/rep/volumes/nfs + mountPropagation: Bidirectional + - name: rpcbind-run + mountPath: /run/rpcbind + securityContext: + privileged: true + runAsUser: 0 + runAsGroup: 0 + {{- if .Values.nfsv3driver.nodeSelector }} + nodeSelector: + {{- toYaml .Values.nfsv3driver.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.nfsv3driver.tolerations }} + tolerations: + {{- toYaml .Values.nfsv3driver.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: voldrivers + hostPath: + path: /var/lib/rep/voldrivers + - name: nfs-mounts + hostPath: + path: /var/lib/rep/volumes/nfs + - name: rpcbind-run + emptyDir: {} +{{- end }} diff --git a/releases/nfs-volume/helm/values.schema.json b/releases/nfs-volume/helm/values.schema.json new file mode 100644 index 00000000..27c3820f --- /dev/null +++ b/releases/nfs-volume/helm/values.schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "global": { + "type": "object" + }, + "nfsbroker": { + "additionalProperties": false, + "properties": { + "certificateSecret": { + "type": "string" + }, + "credhubURL": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "image": { + "additionalProperties": false, + "properties": { + "imagePullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + }, + "type": "object" + }, + "nodeSelector": { + "type": ["object", "null"] + }, + "oauthClientsSecret": { + "type": "string" + }, + "resources": { + "type": ["object", "null"] + }, + "rpcbindStartMode": { + "default": "auto", + "enum": [ + "auto", + "direct", + "service" + ], + "type": "string" + }, + "tolerations": { + "type": ["array", "null"], + "items": { + "additionalProperties": false, + "properties": { + "effect": { + "type": "string" + }, + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "effect", + "key", + "operator", + "value" + ], + "type": "object" + } + } + + }, + "type": "object" + }, + "nfsv3driver": { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "image": { + "additionalProperties": false, + "properties": { + "imagePullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + }, + "type": "object" + }, + "nodeSelector": { + "type": ["object", "null"] + }, + "resources": { + "type": ["object", "null"] + }, + "rpcbindStartMode": { + "default": "auto", + "enum": [ + "auto", + "direct", + "service" + ], + "type": "string" + }, + "tolerations": { + "type": ["array", "null"], + "items": { + "additionalProperties": false, + "properties": { + "effect": { + "type": "string" + }, + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "effect", + "key", + "operator", + "value" + ], + "type": "object" + } + } + }, + "type": "object" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/scripts/create-kind.sh b/scripts/create-kind.sh index 0181f7ae..464a1e49 100755 --- a/scripts/create-kind.sh +++ b/scripts/create-kind.sh @@ -2,6 +2,9 @@ set -e +# Auto-detect Docker or Podman +source "$(dirname "$0")/detect-runtime.sh" + configure_registry_mirror() { local cache_name=$1 local remote_url=$2 @@ -11,10 +14,10 @@ configure_registry_mirror() { echo "Configuring cache ${cache_name} on all nodes..." for node in $(kind get nodes --name cfk8s); do # Create containerd registry config directories - docker exec "$node" mkdir -p /etc/containerd/certs.d/${registry_uri} + ${CONTAINER_RUNTIME} exec "$node" mkdir -p /etc/containerd/certs.d/${registry_uri} # Configure registry to use cache as mirror (expand variables!) - cat < /etc/containerd/certs.d/${registry_uri}/hosts.toml" + cat < /etc/containerd/certs.d/${registry_uri}/hosts.toml" server = "${remote_url}" [host."http://${cache_name}:5000"] @@ -24,9 +27,18 @@ EOF done } +evaluate_progress_option() { + # Podman compose does not support --progress + if [ "${IS_PODMAN}" = "true" ]; then + echo "" + else + echo "--progress plain" + fi +} + setup_registry_caches() { - echo "Starting registry pull-through caches with docker-compose..." - docker compose -p cache -f "${script_full_path}/docker-compose-registries.yaml" --progress plain up -d + echo "Starting registry pull-through caches with ${COMPOSE_CMD}..." + ${COMPOSE_CMD} -p cache -f "${script_full_path}/docker-compose-registries.yaml" $(evaluate_progress_option) up -d configure_registry_mirror "docker-io" "https://registry-1.docker.io" "docker.io" configure_registry_mirror "ghcr-io" "https://ghcr.io" "ghcr.io" @@ -34,18 +46,42 @@ setup_registry_caches() { } setup_nfs() { - echo "Starting NFS server with docker-compose..." - docker compose -p nfs -f "${script_full_path}/docker-compose-nfs.yaml" --progress plain up -d + echo "Starting NFS server with ${COMPOSE_CMD}..." + ${COMPOSE_CMD} -p nfs -f "${script_full_path}/docker-compose-nfs.yaml" $(evaluate_progress_option) up -d } script_full_path=$(dirname "$0") +# Select kind config based on runtime +if [ "${IS_PODMAN}" = "true" ]; then + KIND_CONFIG="$script_full_path/../kind-podman.yaml" +else + KIND_CONFIG="$script_full_path/../kind.yaml" +fi + +# When running under Podman, ensure the Podman VM is ready (NFS modules, inotify, …) +if [ "${IS_PODMAN}" = "true" ]; then + echo "Podman detected – ensuring Podman VM is configured..." + "${script_full_path}/setup-podman-vm.sh" +fi + if kind get clusters | grep -q "cfk8s"; then echo "Kind cluster 'cfk8s' already exists." exit 0 fi -kind create cluster --name "cfk8s" --config="$script_full_path/../kind.yaml" +kind create cluster --name "cfk8s" --config="$KIND_CONFIG" + +# Under rootless Podman each kind node container has its own network namespace +# with the kernel default ip_unprivileged_port_start=1024. HAProxy inside +# cf-tcp-router binds :80 for its health check, so we lower the threshold to 0 +# on every node right after cluster creation. +if [ "${IS_PODMAN}" = "true" ]; then + echo "Setting ip_unprivileged_port_start=0 on all kind nodes..." + for node in $(kind get nodes --name cfk8s); do + ${CONTAINER_RUNTIME} exec "$node" sysctl -w net.ipv4.ip_unprivileged_port_start=0 + done +fi echo "Applying taints to workload nodes..." kubectl taint nodes -l cloudfoundry.org/cell=true cloudfoundry.org/cell=true:NoSchedule --overwrite || true diff --git a/scripts/delete-kind.sh b/scripts/delete-kind.sh index cd373b99..9a7cb1e2 100755 --- a/scripts/delete-kind.sh +++ b/scripts/delete-kind.sh @@ -2,11 +2,18 @@ set -e +# Auto-detect Docker or Podman +source "$(dirname "$0")/detect-runtime.sh" + script_full_path=$(dirname "$0") kind delete cluster --name cfk8s -# Remove registry cache containers using docker-compose +# Remove registry cache containers echo "Deleting registry cache containers..." -docker compose -p cache -f "${script_full_path}/docker-compose-registries.yaml" --progress plain down -docker compose -p nfs -f "${script_full_path}/docker-compose-nfs.yaml" --progress plain down +${COMPOSE_CMD} -p cache -f "${script_full_path}/docker-compose-registries.yaml" down + +if [ "${INSTALL_OPTIONAL_COMPONENTS:-true}" = "true" ]; then + echo "Stopping NFS server..." + ${COMPOSE_CMD} -p nfs -f "${script_full_path}/docker-compose-nfs.yaml" down +fi diff --git a/scripts/detect-runtime.sh b/scripts/detect-runtime.sh new file mode 100755 index 00000000..c7bbdb2b --- /dev/null +++ b/scripts/detect-runtime.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# detect-runtime.sh +# Auto-detects the available container runtime (Docker or Podman). +# Source this file – do NOT execute it directly. +# 1. Honour explicit CONTAINER_RUNTIME env override. +# 2. If 'podman' binary exists → Podman. +# 3. If 'docker' binary exists → Docker (even if it is a Podman socket shim). +# Use CONTAINER_RUNTIME=podman to force Podman when both are present. +# 4. Neither found → error. +# +# NOTE: We intentionally do NOT call 'docker info' / 'podman info' here +# because the Podman machine may not be running yet at detection time. +# The machine is started later by setup-podman-vm.sh. + +if [ -n "${CONTAINER_RUNTIME:-}" ]; then + if [ "${CONTAINER_RUNTIME}" = "podman" ]; then + IS_PODMAN="true" + else + IS_PODMAN="false" + fi +elif command -v podman &>/dev/null; then + CONTAINER_RUNTIME="podman" + IS_PODMAN="true" +elif command -v docker &>/dev/null; then + CONTAINER_RUNTIME="docker" + IS_PODMAN="false" +else + echo "ERROR: Neither 'podman' nor 'docker' binary found in PATH." >&2 + echo " Install Podman Desktop (https://podman-desktop.io) or Docker Desktop." >&2 + exit 1 +fi + +if [ "${IS_PODMAN}" = "true" ]; then + export KIND_EXPERIMENTAL_PROVIDER=podman +fi + +if [ "${IS_PODMAN}" = "true" ] && command -v podman-compose &>/dev/null; then + COMPOSE_CMD="podman-compose" +else + COMPOSE_CMD="${CONTAINER_RUNTIME} compose" +fi + +export CONTAINER_RUNTIME IS_PODMAN COMPOSE_CMD diff --git a/scripts/init.sh b/scripts/init.sh index ea2b4bcd..cbaf3a7f 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -4,8 +4,11 @@ set -euo pipefail mkdir -p temp/certs -OPENSSL="docker run --rm -v $(pwd)/temp/certs:/certs -v $(pwd)/certs/all-in-one.conf:/all-in-one.conf alpine/openssl" -SSH_KEYGEN="docker run --rm -v $(pwd)/temp/certs:/certs --entrypoint /usr/bin/ssh-keygen linuxserver/openssh-server" +# Auto-detect Docker or Podman +source "$(dirname "$0")/detect-runtime.sh" + +OPENSSL="${CONTAINER_RUNTIME} run --rm -v $(pwd)/temp/certs:/certs -v $(pwd)/certs/all-in-one.conf:/all-in-one.conf alpine/openssl" +SSH_KEYGEN="${CONTAINER_RUNTIME} run --rm -v $(pwd)/temp/certs:/certs --entrypoint /usr/bin/ssh-keygen linuxserver/openssh-server" $OPENSSL genrsa -traditional -out /certs/ca.key 4096 $OPENSSL req -x509 -key /certs/ca.key -out /certs/ca.crt -days 365 -noenc -subj "/CN=ca/O=ca" \ diff --git a/scripts/setup-podman-vm.sh b/scripts/setup-podman-vm.sh new file mode 100755 index 00000000..da9066f0 --- /dev/null +++ b/scripts/setup-podman-vm.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# setup-podman-vm.sh +# Called automatically by create-kind.sh / install when Podman is detected. +# Prepares the Podman VM (macOS/Windows) for use with kind and NFS. + +set -euo pipefail + +if [ "$(uname -s)" = "Linux" ]; then + echo "Running on Linux – Podman VM setup not needed (native Podman runtime)." + echo "Loading NFS kernel modules directly on the host..." + + sudo modprobe nfs 2>/dev/null || echo "WARN: Could not load 'nfs' module" + sudo modprobe nfsv3 2>/dev/null || echo "WARN: Could not load 'nfsv3' module" + sudo modprobe nfsd 2>/dev/null || echo "WARN: Could not load 'nfsd' module" + + if grep -qE "^nfsd[[:space:]]" /proc/modules 2>/dev/null; then + echo " nfsd – confirmed in /proc/modules ✓" + else + echo "WARN: nfsd not found in /proc/modules – NFS server may fail to start." + fi + + # On Linux, check if Podman is running rootless + if [ -z "${PODMAN_SOCK:-}" ] && [ ! -w /run/podman/podman.sock ] && [ -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" ]; then + _unpriv_port=$(cat /proc/sys/net/ipv4/ip_unprivileged_port_start 2>/dev/null || echo 1024) + if [ "${_unpriv_port}" -le 80 ]; then + echo "Rootless Podman detected – net.ipv4.ip_unprivileged_port_start=${_unpriv_port} (ports 80/443/2222 are bindable)." + else + echo "WARN: Podman is running in ROOTLESS mode." + echo "WARN: kind will FAIL to bind privileged ports (80, 443, 2222)." + echo "WARN: Fix options:" + echo "WARN: 1. Run this script with sudo: sudo -E make up" + echo "WARN: 2. Or configure unprivileged port start:" + echo "WARN: echo 'net.ipv4.ip_unprivileged_port_start=80' | sudo tee -a /etc/sysctl.conf" + echo "WARN: sudo sysctl -p" + echo "ERROR: Cannot proceed with rootless Podman – kind requires privileged ports." + fi + fi + + echo "Linux NFS setup complete." + exit 0 +fi + +PODMAN_MACHINE="${PODMAN_MACHINE:-podman-machine-default}" + +if ! podman machine list --format '{{.Name}}' 2>/dev/null | grep -qxF "${PODMAN_MACHINE}"; then + echo "Podman machine '${PODMAN_MACHINE}' not found – creating it (rootful, 4 CPU, 8 GB, 60 GB disk)..." + podman machine init \ + --cpus 4 \ + --memory 8192 \ + --disk-size 60 \ + --rootful \ + "${PODMAN_MACHINE}" +fi + +machine_state=$(podman machine inspect "${PODMAN_MACHINE}" --format '{{.State}}' 2>/dev/null || echo "unknown") +if [ "${machine_state}" = "running" ]; then + echo "Podman machine '${PODMAN_MACHINE}' is already running." +else + echo "Starting Podman machine '${PODMAN_MACHINE}' (state: ${machine_state})..." + podman machine start "${PODMAN_MACHINE}" + sleep 5 +fi + + +# Helper +_vm_init_ssh() { + local json + json=$(podman machine inspect "${PODMAN_MACHINE}" 2>/dev/null) \ + || echo "ERROR: Cannot inspect Podman machine '${PODMAN_MACHINE}'." + + VM_SSH_PORT=$(python3 -c " +import sys, json +d = json.loads('''${json}''') +m = d[0] if isinstance(d, list) else d +cfg = m.get('SSHConfig', m) +print(cfg.get('Port', 62522)) +" 2>/dev/null || echo "62522") + + VM_SSH_USER=$(python3 -c " +import sys, json +d = json.loads('''${json}''') +m = d[0] if isinstance(d, list) else d +cfg = m.get('SSHConfig', m) +print(cfg.get('RemoteUsername', 'core')) +" 2>/dev/null || echo "core") + + VM_SSH_KEY=$(python3 -c " +import sys, json +d = json.loads('''${json}''') +m = d[0] if isinstance(d, list) else d +cfg = m.get('SSHConfig', m) +print(cfg.get('IdentityPath', '')) +" 2>/dev/null || echo "") + + export VM_SSH_PORT VM_SSH_USER VM_SSH_KEY + echo "VM SSH: ${VM_SSH_USER}@127.0.0.1:${VM_SSH_PORT} key=${VM_SSH_KEY}" +} + +vm_exec() { + ssh -T \ + -o BatchMode=yes \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o echoLevel=ERROR \ + -o ConnectTimeout=15 \ + ${VM_SSH_KEY:+-i "${VM_SSH_KEY}"} \ + -p "${VM_SSH_PORT}" \ + "${VM_SSH_USER}@127.0.0.1" \ + "$1" +} + +_vm_init_ssh + +echo "Loading NFS kernel modules in the Podman VM..." +vm_exec 'sudo modprobe nfs; sudo modprobe nfsv3; sudo modprobe nfsd; exit 0' \ + || echo "WARN: One or more NFS modprobe calls returned non-zero" + +echo "Verifying NFS kernel support in the Podman VM..." + +nfsd_in_proc=$(vm_exec 'grep -cE "^nfsd[[:space:]]" /proc/modules 2>/dev/null || echo 0') +nfsd_in_proc=$(echo "${nfsd_in_proc}" | tr -d '[:space:]') + +if [ "${nfsd_in_proc:-0}" -gt 0 ]; then + echo " nfsd – confirmed in /proc/modules ✓" +elif vm_exec 'grep -q "nfsd" /proc/filesystems 2>/dev/null'; then + echo " nfsd – built into kernel, injecting synthetic /proc/modules entry..." + vm_exec 'echo "nfsd 0 0 - Live 0x0000000000000000 (builtin)" | sudo tee -a /proc/modules > /dev/null' + echo " nfsd – synthetic entry injected ✓" +else + vm_exec 'grep -iE "nfs" /proc/modules 2>/dev/null || echo " (none)"' + vm_exec 'find /lib/modules/$(uname -r) -name "*nfs*" 2>/dev/null | head -20 || echo " (none found)"' + echo "ERROR: The 'nfsd' kernel module is NOT available in the Podman VM. + Fix: recreate the Podman machine: + podman machine stop ${PODMAN_MACHINE} + podman machine rm ${PODMAN_MACHINE} + make up" +fi + +for mod in nfs nfsv3; do + if vm_exec "grep -qE '^${mod}[[:space:]]' /proc/modules 2>/dev/null || test -d /sys/module/${mod} 2>/dev/null"; then + echo " ${mod} – OK ✓" + else + echo "WARN: ${mod} – not confirmed" + fi +done + +vm_exec 'printf "nfs\nnfsv3\nnfsd\n" | sudo tee /etc/modules-load.d/nfs-kind.conf > /dev/null' +vm_exec 'printf "[Unit]\nDescription=Load NFS kernel modules for CF/kind\nDefaultDependencies=no\nAfter=systemd-modules-load.service\nBefore=network-pre.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/sh -c \"modprobe nfs; modprobe nfsv3; modprobe nfsd; exit 0\"\n\n[Install]\nWantedBy=multi-user.target\n" | sudo tee /etc/systemd/system/nfs-modules-load.service > /dev/null' +vm_exec 'sudo systemctl daemon-reload && sudo systemctl enable nfs-modules-load.service' +echo "NFS module systemd service installed and enabled." + +echo "Configuring inotify limits in the Podman VM..." +vm_exec 'printf "fs.inotify.max_user_instances = 8192\nfs.inotify.max_user_watches = 524288\n" | sudo tee /etc/sysctl.d/99-cf-kind.conf > /dev/null && sudo sysctl --system > /dev/null' + +echo "Podman VM setup complete." +