diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 43de3dcf..90ac4dc4 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -44,4 +44,7 @@ Local dev builds directly from dev stage, no separate initialization needed. **Container won't build** → Ensure Docker is running **Slow rebuild** → Normal, only rebuilds when tools/deps change -See [`docs/COMPLETE_SOLUTION.md`](../docs/COMPLETE_SOLUTION.md) for details. \ No newline at end of file +Kind/DOOD failure findings + DinD migration plan: +[`KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md`](./KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md) + +See [`docs/COMPLETE_SOLUTION.md`](../docs/COMPLETE_SOLUTION.md) for details. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 570704a6..7ada7341 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -66,6 +66,7 @@ "mounts": [ "source=ghconfig,target=/home/vscode/.config/gh,type=volume", "source=${localEnv:HOME}${localEnv:USERPROFILE}/.gitconfig,target=/home/vscode/.gitconfig-host,type=bind,readonly,consistency=cached", + "source=devcontainer-ssh,target=/home/vscode/.ssh,type=volume", "source=gomodcache,target=/go/pkg/mod,type=volume", "source=gobuildcache,target=/home/vscode/.cache/go-build,type=volume", "source=codexconfig,target=/home/vscode/.codex,type=volume" diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 3aa5c29e..952768db 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -6,6 +6,11 @@ log() { echo "[post-create] $*" } +fail() { + echo "[post-create] ERROR: $*" >&2 + exit 1 +} + # Resolve workspace path in a way that works both inside and outside # VS Code-specific shell variable injection. workspace_dir="${1:-${containerWorkspaceFolder:-${WORKSPACE_FOLDER:-$(pwd)}}}" @@ -23,25 +28,42 @@ if [ -f /home/vscode/.gitconfig-host ]; then fi fi -# Ensure SSH commit signing works inside the container -log "Configuring SSH commit signing (Git) inside container" +# Require basic Git identity information. +git_name="$(git config --global --includes --get user.name || true)" +git_email="$(git config --global --includes --get user.email || true)" +if [ -z "${git_name}" ] || [ -z "${git_email}" ]; then + fail "Missing Git identity. Configure both user.name and user.email in your host ~/.gitconfig." +fi + +# Respect existing signing settings (OpenPGP/SSH). Fallback to SSH signing only when missing. +if git config --global --includes --get commit.gpgsign >/dev/null 2>&1 \ + || git config --global --includes --get gpg.format >/dev/null 2>&1 \ + || git config --global --includes --get user.signingkey >/dev/null 2>&1; then + log "Detected existing Git signing configuration; leaving signing settings unchanged" +else + log "No Git signing configuration detected; configuring SSH signing fallback" + + mkdir -p /home/vscode/.ssh -git config --global gpg.format ssh -git config --global commit.gpgsign true + if ! ssh-add -L >/tmp/ssh-agent-keys.out 2>/dev/null || ! grep -qE '^ssh-' /tmp/ssh-agent-keys.out; then + log "No SSH keys found in agent; creating fallback signing key" + if [ ! -f /home/vscode/.ssh/signing_key ]; then + ssh-keygen -t ed25519 -f /home/vscode/.ssh/signing_key -N "" -C "${git_email}" >/dev/null + fi + cp /home/vscode/.ssh/signing_key.pub /tmp/ssh-agent-keys.out + fi -mkdir -p /home/vscode/.config/git /home/vscode/.ssh + first_pubkey="$(head -n 1 /tmp/ssh-agent-keys.out)" + printf "%s %s\n" "${git_email}" "${first_pubkey}" > /home/vscode/.ssh/allowed_signers + printf "%s\n" "${first_pubkey}" > /home/vscode/.ssh/signing_key.pub -# Create allowed signers file if missing -if [ ! -s /home/vscode/.config/git/allowed_signers ]; then - ssh-add -L | head -n 1 | awk '{print "simonkoudijs@gmail.com "$0}' > /home/vscode/.config/git/allowed_signers -fi -git config --global gpg.ssh.allowedSignersFile /home/vscode/.config/git/allowed_signers + git config --global gpg.format ssh + git config --global commit.gpgsign true + git config --global gpg.ssh.allowedSignersFile /home/vscode/.ssh/allowed_signers + git config --global user.signingkey /home/vscode/.ssh/signing_key.pub -# Create a stable public key file to point Git at -if [ ! -s /home/vscode/.ssh/signing_key.pub ]; then - ssh-add -L | head -n 1 > /home/vscode/.ssh/signing_key.pub + rm -f /tmp/ssh-agent-keys.out fi -git config --global user.signingkey /home/vscode/.ssh/signing_key.pub # Ensure Go-related caches exist and are writable by vscode diff --git a/Makefile b/Makefile index fade3a53..2246bca8 100644 --- a/Makefile +++ b/Makefile @@ -307,8 +307,8 @@ check-cert-manager: wait-cert-manager ## Explicit readiness check for cert-manag @echo "✅ cert-manager check passed" ## Smoke test: install from local Helm chart and validate first quickstart flow -.PHONY: test-e2e-install-quickstart -test-e2e-install-quickstart: ## Install + quickstart smoke with E2E_INSTALL_MODE=helm|manifest +.PHONY: test-e2e-quickstart +test-e2e-quickstart: ## Install + quickstart smoke with E2E_INSTALL_MODE=helm|manifest @MODE="$(E2E_INSTALL_MODE)"; \ if [ "$$MODE" != "helm" ] && [ "$$MODE" != "manifest" ]; then \ echo "❌ Invalid E2E_INSTALL_MODE='$$MODE' (expected: helm|manifest)"; \ @@ -327,33 +327,20 @@ test-e2e-install-quickstart: ## Install + quickstart smoke with E2E_INSTALL_MODE KIND_CLUSTER=$(KIND_CLUSTER) PROJECT_IMAGE="$$PROJECT_IMAGE_VALUE" $(MAKE) setup-cluster setup-e2e check-cert-manager e2e-build-load-image; \ fi; \ echo "ℹ️ Running install quickstart smoke mode: $$MODE"; \ - PROJECT_IMAGE="$$PROJECT_IMAGE_VALUE" bash test/e2e/scripts/install-smoke-quickstart.sh "$$MODE"; \ + PROJECT_IMAGE="$$PROJECT_IMAGE_VALUE" bash test/e2e/scripts/run-quickstart.sh "$$MODE"; \ ## Smoke test: install from local Helm chart and validate first quickstart flow -.PHONY: test-e2e-install-quickstart-helm -test-e2e-install-quickstart-helm: - @$(MAKE) test-e2e-install-quickstart E2E_INSTALL_MODE=helm PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" +.PHONY: test-e2e-quickstart-helm +test-e2e-quickstart-helm: + @$(MAKE) test-e2e-quickstart E2E_INSTALL_MODE=helm PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" ## Smoke test: install from generated dist/install.yaml and validate first quickstart flow -.PHONY: test-e2e-install-quickstart-manifest -test-e2e-install-quickstart-manifest: +.PHONY: test-e2e-quickstart-manifest +test-e2e-quickstart-manifest: @if [ -n "$(PROJECT_IMAGE)" ]; then \ - echo "ℹ️ test-e2e-install-quickstart-manifest using existing artifact (PROJECT_IMAGE set, CI/pre-built path)"; \ + echo "ℹ️ test-e2e-quickstart-manifest using existing artifact (PROJECT_IMAGE set, CI/pre-built path)"; \ else \ - echo "ℹ️ test-e2e-install-quickstart-manifest local path: regenerating dist/install.yaml via build-installer"; \ + echo "ℹ️ test-e2e-quickstart-manifest local path: regenerating dist/install.yaml via build-installer"; \ $(MAKE) build-installer; \ fi - @$(MAKE) test-e2e-install-quickstart E2E_INSTALL_MODE=manifest PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" - -## Backward-compatible aliases (kept for one release cycle) -.PHONY: test-e2e-install -test-e2e-install: ## Alias for test-e2e-install-quickstart - @$(MAKE) test-e2e-install-quickstart E2E_INSTALL_MODE="$(E2E_INSTALL_MODE)" PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" - -.PHONY: test-e2e-install-helm -test-e2e-install-helm: ## Alias for test-e2e-install-quickstart-helm - @$(MAKE) test-e2e-install-quickstart-helm PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" - -.PHONY: test-e2e-install-manifest -test-e2e-install-manifest: ## Alias for test-e2e-install-quickstart-manifest - @$(MAKE) test-e2e-install-quickstart-manifest PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" + @$(MAKE) test-e2e-quickstart E2E_INSTALL_MODE=manifest PROJECT_IMAGE="$(PROJECT_IMAGE)" KIND_CLUSTER="$(KIND_CLUSTER)" diff --git a/README.md b/README.md index d90604b9..3de31b48 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,13 @@ spec: path: live-cluster encryption: provider: sops + age: + enabled: true + recipients: + extractFromSecret: true + generateWhenMissing: true secretRef: name: sops-age-key - generateWhenMissing: true --- apiVersion: configbutler.ai/v1alpha1 kind: WatchRule @@ -145,8 +149,9 @@ spec: EOF ``` -When `generateWhenMissing: true` is enabled, GitOps Reverser can create the encryption key Secret automatically. -Back up the generated `SOPS_AGE_KEY` immediately and securely. +When `age.recipients.generateWhenMissing: true` is enabled, GitOps Reverser can create the encryption key Secret +automatically. +Back up the generated `*.agekey` entry immediately and securely. If you lose that private key, existing encrypted `*.sops.yaml` files are unrecoverable. After confirming backup, remove the warning annotation: `kubectl annotate secret sops-age-key -n default configbutler.ai/backup-warning-` @@ -178,8 +183,9 @@ Avoid infinite loops: Do not point GitOps (Argo CD/Flux) and GitOps Reverser at - The container image ships with `/usr/local/bin/sops`. - Per-path `.sops.yaml` files are bootstrapped in the target repo for SOPS-based secret encryption. - If Secret encryption fails, Secret writes are rejected (no plaintext fallback). - - `GitTarget.spec.encryption.generateWhenMissing: true` can auto-generate the referenced encryption Secret when it does not exist. - - Generated Secret data contains one `SOPS_AGE_KEY` (`AGE-SECRET-KEY-...`). + - `GitTarget.spec.encryption.age.recipients.generateWhenMissing: true` can auto-generate a date-based `*.agekey` + in the referenced encryption Secret when no `*.agekey` entry exists. + - Generated Secret data contains one `.agekey` (`AGE-SECRET-KEY-...`). - Generated Secret annotation `configbutler.ai/age-recipient` stores the public age recipient. - Generated Secret annotation `configbutler.ai/backup-warning: REMOVE_AFTER_BACKUP` is set by default. - While `configbutler.ai/backup-warning` remains, gitops-reverser logs a recurring high-visibility backup warning during periodic reconciliation. diff --git a/api/v1alpha1/gitprovider_types.go b/api/v1alpha1/gitprovider_types.go index 0adb11cd..ba4fbea7 100644 --- a/api/v1alpha1/gitprovider_types.go +++ b/api/v1alpha1/gitprovider_types.go @@ -65,10 +65,38 @@ type EncryptionSpec struct { Provider string `json:"provider"` // SecretRef references namespace-local Secret data used by the encryption provider. - SecretRef LocalSecretReference `json:"secretRef"` + // +optional + SecretRef LocalSecretReference `json:"secretRef,omitempty"` + + // Age configures age-specific encryption behavior for SOPS. + // +optional + Age *AgeEncryptionSpec `json:"age,omitempty"` +} + +// AgeEncryptionSpec configures age recipient resolution behavior. +type AgeEncryptionSpec struct { + // Enabled toggles age-based recipient resolution and bootstrap behavior. + // +optional + // +kubebuilder:default=false + Enabled bool `json:"enabled,omitempty"` + + // Recipients defines how recipients are resolved. + // +optional + Recipients AgeRecipientsSpec `json:"recipients,omitempty"` +} + +// AgeRecipientsSpec defines age recipient source and key generation behavior. +type AgeRecipientsSpec struct { + // PublicKeys is a static list of age recipients (age1...). + // +optional + PublicKeys []string `json:"publicKeys,omitempty"` + + // ExtractFromSecret derives recipients from all *.agekey entries in encryption.secretRef. + // +optional + // +kubebuilder:default=false + ExtractFromSecret bool `json:"extractFromSecret,omitempty"` - // GenerateWhenMissing auto-creates the referenced Secret when it does not exist. - // The generated Secret contains one age private key in SOPS_AGE_KEY. + // GenerateWhenMissing creates a date-named *.agekey entry in encryption.secretRef when no *.agekey exists. // +optional // +kubebuilder:default=false GenerateWhenMissing bool `json:"generateWhenMissing,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2251c4e7..2708a6f1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,42 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgeEncryptionSpec) DeepCopyInto(out *AgeEncryptionSpec) { + *out = *in + in.Recipients.DeepCopyInto(&out.Recipients) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgeEncryptionSpec. +func (in *AgeEncryptionSpec) DeepCopy() *AgeEncryptionSpec { + if in == nil { + return nil + } + out := new(AgeEncryptionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgeRecipientsSpec) DeepCopyInto(out *AgeRecipientsSpec) { + *out = *in + if in.PublicKeys != nil { + in, out := &in.PublicKeys, &out.PublicKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgeRecipientsSpec. +func (in *AgeRecipientsSpec) DeepCopy() *AgeRecipientsSpec { + if in == nil { + return nil + } + out := new(AgeRecipientsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterResourceRule) DeepCopyInto(out *ClusterResourceRule) { *out = *in @@ -170,6 +206,11 @@ func (in *ClusterWatchRuleStatus) DeepCopy() *ClusterWatchRuleStatus { func (in *EncryptionSpec) DeepCopyInto(out *EncryptionSpec) { *out = *in out.SecretRef = in.SecretRef + if in.Age != nil { + in, out := &in.Age, &out.Age + *out = new(AgeEncryptionSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionSpec. @@ -409,7 +450,7 @@ func (in *GitTargetSpec) DeepCopyInto(out *GitTargetSpec) { if in.Encryption != nil { in, out := &in.Encryption, &out.Encryption *out = new(EncryptionSpec) - **out = **in + (*in).DeepCopyInto(*out) } } diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index 21023ceb..e2121074 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -31,11 +31,11 @@ kubectl apply -f config/samples/quickstart-watchrule.yaml ``` The quickstart target uses: -- `spec.path` (not `baseFolder`) +- `spec.path` - `spec.encryption.provider: sops` -- `spec.encryption.generateWhenMissing: true` +- `spec.encryption.age.recipients.generateWhenMissing: true` -When the encryption Secret is auto-generated, back up `SOPS_AGE_KEY` immediately. +When the encryption Secret is auto-generated, back up the generated `*.agekey` entry immediately. If you lose it, existing encrypted `*.sops.yaml` files are unrecoverable. After backup verification, remove warning annotation: diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 91139858..4d9dde35 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -48,12 +48,37 @@ spec: description: Encryption defines encryption settings for Secret resource writes. properties: - generateWhenMissing: - default: false - description: |- - GenerateWhenMissing auto-creates the referenced Secret when it does not exist. - The generated Secret contains one age private key in SOPS_AGE_KEY. - type: boolean + age: + description: Age configures age-specific encryption behavior for + SOPS. + properties: + enabled: + default: false + description: Enabled toggles age-based recipient resolution + and bootstrap behavior. + type: boolean + recipients: + description: Recipients defines how recipients are resolved. + properties: + extractFromSecret: + default: false + description: ExtractFromSecret derives recipients from + all *.agekey entries in encryption.secretRef. + type: boolean + generateWhenMissing: + default: false + description: GenerateWhenMissing creates a date-named + *.agekey entry in encryption.secretRef when no *.agekey + exists. + type: boolean + publicKeys: + description: PublicKeys is a static list of age recipients + (age1...). + items: + type: string + type: array + type: object + type: object provider: default: sops description: Provider selects the encryption provider. @@ -83,7 +108,6 @@ spec: type: object required: - provider - - secretRef type: object path: description: Path within the repository to write resources to. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 512be9a1..f1fec8b7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -20,6 +20,8 @@ rules: - create - get - list + - patch + - update - watch - apiGroups: - '*' diff --git a/config/samples/quickstart-gittarget.yaml b/config/samples/quickstart-gittarget.yaml index 16eba113..43429795 100644 --- a/config/samples/quickstart-gittarget.yaml +++ b/config/samples/quickstart-gittarget.yaml @@ -10,6 +10,10 @@ spec: path: live-cluster encryption: provider: sops + age: + enabled: true + recipients: + extractFromSecret: true + generateWhenMissing: true secretRef: name: sops-age-key - generateWhenMissing: true diff --git a/docs/SOPS_AGE_GUIDE.md b/docs/SOPS_AGE_GUIDE.md index 50b2b183..089062b7 100644 --- a/docs/SOPS_AGE_GUIDE.md +++ b/docs/SOPS_AGE_GUIDE.md @@ -11,7 +11,7 @@ This is a practical reference for how `sops` and `age` fit together in this repo For this repo: - `.sops.yaml` contains recipient rules (public info). -- Private key material is provided at runtime via Kubernetes Secret env entries (for example `SOPS_AGE_KEY`). +- Private key material for Flux-compatible secrets is stored under `*.agekey` keys (for example `identity.agekey`). - Secret manifests written to Git are stored as `*.sops.yaml`. ## Important assets @@ -67,6 +67,7 @@ Notes: - `path_regex` controls which files are encrypted with this rule. - `encrypted_regex` keeps Kubernetes Secret payload fields encrypted. - You can add multiple recipients under `age:`. +- SOPS encrypts to all listed recipients; it does not select only one. ## Encrypt/decrypt with sops @@ -93,12 +94,12 @@ sops --decrypt secret.sops.yaml ## Key material in Kubernetes -In this project, encryption env vars are sourced from `GitTarget.spec.encryption.secretRef`. +In this project, age identities are sourced from `GitTarget.spec.encryption.secretRef` using `*.agekey` entries. Create the runtime secret like this (namespace must match the `GitTarget` namespace): ```bash kubectl -n sut create secret generic sops-age-key \ - --from-literal=SOPS_AGE_KEY="$(grep '^AGE-SECRET-KEY-' age-key.txt)" + --from-literal=identity.agekey="$(cat age-key.txt)" ``` Then reference it from `GitTarget.spec.encryption.secretRef.name`. @@ -109,9 +110,13 @@ Optional bootstrap mode: spec: encryption: provider: sops + age: + enabled: true + recipients: + extractFromSecret: true + generateWhenMissing: true secretRef: name: sops-age-key - generateWhenMissing: true ``` When enabled and the secret is missing, gitops-reverser generates one age key and @@ -154,3 +159,8 @@ The bootstrap template currently contains a static recipient in Moved to a dedicated document: [`docs/SOPS_GENERATE_WHEN_MISSING_PLAN.md`](docs/SOPS_GENERATE_WHEN_MISSING_PLAN.md) +## Design plan: Flux-aligned key handling + +Implications and proposal for optional secretless encryption, Flux-compatible +`*.agekey` generation, and multi-key support: +[`docs/SOPS_FLUX_AGE_KEY_ALIGNMENT_PLAN.md`](docs/SOPS_FLUX_AGE_KEY_ALIGNMENT_PLAN.md) diff --git a/docs/SOPS_AGE_KEY_TOOLING_STRATEGY.md b/docs/SOPS_AGE_KEY_TOOLING_STRATEGY.md new file mode 100644 index 00000000..3b7fff7a --- /dev/null +++ b/docs/SOPS_AGE_KEY_TOOLING_STRATEGY.md @@ -0,0 +1,196 @@ +# SOPS Age Key Tooling Strategy (CLI vs Operator) + +## Purpose + +Define how Age key lifecycle tooling should integrate with GitOps Reverser and Flux, with a focus on: +- first-time user experience, +- operational safety, +- clear component boundaries. + +## Problem statement + +GitOps Reverser should stay focused on: +- capturing Kubernetes changes, +- encrypting with resolved recipients, +- bootstrapping `.sops.yaml` recipient policy. + +Private-key lifecycle workflows are separate concerns: +- generating and storing Age identities, +- rotating identities, +- validating whether old recipients are still in use. + +## Design principles + +1. Keep key management explicit and auditable. +2. Keep controller responsibilities small. +3. Prefer least-privilege RBAC for key operations. +4. Make onboarding one-command simple for first-time users. +5. Keep Flux compatibility first-class (`*.agekey`). + +## Option A: Companion CLI + +### Model + +A standalone CLI (or `kubectl` plugin) performs explicit key lifecycle actions on demand. + +### Pros + +- Best first-time UX (guided init flows). +- Least controller complexity. +- Lower persistent RBAC footprint than always-on automation. +- Easy to use in local dev, CI, and GitOps pipelines. +- Works for Flux-only users (no GitOps Reverser dependency). + +### Cons + +- Requires users/CI to run commands intentionally. +- Rotation can be forgotten without policy reminders. +- No continuous reconciliation loop for drift correction. + +### Best-fit use cases + +- Initial setup. +- Periodic manual rotations. +- Platform teams that prefer explicit, reviewable key changes. + +## Option B: Key Lifecycle Operator + +### Model + +A dedicated Kubernetes operator reconciles desired key lifecycle state continuously. + +### Pros + +- Strong automation for recurring tasks. +- Can enforce org policy (age limits, rotation windows, stale-key alerts). +- Reduces manual operational burden at larger scale. + +### Cons + +- Higher implementation and maintenance complexity. +- Higher security risk surface (long-lived RBAC over secrets and repos). +- Harder failure modes (partial rewraps, broad commit churn, rollback complexity). +- Can blur boundaries between config reconciliation and key lifecycle governance. + +### Best-fit use cases + +- Large multi-team environments with strict compliance automation requirements. +- Mature installations after CLI workflows are proven. + +## Recommendation + +Start with a companion CLI, not an operator. + +Rationale: +- Faster path to high-quality onboarding. +- Cleaner architecture boundary for GitOps Reverser. +- Lower risk while SOPS support is pre-release. + +Re-evaluate operator path later if policy automation demand becomes strong. + +## Existing work and reusable tools + +### `age` CLI + +Useful primitives: +- `age-keygen` for identity generation. +- `age-keygen -y` for recipient derivation from private key files. + +Why it matters: +- battle-tested key format and generation workflow, +- no need to reinvent cryptographic key generation. + +### `sops` CLI + +Useful primitives: +- encryption/decryption workflows, +- `sops updatekeys` for rewrapping existing encrypted files to new recipient sets. + +Why it matters: +- key update operations can stay explicit and operator-driven at first. + +### Flux SOPS integration + +Useful contract: +- Flux decryption Secret detection via `*.agekey` entries. + +Why it matters: +- shared format between GitOps Reverser and Flux avoids duplicated key material models. + +References: +- https://fluxcd.io/flux/components/kustomize/kustomizations/ +- https://getsops.io/docs/ +- https://github.com/FiloSottile/age + +## Interface contract with GitOps Reverser + +The helper tool should integrate through stable data contracts, not internal controller hooks. + +### Inputs expected by GitOps Reverser + +1. `GitTarget.spec.encryption.provider` (for this design: `sops`). +2. `GitTarget.spec.encryption.age.enabled`. +3. `GitTarget.spec.encryption.age.recipients.publicKeys`. +4. `GitTarget.spec.encryption.age.recipients.extractFromSecret`. +5. `GitTarget.spec.encryption.age.recipients.generateWhenMissing`. +6. `GitTarget.spec.encryption.secretRef` pointing to Secret entries ending with `.agekey`. + +### Outputs produced by helper tool + +1. Kubernetes Secret (Flux-compatible): +- `type: Opaque` +- `stringData.identity.agekey: AGE-SECRET-KEY-...` + +2. Recipient list: +- derived `age1...` values for `age.recipients.publicKeys`. + +3. Optional patch/apply action: +- update target `GitTarget` encryption fields under `spec.encryption`. + +### Behavior boundary + +- GitOps Reverser consumes recipients and bootstraps `.sops.yaml`. +- Helper tool owns key generation and lifecycle workflows. +- Rewrap (`sops updatekeys`) stays in helper/manual operations, not controller reconcile loops. + +## First-time user experience design + +### Goal + +Enable safe setup in one guided flow with minimal crypto knowledge. + +### Suggested MVP flow + +1. User chooses target boundary (`namespace/name` of `GitTarget`). +2. Tool generates one Age identity for that boundary. +3. Tool creates Secret with `identity.agekey`. +4. Tool derives and prints recipient (`age1...`). +5. Tool patches/applies `spec.encryption` (provider + age config + secretRef). +6. Tool runs validation checks and prints next steps. + +### Why this improves onboarding + +- Avoids manual Secret formatting mistakes. +- Avoids confusion around recipient vs private key. +- Produces Flux-compatible artifacts from day one. +- Keeps an explicit audit trail of key lifecycle actions. + +## Key usage counters (future helper capability) + +### What to count + +Count recipient (`age1...`) references in SOPS metadata across `*.sops.yaml` files. + +Important: +- count by recipient public key, +- not by Secret entry name (`identity.agekey`). + +### Boundary rules + +- default boundary: one keypair per GitTarget, +- count only repos/paths owned by that boundary. + +### Retirement signal + +A recipient is retirement-candidate only when usage count is zero in the chosen boundary. +Initially treat this as report-only guidance. diff --git a/docs/SOPS_FLUX_AGE_KEY_ALIGNMENT_PLAN.md b/docs/SOPS_FLUX_AGE_KEY_ALIGNMENT_PLAN.md new file mode 100644 index 00000000..a4c7b9a9 --- /dev/null +++ b/docs/SOPS_FLUX_AGE_KEY_ALIGNMENT_PLAN.md @@ -0,0 +1,258 @@ +# SOPS Age Key Alignment Plan (GitOps Reverser + Flux) + +## Goal + +Define a minimal and stable SOPS/Age configuration model that: +- keeps `encryption.provider` open for future providers, +- keeps Age-specific controls under `encryption.age`, +- supports Flux-compatible secret handling. + +## Target configuration model + +```yaml +apiVersion: configbutler.ai/v1alpha1 +kind: GitTarget +metadata: + name: byok + namespace: default +spec: + providerRef: + name: my-provider + branch: main + path: clusters/prod + encryption: + provider: sops + age: + enabled: true + recipients: + publicKeys: [] + extractFromSecret: true + generateWhenMissing: true # Requires right to create/update the referenced Secret + secretRef: + name: flux-age +``` + +## Why this shape + +- `encryption.provider` stays generic for future encryption methods. +- bootstrap policy file is implicit and fixed to `.sops.yaml` for now. +- `age.enabled` explicitly gates Age behavior. +- `secretRef` remains at `encryption` level so future providers can reuse it if needed. +- `age.recipients.*` keeps recipient source and bootstrap behavior explicit. + +## Behavior contract + +1. Recipient resolution: +- Start from `encryption.age.recipients.publicKeys`. +- If `extractFromSecret=true`, derive recipients from all `*.agekey` entries in `encryption.secretRef`. +- Union + deduplicate recipients. + +2. Missing key behavior: +- If `generateWhenMissing=true` and Secret does not exist, create `encryption.secretRef` with one `identity.agekey`. +- If `generateWhenMissing=true` and Secret exists but has no `*.agekey` entry, add one `identity.agekey`. +- If Secret already has at least one `*.agekey`, do not generate an additional key. + +3. Bootstrap behavior: +- Use `.sops.yaml` as the implicit bootstrap target. +- SOPS encrypts to all listed recipients, not one selected recipient. + +4. Validation: +- If `encryption.provider=sops` and `age.enabled=true`, recipient set must resolve non-empty. +- If `extractFromSecret=true` or `generateWhenMissing=true`, `encryption.secretRef.name` is required. + +5. Security: +- Never log private key material. +- Never write private keys to Git. + +## Key update policy (for now) + +Automatic rewrap/rotation remains out of scope for controller logic. + +For now: +1. Update recipient policy. +2. Run `sops updatekeys` manually. +3. Commit rewrapped files explicitly. + +## Concrete implementation actions (source-code mapped) + +This section maps the target design to concrete code changes in the current codebase. + +### 1. Extend API model in `GitTarget` encryption spec + +Current state: +- `EncryptionSpec` exists in `api/v1alpha1/gitprovider_types.go` and currently has + `provider`, `secretRef`, and top-level `generateWhenMissing`. + +Actions: +1. Add `spec.encryption.age.enabled`. +2. Add `spec.encryption.age.recipients.publicKeys`. +3. Add `spec.encryption.age.recipients.extractFromSecret`. +4. Add `spec.encryption.age.recipients.generateWhenMissing`. +5. Keep `spec.encryption.secretRef` at encryption level. +6. Remove/deprecate top-level `spec.encryption.generateWhenMissing` in favor of recipient-scoped flag. +7. Regenerate API artifacts (`make generate`, `make manifests`) when implementing. + +Notes: +- Because SOPS support is not released yet, implement this as a clean shape update (no migration path required). + +### 2. Validation and defaults + +Current state: +- Validation for uniqueness exists in `internal/webhook/gittarget_validator.go`. +- Encryption field validation is mostly runtime/controller-side. + +Actions: +1. Add schema/webhook validation for: +- If `age.enabled=true`, recipient resolution inputs must be configured (`publicKeys` and/or `extractFromSecret`). +- If `extractFromSecret=true` or `generateWhenMissing=true`, `encryption.secretRef.name` is required. +- If `generateWhenMissing=true`, `extractFromSecret` must also be `true`. +2. Keep failure reasons actionable so users can fix spec quickly. + +### 3. Update encryption secret management to Flux key format + +Current state: +- `internal/controller/gittarget_controller.go` (`ensureEncryptionSecret`) creates Secret data key `SOPS_AGE_KEY`. +- RBAC marker currently allows Secret `create` but not `update/patch`. + +Actions: +1. Change generated Secret content to Flux-compatible `*.agekey` entries (default `identity.agekey`). +2. Keep generation behavior single-key: +- Missing Secret: create with one `identity.agekey`. +- Existing Secret with no `*.agekey`: add one `identity.agekey`. +- Existing Secret with `*.agekey`: do nothing. +3. Update controller RBAC marker to allow Secret updates (needed when adding `identity.agekey` to an existing Secret). +4. Keep backup-warning annotation behavior. + +### 4. Refactor encryption resolution for hybrid mode + +Current state: +- `internal/git/encryption.go` expects `secretRef` and resolves env vars from Secret data. +- Runtime assumes `SOPS_AGE_KEY` is present for age recipient derivation. + +Actions: +1. Split recipient resolution from private-key/env resolution: +- Recipient set = `publicKeys` union extracted recipients from `*.agekey` (if enabled). +2. Make secret optional for encryption-only mode: +- If `extractFromSecret=false` and `generateWhenMissing=false`, allow public-key-only path. +3. Maintain deterministic recipient output (sort + deduplicate). +4. Keep strict parsing for `*.agekey` values and never log private key material. + +### 5. Bootstrap `.sops.yaml` using resolved recipients + +Current state: +- `internal/git/branch_worker.go` derives one recipient from one key. +- `internal/git/bootstrapped_repo_template.go` template data has `AgeRecipient string`. +- `internal/git/bootstrapped-repo-template/.sops.yaml` renders one age recipient. + +Actions: +1. Change template data to `AgeRecipients []string`. +2. Render all resolved recipients in `.sops.yaml` (`key_groups[].age[]`). +3. Keep bootstrap target file fixed to `.sops.yaml` for now. +4. Ensure bootstrap still skips `.sops.yaml` creation when encryption/age is disabled. + +### 6. Tests to add/update during implementation + +Actions: +1. Update API/validation tests: +- `internal/webhook/gittarget_validator_test.go` +2. Update controller tests for generation and existing-secret update behavior: +- `internal/controller/gittarget_controller_test.go` +3. Update recipient/env resolution and parsing tests: +- `internal/git/encryption_test.go` +4. Update bootstrap rendering tests for multi-recipient output: +- `internal/git/branch_worker_test.go` +- `internal/git/bootstrapped_repo_template.go` companion tests (if added) +5. Update docs/examples after code lands: +- `docs/SOPS_AGE_GUIDE.md` +- `internal/git/bootstrapped-repo-template/README.md` + +## Configuration examples + +### Example A: Reverser-only (public keys only) + +```yaml +apiVersion: configbutler.ai/v1alpha1 +kind: GitTarget +metadata: + name: encrypt-only + namespace: default +spec: + providerRef: + name: my-provider + branch: main + path: clusters/dev + encryption: + provider: sops + age: + enabled: true + recipients: + publicKeys: + - age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq7k8m6 + extractFromSecret: false + generateWhenMissing: false +``` + +### Example B: Hybrid with secret extraction + generation + +```yaml +apiVersion: configbutler.ai/v1alpha1 +kind: GitTarget +metadata: + name: hybrid-autogen + namespace: default +spec: + providerRef: + name: my-provider + branch: main + path: clusters/dev + encryption: + provider: sops + age: + enabled: true + recipients: + publicKeys: + - age1yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyv0r2a + extractFromSecret: true + generateWhenMissing: true + secretRef: + name: sops-age-key +``` + +### Example C: BYOK Flux secret, no generation + +```yaml +apiVersion: configbutler.ai/v1alpha1 +kind: GitTarget +metadata: + name: byok + namespace: default +spec: + providerRef: + name: my-provider + branch: main + path: clusters/prod + encryption: + provider: sops + age: + enabled: true + recipients: + publicKeys: [] + extractFromSecret: true + generateWhenMissing: false + secretRef: + name: flux-age +``` + +Example Secret shape: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: flux-age + namespace: default +type: Opaque +stringData: + identity.agekey: | + AGE-SECRET-KEY-1XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` diff --git a/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md b/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md new file mode 100644 index 00000000..a689f581 --- /dev/null +++ b/docs/ci/KIND_DOOD_FINDINGS_AND_DIND_TRANSITION_PLAN.md @@ -0,0 +1,133 @@ +# Kind on Docker-Outside-of-Docker: Findings and DinD Transition Plan + +Date: February 21, 2026 + +## Scope + +This document records reproducible findings from the current devcontainer setup and proposes a clean transition plan +from Docker-outside-of-Docker (DOOD, host socket mount) to Docker-in-Docker (DinD) for local Kind-based e2e flows. + +## Current Setup + +- `.devcontainer/devcontainer.json` uses: + - `ghcr.io/devcontainers/features/docker-outside-of-docker:1` + - `HOST_PROJECT_PATH=${localWorkspaceFolder}` +- Kind version in container: `v0.31.0` +- Docker engine in host environment: `29.2.1` + +## Reproduced Behavior + +All checks below were executed from inside the devcontainer. + +1. Docker daemon is reachable: + - `docker info` succeeds +2. Plain Kind single-node cluster fails: + - `kind create cluster --name repro-one --image kindest/node:v1.35.0 --wait 2m` + - Failure: `failed to remove control plane taint ... connect: connection refused` +3. Plain Kind two-node cluster fails: + - `kind create cluster --name repro-two --config /tmp/repro-two-node.yaml --wait 2m` + - Failure: `Installing CNI ... failed to download openapi ... connect: connection refused` +4. Project cluster setup fails the same way: + - `make setup-cluster` + - Failure: `Installing CNI ... failed to download openapi: unknown` + +Conclusion: this is reproducible outside repository-specific logic and is not limited to `test-e2e-quickstart-helm`. + +## Key Finding + +The failure reproduces with plain `kind create cluster` in this DOOD model. That points to a runtime/interaction issue +between Kind and Docker socket usage from inside a container, not to Helm chart logic or project e2e scripts. + +This matches known upstream issue patterns for Kind in DOOD-style environments: + +- https://github.com/kubernetes-sigs/kind/issues/2867 + +## Implemented Interim Workaround (DOOD) + +`test/e2e/kind/start-cluster.sh` now includes a compact self-heal path for known DOOD bootstrap flakes: +create with `--retain`, wait for API readiness, apply Kind default CNI/storage with `--validate=false`, then wait for +`kindnet` and node readiness. + +- Known issue: https://github.com/kubernetes-sigs/kind/issues/2867 (Kind bootstrap race in DOOD-style setups) + +This keeps DOOD usable while preserving a clear migration path to DinD for long-term reliability. + +## Notes About Existing Troubleshooting Doc + +`.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md` focuses on path mapping and audit mount content. That diagnosis can be +valid in some setups, but current observed failures still occur in plain Kind flows where those repo mounts are not +involved. The dominant issue in this environment is now Kind bootstrap instability in DOOD. + +## Transition Goal + +Move local devcontainer e2e flows to DinD so Kind interacts with a Docker daemon running in the same container +environment, removing host-socket coupling for local cluster lifecycle. + +## Clean Transition Plan to DinD + +### Phase 1: Devcontainer Runtime Switch + +1. In `.devcontainer/devcontainer.json`: + - Replace `docker-outside-of-docker` with `docker-in-docker` + - Remove DOOD-specific host coupling where no longer needed: + - `--add-host=host.docker.internal:host-gateway` (only keep if still required by other workflows) +2. Keep `workspaceMount` as-is so project source remains bind-mounted. +3. Set path env to container-visible path for Kind mounts: + - `HOST_PROJECT_PATH=/workspaces/${localWorkspaceFolderBasename}` + - Keep `PROJECT_PATH` unchanged + +### Phase 2: Kind Config Alignment + +1. Keep `test/e2e/kind/cluster-template.yaml` mount source aligned with container-visible path. +2. Review `test/e2e/kind/start-cluster.sh` kubeconfig rewrite logic: + - Current rewrite to `host.docker.internal` is DOOD-oriented + - In DinD, direct endpoint usage is preferred unless a concrete networking reason remains +3. Validate audit mounts from inside Kind control-plane: + - `/etc/kubernetes/audit/policy.yaml` + - `/etc/kubernetes/audit/webhook-config.yaml` + +### Phase 3: Docs and Developer UX + +1. Update `.devcontainer/README.md` to explicitly state DinD mode. +2. Update `.devcontainer/SETUP_CLUSTER_TROUBLESHOOTING.md` with: + - DinD-first flow + - DOOD as legacy/optional path (if still supported) +3. Add one short "mode check" snippet: + - `docker info` + - `kind create cluster --name smoke --wait 2m && kind delete cluster --name smoke` + +### Phase 4: Rollout and Validation + +1. Rebuild devcontainer from scratch. +2. Validate cluster bootstrap reliability: + - Plain Kind single-node create/delete + - Plain Kind two-node create/delete +3. Validate project targets: + - `make setup-cluster` + - `make test-e2e-quickstart-helm` +4. Validate mandatory repo checks after migration: + - `make lint` + - `make test` + - `make test-e2e` + +### Phase 5: Rollback Strategy + +1. Keep a saved DOOD config variant during transition (for example `devcontainer.dood.json` in docs/reference). +2. If DinD regression appears: + - Restore previous `devcontainer.json` + - Rebuild container +3. Keep migration changes scoped and isolated so rollback is one-file plus documentation updates. + +## Risks and Mitigations + +- Risk: DinD networking differences break port-forward expectations. + - Mitigation: validate forwarded ports (`13000`, `19090`) in smoke checks. +- Risk: kubeconfig endpoint rewriting becomes incorrect. + - Mitigation: gate rewrite logic by detected endpoint pattern; prefer no rewrite in DinD. +- Risk: confusion between local DinD and CI setup. + - Mitigation: document local (DinD) vs CI (host runner) separation explicitly. + +## Recommendation + +Adopt DinD as the default local devcontainer mode for Kind-driven e2e testing and treat DOOD as unsupported for +reliable Kind lifecycle in this repository. diff --git a/docs/config-kustomize-simplification-findings.md b/docs/config-kustomize-simplification-findings.md deleted file mode 100644 index e1875b05..00000000 --- a/docs/config-kustomize-simplification-findings.md +++ /dev/null @@ -1,202 +0,0 @@ -# Config Kustomize Review: What Is Needed vs. What Can Be Simpler - -## Scope reviewed -- `config/default/kustomization.yaml` -- `config/default/manager_webhook_patch.yaml` -- `config/default/cert_metrics_manager_patch.yaml` -- `config/webhook/*` -- `config/certmanager/*` -- `cmd/main.go` -- `test/e2e/*` (especially namespace/cert assumptions) - -## Executive summary -- The certs are **already rendered into `sut`**, not `system`, when deploying via `config/default`. -- For webhook TLS + cert-manager CA injection, some kustomize wiring is genuinely required. -- There is also clear scaffolding/legacy complexity that can be reduced (especially commented replacement blocks and currently-unused metrics cert flow). - -## What is definitely useful / required - -### 1. `manager_webhook_patch.yaml` is required for current runtime behavior -Why: -- Your manager needs `--webhook-cert-path` and `--audit-cert-path`, plus mounted secrets and container ports (`9443`, `9444`). -- Without this patch, the cert secrets are not mounted where `cmd/main.go` expects them. - -References: -- `config/default/manager_webhook_patch.yaml:5` -- `config/default/manager_webhook_patch.yaml:27` -- `cmd/main.go:365` -- `cmd/main.go:522` - -### 2. Webhook kustomize namespace/name rewriting is required -Why: -- `ValidatingWebhookConfiguration` is cluster-scoped, but it embeds `clientConfig.service.name/namespace` fields. -- Kustomize needs explicit field specs to rewrite those embedded fields with your prefix/namespace. - -References: -- `config/webhook/kustomizeconfig.yaml:1` -- `config/webhook/webhook_service_name_patch.yaml:1` - -### 3. CA injection annotation wiring for cert-manager is required (if using cert-manager) -Why: -- API server must trust the serving cert for the validating webhook. -- `cert-manager.io/inject-ca-from` annotation on `ValidatingWebhookConfiguration` is the mechanism you currently use. - -References: -- `config/default/kustomization.yaml:187` -- Rendered output includes: `cert-manager.io/inject-ca-from: sut/gitops-reverser-admission-server-cert` - -### 4. `certmanager/kustomizeconfig.yaml` is required with `namePrefix` -Why: -- `namePrefix: gitops-reverser-` renames `Issuer` metadata.name. -- `Certificate.spec.issuerRef.name` must be rewritten to match, otherwise cert issuance breaks. - -References: -- `config/default/kustomization.yaml:9` -- `config/certmanager/kustomizeconfig.yaml:1` - -## What is currently over-complicated / likely removable - -### 1. Huge commented replacement blocks in `config/default/kustomization.yaml` -- Most of the metrics/servicemonitor replacement blocks are commented and unused in your current default/e2e flow. -- Keeping them bloats maintenance and confuses intent. - -Reference: -- `config/default/kustomization.yaml:53` - -### 2. Mutating webhook CA injection replacements appear unused -- You only have `ValidatingWebhookConfiguration` in `config/webhook/manifests.yaml`. -- Replacement entries targeting `MutatingWebhookConfiguration` look like kubebuilder scaffold leftovers. - -References: -- `config/default/kustomization.yaml:218` -- `config/webhook/manifests.yaml:3` - -### 3. Metrics certificate is created but not mounted by default -- `metrics-server-cert.yaml` is included in resources. -- But `cert_metrics_manager_patch.yaml` is commented out, so manager does not mount/use `metrics-server-cert` by default. -- E2E Prometheus scrape uses `insecure_skip_verify: true` anyway. - -References: -- `config/certmanager/kustomization.yaml:5` -- `config/default/kustomization.yaml:40` -- `test/e2e/prometheus/deployment.yaml:24` - -## Certificate flow (how certs are used today) - -### Admission webhook cert (`admission-server-cert` secret) -1. `Certificate` resource requests cert for service DNS. -2. cert-manager writes secret `admission-server-cert`. -3. Deployment mounts that secret and passes `--webhook-cert-path`. -4. admission-server listener serves TLS on `9443` using cert watcher. -5. cert-manager injects CA into `ValidatingWebhookConfiguration` annotation target. -6. kube-apiserver calls webhook via Service over TLS and trusts injected CA. - -Key refs: -- `config/certmanager/admission-server-cert.yaml:18` -- `config/default/manager_webhook_patch.yaml:52` -- `config/default/kustomization.yaml:187` - -### Audit ingress cert (`audit-server-cert` secret) -1. Separate `Certificate` resource issues audit cert. -2. Secret `audit-server-cert` is mounted. -3. Manager serves HTTPS audit endpoint on `9444` using `--audit-cert-path`. -4. In e2e, kube-apiserver audit webhook config uses `insecure-skip-tls-verify: true` (so CA pinning is not enforced in test). - -Key refs: -- `config/certmanager/audit-server-cert.yaml:17` -- `config/default/manager_webhook_patch.yaml:60` -- `test/e2e/kind/audit/webhook-config.yaml:14` - -### Metrics cert (`metrics-server-cert` secret) -- Issued by cert-manager, but only actively used if you also enable metrics cert patch and corresponding monitor TLS config. - -Refs: -- `config/certmanager/metrics-server-cert.yaml:20` -- `config/default/cert_metrics_manager_patch.yaml:12` -- `config/prometheus/monitor_tls_patch.yaml:1` - -## Your namespace question: `sut` vs `system` - -Short answer: -- `system` is **not required** for kube-api webhooks. -- Certs should live in the same namespace as the workload/service that uses them. -- In your current default deployment, that namespace is effectively `sut`. - -Important detail: -- Source files still show `namespace: system` in some places, but `config/default/kustomization.yaml` applies `namespace: sut` globally. -- Rendered manifests confirm certs, issuer, service, deployment are in `sut`. - -## Recommended simplification plan (test-focused) - -### Phase 1 (safe cleanup, behavior unchanged) -1. Remove large commented blocks in `config/default/kustomization.yaml` (keep only active replacements). -2. Remove unused mutating-webhook replacement entries if you do not plan mutating webhooks. -3. Add a short comment block at top: "test profile: single service + validating webhook + audit ingress". - -### Phase 2 (decide metrics cert strategy) -Choose one: -1. Keep metrics cert end-to-end: enable `cert_metrics_manager_patch.yaml` and proper monitor TLS usage. -2. Or simplify: remove `metrics-server-cert.yaml` from `config/certmanager/kustomization.yaml` and stop waiting for `metrics-server-cert` in e2e helper. - -Given current e2e (`insecure_skip_verify: true`), option 2 is simpler and consistent. - -### Phase 3 (optional bigger simplification) -If these manifests are truly test-only and namespace/prefix are fixed: -1. Replace dynamic cert DNS replacements with explicit static DNS names. -2. Replace dynamic `inject-ca-from` replacements with static annotation value. - -Tradeoff: -- Less kustomize complexity, but less reusable/generic. - -## Extra note on fixed ClusterIP -- The fixed service ClusterIP (`10.96.200.200`) is coupled to Kind audit webhook bootstrap (API server before DNS). -- Keep it if you depend on that startup behavior in e2e. - -Refs: -- `config/webhook/service.yaml:10` -- `test/e2e/kind/audit/webhook-config.yaml:12` - -## Bold strategy (essence-first): freeze rendered output, delete most kustomize machinery - -### What you mean in practice -1. Render today’s desired install profile once (`kustomize build config/default`). -2. Split that output into plain, human-owned files by concern (for example `namespace.yaml`, `crds.yaml`, `rbac.yaml`, `deployment.yaml`, `service.yaml`, `certificates.yaml`, `webhook.yaml`). -3. Remove the current deep transformer/replacement structure from `config/`. -4. Keep either: - - no kustomize at all (apply a folder of plain YAML in order), or - - one tiny `kustomization.yaml` that just lists resources with zero patches/replacements. - -This is a valid strategy if your goal is readability and low cognitive overhead over portability. - -### Why this can be good -1. You get back to essentials: explicit manifests, no hidden transformations. -2. Refactoring confidence improves because object names/refs are visible directly. -3. New contributors can reason about install behavior without learning kustomize tricks. -4. Debugging production/test drift is easier because rendered state is source of truth. - -### What you lose -1. Easy rebasing of namespace/namePrefix/env variants. -2. Automatic reference rewriting (`Issuer` name, webhook service references, CA injection path assembly). -3. Scaffold compatibility with future kubebuilder regeneration patterns. - -### Where this can hurt later -1. If you later need a second profile (for example non-e2e namespace or no fixed ClusterIP), you will duplicate YAML or re-introduce templating. -2. If cert naming/service naming changes, all references must be manually updated everywhere. -3. Large CRD/regenerated sections can become noisy unless you keep strict ownership boundaries. - -### Guardrails to keep this maintainable -1. Declare one supported raw-manifest profile explicitly (for example: `sut` test profile). -2. Keep clear file boundaries: - - `config/raw/00-namespace.yaml` - - `config/raw/10-crds.yaml` - - `config/raw/20-rbac.yaml` - - `config/raw/30-manager.yaml` - - `config/raw/40-service.yaml` - - `config/raw/50-certificates.yaml` - - `config/raw/60-webhook.yaml` -3. If you keep minimal kustomize, allow only `resources:` entries (no `patches`, no `replacements`, no `configurations`). -4. Add a lightweight validation target (for example `kubectl apply --dry-run=server -f config/raw` in CI). - -### Recommendation for your repo -- If these manifests are primarily for e2e and internal testing, this essence-first model is reasonable and likely worth it. -- If you want `config/` to be a broadly reusable install path, keep some kustomize composition and instead prune it aggressively (not fully remove it). diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 766eb6df..8dba237b 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -88,7 +88,6 @@ const ( ) const ( - sopsAgeKeySecretKey = "SOPS" + "_AGE_KEY" encryptionSecretRecipientAnnoKey = "configbutler.ai/age-recipient" encryptionSecretBackupWarningAnno = "configbutler.ai/backup-warning" encryptionSecretBackupWarningValue = "REMOVE_AFTER_BACKUP" @@ -106,7 +105,7 @@ type GitTargetReconciler struct { // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=configbutler.ai,resources=gitproviders,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch // Reconcile validates GitTarget references and drives startup lifecycle gates. // @@ -366,13 +365,13 @@ func (r *GitTargetReconciler) evaluateEncryptionGate( target *configbutleraiv1alpha1.GitTarget, log logr.Logger, ) (bool, string, time.Duration) { - if target.Spec.Encryption == nil { + if !isTargetAgeEncryptionEnabled(target) { r.setCondition( target, GitTargetConditionEncryptionConfigured, metav1.ConditionTrue, GitTargetReasonNotRequired, - "Encryption is not configured for this GitTarget", + "SOPS age encryption is not enabled for this GitTarget", ) return true, "", 0 } @@ -388,6 +387,14 @@ func (r *GitTargetReconciler) evaluateEncryptionGate( r.setCondition(target, GitTargetConditionEncryptionConfigured, metav1.ConditionFalse, reason, err.Error()) return false, fmt.Sprintf("EncryptionConfigured gate failed: %s", reason), RequeueMediumInterval } + if _, err := git.ResolveTargetEncryption(ctx, r.Client, target); err != nil { + reason := GitTargetReasonInvalidConfig + if strings.Contains(err.Error(), "failed to fetch encryption secret") { + reason = GitTargetReasonMissingSecret + } + r.setCondition(target, GitTargetConditionEncryptionConfigured, metav1.ConditionFalse, reason, err.Error()) + return false, fmt.Sprintf("EncryptionConfigured gate failed: %s", reason), RequeueMediumInterval + } r.setCondition( target, @@ -710,42 +717,64 @@ func (r *GitTargetReconciler) ensureEncryptionSecret( target *configbutleraiv1alpha1.GitTarget, log logr.Logger, ) error { - if target.Spec.Encryption == nil { + if !shouldGenerateAgeKey(target) { return nil } - secretName := strings.TrimSpace(target.Spec.Encryption.SecretRef.Name) - if secretName == "" { - return errors.New("encryption.secretRef.name must be set when encryption is configured") + secretKey, err := secretKeyForGeneratedEncryption(target) + if err != nil { + return err } - secretKey := k8stypes.NamespacedName{Name: secretName, Namespace: target.Namespace} var existing corev1.Secret - if err := r.Get(ctx, secretKey, &existing); err == nil { - if existing.Annotations[encryptionSecretBackupWarningAnno] == encryptionSecretBackupWarningValue { - log.Info("ENCRYPTION KEY BACKUP REQUIRED: remove annotation after backup is completed", - "secret", secretKey.String(), - "annotation", encryptionSecretBackupWarningAnno) + if err := r.Get(ctx, secretKey, &existing); err != nil { + if apierrors.IsNotFound(err) { + return r.createGeneratedEncryptionSecret(ctx, target.Namespace, secretKey, log) } - return nil - } else if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to fetch encryption secret %s: %w", secretKey.String(), err) } - if !target.Spec.Encryption.GenerateWhenMissing { - return fmt.Errorf("encryption secret %s is missing and generateWhenMissing is disabled", secretKey.String()) + if hasAgeKeyEntry(existing.Data) { + logEncryptionBackupWarning(log, secretKey, existing.Annotations) + return nil } - identity, err := age.GenerateX25519Identity() + identity, recipient, err := generateAgeIdentity() if err != nil { return fmt.Errorf("failed to generate age identity for encryption secret %s: %w", secretKey.String(), err) } - recipient := identity.Recipient().String() + ageKeyDataKey := currentDateAgeKeySecretDataKey() + ensureAgeSecretDataAndAnnotations(&existing, ageKeyDataKey, identity.String(), recipient) + if err := r.Update(ctx, &existing); err != nil { + return fmt.Errorf("failed to update encryption secret %s: %w", secretKey.String(), err) + } + + log.Info( + "Added missing .agekey entry to encryption secret. Back up the private key and remove warning annotation.", + "secret", secretKey.String(), + "secretDataKey", ageKeyDataKey, + "recipient", recipient, + "warningAnnotation", encryptionSecretBackupWarningAnno, + ) + return nil +} + +func (r *GitTargetReconciler) createGeneratedEncryptionSecret( + ctx context.Context, + namespace string, + secretKey k8stypes.NamespacedName, + log logr.Logger, +) error { + identity, recipient, err := generateAgeIdentity() + if err != nil { + return fmt.Errorf("failed to generate age identity for encryption secret %s: %w", secretKey.String(), err) + } + ageKeyDataKey := currentDateAgeKeySecretDataKey() secret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: target.Namespace, + Name: secretKey.Name, + Namespace: namespace, Annotations: map[string]string{ encryptionSecretRecipientAnnoKey: recipient, encryptionSecretBackupWarningAnno: encryptionSecretBackupWarningValue, @@ -753,10 +782,9 @@ func (r *GitTargetReconciler) ensureEncryptionSecret( }, Type: corev1.SecretTypeOpaque, StringData: map[string]string{ - sopsAgeKeySecretKey: identity.String(), + ageKeyDataKey: identity.String(), }, } - if err := r.Create(ctx, &secret); err != nil { if apierrors.IsAlreadyExists(err) { return nil @@ -767,13 +795,92 @@ func (r *GitTargetReconciler) ensureEncryptionSecret( log.Info( "Generated missing encryption secret with age key. Back up the private key and remove warning annotation.", "secret", secretKey.String(), + "secretDataKey", ageKeyDataKey, "recipient", recipient, "warningAnnotation", encryptionSecretBackupWarningAnno, ) - return nil } +func shouldGenerateAgeKey(target *configbutleraiv1alpha1.GitTarget) bool { + return isTargetAgeEncryptionEnabled(target) && target.Spec.Encryption.Age.Recipients.GenerateWhenMissing +} + +func secretKeyForGeneratedEncryption(target *configbutleraiv1alpha1.GitTarget) (k8stypes.NamespacedName, error) { + if !target.Spec.Encryption.Age.Recipients.ExtractFromSecret { + return k8stypes.NamespacedName{}, + errors.New("encryption.age.recipients.generateWhenMissing=true requires extractFromSecret=true") + } + + secretName := strings.TrimSpace(target.Spec.Encryption.SecretRef.Name) + if secretName == "" { + return k8stypes.NamespacedName{}, errors.New( + "encryption.secretRef.name must be set when encryption is configured", + ) + } + + return k8stypes.NamespacedName{Name: secretName, Namespace: target.Namespace}, nil +} + +func generateAgeIdentity() (*age.X25519Identity, string, error) { + identity, err := age.GenerateX25519Identity() + if err != nil { + return nil, "", err + } + return identity, identity.Recipient().String(), nil +} + +func ensureAgeSecretDataAndAnnotations(secret *corev1.Secret, ageKeyDataKey, identityValue, recipient string) { + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + secret.Data[ageKeyDataKey] = []byte(identityValue) + + if secret.Annotations == nil { + secret.Annotations = make(map[string]string) + } + secret.Annotations[encryptionSecretRecipientAnnoKey] = recipient + secret.Annotations[encryptionSecretBackupWarningAnno] = encryptionSecretBackupWarningValue +} + +func currentDateAgeKeySecretDataKey() string { + now := time.Now().UTC() + return fmt.Sprintf("%04d%02d%d.agekey", now.Year(), now.Month(), now.Day()) +} + +func logEncryptionBackupWarning(log logr.Logger, secretKey k8stypes.NamespacedName, annotations map[string]string) { + if annotations[encryptionSecretBackupWarningAnno] != encryptionSecretBackupWarningValue { + return + } + log.Info("ENCRYPTION KEY BACKUP REQUIRED: remove annotation after backup is completed", + "secret", secretKey.String(), + "annotation", encryptionSecretBackupWarningAnno) +} + +func isTargetAgeEncryptionEnabled(target *configbutleraiv1alpha1.GitTarget) bool { + if target == nil || target.Spec.Encryption == nil { + return false + } + + providerName := strings.TrimSpace(target.Spec.Encryption.Provider) + if providerName == "" { + providerName = git.EncryptionProviderSOPS + } + if providerName != git.EncryptionProviderSOPS { + return false + } + return target.Spec.Encryption.Age != nil && target.Spec.Encryption.Age.Enabled +} + +func hasAgeKeyEntry(data map[string][]byte) bool { + for key := range data { + if strings.HasSuffix(key, ".agekey") { + return true + } + } + return false +} + func (r *GitTargetReconciler) updateRepositoryStatus( ctx context.Context, target *configbutleraiv1alpha1.GitTarget, diff --git a/internal/controller/gittarget_controller_test.go b/internal/controller/gittarget_controller_test.go index 4a2a2006..04501843 100644 --- a/internal/controller/gittarget_controller_test.go +++ b/internal/controller/gittarget_controller_test.go @@ -20,8 +20,10 @@ package controller import ( "context" + "strings" "time" + "filippo.io/age" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -583,7 +585,13 @@ var _ = Describe("GitTarget Controller Security", func() { SecretRef: configbutleraiv1alpha1.LocalSecretReference{ Name: "generated-sops-age-key", }, - GenerateWhenMissing: true, + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + GenerateWhenMissing: true, + }, + }, }, }, } @@ -595,8 +603,10 @@ var _ = Describe("GitTarget Controller Security", func() { err := k8sClient.Get(ctx, secretKey, &secret) g.Expect(err).NotTo(HaveOccurred()) g.Expect(secret.Type).To(Equal(corev1.SecretTypeOpaque)) - g.Expect(secret.Data).To(HaveKey("SOPS_AGE_KEY")) - g.Expect(string(secret.Data["SOPS_AGE_KEY"])).To(ContainSubstring("AGE-SECRET-KEY-")) + ageKeyName, ageKeyValue := findFirstAgeKeyEntry(secret.Data) + g.Expect(ageKeyName).NotTo(BeEmpty()) + g.Expect(ageKeyName).To(Equal(currentDateAgeKeySecretDataKey())) + g.Expect(string(ageKeyValue)).To(ContainSubstring("AGE-SECRET-KEY-")) g.Expect(secret.Annotations).To(HaveKey(encryptionSecretRecipientAnnoKey)) g.Expect(secret.Annotations[encryptionSecretRecipientAnnoKey]).To(HavePrefix("age1")) g.Expect(secret.Annotations).To(HaveKeyWithValue( @@ -641,6 +651,13 @@ var _ = Describe("GitTarget Controller Security", func() { SecretRef: configbutleraiv1alpha1.LocalSecretReference{ Name: "missing-sops-age-key", }, + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + GenerateWhenMissing: false, + }, + }, }, }, } @@ -670,12 +687,180 @@ var _ = Describe("GitTarget Controller Security", func() { } } g.Expect(encryptionCondition).NotTo(BeNil()) - g.Expect(encryptionCondition.Reason).To(Equal(GitTargetReasonSecretCreateDisabled)) - g.Expect(encryptionCondition.Message).To(ContainSubstring("generateWhenMissing is disabled")) + g.Expect(encryptionCondition.Reason).To(Equal(GitTargetReasonMissingSecret)) + g.Expect(encryptionCondition.Message).To(ContainSubstring("failed to fetch encryption secret")) + }, timeout, interval).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, target)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, gitProvider)).Should(Succeed()) + }) + + It("Should add one .agekey entry when secret exists without any .agekey entries", func() { + ctx := context.Background() + + gitProvider := &configbutleraiv1alpha1.GitProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provider-update-enc-secret", + Namespace: "default", + }, + Spec: configbutleraiv1alpha1.GitProviderSpec{ + URL: "https://github.com/test-org/test-repo.git", + AllowedBranches: []string{"main"}, + }, + } + Expect(k8sClient.Create(ctx, gitProvider)).Should(Succeed()) + + seedSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-sops-secret", + Namespace: "default", + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "OTHER_ENV": []byte("value"), + }, + } + Expect(k8sClient.Create(ctx, seedSecret)).Should(Succeed()) + + target := &configbutleraiv1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-target-update-enc-secret", + Namespace: "default", + }, + Spec: configbutleraiv1alpha1.GitTargetSpec{ + ProviderRef: configbutleraiv1alpha1.GitProviderReference{ + Name: "test-provider-update-enc-secret", + Kind: "GitProvider", + }, + Branch: "main", + Path: "test-path", + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + SecretRef: configbutleraiv1alpha1.LocalSecretReference{ + Name: "existing-sops-secret", + }, + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + GenerateWhenMissing: true, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, target)).Should(Succeed()) + + secretKey := types.NamespacedName{Name: "existing-sops-secret", Namespace: "default"} + Eventually(func(g Gomega) { + var secret corev1.Secret + err := k8sClient.Get(ctx, secretKey, &secret) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(secret.Data).To(HaveKey("OTHER_ENV")) + ageKeyName, ageKeyValue := findFirstAgeKeyEntry(secret.Data) + g.Expect(ageKeyName).NotTo(BeEmpty()) + g.Expect(ageKeyName).To(Equal(currentDateAgeKeySecretDataKey())) + g.Expect(string(ageKeyValue)).To(ContainSubstring("AGE-SECRET-KEY-")) }, timeout, interval).Should(Succeed()) Expect(k8sClient.Delete(ctx, target)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, seedSecret)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, gitProvider)).Should(Succeed()) + }) + + It("Should not overwrite existing .agekey values when one already exists", func() { + ctx := context.Background() + + gitProvider := &configbutleraiv1alpha1.GitProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-provider-existing-agekey", + Namespace: "default", + }, + Spec: configbutleraiv1alpha1.GitProviderSpec{ + URL: "https://github.com/test-org/test-repo.git", + AllowedBranches: []string{"main"}, + }, + } + Expect(k8sClient.Create(ctx, gitProvider)).Should(Succeed()) + + existingIdentity, err := age.GenerateX25519Identity() + Expect(err).NotTo(HaveOccurred()) + existingAgeKey := existingIdentity.String() + seedSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-agekey-secret", + Namespace: "default", + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "20260221.agekey": []byte(existingAgeKey), + "OTHER_ENV": []byte("value"), + }, + } + Expect(k8sClient.Create(ctx, seedSecret)).Should(Succeed()) + + target := &configbutleraiv1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-target-existing-agekey", + Namespace: "default", + }, + Spec: configbutleraiv1alpha1.GitTargetSpec{ + ProviderRef: configbutleraiv1alpha1.GitProviderReference{ + Name: "test-provider-existing-agekey", + Kind: "GitProvider", + }, + Branch: "main", + Path: "test-path", + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + SecretRef: configbutleraiv1alpha1.LocalSecretReference{ + Name: "existing-agekey-secret", + }, + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + GenerateWhenMissing: true, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, target)).Should(Succeed()) + + secretKey := types.NamespacedName{Name: "existing-agekey-secret", Namespace: "default"} + Eventually(func(g Gomega) { + var secret corev1.Secret + err := k8sClient.Get(ctx, secretKey, &secret) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(secret.Data).To(HaveKey("OTHER_ENV")) + g.Expect(secret.Data).To(HaveKey("20260221.agekey")) + g.Expect(string(secret.Data["20260221.agekey"])).To(Equal(existingAgeKey)) + g.Expect(countAgeKeyEntries(secret.Data)).To(Equal(1)) + }, timeout, interval).Should(Succeed()) + + Expect(k8sClient.Delete(ctx, target)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, seedSecret)).Should(Succeed()) Expect(k8sClient.Delete(ctx, gitProvider)).Should(Succeed()) }) }) }) + +func findFirstAgeKeyEntry(data map[string][]byte) (string, []byte) { + for key, value := range data { + if strings.HasSuffix(key, ".agekey") { + return key, value + } + } + return "", nil +} + +func countAgeKeyEntries(data map[string][]byte) int { + count := 0 + for key := range data { + if strings.HasSuffix(key, ".agekey") { + count++ + } + } + return count +} diff --git a/internal/git/bootstrapped-repo-template/.sops.yaml b/internal/git/bootstrapped-repo-template/.sops.yaml index f0c02e43..ce634ec3 100644 --- a/internal/git/bootstrapped-repo-template/.sops.yaml +++ b/internal/git/bootstrapped-repo-template/.sops.yaml @@ -5,4 +5,6 @@ creation_rules: encrypted_regex: '^(data|stringData)$' key_groups: - age: - - "{{ .AgeRecipient }}" +{{- range .AgeRecipients }} + - "{{ . }}" +{{- end }} diff --git a/internal/git/bootstrapped-repo-template/README.md b/internal/git/bootstrapped-repo-template/README.md index 3da2012e..ad9527bf 100644 --- a/internal/git/bootstrapped-repo-template/README.md +++ b/internal/git/bootstrapped-repo-template/README.md @@ -1,6 +1,6 @@ # GitOps Reverser Bootstrap Files -This repository was initialized by GitOps Reverser. +This path was initialized by GitOps Reverser. -The `.sops.yaml` file is required for SOPS-based secret encryption. +The `.sops.yaml` file is required for SOPS-based secret encryption and includes all resolved age recipients. If you remove or change these bootstrap files, secret mirroring may stop working. diff --git a/internal/git/bootstrapped_repo_template.go b/internal/git/bootstrapped_repo_template.go index 9e618813..b54358ea 100644 --- a/internal/git/bootstrapped_repo_template.go +++ b/internal/git/bootstrapped_repo_template.go @@ -51,7 +51,7 @@ const ( var bootstrapTemplateFS embed.FS type bootstrapTemplateData struct { - AgeRecipient string + AgeRecipients []string } type pathBootstrapOptions struct { @@ -234,8 +234,13 @@ func stageBootstrapFile(worktree *gogit.Worktree, targetPath, entryName string) } func renderSOPSBootstrapTemplate(raw []byte, data bootstrapTemplateData) ([]byte, error) { - if strings.TrimSpace(data.AgeRecipient) == "" { - return nil, fmt.Errorf("failed to render bootstrap file %s: missing age recipient", sopsConfigFileName) + if len(data.AgeRecipients) == 0 { + return nil, fmt.Errorf("failed to render bootstrap file %s: missing age recipients", sopsConfigFileName) + } + for _, recipient := range data.AgeRecipients { + if strings.TrimSpace(recipient) == "" { + return nil, fmt.Errorf("failed to render bootstrap file %s: empty age recipient", sopsConfigFileName) + } } tmpl, err := template.New(sopsConfigFileName).Parse(string(raw)) diff --git a/internal/git/bootstrapped_repo_template_test.go b/internal/git/bootstrapped_repo_template_test.go new file mode 100644 index 00000000..246cb753 --- /dev/null +++ b/internal/git/bootstrapped_repo_template_test.go @@ -0,0 +1,51 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRenderSOPSBootstrapTemplate_MultipleRecipients(t *testing.T) { + raw, err := bootstrapTemplateFS.ReadFile(path.Join(bootstrapTemplateDir, sopsConfigFileName)) + require.NoError(t, err) + + rendered, err := renderSOPSBootstrapTemplate(raw, bootstrapTemplateData{ + AgeRecipients: []string{ + "age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq7k8m6", + "age1yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyv0r2a", + }, + }) + require.NoError(t, err) + assert.Contains(t, string(rendered), "age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq7k8m6") + assert.Contains(t, string(rendered), "age1yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyv0r2a") +} + +func TestRenderSOPSBootstrapTemplate_MissingRecipients(t *testing.T) { + raw, err := bootstrapTemplateFS.ReadFile(path.Join(bootstrapTemplateDir, sopsConfigFileName)) + require.NoError(t, err) + + _, err = renderSOPSBootstrapTemplate(raw, bootstrapTemplateData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing age recipients") +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index ddae734e..474f5f33 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -261,17 +261,8 @@ func (w *BranchWorker) resolveBootstrapOptions( return pathBootstrapOptions{}, nil } - sopsKey := strings.TrimSpace(encryptionConfig.Environment[sopsAgeKeyEnvVar]) - if sopsKey == "" { - w.Log.Info("Skipping SOPS bootstrap due to missing encryption key in target secret", - "target", targetKey.String(), - "requiredKey", sopsAgeKeyEnvVar) - return pathBootstrapOptions{}, nil - } - - recipient, err := deriveAgeRecipientFromSOPSKey(sopsKey) - if err != nil { - w.Log.Error(err, "Skipping SOPS bootstrap due to invalid encryption key", + if len(encryptionConfig.AgeRecipients) == 0 { + w.Log.Info("Skipping SOPS bootstrap due to missing resolved age recipients", "target", targetKey.String()) return pathBootstrapOptions{}, nil } @@ -279,7 +270,7 @@ func (w *BranchWorker) resolveBootstrapOptions( return pathBootstrapOptions{ IncludeSOPSConfig: true, TemplateData: bootstrapTemplateData{ - AgeRecipient: recipient, + AgeRecipients: encryptionConfig.AgeRecipients, }, }, nil } diff --git a/internal/git/branch_worker_test.go b/internal/git/branch_worker_test.go index f127dd0a..b36d1423 100644 --- a/internal/git/branch_worker_test.go +++ b/internal/git/branch_worker_test.go @@ -490,7 +490,7 @@ func TestBranchWorker_EnsurePathBootstrapped_InvalidEncryptionSecretSkipsSOPSCon "main", "clusters/dev", map[string][]byte{ - sopsAgeKeyEnvVar: []byte("not-an-age-identity"), + "identity.agekey": []byte("not-an-age-identity"), }, ) @@ -552,7 +552,84 @@ func TestBranchWorker_EnsurePathBootstrapped_MissingSOPSKeySkipsSOPSConfig(t *te _, err = os.Stat(filepath.Join(clonePath, "clusters/dev", "README.md")) require.NoError(t, err) _, err = os.Stat(filepath.Join(clonePath, "clusters/dev", sopsConfigFileName)) - assert.True(t, os.IsNotExist(err), "Bootstrap SOPS config should be skipped when SOPS_AGE_KEY is missing") + assert.True(t, os.IsNotExist(err), "Bootstrap SOPS config should be skipped when no .agekey entry is present") +} + +func TestBranchWorker_EnsurePathBootstrapped_RendersAllResolvedRecipients(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + remotePath := filepath.Join(tempDir, "remote.git") + remoteURL := "file://" + remotePath + _ = createBareRepo(t, remotePath) + + secretIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + secondaryIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + publicOnlyIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + + scheme := runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = configv1alpha1.AddToScheme(scheme) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + + provider := &configv1alpha1.GitProvider{ + Spec: configv1alpha1.GitProviderSpec{ + URL: remoteURL, + }, + } + provider.Name = "test-repo" + provider.Namespace = "default" + require.NoError(t, k8sClient.Create(ctx, provider)) + + encryptionSecret := &corev1.Secret{} + encryptionSecret.Name = "sops-age-key" + encryptionSecret.Namespace = "default" + encryptionSecret.Data = map[string][]byte{ + "identity.agekey": []byte(secretIdentity.String()), + "backup.agekey": []byte(secondaryIdentity.String()), + } + require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) + + target := &configv1alpha1.GitTarget{} + target.Name = "bootstrap-target" + target.Namespace = "default" + target.Spec.ProviderRef = configv1alpha1.GitProviderReference{ + Kind: "GitProvider", + Name: "test-repo", + } + target.Spec.Branch = "main" + target.Spec.Path = "clusters/dev" + target.Spec.Encryption = &configv1alpha1.EncryptionSpec{ + Provider: "sops", + SecretRef: configv1alpha1.LocalSecretReference{ + Name: encryptionSecret.Name, + }, + Age: &configv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configv1alpha1.AgeRecipientsSpec{ + PublicKeys: []string{ + publicOnlyIdentity.Recipient().String(), + }, + ExtractFromSecret: true, + }, + }, + } + require.NoError(t, k8sClient.Create(ctx, target)) + + worker := NewBranchWorker(k8sClient, logr.Discard(), "test-repo", "default", "main", nil) + require.NoError(t, worker.EnsurePathBootstrapped("clusters/dev", "bootstrap-target", "default")) + + clonePath := filepath.Join(tempDir, "inspect") + _, err = PrepareBranch(ctx, remoteURL, clonePath, "main", nil) + require.NoError(t, err) + + sopsConfig, err := os.ReadFile(filepath.Join(clonePath, "clusters/dev", sopsConfigFileName)) + require.NoError(t, err) + assert.Contains(t, string(sopsConfig), secretIdentity.Recipient().String()) + assert.Contains(t, string(sopsConfig), secondaryIdentity.Recipient().String()) + assert.Contains(t, string(sopsConfig), publicOnlyIdentity.Recipient().String()) } func createTargetWithEncryption( @@ -569,7 +646,7 @@ func createTargetWithEncryption( encryptionSecret.Name = "sops-age-key" encryptionSecret.Namespace = namespace encryptionSecret.Data = map[string][]byte{ - sopsAgeKeyEnvVar: []byte(identity.String()), + "identity.agekey": []byte(identity.String()), } require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) @@ -587,6 +664,12 @@ func createTargetWithEncryption( SecretRef: configv1alpha1.LocalSecretReference{ Name: "sops-age-key", }, + Age: &configv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, } require.NoError(t, k8sClient.Create(ctx, target)) } @@ -639,6 +722,12 @@ func createTargetWithEncryptionSecretData( SecretRef: configv1alpha1.LocalSecretReference{ Name: encryptionSecret.Name, }, + Age: &configv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, } require.NoError(t, k8sClient.Create(ctx, target)) } @@ -657,7 +746,7 @@ func attachEncryptionToTarget( encryptionSecret.Name = "sops-age-key" encryptionSecret.Namespace = targetNamespace encryptionSecret.Data = map[string][]byte{ - sopsAgeKeyEnvVar: []byte(identity.String()), + "identity.agekey": []byte(identity.String()), } require.NoError(t, k8sClient.Create(ctx, encryptionSecret)) @@ -668,6 +757,12 @@ func attachEncryptionToTarget( SecretRef: configv1alpha1.LocalSecretReference{ Name: encryptionSecret.Name, }, + Age: &configv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, } require.NoError(t, k8sClient.Update(ctx, target)) } diff --git a/internal/git/encryption.go b/internal/git/encryption.go index 60c9b711..627d4aa7 100644 --- a/internal/git/encryption.go +++ b/internal/git/encryption.go @@ -20,8 +20,14 @@ package git import ( "context" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" + "os" + "path/filepath" "regexp" + "sort" "strings" "filippo.io/age" @@ -51,16 +57,22 @@ const ( EncryptionProviderSOPS = "sops" // defaultSOPSBinaryPath is resolved from PATH in the controller runtime image. defaultSOPSBinaryPath = "sops" - // sopsAgeKeyEnvVar is the required secret key for age-based SOPS encryption. - sopsAgeKeyEnvVar = "SOPS_AGE_KEY" + // sopsAgeKeyFileEnvVar points SOPS to a file containing age private identities. + sopsAgeKeyFileEnvVar = "SOPS" + "_AGE_KEY_FILE" + // ageSecretKeySuffix identifies Flux-compatible age private-key entries. + ageSecretKeySuffix = ".agekey" + // ageIdentityFileDir is the temp directory used for SOPS age identity files. + ageIdentityFileDir = "gitops-reverser-age-identities" ) var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) // ResolvedEncryptionConfig contains runtime encryption settings resolved from GitTarget spec. type ResolvedEncryptionConfig struct { - Provider string - Environment map[string]string + Provider string + Environment map[string]string + AgeRecipients []string + AgeIdentities []string } // ResolveTargetEncryption resolves and validates GitTarget encryption configuration. @@ -74,45 +86,118 @@ func ResolveTargetEncryption( } encryptionSpec := target.Spec.Encryption + providerName, err := resolveEncryptionProvider(encryptionSpec) + if err != nil { + return nil, err + } + ageSpec := encryptionSpec.Age + if ageSpec == nil || !ageSpec.Enabled { + return nil, nil //nolint:nilnil // nil means encryption disabled for current provider implementation + } + + publicRecipients, err := normalizePublicAgeRecipients(ageSpec.Recipients.PublicKeys) + if err != nil { + return nil, err + } + secretRecipients, secretIdentities, environment, err := resolveSecretRecipientsAndEnvironment( + ctx, + k8sClient, + target, + encryptionSpec, + ) + if err != nil { + return nil, err + } + + resolvedRecipients := dedupeAndSortRecipients(append(publicRecipients, secretRecipients...)) + if len(resolvedRecipients) == 0 { + return nil, errors.New( + "encryption.age.enabled=true requires at least one resolved recipient from age.recipients.publicKeys or secret *.agekey entries", + ) + } + + environment = normalizeEnvironment(environment) + return &ResolvedEncryptionConfig{ + Provider: providerName, + Environment: environment, + AgeRecipients: resolvedRecipients, + AgeIdentities: secretIdentities, + }, nil +} + +func resolveEncryptionProvider(encryptionSpec *v1alpha1.EncryptionSpec) (string, error) { providerName := strings.TrimSpace(encryptionSpec.Provider) if providerName == "" { providerName = EncryptionProviderSOPS } if providerName != EncryptionProviderSOPS { - return nil, fmt.Errorf("unsupported encryption provider %q", encryptionSpec.Provider) + return "", fmt.Errorf("unsupported encryption provider %q", encryptionSpec.Provider) + } + return providerName, nil +} + +func resolveSecretRecipientsAndEnvironment( + ctx context.Context, + k8sClient client.Client, + target *v1alpha1.GitTarget, + encryptionSpec *v1alpha1.EncryptionSpec, +) ([]string, []string, map[string]string, error) { + if encryptionSpec.Age == nil || !encryptionSpec.Age.Recipients.ExtractFromSecret { + return nil, nil, nil, nil } secretKind := strings.TrimSpace(encryptionSpec.SecretRef.Kind) if secretKind != "" && secretKind != "Secret" { - return nil, fmt.Errorf("encryption.secretRef.kind must be Secret, got %q", encryptionSpec.SecretRef.Kind) + return nil, nil, nil, fmt.Errorf( + "encryption.secretRef.kind must be Secret, got %q", + encryptionSpec.SecretRef.Kind, + ) } secretName := strings.TrimSpace(encryptionSpec.SecretRef.Name) if secretName == "" { - return nil, fmt.Errorf("encryption.secretRef.name must be set for provider %q", providerName) + return nil, nil, nil, errors.New( + "encryption.secretRef.name must be set when age.recipients.extractFromSecret=true", + ) } - var secret corev1.Secret + secret, secretKey, err := getEncryptionSecret(ctx, k8sClient, target.Namespace, secretName) + if err != nil { + return nil, nil, nil, err + } + + secretRecipients, secretIdentities, err := resolveAgeRecipientsFromSecret(secret.Data) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to resolve recipients from encryption secret %s: %w", secretKey, err) + } + + environment := toSOPSEnvironment(secret.Data) + return secretRecipients, secretIdentities, environment, nil +} + +func getEncryptionSecret( + ctx context.Context, + k8sClient client.Client, + namespace string, + secretName string, +) (*corev1.Secret, k8stypes.NamespacedName, error) { secretKey := k8stypes.NamespacedName{ Name: secretName, - Namespace: target.Namespace, + Namespace: namespace, } + var secret corev1.Secret if err := k8sClient.Get(ctx, secretKey, &secret); err != nil { - return nil, fmt.Errorf("failed to fetch encryption secret %s: %w", secretKey, err) + return nil, secretKey, fmt.Errorf("failed to fetch encryption secret %s: %w", secretKey, err) } - environment := toSOPSEnvironment(secret.Data) + return &secret, secretKey, nil +} + +func normalizeEnvironment(environment map[string]string) map[string]string { if len(environment) == 0 { - return nil, fmt.Errorf( - "encryption secret %s must contain at least one valid environment variable entry", - secretKey, - ) + return nil } - - return &ResolvedEncryptionConfig{ - Provider: providerName, - Environment: environment, - }, nil + return environment } func configureSecretEncryptionWriter( @@ -127,13 +212,66 @@ func configureSecretEncryptionWriter( switch cfg.Provider { case EncryptionProviderSOPS: - writer.setEncryptor(NewSOPSEncryptorWithEnv(defaultSOPSBinaryPath, "", workDir, cfg.Environment)) + environment, err := buildSOPSEnvironment(workDir, cfg) + if err != nil { + return err + } + writer.setEncryptor(NewSOPSEncryptorWithEnv(defaultSOPSBinaryPath, "", workDir, environment)) return nil default: return fmt.Errorf("unsupported encryption provider %q", cfg.Provider) } } +func buildSOPSEnvironment(workDir string, cfg *ResolvedEncryptionConfig) (map[string]string, error) { + environment := cloneEnvironment(cfg.Environment) + if len(cfg.AgeIdentities) == 0 { + return environment, nil + } + + ageKeyFilePath, err := writeAgeIdentityFile(workDir, cfg.AgeIdentities) + if err != nil { + return nil, fmt.Errorf("failed to write SOPS age identity file: %w", err) + } + if environment == nil { + environment = make(map[string]string, 1) + } + environment[sopsAgeKeyFileEnvVar] = ageKeyFilePath + return environment, nil +} + +func cloneEnvironment(environment map[string]string) map[string]string { + if len(environment) == 0 { + return nil + } + cloned := make(map[string]string, len(environment)) + for key, value := range environment { + cloned[key] = value + } + return cloned +} + +func writeAgeIdentityFile(workDir string, identities []string) (string, error) { + if len(identities) == 0 { + return "", errors.New("no age identities provided") + } + + dirPath := filepath.Join(os.TempDir(), ageIdentityFileDir) + if err := os.MkdirAll(dirPath, 0700); err != nil { + return "", fmt.Errorf("create age identity directory: %w", err) + } + + keyHash := sha256.Sum256([]byte(strings.TrimSpace(workDir))) + fileName := hex.EncodeToString(keyHash[:8]) + ageSecretKeySuffix + filePath := filepath.Join(dirPath, fileName) + fileContent := strings.Join(identities, "\n") + "\n" + if err := os.WriteFile(filePath, []byte(fileContent), 0600); err != nil { + return "", fmt.Errorf("write age identity file: %w", err) + } + + return filePath, nil +} + func toSOPSEnvironment(secretData map[string][]byte) map[string]string { if len(secretData) == 0 { return nil @@ -154,7 +292,52 @@ func toSOPSEnvironment(secretData map[string][]byte) map[string]string { return environment } -func deriveAgeRecipientFromSOPSKey(secretValue string) (string, error) { +func resolveAgeRecipientsFromSecret(secretData map[string][]byte) ([]string, []string, error) { + if len(secretData) == 0 { + return nil, nil, nil + } + + recipients := make([]string, 0, len(secretData)) + identities := make([]string, 0, len(secretData)) + for key, value := range secretData { + if !strings.HasSuffix(key, ageSecretKeySuffix) { + continue + } + + recipient, identity, err := deriveAgeRecipientFromSecretEntry(key, string(value)) + if err != nil { + return nil, nil, err + } + recipients = append(recipients, recipient) + identities = append(identities, identity) + } + + return dedupeAndSortRecipients(recipients), dedupeAndSortIdentities(identities), nil +} + +func normalizePublicAgeRecipients(publicKeys []string) ([]string, error) { + if len(publicKeys) == 0 { + return nil, nil + } + + recipients := make([]string, 0, len(publicKeys)) + for i, publicKey := range publicKeys { + trimmed := strings.TrimSpace(publicKey) + if trimmed == "" { + continue + } + + recipient, err := age.ParseX25519Recipient(trimmed) + if err != nil { + return nil, fmt.Errorf("invalid age recipient in age.recipients.publicKeys[%d]: %w", i, err) + } + recipients = append(recipients, recipient.String()) + } + + return dedupeAndSortRecipients(recipients), nil +} + +func deriveAgeRecipientFromSecretEntry(secretKey, secretValue string) (string, string, error) { lines := strings.Split(secretValue, "\n") identities := make([]string, 0, len(lines)) for _, line := range lines { @@ -166,19 +349,71 @@ func deriveAgeRecipientFromSOPSKey(secretValue string) (string, error) { } if len(identities) == 0 { - return "", fmt.Errorf("%s must contain one AGE-SECRET-KEY identity", sopsAgeKeyEnvVar) + return "", "", fmt.Errorf("%s must contain one AGE-SECRET-KEY identity", secretKey) } if len(identities) > 1 { - return "", fmt.Errorf("%s must contain exactly one AGE-SECRET-KEY identity", sopsAgeKeyEnvVar) + return "", "", fmt.Errorf("%s must contain exactly one AGE-SECRET-KEY identity", secretKey) } if !strings.HasPrefix(identities[0], "AGE-SECRET-KEY-") { - return "", fmt.Errorf("%s must contain AGE-SECRET-KEY identity", sopsAgeKeyEnvVar) + return "", "", fmt.Errorf("%s must contain AGE-SECRET-KEY identity", secretKey) } identity, err := age.ParseX25519Identity(identities[0]) if err != nil { - return "", fmt.Errorf("invalid %s identity: %w", sopsAgeKeyEnvVar, err) + return "", "", fmt.Errorf("invalid %s identity: %w", secretKey, err) + } + + return identity.Recipient().String(), identity.String(), nil +} + +func dedupeAndSortRecipients(recipients []string) []string { + if len(recipients) == 0 { + return nil } - return identity.Recipient().String(), nil + uniq := make(map[string]struct{}, len(recipients)) + for _, recipient := range recipients { + trimmed := strings.TrimSpace(recipient) + if trimmed == "" { + continue + } + uniq[trimmed] = struct{}{} + } + + if len(uniq) == 0 { + return nil + } + + result := make([]string, 0, len(uniq)) + for recipient := range uniq { + result = append(result, recipient) + } + sort.Strings(result) + return result +} + +func dedupeAndSortIdentities(identities []string) []string { + if len(identities) == 0 { + return nil + } + + uniq := make(map[string]struct{}, len(identities)) + for _, identity := range identities { + trimmed := strings.TrimSpace(identity) + if trimmed == "" { + continue + } + uniq[trimmed] = struct{}{} + } + + if len(uniq) == 0 { + return nil + } + + result := make([]string, 0, len(uniq)) + for identity := range uniq { + result = append(result, identity) + } + sort.Strings(result) + return result } diff --git a/internal/git/encryption_test.go b/internal/git/encryption_test.go index b8c242b6..f2167c6b 100644 --- a/internal/git/encryption_test.go +++ b/internal/git/encryption_test.go @@ -20,7 +20,9 @@ package git import ( "context" - "strings" + "os" + "path/filepath" + "sort" "testing" "filippo.io/age" @@ -51,6 +53,22 @@ func TestResolveTargetEncryption(t *testing.T) { assert.Nil(t, resolved) }) + t.Run("returns nil when age is disabled", func(t *testing.T) { + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + target := &v1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default"}, + Spec: v1alpha1.GitTargetSpec{ + Encryption: &v1alpha1.EncryptionSpec{ + Provider: EncryptionProviderSOPS, + }, + }, + } + + resolved, err := ResolveTargetEncryption(context.Background(), k8sClient, target) + require.NoError(t, err) + assert.Nil(t, resolved) + }) + t.Run("fails when provider is unsupported", func(t *testing.T) { k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() target := &v1alpha1.GitTarget{ @@ -58,8 +76,8 @@ func TestResolveTargetEncryption(t *testing.T) { Spec: v1alpha1.GitTargetSpec{ Encryption: &v1alpha1.EncryptionSpec{ Provider: "kms", - SecretRef: v1alpha1.LocalSecretReference{ - Name: "enc-secret", + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, }, }, }, @@ -70,32 +88,37 @@ func TestResolveTargetEncryption(t *testing.T) { assert.Contains(t, err.Error(), "unsupported encryption provider") }) - t.Run("fails when secret name is missing", func(t *testing.T) { + t.Run("fails when public key is invalid", func(t *testing.T) { k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() target := &v1alpha1.GitTarget{ ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default"}, Spec: v1alpha1.GitTargetSpec{ Encryption: &v1alpha1.EncryptionSpec{ - Provider: EncryptionProviderSOPS, - SecretRef: v1alpha1.LocalSecretReference{}, + Provider: EncryptionProviderSOPS, + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: v1alpha1.AgeRecipientsSpec{ + PublicKeys: []string{"invalid-recipient"}, + }, + }, }, }, } _, err := ResolveTargetEncryption(context.Background(), k8sClient, target) require.Error(t, err) - assert.Contains(t, err.Error(), "encryption.secretRef.name must be set") + assert.Contains(t, err.Error(), "invalid age recipient") }) - t.Run("fails when secret is missing", func(t *testing.T) { + t.Run("fails when no recipient resolves", func(t *testing.T) { k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() target := &v1alpha1.GitTarget{ ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default"}, Spec: v1alpha1.GitTargetSpec{ Encryption: &v1alpha1.EncryptionSpec{ Provider: EncryptionProviderSOPS, - SecretRef: v1alpha1.LocalSecretReference{ - Name: "enc-secret", + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, }, }, }, @@ -103,16 +126,59 @@ func TestResolveTargetEncryption(t *testing.T) { _, err := ResolveTargetEncryption(context.Background(), k8sClient, target) require.Error(t, err) - assert.Contains(t, err.Error(), "failed to fetch encryption secret") + assert.Contains(t, err.Error(), "requires at least one resolved recipient") }) - t.Run("fails when secret has no valid environment keys", func(t *testing.T) { - k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "enc-secret", Namespace: "default"}, - Data: map[string][]byte{ - "invalid.key": []byte("value"), + t.Run("resolves public key only mode without secret", func(t *testing.T) { + identity, err := age.GenerateX25519Identity() + require.NoError(t, err) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + target := &v1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default"}, + Spec: v1alpha1.GitTargetSpec{ + Encryption: &v1alpha1.EncryptionSpec{ + Provider: EncryptionProviderSOPS, + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: v1alpha1.AgeRecipientsSpec{ + PublicKeys: []string{identity.Recipient().String()}, + }, + }, + }, }, - }).Build() + } + + resolved, err := ResolveTargetEncryption(context.Background(), k8sClient, target) + require.NoError(t, err) + require.NotNil(t, resolved) + assert.Equal(t, []string{identity.Recipient().String()}, resolved.AgeRecipients) + assert.Nil(t, resolved.Environment) + }) + + t.Run("fails when extractFromSecret is enabled and secret name is empty", func(t *testing.T) { + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + target := &v1alpha1.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default"}, + Spec: v1alpha1.GitTargetSpec{ + Encryption: &v1alpha1.EncryptionSpec{ + Provider: EncryptionProviderSOPS, + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: v1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, + }, + }, + } + + _, err := ResolveTargetEncryption(context.Background(), k8sClient, target) + require.Error(t, err) + assert.Contains(t, err.Error(), "encryption.secretRef.name must be set") + }) + + t.Run("fails when extracted secret is missing", func(t *testing.T) { + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() target := &v1alpha1.GitTarget{ ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "default"}, Spec: v1alpha1.GitTargetSpec{ @@ -121,20 +187,26 @@ func TestResolveTargetEncryption(t *testing.T) { SecretRef: v1alpha1.LocalSecretReference{ Name: "enc-secret", }, + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: v1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, }, }, } _, err := ResolveTargetEncryption(context.Background(), k8sClient, target) require.Error(t, err) - assert.Contains(t, err.Error(), "must contain at least one valid environment variable entry") + assert.Contains(t, err.Error(), "failed to fetch encryption secret") }) - t.Run("returns resolved environment for valid sops config", func(t *testing.T) { + t.Run("fails when extracted agekey data is invalid", func(t *testing.T) { k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "enc-secret", Namespace: "default"}, Data: map[string][]byte{ - "SOPS_AGE_KEY": []byte("AGE-SECRET-KEY-1example"), + "identity.agekey": []byte("invalid"), }, }).Build() target := &v1alpha1.GitTarget{ @@ -145,22 +217,35 @@ func TestResolveTargetEncryption(t *testing.T) { SecretRef: v1alpha1.LocalSecretReference{ Name: "enc-secret", }, + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: v1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, }, }, } - resolved, err := ResolveTargetEncryption(context.Background(), k8sClient, target) - require.NoError(t, err) - require.NotNil(t, resolved) - assert.Equal(t, EncryptionProviderSOPS, resolved.Provider) - assert.Equal(t, "AGE-SECRET-KEY-1example", resolved.Environment["SOPS_AGE_KEY"]) + _, err := ResolveTargetEncryption(context.Background(), k8sClient, target) + require.Error(t, err) + assert.Contains(t, err.Error(), "identity.agekey must contain AGE-SECRET-KEY identity") }) - t.Run("defaults provider to sops when omitted", func(t *testing.T) { + t.Run("resolves recipients from public keys and secret entries", func(t *testing.T) { + firstIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + secondIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + thirdIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "enc-secret", Namespace: "default"}, Data: map[string][]byte{ - "SOPS_AGE_KEY": []byte("AGE-SECRET-KEY-1example"), + "identity.agekey": []byte(firstIdentity.String()), + "backup.agekey": []byte(secondIdentity.String()), + "SOPS_KMS_ARN": []byte("kms-arn"), }, }).Build() target := &v1alpha1.GitTarget{ @@ -170,6 +255,16 @@ func TestResolveTargetEncryption(t *testing.T) { SecretRef: v1alpha1.LocalSecretReference{ Name: "enc-secret", }, + Age: &v1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: v1alpha1.AgeRecipientsSpec{ + PublicKeys: []string{ + thirdIdentity.Recipient().String(), + firstIdentity.Recipient().String(), + }, + ExtractFromSecret: true, + }, + }, }, }, } @@ -178,29 +273,89 @@ func TestResolveTargetEncryption(t *testing.T) { require.NoError(t, err) require.NotNil(t, resolved) assert.Equal(t, EncryptionProviderSOPS, resolved.Provider) + + expectedRecipients := []string{ + firstIdentity.Recipient().String(), + secondIdentity.Recipient().String(), + thirdIdentity.Recipient().String(), + } + sort.Strings(expectedRecipients) + assert.Equal(t, expectedRecipients, resolved.AgeRecipients) + assert.Equal(t, "kms-arn", resolved.Environment["SOPS_KMS_ARN"]) + + expectedIdentities := []string{ + firstIdentity.String(), + secondIdentity.String(), + } + sort.Strings(expectedIdentities) + assert.Equal(t, expectedIdentities, resolved.AgeIdentities) + }) +} + +func TestBuildSOPSEnvironment(t *testing.T) { + t.Run("returns base environment when no age identities exist", func(t *testing.T) { + cfg := &ResolvedEncryptionConfig{ + Environment: map[string]string{ + "SOPS_KMS_ARN": "kms-arn", + }, + } + + env, err := buildSOPSEnvironment(t.TempDir(), cfg) + require.NoError(t, err) + assert.Equal(t, "kms-arn", env["SOPS_KMS_ARN"]) + _, hasKeyFile := env[sopsAgeKeyFileEnvVar] + assert.False(t, hasKeyFile) + }) + + t.Run("writes age identities to temp file and sets SOPS age key file env", func(t *testing.T) { + firstIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + secondIdentity, err := age.GenerateX25519Identity() + require.NoError(t, err) + + workDir := t.TempDir() + cfg := &ResolvedEncryptionConfig{ + Environment: map[string]string{ + "SOPS_KMS_ARN": "kms-arn", + }, + AgeIdentities: []string{firstIdentity.String(), secondIdentity.String()}, + } + + env, err := buildSOPSEnvironment(workDir, cfg) + require.NoError(t, err) + keyFilePath := env[sopsAgeKeyFileEnvVar] + require.NotEmpty(t, keyFilePath) + assert.Equal(t, ageIdentityFileDir, filepath.Base(filepath.Dir(keyFilePath))) + + content, err := os.ReadFile(keyFilePath) + require.NoError(t, err) + assert.Contains(t, string(content), firstIdentity.String()) + assert.Contains(t, string(content), secondIdentity.String()) + assert.Equal(t, "kms-arn", env["SOPS_KMS_ARN"]) }) } -func TestDeriveAgeRecipientFromSOPSKey(t *testing.T) { +func TestDeriveAgeRecipientFromSecretEntry(t *testing.T) { t.Run("returns recipient for single valid identity", func(t *testing.T) { identity, err := age.GenerateX25519Identity() require.NoError(t, err) - recipient, err := deriveAgeRecipientFromSOPSKey(identity.String()) + recipient, parsedIdentity, err := deriveAgeRecipientFromSecretEntry("identity.agekey", identity.String()) require.NoError(t, err) assert.Equal(t, identity.Recipient().String(), recipient) + assert.Equal(t, identity.String(), parsedIdentity) }) t.Run("fails when identity is missing", func(t *testing.T) { - _, err := deriveAgeRecipientFromSOPSKey("") + _, _, err := deriveAgeRecipientFromSecretEntry("identity.agekey", "") require.Error(t, err) - assert.Contains(t, err.Error(), "must contain one AGE-SECRET-KEY identity") + assert.Contains(t, err.Error(), "identity.agekey must contain one AGE-SECRET-KEY identity") }) t.Run("fails when identity is invalid", func(t *testing.T) { - _, err := deriveAgeRecipientFromSOPSKey("AGE-SECRET-KEY-1invalid") + _, _, err := deriveAgeRecipientFromSecretEntry("identity.agekey", "AGE-SECRET-KEY-1invalid") require.Error(t, err) - assert.Contains(t, err.Error(), "invalid SOPS_AGE_KEY identity") + assert.Contains(t, err.Error(), "invalid identity.agekey identity") }) t.Run("fails when multiple identities are present", func(t *testing.T) { @@ -209,8 +364,8 @@ func TestDeriveAgeRecipientFromSOPSKey(t *testing.T) { second, err := age.GenerateX25519Identity() require.NoError(t, err) - combined := strings.Join([]string{first.String(), second.String()}, "\n") - _, err = deriveAgeRecipientFromSOPSKey(combined) + combined := first.String() + "\n" + second.String() + _, _, err = deriveAgeRecipientFromSecretEntry("identity.agekey", combined) require.Error(t, err) assert.Contains(t, err.Error(), "must contain exactly one AGE-SECRET-KEY identity") }) diff --git a/internal/webhook/gittarget_validator.go b/internal/webhook/gittarget_validator.go index d6dbcfdc..e466aba5 100644 --- a/internal/webhook/gittarget_validator.go +++ b/internal/webhook/gittarget_validator.go @@ -68,6 +68,10 @@ func (v *GitTargetValidator) ValidateCreate( "branch", target.Spec.Branch, "path", target.Spec.Path) + if err := validateGitTargetEncryptionSpec(target); err != nil { + return nil, err + } + return v.validateUniqueness(ctx, target, nil) } @@ -88,6 +92,10 @@ func (v *GitTargetValidator) ValidateUpdate( "branch", newTarget.Spec.Branch, "path", newTarget.Spec.Path) + if err := validateGitTargetEncryptionSpec(newTarget); err != nil { + return nil, err + } + return v.validateUniqueness(ctx, newTarget, oldTarget) } @@ -290,3 +298,41 @@ func createTargetIdentifier(normalizedRepoURL, branch, path string) string { hash := sha256.Sum256([]byte(data)) return hex.EncodeToString(hash[:]) } + +func validateGitTargetEncryptionSpec(target *configbutleraiv1alpha1.GitTarget) error { + if target.Spec.Encryption == nil { + return nil + } + + encryption := target.Spec.Encryption + providerName := strings.TrimSpace(encryption.Provider) + if providerName == "" { + providerName = "sops" + } + if providerName != "sops" { + return nil + } + if encryption.Age == nil || !encryption.Age.Enabled { + return nil + } + + recipients := encryption.Age.Recipients + if len(recipients.PublicKeys) == 0 && !recipients.ExtractFromSecret { + return errors.New( + "spec.encryption.age.enabled=true requires recipients via age.recipients.publicKeys or age.recipients.extractFromSecret=true", + ) + } + if recipients.GenerateWhenMissing && !recipients.ExtractFromSecret { + return errors.New( + "spec.encryption.age.recipients.generateWhenMissing=true requires age.recipients.extractFromSecret=true", + ) + } + if (recipients.ExtractFromSecret || recipients.GenerateWhenMissing) && + strings.TrimSpace(encryption.SecretRef.Name) == "" { + return errors.New( + "spec.encryption.secretRef.name is required when age.recipients.extractFromSecret or age.recipients.generateWhenMissing is enabled", + ) + } + + return nil +} diff --git a/internal/webhook/gittarget_validator_test.go b/internal/webhook/gittarget_validator_test.go index 14d40f0f..a6ba153f 100644 --- a/internal/webhook/gittarget_validator_test.go +++ b/internal/webhook/gittarget_validator_test.go @@ -460,6 +460,102 @@ func TestGitTargetValidator_NilObject(t *testing.T) { }) } +func TestValidateGitTargetEncryptionSpec(t *testing.T) { + t.Run("passes when encryption is nil", func(t *testing.T) { + target := &configbutleraiv1alpha1.GitTarget{} + require.NoError(t, validateGitTargetEncryptionSpec(target)) + }) + + t.Run("passes when age is disabled", func(t *testing.T) { + target := &configbutleraiv1alpha1.GitTarget{ + Spec: configbutleraiv1alpha1.GitTargetSpec{ + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + }, + }, + } + require.NoError(t, validateGitTargetEncryptionSpec(target)) + }) + + t.Run("fails when age enabled without recipient sources", func(t *testing.T) { + target := &configbutleraiv1alpha1.GitTarget{ + Spec: configbutleraiv1alpha1.GitTargetSpec{ + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + }, + }, + }, + } + + err := validateGitTargetEncryptionSpec(target) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires recipients") + }) + + t.Run("fails when generateWhenMissing is set without extractFromSecret", func(t *testing.T) { + target := &configbutleraiv1alpha1.GitTarget{ + Spec: configbutleraiv1alpha1.GitTargetSpec{ + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + PublicKeys: []string{ + "age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq7k8m6", + }, + GenerateWhenMissing: true, + }, + }, + }, + }, + } + + err := validateGitTargetEncryptionSpec(target) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires age.recipients.extractFromSecret=true") + }) + + t.Run("fails when secretRef is missing for extractFromSecret", func(t *testing.T) { + target := &configbutleraiv1alpha1.GitTarget{ + Spec: configbutleraiv1alpha1.GitTargetSpec{ + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + ExtractFromSecret: true, + }, + }, + }, + }, + } + + err := validateGitTargetEncryptionSpec(target) + require.Error(t, err) + assert.Contains(t, err.Error(), "spec.encryption.secretRef.name is required") + }) + + t.Run("passes for public key only configuration", func(t *testing.T) { + target := &configbutleraiv1alpha1.GitTarget{ + Spec: configbutleraiv1alpha1.GitTargetSpec{ + Encryption: &configbutleraiv1alpha1.EncryptionSpec{ + Provider: "sops", + Age: &configbutleraiv1alpha1.AgeEncryptionSpec{ + Enabled: true, + Recipients: configbutleraiv1alpha1.AgeRecipientsSpec{ + PublicKeys: []string{"age1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq7k8m6"}, + }, + }, + }, + }, + } + + require.NoError(t, validateGitTargetEncryptionSpec(target)) + }) +} + // Mock client that returns errors for testing error paths. type errorClient struct { client.Client diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index b3ca2502..d9288c7f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -613,7 +613,7 @@ var _ = Describe("Manager", Ordered, func() { cleanupGitTarget(destName, namespace) }) - It("should generate missing SOPS age secret when generateWhenMissing is enabled", func() { + It("should generate missing SOPS age secret when age.recipients.generateWhenMissing is enabled", func() { gitProviderName := "gitprovider-normal" watchRuleName := "watchrule-secret-autogen-test" secretName := "test-secret-autogen" @@ -623,7 +623,7 @@ var _ = Describe("Manager", Ordered, func() { _, _ = utils.Run(exec.Command("kubectl", "delete", "secret", generatedSecretName, "-n", namespace, "--ignore-not-found=true")) - By("creating GitTarget with generateWhenMissing enabled") + By("creating GitTarget with age recipient auto-generation enabled") destName := watchRuleName + "-dest" createGitTargetWithEncryptionOptions( destName, @@ -668,9 +668,17 @@ var _ = Describe("Manager", Ordered, func() { g.Expect(annotations).To(HaveKeyWithValue("configbutler.ai/backup-warning", "REMOVE_AFTER_BACKUP")) generatedRecipient = annotations["configbutler.ai/age-recipient"] - sopsAgeKeyB64, found, keyErr := unstructured.NestedString(secretObj, "data", "SOPS_AGE_KEY") + secretData, found, keyErr := unstructured.NestedStringMap(secretObj, "data") g.Expect(keyErr).NotTo(HaveOccurred()) g.Expect(found).To(BeTrue()) + var sopsAgeKeyB64 string + for key, value := range secretData { + if strings.HasSuffix(key, ".agekey") { + sopsAgeKeyB64 = value + break + } + } + g.Expect(sopsAgeKeyB64).NotTo(BeEmpty()) keyBytes, decodeErr := base64.StdEncoding.DecodeString(sopsAgeKeyB64) g.Expect(decodeErr).NotTo(HaveOccurred()) diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index d0f726ac..fae8517c 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -299,7 +299,7 @@ metadata: namespace: %s type: Opaque stringData: - SOPS_AGE_KEY: %q + identity.agekey: %q `, e2eEncryptionRefName, namespace, identity.String()) ctx := context.Background() diff --git a/test/e2e/kind/start-cluster.sh b/test/e2e/kind/start-cluster.sh index 42963085..bd3ba28f 100755 --- a/test/e2e/kind/start-cluster.sh +++ b/test/e2e/kind/start-cluster.sh @@ -1,11 +1,13 @@ #!/bin/bash # Script to create Kind cluster with proper host path substitution for Docker-in-Docker -set -e +set -euo pipefail CLUSTER_NAME="${KIND_CLUSTER:-gitops-reverser-test-e2e}" TEMPLATE_FILE="test/e2e/kind/cluster-template.yaml" CONFIG_FILE="test/e2e/kind/cluster.ignore.yaml" +KIND_CREATE_LOG_FILE="${TMPDIR:-/tmp}/kind-create-${CLUSTER_NAME}.log" +POD_SUBNET="${KIND_POD_SUBNET:-10.244.0.0/16}" # Check if HOST_PROJECT_PATH is set if [ -z "$HOST_PROJECT_PATH" ]; then @@ -24,12 +26,72 @@ echo "✅ Generated configuration:" cat "$CONFIG_FILE" echo "" +create_primary_cluster() { + kind create cluster --name "$CLUSTER_NAME" --config "$CONFIG_FILE" --wait 5m --retain 2>&1 | tee "$KIND_CREATE_LOG_FILE" +} + +# Known Kind bootstrap race in DOOD setups: https://github.com/kubernetes-sigs/kind/issues/2867 +is_known_kind_bootstrap_flake() { + grep -Eq \ + "failed to apply overlay network|failed to remove control plane taint|failed to remove control plane load balancer label|failed to download openapi|couldn't get current server API group list|The connection to the server .*:6443 was refused" \ + "$KIND_CREATE_LOG_FILE" +} + +run_dood_self_heal() { + local control_plane_container="${CLUSTER_NAME}-control-plane" + docker inspect "$control_plane_container" >/dev/null 2>&1 || return 1 + + local ready=false + for _ in $(seq 1 120); do + if docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw=/readyz >/dev/null 2>&1; then + ready=true + break + fi + sleep 2 + done + if [ "$ready" != "true" ]; then + docker exec "$control_plane_container" crictl ps -a || true + return 1 + fi + + local applied=false + for _ in $(seq 1 40); do + if docker exec "$control_plane_container" sh -lc \ + "sed 's|{{ \\.PodSubnet }}|${POD_SUBNET}|g' /kind/manifests/default-cni.yaml > /tmp/default-cni-rendered.yaml && \ + kubectl --kubeconfig=/etc/kubernetes/admin.conf apply --validate=false -f /tmp/default-cni-rendered.yaml && \ + kubectl --kubeconfig=/etc/kubernetes/admin.conf apply --validate=false -f /kind/manifests/default-storage.yaml"; then + applied=true + break + fi + sleep 2 + done + [ "$applied" = "true" ] || return 1 + + docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ + taint nodes --all node-role.kubernetes.io/control-plane- >/dev/null 2>&1 || true + docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ + label nodes --all node.kubernetes.io/exclude-from-external-load-balancers- >/dev/null 2>&1 || true + + docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ + -n kube-system rollout status daemonset/kindnet --timeout=300s + + docker exec "$control_plane_container" kubectl --kubeconfig=/etc/kubernetes/admin.conf \ + wait --for=condition=Ready nodes --all --timeout=300s +} + if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then echo "♻️ Reusing existing Kind cluster '$CLUSTER_NAME' (no delete/recreate)" else echo "🚀 Creating Kind cluster '$CLUSTER_NAME' with audit webhook support..." - kind create cluster --name "$CLUSTER_NAME" --config "$CONFIG_FILE" --wait 5m - echo "✅ Kind cluster created successfully" + if create_primary_cluster; then + echo "✅ Kind cluster created successfully" + elif is_known_kind_bootstrap_flake && run_dood_self_heal; then + echo "✅ Kind cluster self-healed successfully" + else + echo "❌ Kind cluster creation failed." + echo "📄 See log: $KIND_CREATE_LOG_FILE" + exit 1 + fi fi echo "📋 Configuring kubeconfig for cluster '$CLUSTER_NAME'..." diff --git a/test/e2e/scripts/run-quickstart.sh b/test/e2e/scripts/run-quickstart.sh index a20c9064..b2f046bc 100755 --- a/test/e2e/scripts/run-quickstart.sh +++ b/test/e2e/scripts/run-quickstart.sh @@ -373,10 +373,12 @@ extract_generated_age_key() { local key_data age_key key_data="$( - kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${secret_name}" -o jsonpath='{.data.SOPS_AGE_KEY}' 2>/dev/null || true + kubectl -n "${QUICKSTART_NAMESPACE}" get secret "${secret_name}" \ + -o go-template='{{ range $k, $v := .data }}{{ printf "%s=%s\n" $k $v }}{{ end }}' 2>/dev/null | + grep '\.agekey=' | head -n1 | cut -d'=' -f2- || true )" if [[ -z "${key_data}" ]]; then - echo "SOPS_AGE_KEY is missing from secret ${QUICKSTART_NAMESPACE}/${secret_name}" + echo "No .agekey entry found in secret ${QUICKSTART_NAMESPACE}/${secret_name}" return 1 fi @@ -444,9 +446,13 @@ spec: path: live-cluster encryption: provider: sops + age: + enabled: true + recipients: + extractFromSecret: true + generateWhenMissing: true secretRef: name: ${ENCRYPTION_SECRET_NAME} - generateWhenMissing: true --- apiVersion: configbutler.ai/v1alpha1 kind: WatchRule diff --git a/test/e2e/templates/gittarget.tmpl b/test/e2e/templates/gittarget.tmpl index b5c930bc..0d7a10b9 100644 --- a/test/e2e/templates/gittarget.tmpl +++ b/test/e2e/templates/gittarget.tmpl @@ -11,8 +11,12 @@ spec: path: {{ .Path }} encryption: provider: sops - secretRef: - name: {{ .EncryptionSecretName }} + age: + enabled: true + recipients: + extractFromSecret: true {{- if .GenerateWhenMissing }} - generateWhenMissing: true + generateWhenMissing: true {{- end }} + secretRef: + name: {{ .EncryptionSecretName }}