fix(deploy): keep the scaffold's pod spread in lockstep with the platform

.github/workflows/validate-scaffold.yaml

@@ -510,3 +510,154 @@ jobs:
             tenant-clustersecretstore.yaml \
             --values-file tenant-values.yaml \
             --userinfo platform-user-info.yaml
+      - name: 🔗 Assert the spread-pods lockstep
+        id: assert-spread-lockstep
+        # The platform's `spread-pods` policy injects its topology spread
+        # constraint through Kyverno's ADD-IF-NOT-PRESENT anchor
+        # (`+(topologySpreadConstraints)`). Because the scaffold authors its own
+        # constraint, that mutation never fires — the scaffold's block is what
+        # actually runs, and any guard the platform adds to its version (notably
+        # `matchLabelKeys: [pod-template-hash]`, which stops a rolling update
+        # deadlocking under `DoNotSchedule`) is silently NOT inherited.
+        #
+        # So assert the two agree, reading the expectation from the platform's
+        # live policy rather than a copy: if the platform changes its remedy,
+        # this goes red and the scaffold gets updated. Same philosophy as the
+        # rest of this job — drift IS the signal.
+        run: |
+          set -euo pipefail
+
+          kubectl kustomize deploy/ > rendered.yaml
+          kubectl kustomize .platform/k8s/bases/infrastructure/cluster-policies \
+            > spread-policies.yaml
+
+          # Compare the WHOLE constraint LIST, not one element and not a field
+          # allowlist. Indexing `[0]` would let the platform add a second
+          # constraint (say, a zone spread) that the scaffold never inherits
+          # while this stayed green; an allowlist silently drops any field it
+          # doesn't name, so the day the platform adds `minDomains`,
+          # `nodeAffinityPolicy` or `nodeTaintsPolicy` the gate would stay green
+          # while the scaffold diverged. Both are the exact drift this gate
+          # exists to catch. `labelSelector` is the ONE field held out here —
+          # the platform templates it from the admission request
+          # (matchExpressions over a Kyverno variable) while the scaffold
+          # hard-codes the same app via matchLabels, so comparing them literally
+          # would be permanent false drift — and it is asserted separately
+          # below rather than discarded. `sort_keys` makes the comparison
+          # independent of authoring order on either side.
+          normalise='del(.[].labelSelector) | sort_keys(..)'
+
+          expected=$(yq eval-all "
+            select(.kind == \"ClusterPolicy\" and .metadata.name == \"spread-pods\")
+            | .spec.rules[0].mutate.patchStrategicMerge.spec.template.spec.\"+(topologySpreadConstraints)\"
+            | ${normalise}
+          " spread-policies.yaml)
+
+          actual=$(yq eval-all "
+            select(.kind == \"Deployment\")
+            | .spec.template.spec.topologySpreadConstraints
+            | ${normalise}
+          " rendered.yaml)
+
+          # Fail closed on a read that found nothing: an empty document, a bare
+          # `null`, or an empty list means the policy was renamed, the anchor
+          # changed, or the block was deleted outright — none of which may pass
+          # green. A read that found constraints but DIFFERENT ones falls
+          # through to the diff below, which shows both sides.
+          for pair in "platform:${expected}" "scaffold:${actual}"; do
+            side=${pair%%:*}
+            doc=${pair#*:}
+            if [ -z "${doc}" ] || [ "${doc}" = "null" ] || [ "${doc}" = "[]" ]; then
+              echo "::error::${side}: found no topology spread constraint at all — the policy or the scaffold moved; fix the reader rather than trusting a vacuous green"
+              printf '%s\n' "${doc}"
+              exit 1
+            fi
+          done
+
+          if [ "${expected}" != "${actual}" ]; then
+            echo "::error::the scaffold's topologySpreadConstraints no longer matches what the platform's spread-pods policy would inject"
+            echo "--- platform (spread-pods) ---"
+            printf '%s\n' "${expected}"
+            echo "--- scaffold (deploy/deployment.yaml) ---"
+            printf '%s\n' "${actual}"
+            exit 1
+          fi
+
+          # The selectors are held out above because they are SPELLED
+          # differently, not because they are unimportant: the selector decides
+          # which pods count toward the skew, so a platform change there
+          # silently changes the scaffold's scheduling guarantee. Rather than
+          # canonicalising two shapes into one — which re-introduces the same
+          # blindness the moment a shape gains a field — pin each side to the
+          # exact form that was hand-verified equivalent: the platform selects
+          # `app.kubernetes.io/name In [<the admitted object's own name>]`, the
+          # scaffold selects `app.kubernetes.io/name = <its own name>`. Same
+          # selection, different spelling. If EITHER shape changes at all — a
+          # second matchExpression, a different key, a broadened operator —
+          # this goes red and a human re-derives the equivalence instead of a
+          # script guessing at it.
+          platform_selectors=$(yq eval-all "
+            select(.kind == \"ClusterPolicy\" and .metadata.name == \"spread-pods\")
+            | [ .spec.rules[0].mutate.patchStrategicMerge.spec.template.spec.\"+(topologySpreadConstraints)\"[].labelSelector ]
+            | sort_keys(..)
+          " spread-policies.yaml)
+
+          expected_platform_selectors=$(yq eval 'sort_keys(..)' - <<'PLATFORM_SELECTOR'
+          - matchExpressions:
+              - key: app.kubernetes.io/name
+                operator: In
+                values:
+                  - '{{request.object.spec.template.metadata.labels."app.kubernetes.io/name"}}'
+          PLATFORM_SELECTOR
+          )
+
+          if [ "${platform_selectors}" != "${expected_platform_selectors}" ]; then
+            echo "::error::the platform's spread-pods labelSelector is no longer the shape this scaffold was verified equivalent to — re-derive the equivalence and update both sides"
+            echo "--- platform (spread-pods) ---"
+            printf '%s\n' "${platform_selectors}"
+            echo "--- verified-equivalent shape ---"
+            printf '%s\n' "${expected_platform_selectors}"
+            exit 1
+          fi
+
+          # Asserting the scaffold's selector against the Deployment's OWN app
+          # name — rather than a hardcoded string — is what makes it the same
+          # statement as the platform's request-derived variable. `strenv`
+          # interpolates the name without it ever being YAML text, so a value
+          # needing quoting cannot change the document's shape.
+          scaffold_app=$(yq eval-all '
+            select(.kind == "Deployment")
+            | .spec.template.metadata.labels."app.kubernetes.io/name"
+          ' rendered.yaml)
+
+          if [ -z "${scaffold_app}" ] || [ "${scaffold_app}" = "null" ]; then
+            echo "::error::scaffold: the Deployment has no app.kubernetes.io/name template label, so its spread selector cannot be checked against its own identity"
+            exit 1
+          fi
+
+          scaffold_selectors=$(yq eval-all "
+            select(.kind == \"Deployment\")
+            | [ .spec.template.spec.topologySpreadConstraints[].labelSelector ]
+            | sort_keys(..)
+          " rendered.yaml)
+
+          expected_scaffold_selectors=$(SCAFFOLD_APP="${scaffold_app}" yq eval '
+            .[0].matchLabels."app.kubernetes.io/name" = strenv(SCAFFOLD_APP) | sort_keys(..)
+          ' - <<'SCAFFOLD_SELECTOR'
+          - matchLabels:
+              app.kubernetes.io/name: PLACEHOLDER
+          SCAFFOLD_SELECTOR
+          )
+
+          if [ "${scaffold_selectors}" != "${expected_scaffold_selectors}" ]; then
+            echo "::error::the scaffold's spread labelSelector no longer selects exactly its own app, so it no longer matches the platform's request-derived selection"
+            echo "--- scaffold ---"
+            printf '%s\n' "${scaffold_selectors}"
+            echo "--- expected (selects its own app.kubernetes.io/name) ---"
+            printf '%s\n' "${expected_scaffold_selectors}"
+            exit 1
+          fi
+
+          echo "spread-pods lockstep OK:"
+          printf '%s\n' "${actual}"
+          echo "selectors verified equivalent (platform request-derived In; scaffold matchLabels ${scaffold_app})"

deploy/deployment.yaml

@@ -62,10 +62,25 @@ spec:
       # Spread replicas across nodes so a scale-up to 2-3 replicas is actually
       # drain-safe: without this, both replicas can land on one node and defeat
       # the `maxUnavailable: 1` PDB. A single replica satisfies it trivially.
+      #
+      # ⚠️ The platform's `spread-pods` ClusterPolicy would inject an equivalent
+      # constraint — but its mutation uses Kyverno's add-if-not-present anchor
+      # (`+(topologySpreadConstraints)`), so authoring one here means the
+      # platform's version is skipped entirely and this block is what runs.
+      # Keep it in lockstep with the platform's; `matchLabelKeys` in particular
+      # is load-bearing, not optional (see below). The `Platform Admissibility`
+      # CI job asserts the two still agree.
       topologySpreadConstraints:
         - maxSkew: 1
           topologyKey: kubernetes.io/hostname
           whenUnsatisfiable: DoNotSchedule
+          # Scope skew per ReplicaSet revision. Without this, a rolling update's
+          # surge pods (new revision) count against the old revision's skew, and
+          # under `DoNotSchedule` the new ReplicaSet can be left unschedulable —
+          # the rollout deadlocks. Only bites at 2+ replicas, which is exactly
+          # when you least want to discover it. Requires k8s >= 1.27 (GA 1.34).
+          matchLabelKeys:
+            - pod-template-hash
           labelSelector:
             matchLabels:
               app.kubernetes.io/name: app