Add Helm resilience profile

Makefile

@@ -5,7 +5,7 @@ BIN_DIR := ./bin
 VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 LDFLAGS := -ldflags "-s -w -X main.Version=$(VERSION)"
 
-.PHONY: help build server cli install clean fmt lint vet test cover mut sec bench tools ci
+.PHONY: help build server cli install clean fmt lint vet test cover mut sec bench helm-test tools ci
 
 help: ## List available targets.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-12s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -73,4 +73,7 @@ sec: ## Run govulncheck, gosec, osv-scanner.
 bench: ## Run benchmarks across all packages.
 	go test -run=^$$ -bench=. -benchmem ./...
 
+helm-test: ## Render-test Helm resilience profiles.
+	scripts/test_helm_resilience.sh
+
 ci: vet lint test cover sec ## Full quality gate (mirror of CI workflow).

README.md

@@ -90,6 +90,12 @@ For the full flag surface:
 ./bin/cefas   --help
 ```
 
+### Kubernetes
+
+The Helm chart includes an RF=3 resilience profile for StatefulSet
+placement, PVC-backed storage, disruption control, and database resource
+policy. See [`docs/helm-resilience.md`](docs/helm-resilience.md).
+
 ## APIs
 
 CefasDB exposes the same surface on three transports:

dist/helm/cefas/templates/_helpers.tpl

@@ -29,3 +29,90 @@ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
 app.kubernetes.io/name: {{ include "cefas.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end -}}
+
+{{- define "cefas.resolvedReplicaCount" -}}
+{{- $replicas := int .Values.replicaCount -}}
+{{- if .Values.resilience.enabled -}}
+{{- $resilienceReplicas := int .Values.resilience.replicas -}}
+{{- if gt $resilienceReplicas $replicas -}}
+{{- $resilienceReplicas -}}
+{{- else -}}
+{{- $replicas -}}
+{{- end -}}
+{{- else -}}
+{{- $replicas -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "cefas.resolvedReplicationFactor" -}}
+{{- if gt (int .Values.cluster.replicationFactor) 0 -}}
+{{- int .Values.cluster.replicationFactor -}}
+{{- else if .Values.resilience.enabled -}}
+{{- int .Values.resilience.replicationFactor -}}
+{{- else -}}
+0
+{{- end -}}
+{{- end -}}
+
+{{- define "cefas.podDNS" -}}
+{{- $root := index . 0 -}}
+{{- $ordinal := index . 1 -}}
+{{ include "cefas.fullname" $root }}-{{ $ordinal }}.{{ include "cefas.headless" $root }}.{{ $root.Release.Namespace }}.svc.cluster.local
+{{- end -}}
+
+{{- define "cefas.startupProbe" -}}
+httpGet:
+{{- toYaml .Values.startupProbe.httpGet | nindent 2 }}
+{{- if .Values.resilience.enabled }}
+failureThreshold: {{ max (int .Values.startupProbe.failureThreshold) (int .Values.resilience.startupProbe.failureThreshold) }}
+periodSeconds: {{ max (int .Values.startupProbe.periodSeconds) (int .Values.resilience.startupProbe.periodSeconds) }}
+{{- else }}
+{{- omit .Values.startupProbe "httpGet" | toYaml | nindent 0 }}
+{{- end }}
+{{- end -}}
+
+{{- define "cefas.dbResources" -}}
+{{- $requests := .Values.resources.requests | default dict -}}
+{{- $limits := .Values.resources.limits | default dict -}}
+{{- if $requests }}
+requests:
+{{- toYaml $requests | nindent 2 }}
+{{- end }}
+{{- if or $limits.cpu $limits.memory }}
+limits:
+{{- if and $limits.cpu (not .Values.resourcePolicy.disableCPULimits) }}
+  cpu: {{ $limits.cpu | quote }}
+{{- end }}
+{{- if $limits.memory }}
+  memory: {{ $limits.memory | quote }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{- define "cefas.validate" -}}
+{{- if .Values.resilience.enabled -}}
+{{- $replicas := include "cefas.resolvedReplicaCount" . | int -}}
+{{- $rf := include "cefas.resolvedReplicationFactor" . | int -}}
+{{- if lt (int .Values.resilience.replicas) 3 -}}
+{{- fail "resilience.replicas must be >= 3 for the RF=3 resilience profile" -}}
+{{- end -}}
+{{- if lt $replicas 3 -}}
+{{- fail "resilience.enabled=true requires at least 3 rendered database replicas" -}}
+{{- end -}}
+{{- if lt $rf 3 -}}
+{{- fail "resilience.enabled=true requires cluster.replicationFactor or resilience.replicationFactor >= 3" -}}
+{{- end -}}
+{{- if gt $rf $replicas -}}
+{{- fail "replication factor cannot exceed rendered database replicas" -}}
+{{- end -}}
+{{- if and .Values.resilience.requirePersistentStorage (not .Values.persistence.enabled) (not .Values.resilience.allowEphemeralStorage) -}}
+{{- fail "resilience.enabled=true requires persistence.enabled=true unless resilience.allowEphemeralStorage=true is explicitly set" -}}
+{{- end -}}
+{{- if and .Values.resourcePolicy.requireMemoryLimit (not .Values.resources.limits.memory) -}}
+{{- fail "resilience.enabled=true requires resources.limits.memory to preserve memory safeguards" -}}
+{{- end -}}
+{{- if lt (int .Values.startupProbe.failureThreshold) 1 -}}
+{{- fail "startupProbe.failureThreshold must be >= 1" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

dist/helm/cefas/templates/configmap.yaml

@@ -20,8 +20,24 @@ data:
       fsyncOnCommit: {{ .Values.cluster.fsyncOnCommit }}
     cluster:
       shards: {{ .Values.cluster.shards }}
+      replicationFactor: {{ include "cefas.resolvedReplicationFactor" . }}
       muxAddr: ":{{ .Values.service.mux }}"
       bootstrap: {{ .Values.cluster.bootstrap }}
+      {{- $replicas := include "cefas.resolvedReplicaCount" . | int }}
+      {{- if gt $replicas 1 }}
+      peers:
+        {{- range $i := until $replicas }}
+        {{ printf "%s-%d" (include "cefas.fullname" $) $i | quote }}: {{ printf "%s:%v" (include "cefas.podDNS" (list $ $i)) $.Values.service.mux | quote }}
+        {{- end }}
+      httpPeers:
+        {{- range $i := until $replicas }}
+        {{ printf "%s-%d" (include "cefas.fullname" $) $i | quote }}: {{ printf "http://%s:%v" (include "cefas.podDNS" (list $ $i)) $.Values.service.http | quote }}
+        {{- end }}
+      grpcPeers:
+        {{- range $i := until $replicas }}
+        {{ printf "%s-%d" (include "cefas.fullname" $) $i | quote }}: {{ printf "%s:%v" (include "cefas.podDNS" (list $ $i)) $.Values.service.grpc | quote }}
+        {{- end }}
+      {{- end }}
     metrics:
       enabled: {{ .Values.metrics.enabled }}
     identity:

dist/helm/cefas/templates/manager-deployment.yaml

@@ -10,12 +10,14 @@ spec:
   replicas: {{ .Values.manager.replicaCount }}
   selector:
     matchLabels:
-      {{- include "cefas.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/name: {{ include "cefas.name" . }}-manager
+      app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: manager
   template:
     metadata:
       labels:
-        {{- include "cefas.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/name: {{ include "cefas.name" . }}-manager
+        app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: manager
     spec:
       serviceAccountName: {{ include "cefas.serviceAccountName" . }}

dist/helm/cefas/templates/pdb.yaml

@@ -0,0 +1,17 @@
+{{- if and .Values.resilience.enabled .Values.resilience.podDisruptionBudget.enabled -}}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "cefas.fullname" . }}
+  labels:
+    {{- include "cefas.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.resilience.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.resilience.podDisruptionBudget.minAvailable }}
+  {{- else }}
+  maxUnavailable: {{ .Values.resilience.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "cefas.selectorLabels" . | nindent 6 }}
+{{- end }}

dist/helm/cefas/templates/statefulset.yaml

@@ -6,7 +6,7 @@ metadata:
     {{- include "cefas.labels" . | nindent 4 }}
 spec:
   serviceName: {{ include "cefas.headless" . }}
-  replicas: {{ .Values.replicaCount }}
+  replicas: {{ include "cefas.resolvedReplicaCount" . }}
   podManagementPolicy: Parallel
   selector:
     matchLabels:
@@ -21,6 +21,42 @@ spec:
       serviceAccountName: {{ include "cefas.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.securityContext | nindent 8 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.affinity }}
+      affinity:
+        {{- toYaml .Values.affinity | nindent 8 }}
+      {{- else if and .Values.resilience.enabled .Values.resilience.scheduling.antiAffinity.enabled }}
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: {{ .Values.resilience.scheduling.antiAffinity.weight }}
+              podAffinityTerm:
+                topologyKey: {{ .Values.resilience.scheduling.antiAffinity.topologyKey | quote }}
+                labelSelector:
+                  matchLabels:
+                    {{- include "cefas.selectorLabels" . | nindent 20 }}
+      {{- end }}
+      {{- if .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
+      {{- else if and .Values.resilience.enabled .Values.resilience.scheduling.topologySpread.enabled }}
+      topologySpreadConstraints:
+        {{- range .Values.resilience.scheduling.topologySpread.topologyKeys }}
+        - maxSkew: {{ $.Values.resilience.scheduling.topologySpread.maxSkew }}
+          topologyKey: {{ . | quote }}
+          whenUnsatisfiable: {{ $.Values.resilience.scheduling.topologySpread.whenUnsatisfiable | quote }}
+          labelSelector:
+            matchLabels:
+              {{- include "cefas.selectorLabels" $ | nindent 14 }}
+        {{- end }}
+      {{- end }}
       terminationGracePeriodSeconds: {{ .Values.lifecycle.terminationGracePeriodSeconds }}
       containers:
         - name: cefas
@@ -67,15 +103,15 @@ spec:
           readinessProbe:
             {{- toYaml .Values.readinessProbe | nindent 12 }}
           startupProbe:
-            {{- toYaml .Values.startupProbe | nindent 12 }}
+            {{- include "cefas.startupProbe" . | nindent 12 }}
           volumeMounts:
             - name: data
               mountPath: /var/lib/cefas
             - name: config
               mountPath: /etc/cefas
               readOnly: true
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            {{- include "cefas.dbResources" . | nindent 12 }}
       volumes:
         - name: config
           configMap:

dist/helm/cefas/templates/validation.yaml

@@ -0,0 +1 @@
+{{- include "cefas.validate" . -}}

dist/helm/cefas/values.yaml

@@ -1,8 +1,7 @@
 # cefas Helm chart values.
 #
-# Defaults bring up a single replica StatefulSet on hostPort-style
-# Services with persistent storage. Multi-node clusters require
-# replicaCount ≥ 3 and the multi-Raft flags below.
+# Defaults bring up a single replica StatefulSet for local development.
+# Set resilience.enabled=true for an RF=3-safe Kubernetes profile.
 
 image:
   repository: ghcr.io/cefasdb/cefasdb
@@ -15,6 +14,37 @@ serviceAccount:
   create: true
   name: ""
 
+resilience:
+  enabled: false
+  # Used when resilience.enabled=true. The rendered StatefulSet uses
+  # max(replicaCount, resilience.replicas) so `--set resilience.enabled=true`
+  # is enough to render three database Pods.
+  replicas: 3
+  replicationFactor: 3
+  requirePersistentStorage: true
+  allowEphemeralStorage: false
+  startupProbe:
+    # 120*2s gives recovering Pods up to four minutes for bootstrap,
+    # log replay, and shard startup before kubelet restarts the process.
+    failureThreshold: 120
+    periodSeconds: 2
+  podDisruptionBudget:
+    enabled: true
+    maxUnavailable: 1
+    minAvailable: ""
+  scheduling:
+    antiAffinity:
+      enabled: true
+      topologyKey: kubernetes.io/hostname
+      weight: 100
+    topologySpread:
+      enabled: true
+      maxSkew: 1
+      whenUnsatisfiable: ScheduleAnyway
+      topologyKeys:
+        - kubernetes.io/hostname
+        - cefasdb.io/failure-domain
+
 manager:
   enabled: true
   replicaCount: 2
@@ -34,6 +64,13 @@ manager:
       cpu: "500m"
       memory: "512Mi"
 
+resourcePolicy:
+  # Database performance tests and dedicated DB nodes usually should not
+  # use CPU limits; set this true to omit resources.limits.cpu while
+  # retaining memory limits.
+  disableCPULimits: false
+  requireMemoryLimit: true
+
 resources:
   requests:
     cpu: "200m"
@@ -67,6 +104,9 @@ lifecycle:
 # Multi-Raft sharding. shards>1 turns on the mux transport.
 cluster:
   shards: 1
+  # 0 means "use every peer". With resilience.enabled=true this resolves
+  # to resilience.replicationFactor unless explicitly set to >=3.
+  replicationFactor: 0
   bootstrap: true
   fsyncOnCommit: false
 
@@ -111,6 +151,11 @@ podAnnotations:
   prometheus.io/port: "8080"
   prometheus.io/path: "/metrics"
 
+nodeSelector: {}
+tolerations: []
+affinity: {}
+topologySpreadConstraints: []
+
 securityContext:
   runAsNonRoot: true
   runAsUser: 65532