From b5d27c0838ddac6f310a521fca0dc90595165e95 Mon Sep 17 00:00:00 2001 From: Osvaldo Andrade Date: Thu, 25 Jun 2026 17:46:23 -0300 Subject: [PATCH] Add Helm resilience profile --- Makefile | 5 +- README.md | 6 ++ dist/helm/cefas/templates/_helpers.tpl | 87 ++++++++++++++++ dist/helm/cefas/templates/configmap.yaml | 16 +++ .../cefas/templates/manager-deployment.yaml | 6 +- dist/helm/cefas/templates/pdb.yaml | 17 ++++ dist/helm/cefas/templates/statefulset.yaml | 42 +++++++- dist/helm/cefas/templates/validation.yaml | 1 + dist/helm/cefas/values.yaml | 51 +++++++++- docs/helm-resilience.md | 98 +++++++++++++++++++ scripts/test_helm_resilience.sh | 72 ++++++++++++++ 11 files changed, 392 insertions(+), 9 deletions(-) create mode 100644 dist/helm/cefas/templates/pdb.yaml create mode 100644 dist/helm/cefas/templates/validation.yaml create mode 100644 docs/helm-resilience.md create mode 100755 scripts/test_helm_resilience.sh diff --git a/Makefile b/Makefile index b5b60f0..0e61f7a 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BIN_DIR := ./bin VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -ldflags "-s -w -X main.Version=$(VERSION)" -.PHONY: help build server cli install clean fmt lint vet test cover mut sec bench tools ci +.PHONY: help build server cli install clean fmt lint vet test cover mut sec bench helm-test tools ci help: ## List available targets. @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-12s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -73,4 +73,7 @@ sec: ## Run govulncheck, gosec, osv-scanner. bench: ## Run benchmarks across all packages. go test -run=^$$ -bench=. -benchmem ./... +helm-test: ## Render-test Helm resilience profiles. + scripts/test_helm_resilience.sh + ci: vet lint test cover sec ## Full quality gate (mirror of CI workflow). diff --git a/README.md b/README.md index b597267..9260904 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,12 @@ For the full flag surface: ./bin/cefas --help ``` +### Kubernetes + +The Helm chart includes an RF=3 resilience profile for StatefulSet +placement, PVC-backed storage, disruption control, and database resource +policy. See [`docs/helm-resilience.md`](docs/helm-resilience.md). + ## APIs CefasDB exposes the same surface on three transports: diff --git a/dist/helm/cefas/templates/_helpers.tpl b/dist/helm/cefas/templates/_helpers.tpl index 20875e7..fd11301 100644 --- a/dist/helm/cefas/templates/_helpers.tpl +++ b/dist/helm/cefas/templates/_helpers.tpl @@ -29,3 +29,90 @@ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/name: {{ include "cefas.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} + +{{- define "cefas.resolvedReplicaCount" -}} +{{- $replicas := int .Values.replicaCount -}} +{{- if .Values.resilience.enabled -}} +{{- $resilienceReplicas := int .Values.resilience.replicas -}} +{{- if gt $resilienceReplicas $replicas -}} +{{- $resilienceReplicas -}} +{{- else -}} +{{- $replicas -}} +{{- end -}} +{{- else -}} +{{- $replicas -}} +{{- end -}} +{{- end -}} + +{{- define "cefas.resolvedReplicationFactor" -}} +{{- if gt (int .Values.cluster.replicationFactor) 0 -}} +{{- int .Values.cluster.replicationFactor -}} +{{- else if .Values.resilience.enabled -}} +{{- int .Values.resilience.replicationFactor -}} +{{- else -}} +0 +{{- end -}} +{{- end -}} + +{{- define "cefas.podDNS" -}} +{{- $root := index . 0 -}} +{{- $ordinal := index . 1 -}} +{{ include "cefas.fullname" $root }}-{{ $ordinal }}.{{ include "cefas.headless" $root }}.{{ $root.Release.Namespace }}.svc.cluster.local +{{- end -}} + +{{- define "cefas.startupProbe" -}} +httpGet: +{{- toYaml .Values.startupProbe.httpGet | nindent 2 }} +{{- if .Values.resilience.enabled }} +failureThreshold: {{ max (int .Values.startupProbe.failureThreshold) (int .Values.resilience.startupProbe.failureThreshold) }} +periodSeconds: {{ max (int .Values.startupProbe.periodSeconds) (int .Values.resilience.startupProbe.periodSeconds) }} +{{- else }} +{{- omit .Values.startupProbe "httpGet" | toYaml | nindent 0 }} +{{- end }} +{{- end -}} + +{{- define "cefas.dbResources" -}} +{{- $requests := .Values.resources.requests | default dict -}} +{{- $limits := .Values.resources.limits | default dict -}} +{{- if $requests }} +requests: +{{- toYaml $requests | nindent 2 }} +{{- end }} +{{- if or $limits.cpu $limits.memory }} +limits: +{{- if and $limits.cpu (not .Values.resourcePolicy.disableCPULimits) }} + cpu: {{ $limits.cpu | quote }} +{{- end }} +{{- if $limits.memory }} + memory: {{ $limits.memory | quote }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "cefas.validate" -}} +{{- if .Values.resilience.enabled -}} +{{- $replicas := include "cefas.resolvedReplicaCount" . | int -}} +{{- $rf := include "cefas.resolvedReplicationFactor" . | int -}} +{{- if lt (int .Values.resilience.replicas) 3 -}} +{{- fail "resilience.replicas must be >= 3 for the RF=3 resilience profile" -}} +{{- end -}} +{{- if lt $replicas 3 -}} +{{- fail "resilience.enabled=true requires at least 3 rendered database replicas" -}} +{{- end -}} +{{- if lt $rf 3 -}} +{{- fail "resilience.enabled=true requires cluster.replicationFactor or resilience.replicationFactor >= 3" -}} +{{- end -}} +{{- if gt $rf $replicas -}} +{{- fail "replication factor cannot exceed rendered database replicas" -}} +{{- end -}} +{{- if and .Values.resilience.requirePersistentStorage (not .Values.persistence.enabled) (not .Values.resilience.allowEphemeralStorage) -}} +{{- fail "resilience.enabled=true requires persistence.enabled=true unless resilience.allowEphemeralStorage=true is explicitly set" -}} +{{- end -}} +{{- if and .Values.resourcePolicy.requireMemoryLimit (not .Values.resources.limits.memory) -}} +{{- fail "resilience.enabled=true requires resources.limits.memory to preserve memory safeguards" -}} +{{- end -}} +{{- if lt (int .Values.startupProbe.failureThreshold) 1 -}} +{{- fail "startupProbe.failureThreshold must be >= 1" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/dist/helm/cefas/templates/configmap.yaml b/dist/helm/cefas/templates/configmap.yaml index 27fe1aa..e091960 100644 --- a/dist/helm/cefas/templates/configmap.yaml +++ b/dist/helm/cefas/templates/configmap.yaml @@ -20,8 +20,24 @@ data: fsyncOnCommit: {{ .Values.cluster.fsyncOnCommit }} cluster: shards: {{ .Values.cluster.shards }} + replicationFactor: {{ include "cefas.resolvedReplicationFactor" . }} muxAddr: ":{{ .Values.service.mux }}" bootstrap: {{ .Values.cluster.bootstrap }} + {{- $replicas := include "cefas.resolvedReplicaCount" . | int }} + {{- if gt $replicas 1 }} + peers: + {{- range $i := until $replicas }} + {{ printf "%s-%d" (include "cefas.fullname" $) $i | quote }}: {{ printf "%s:%v" (include "cefas.podDNS" (list $ $i)) $.Values.service.mux | quote }} + {{- end }} + httpPeers: + {{- range $i := until $replicas }} + {{ printf "%s-%d" (include "cefas.fullname" $) $i | quote }}: {{ printf "http://%s:%v" (include "cefas.podDNS" (list $ $i)) $.Values.service.http | quote }} + {{- end }} + grpcPeers: + {{- range $i := until $replicas }} + {{ printf "%s-%d" (include "cefas.fullname" $) $i | quote }}: {{ printf "%s:%v" (include "cefas.podDNS" (list $ $i)) $.Values.service.grpc | quote }} + {{- end }} + {{- end }} metrics: enabled: {{ .Values.metrics.enabled }} identity: diff --git a/dist/helm/cefas/templates/manager-deployment.yaml b/dist/helm/cefas/templates/manager-deployment.yaml index c21d7e3..a90a475 100644 --- a/dist/helm/cefas/templates/manager-deployment.yaml +++ b/dist/helm/cefas/templates/manager-deployment.yaml @@ -10,12 +10,14 @@ spec: replicas: {{ .Values.manager.replicaCount }} selector: matchLabels: - {{- include "cefas.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ include "cefas.name" . }}-manager + app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: manager template: metadata: labels: - {{- include "cefas.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ include "cefas.name" . }}-manager + app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: manager spec: serviceAccountName: {{ include "cefas.serviceAccountName" . }} diff --git a/dist/helm/cefas/templates/pdb.yaml b/dist/helm/cefas/templates/pdb.yaml new file mode 100644 index 0000000..16643f0 --- /dev/null +++ b/dist/helm/cefas/templates/pdb.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.resilience.enabled .Values.resilience.podDisruptionBudget.enabled -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "cefas.fullname" . }} + labels: + {{- include "cefas.labels" . | nindent 4 }} +spec: + {{- if .Values.resilience.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.resilience.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.resilience.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "cefas.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/dist/helm/cefas/templates/statefulset.yaml b/dist/helm/cefas/templates/statefulset.yaml index 66240d4..2cb6f3c 100644 --- a/dist/helm/cefas/templates/statefulset.yaml +++ b/dist/helm/cefas/templates/statefulset.yaml @@ -6,7 +6,7 @@ metadata: {{- include "cefas.labels" . | nindent 4 }} spec: serviceName: {{ include "cefas.headless" . }} - replicas: {{ .Values.replicaCount }} + replicas: {{ include "cefas.resolvedReplicaCount" . }} podManagementPolicy: Parallel selector: matchLabels: @@ -21,6 +21,42 @@ spec: serviceAccountName: {{ include "cefas.serviceAccountName" . }} securityContext: {{- toYaml .Values.securityContext | nindent 8 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + {{- else if and .Values.resilience.enabled .Values.resilience.scheduling.antiAffinity.enabled }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: {{ .Values.resilience.scheduling.antiAffinity.weight }} + podAffinityTerm: + topologyKey: {{ .Values.resilience.scheduling.antiAffinity.topologyKey | quote }} + labelSelector: + matchLabels: + {{- include "cefas.selectorLabels" . | nindent 20 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + {{- else if and .Values.resilience.enabled .Values.resilience.scheduling.topologySpread.enabled }} + topologySpreadConstraints: + {{- range .Values.resilience.scheduling.topologySpread.topologyKeys }} + - maxSkew: {{ $.Values.resilience.scheduling.topologySpread.maxSkew }} + topologyKey: {{ . | quote }} + whenUnsatisfiable: {{ $.Values.resilience.scheduling.topologySpread.whenUnsatisfiable | quote }} + labelSelector: + matchLabels: + {{- include "cefas.selectorLabels" $ | nindent 14 }} + {{- end }} + {{- end }} terminationGracePeriodSeconds: {{ .Values.lifecycle.terminationGracePeriodSeconds }} containers: - name: cefas @@ -67,7 +103,7 @@ spec: readinessProbe: {{- toYaml .Values.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- include "cefas.startupProbe" . | nindent 12 }} volumeMounts: - name: data mountPath: /var/lib/cefas @@ -75,7 +111,7 @@ spec: mountPath: /etc/cefas readOnly: true resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- include "cefas.dbResources" . | nindent 12 }} volumes: - name: config configMap: diff --git a/dist/helm/cefas/templates/validation.yaml b/dist/helm/cefas/templates/validation.yaml new file mode 100644 index 0000000..952e4fc --- /dev/null +++ b/dist/helm/cefas/templates/validation.yaml @@ -0,0 +1 @@ +{{- include "cefas.validate" . -}} diff --git a/dist/helm/cefas/values.yaml b/dist/helm/cefas/values.yaml index 0c0fe40..af6585e 100644 --- a/dist/helm/cefas/values.yaml +++ b/dist/helm/cefas/values.yaml @@ -1,8 +1,7 @@ # cefas Helm chart values. # -# Defaults bring up a single replica StatefulSet on hostPort-style -# Services with persistent storage. Multi-node clusters require -# replicaCount ≥ 3 and the multi-Raft flags below. +# Defaults bring up a single replica StatefulSet for local development. +# Set resilience.enabled=true for an RF=3-safe Kubernetes profile. image: repository: ghcr.io/cefasdb/cefasdb @@ -15,6 +14,37 @@ serviceAccount: create: true name: "" +resilience: + enabled: false + # Used when resilience.enabled=true. The rendered StatefulSet uses + # max(replicaCount, resilience.replicas) so `--set resilience.enabled=true` + # is enough to render three database Pods. + replicas: 3 + replicationFactor: 3 + requirePersistentStorage: true + allowEphemeralStorage: false + startupProbe: + # 120*2s gives recovering Pods up to four minutes for bootstrap, + # log replay, and shard startup before kubelet restarts the process. + failureThreshold: 120 + periodSeconds: 2 + podDisruptionBudget: + enabled: true + maxUnavailable: 1 + minAvailable: "" + scheduling: + antiAffinity: + enabled: true + topologyKey: kubernetes.io/hostname + weight: 100 + topologySpread: + enabled: true + maxSkew: 1 + whenUnsatisfiable: ScheduleAnyway + topologyKeys: + - kubernetes.io/hostname + - cefasdb.io/failure-domain + manager: enabled: true replicaCount: 2 @@ -34,6 +64,13 @@ manager: cpu: "500m" memory: "512Mi" +resourcePolicy: + # Database performance tests and dedicated DB nodes usually should not + # use CPU limits; set this true to omit resources.limits.cpu while + # retaining memory limits. + disableCPULimits: false + requireMemoryLimit: true + resources: requests: cpu: "200m" @@ -67,6 +104,9 @@ lifecycle: # Multi-Raft sharding. shards>1 turns on the mux transport. cluster: shards: 1 + # 0 means "use every peer". With resilience.enabled=true this resolves + # to resilience.replicationFactor unless explicitly set to >=3. + replicationFactor: 0 bootstrap: true fsyncOnCommit: false @@ -111,6 +151,11 @@ podAnnotations: prometheus.io/port: "8080" prometheus.io/path: "/metrics" +nodeSelector: {} +tolerations: [] +affinity: {} +topologySpreadConstraints: [] + securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/docs/helm-resilience.md b/docs/helm-resilience.md new file mode 100644 index 0000000..2a9a17a --- /dev/null +++ b/docs/helm-resilience.md @@ -0,0 +1,98 @@ +# Helm Resilience Profile + +The default chart remains a single-Pod development install. Enable the +resilience profile for an RF=3 Kubernetes deployment: + +```sh +helm upgrade --install cefas dist/helm/cefas \ + --set resilience.enabled=true \ + --set cluster.shards=24 +``` + +With `resilience.enabled=true`, the chart renders: + +- at least three `cefasdb` StatefulSet replicas +- `cluster.replicationFactor=3` unless explicitly set to a higher safe value +- generated Raft, HTTP, and gRPC peer maps for every StatefulSet ordinal +- PVC-backed storage unless `resilience.allowEphemeralStorage=true` is explicit +- a PodDisruptionBudget with one voluntary disruption at a time +- preferred pod anti-affinity by `kubernetes.io/hostname` +- topology spread constraints for `kubernetes.io/hostname` and + `cefasdb.io/failure-domain` +- a longer startup probe window for bootstrap and Raft log replay + +## Failure Domains + +Label physical hosts when the cluster spans local machines: + +```sh +kubectl label node m1-node cefasdb.io/failure-domain=m1 +kubectl label node m4-node cefasdb.io/failure-domain=m4 +``` + +The default spread rule is `ScheduleAnyway`: it prefers distribution when the +labels exist, but it does not make local test clusters unschedulable while a +node label is missing. + +## Resource Policy + +Database Pods should have explicit memory safeguards. CPU limits are optional: +they are useful for shared clusters, but they can hide available hardware and +make performance tests misleading through throttling. + +Production baseline: + +```yaml +resilience: + enabled: true +resources: + requests: + cpu: "2000m" + memory: "8Gi" + limits: + cpu: "4000m" + memory: "8Gi" +resourcePolicy: + disableCPULimits: false + requireMemoryLimit: true +``` + +Dedicated performance or local M1/M4 test nodes: + +```yaml +resilience: + enabled: true +cluster: + shards: 24 +resources: + requests: + cpu: "1000m" + memory: "4Gi" + limits: + memory: "8Gi" +resourcePolicy: + disableCPULimits: true + requireMemoryLimit: true +``` + +When CPU limits are disabled, use metrics and alerts to detect saturation +instead of relying on throttling as the control plane. Watch Kubernetes CPU +usage/throttling, `process_cpu_seconds_total`, Go runtime metrics, +`cefas_storage_lane_*`, `cefas_backpressure_*`, and Raft lag/leadership +metrics. + +## Validation + +Unsafe combinations fail during Helm rendering: + +- `resilience.replicas < 3` +- replication factor below 3 +- replication factor greater than rendered replicas +- disabled persistence without `resilience.allowEphemeralStorage=true` +- missing memory limit when `resourcePolicy.requireMemoryLimit=true` + +Run the chart smoke tests: + +```sh +scripts/test_helm_resilience.sh +``` diff --git a/scripts/test_helm_resilience.sh b/scripts/test_helm_resilience.sh new file mode 100755 index 0000000..475a731 --- /dev/null +++ b/scripts/test_helm_resilience.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +CHART="${CHART:-dist/helm/cefas}" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +require_grep() { + local pattern="$1" + local file="$2" + if ! grep -q "$pattern" "$file"; then + echo "missing pattern: $pattern" >&2 + echo "file: $file" >&2 + exit 1 + fi +} + +expect_helm_failure() { + local message="$1" + shift + local err_file="$TMP_DIR/error.log" + if helm template cefas-bad "$CHART" "$@" >"$TMP_DIR/bad.yaml" 2>"$err_file"; then + echo "helm template unexpectedly passed: $*" >&2 + exit 1 + fi + require_grep "$message" "$err_file" +} + +resilient="$TMP_DIR/resilient.yaml" +helm template cefas-resilient "$CHART" \ + --namespace cefas-test \ + --set resilience.enabled=true \ + --set cluster.shards=24 \ + >"$resilient" + +require_grep "kind: PodDisruptionBudget" "$resilient" +require_grep "replicas: 3" "$resilient" +require_grep "replicationFactor: 3" "$resilient" +require_grep "peers:" "$resilient" +require_grep "httpPeers:" "$resilient" +require_grep "grpcPeers:" "$resilient" +require_grep "podAntiAffinity:" "$resilient" +require_grep "topologySpreadConstraints:" "$resilient" +require_grep "cefasdb.io/failure-domain" "$resilient" +require_grep "failureThreshold: 120" "$resilient" +require_grep "app.kubernetes.io/name: cefas-manager" "$resilient" + +no_cpu_limit="$TMP_DIR/no-cpu-limit.yaml" +helm template cefas-perf "$CHART" \ + --namespace cefas-test \ + --set resilience.enabled=true \ + --set resourcePolicy.disableCPULimits=true \ + >"$no_cpu_limit" +if grep -q 'cpu: "1000m"' "$no_cpu_limit"; then + echo "database CPU limit should be omitted when resourcePolicy.disableCPULimits=true" >&2 + exit 1 +fi +require_grep 'memory: "2Gi"' "$no_cpu_limit" + +expect_helm_failure "resilience.replicas must be >= 3" \ + --set resilience.enabled=true \ + --set resilience.replicas=2 + +expect_helm_failure "requires persistence.enabled=true" \ + --set resilience.enabled=true \ + --set persistence.enabled=false + +expect_helm_failure "replication factor cannot exceed rendered database replicas" \ + --set resilience.enabled=true \ + --set cluster.replicationFactor=4 + +echo "helm resilience chart tests passed"