Closes: #399 - Migrate away from bitnami/elasticsearch

.editorconfig

@@ -0,0 +1,5 @@
+root = true
+
+[*.sh]
+indent_style = space
+indent_size = 2

.github/ct.yaml

@@ -2,6 +2,7 @@
 chart-dirs: .
 chart-repos:
   - bitnami=https://charts.bitnami.com/bitnami
+  - elastic=https://helm.elastic.co
 check-version-increment: true
 debug: true
 # upgrade: true # Causes issues because of low performance on default GitHub actions servers.

.github/kubeconform.sh

@@ -13,10 +13,15 @@ curl --silent --show-error --fail --location --output /tmp/kubeconform.tar.gz ht
 sudo tar -C /usr/local/bin -xf /tmp/kubeconform.tar.gz kubeconform
 
 # validate charts
-for CHART_DIR in ${CHART_DIRS};do
+for CHART_DIR in ${CHART_DIRS}; do
   echo "helm dependency build…"
   helm dependency build "${CHART_DIR}"
 
   echo "kubeconform(ing) ${CHART_DIR##charts/} chart…"
-  helm template "${CHART_DIR}" | kubeconform --strict --verbose --kubernetes-version "${KUBERNETES_VERSION#v}"
+  # The second schema location resolves CRD-based kinds (e.g. the ECK-managed
+  # Elasticsearch resource) from the datree CRDs catalog.
+  helm template "${CHART_DIR}" | kubeconform --strict --verbose \
+    --schema-location default \
+    --schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' \
+    --kubernetes-version "${KUBERNETES_VERSION#v}"
 done

.github/workflows/ci.yaml

@@ -123,6 +123,14 @@ jobs:
           config: .github/kind-config.yaml
           node_image: kindest/node:${{ matrix.k8s }}
 
+      - name: Install ECK operator (CRDs + operator)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          helm repo add elastic https://helm.elastic.co
+          helm repo update
+          helm install elastic-operator elastic/eck-operator \
+            --namespace elastic-system --create-namespace --wait
+
       - name: Create Namespace 'zammad'
         run: kubectl create namespace zammad
 

zammad/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: zammad
-version: 16.4.4
+version: 17.0.0
 appVersion: 7.1.1-0009
 description: Zammad is a web based open source helpdesk/customer support system with many features to manage customer communication via several channels like telephone, facebook, twitter, chat and e-mails.
 home: https://zammad.org
@@ -14,9 +14,14 @@ maintainers:
   - name: mgruner
     email: enjoy@zammad.com
 dependencies:
-  - name: elasticsearch
-    repository: https://charts.bitnami.com/bitnami
-    version: 21.6.3
+  # Official Elastic way to run Elasticsearch: the eck-elasticsearch chart deploys
+  # an Elasticsearch custom resource that is reconciled by the ECK operator.
+  # The ECK operator (+ CRDs) must be installed cluster-wide as a prerequisite,
+  # see README. Aliased to "elasticsearch" to keep the values key stable.
+  - name: eck-elasticsearch
+    alias: elasticsearch
+    repository: https://helm.elastic.co
+    version: 0.19.1
     condition: zammadConfig.elasticsearch.enabled
   - name: minio
     version: 14.10.5

zammad/README.md

@@ -41,9 +41,24 @@ Be aware that the Zammad Helm chart version is different from the actual Zammad
 
 ## Prerequisites
 
-- Kubernetes 1.19+
+- Kubernetes 1.21+
 - Helm 3.2.0+
 - Cluster with at least 4GB of free RAM
+- The [ECK operator](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-install-helm.html)
+  must be installed in the cluster when using the bundled Elasticsearch
+  (`zammadConfig.elasticsearch.enabled: true`, the default). The bundled
+  Elasticsearch is deployed via the official `eck-elasticsearch` chart, which
+  creates an `Elasticsearch` custom resource that is reconciled by the operator.
+  This is **not** required when connecting to an external Elasticsearch service.
+
+  Install the operator once per cluster:
+
+  ```console
+  helm repo add elastic https://helm.elastic.co
+  helm repo update
+  helm install elastic-operator elastic/eck-operator \
+    --namespace elastic-system --create-namespace
+  ```
 
 ## Installing the Chart
 
@@ -114,7 +129,11 @@ To deploy on OpenShift unprivileged and with [arbitrary UIDs and GIDs](https://c
 - [Delete the default key](https://helm.sh/docs/chart_template_guide/values_files/#deleting-a-default-key) `securityContext` and `zammadConfig.initContainers.zammad.securityContext.runAsUser` with `null`.
 - Disable if used:
   - also `podSecurityContext` in all subcharts.
-  - the privileged [sysctlImage](https://github.com/bitnami/charts/tree/main/bitnami/elasticsearch#default-kernel-settings) in elasticsearch subchart.
+
+The bundled Elasticsearch uses `node.store.allow_mmap: false` by default (see
+`elasticsearch.nodeSets[].config`), so no privileged sysctl init container is
+required. If you raise `vm.max_map_count` for production instead, do so on the
+node level rather than via a privileged init container.
 
 ```yaml
 securityContext: null
@@ -130,15 +149,6 @@ zammadConfig:
     emptyDir:
       medium: memory
 
-elasticsearch:
-  sysctlImage:
-    enabled: false
-  master:
-    podSecurityContext:
-      enabled: false
-    containerSecurityContext:
-      enabled: false
-
 memcached:
   podSecurityContext:
     enabled: false
@@ -198,6 +208,32 @@ and `zammadConfig.cronJob.reindex.schedule` if you want to run it periodically.
 
 ## Upgrading
 
+### From Chart Version 16.x to 17.0.0
+
+- The bundled Elasticsearch was migrated from the `bitnami/elasticsearch` subchart to the
+  official [`eck-elasticsearch`](https://artifacthub.io/packages/helm/elastic/eck-elasticsearch)
+  chart, which is managed by the [ECK operator](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-overview.html).
+  This is the deployment method officially supported by Elastic.
+- **The ECK operator (and its CRDs) must now be installed in the cluster** before upgrading
+  when `zammadConfig.elasticsearch.enabled` is `true` (the default). See [Prerequisites](#prerequisites).
+- The `elasticsearch.*` values changed completely. The previous bitnami values (`master`, `data`,
+  `ingest`, `coordinating`, `sysctlImage`, `image`, `global.security.allowInsecureImages`, …) no
+  longer apply. Review the new `elasticsearch.*` block in `values.yaml` (`version`, `nodeSets`,
+  `http`, …).
+- Authentication is now mandatory: ECK creates a superuser `elastic` and stores its password in
+  the auto-generated secret `{{ .Release.Name }}-elasticsearch-es-elastic-user`. The chart wires
+  this up automatically, so `secrets.elasticsearch.*` is only relevant for an **external**
+  Elasticsearch now.
+- The in-cluster service name changed from `{{ .Release.Name }}-elasticsearch` to
+  `{{ .Release.Name }}-elasticsearch-es-http`.
+- TLS on the HTTP layer is disabled by default (`elasticsearch.http.tls.selfSignedCertificate.disabled: true`)
+  to keep the plain-`http` connection behaviour. Set it to `false` and `zammadConfig.elasticsearch.schema: https`
+  to use the operator-managed self-signed certificate.
+- The StatefulSet and its `PersistentVolumeClaim`s are recreated under new names. As the search
+  index can be rebuilt from PostgreSQL, the recommended path is to delete the old bitnami
+  Elasticsearch PVCs and let the index be recreated. To force a full rebuild after the upgrade,
+  set `zammadConfig.elasticsearch.reindex: true` for the upgrade run.
+
 ### From Chart Version 15.x to 16.0.0
 
 - The bitnami memcached subchart was replaced with [cloudpirates-memcached/memcached](https://artifacthub.io/packages/helm/cloudpirates-memcached/memcached).

zammad/ci/full-values.yaml

@@ -24,8 +24,7 @@ autoWizard:
   enabled: true
 
 elasticsearch:
-  security:
-    existingSecret: elasticsearch-existing-secret
+  nameOverride: my-local-es
 
 ingress:
   enabled: true

zammad/templates/_helpers.tpl

@@ -105,17 +105,71 @@ autowizard secret name
 {{- end -}}
 {{- end -}}
 
+{{/*
+Name of the ECK Elasticsearch resource. Mirrors the eck-elasticsearch chart's
+"elasticsearch.fullname" logic so that ECK-derived names (services, secrets)
+stay correct even if elasticsearch.nameOverride/fullnameOverride are changed.
+*/}}
+{{- define "zammad.elasticsearchName" -}}
+{{- $es := .Values.elasticsearch | default dict -}}
+{{- if $es.fullnameOverride -}}
+{{- $es.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default "eck-elasticsearch" $es.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+elasticsearch service host (in-cluster ECK service or external host)
+*/}}
+{{- define "zammad.elasticsearchHost" -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+{{ include "zammad.elasticsearchName" . }}-es-http
+{{- else -}}
+{{ .Values.zammadConfig.elasticsearch.host }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+elasticsearch user (ECK creates the "elastic" superuser for the in-cluster service)
+*/}}
+{{- define "zammad.elasticsearchUser" -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+elastic
+{{- else -}}
+{{ .Values.zammadConfig.elasticsearch.user }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 elasticsearch secret name
 */}}
 {{- define "zammad.elasticsearchSecretName" -}}
-{{- if .Values.secrets.elasticsearch.useExisting -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+{{ include "zammad.elasticsearchName" . }}-es-elastic-user
+{{- else if .Values.secrets.elasticsearch.useExisting -}}
 {{ .Values.secrets.elasticsearch.secretName }}
 {{- else -}}
 {{ include "zammad.fullname" . }}-{{ .Values.secrets.elasticsearch.secretName }}
 {{- end -}}
 {{- end -}}
 
+{{/*
+elasticsearch secret key (ECK stores the password under the user name "elastic")
+*/}}
+{{- define "zammad.elasticsearchSecretKey" -}}
+{{- if .Values.zammadConfig.elasticsearch.enabled -}}
+elastic
+{{- else -}}
+{{ .Values.secrets.elasticsearch.secretKey }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 postgresql secret name
 */}}

zammad/templates/configmap-init.yaml

@@ -45,10 +45,10 @@ data:
     #!/bin/bash
     set -e
 
-    ELASTICSEARCH_URL={{ .Values.zammadConfig.elasticsearch.schema }}://{{ if .Values.zammadConfig.elasticsearch.enabled }}{{ .Release.Name }}-elasticsearch{{ else }}{{ .Values.zammadConfig.elasticsearch.host }}{{ end }}:{{ .Values.zammadConfig.elasticsearch.port }}
+    ELASTICSEARCH_URL={{ .Values.zammadConfig.elasticsearch.schema }}://{{ include "zammad.elasticsearchHost" . }}:{{ .Values.zammadConfig.elasticsearch.port }}
     bundle exec rails r "Setting.set('es_url', '${ELASTICSEARCH_URL}')"
 
-    ELASTICSEARCH_USER=${ELASTICSEARCH_USER:-{{ .Values.zammadConfig.elasticsearch.user }}}
+    ELASTICSEARCH_USER=${ELASTICSEARCH_USER:-{{ include "zammad.elasticsearchUser" . }}}
     if [ -n "${ELASTICSEARCH_USER}" ] && [ -n "${ELASTICSEARCH_PASSWORD}" ]; then
         bundle exec rails r "Setting.set('es_user', '${ELASTICSEARCH_USER}'); Setting.set('es_password', '${ELASTICSEARCH_PASSWORD}')"
     fi

zammad/templates/job-init.yaml

@@ -90,12 +90,12 @@ spec:
           env:
             {{- include "zammad.env" . | nindent 12 }}
             {{- include "zammad.env.failOnPendingMigrations" . | nindent 12 }}
-          {{- if or .Values.zammadConfig.elasticsearch.pass .Values.secrets.elasticsearch.useExisting }}
+          {{- if or .Values.zammadConfig.elasticsearch.enabled .Values.zammadConfig.elasticsearch.pass .Values.secrets.elasticsearch.useExisting }}
             - name: ELASTICSEARCH_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: {{ include "zammad.elasticsearchSecretName" . }}
-                  key: {{ .Values.secrets.elasticsearch.secretKey }}
+                  key: {{ include "zammad.elasticsearchSecretKey" . }}
           {{- end }}
           volumeMounts:
             {{- include "zammad.volumeMounts" . | nindent 12 }}

zammad/templates/secrets.yaml

@@ -12,7 +12,7 @@ type: Opaque
 data:
   {{ .Values.secrets.autowizard.secretKey }}: {{ .Values.autoWizard.config | b64enc | quote }}
 {{ end }}
-{{ if and .Values.zammadConfig.elasticsearch.pass (not .Values.secrets.elasticsearch.useExisting) }}
+{{ if and (not .Values.zammadConfig.elasticsearch.enabled) .Values.zammadConfig.elasticsearch.pass (not .Values.secrets.elasticsearch.useExisting) }}
 ---
 apiVersion: v1
 kind: Secret

zammad/values.yaml

@@ -85,10 +85,13 @@ podDisruptionBudget:
 
 zammadConfig:
   elasticsearch:
-    # enable/disable elasticsearch chart dependency
+    # enable/disable the bundled elasticsearch (ECK) chart dependency
     enabled: true
-    # host env var is only used when zammadConfig.elasticsearch.enabled is false
-    host: zammad-elasticsearch-master
+    # host, user and pass are only used when zammadConfig.elasticsearch.enabled is
+    # false (i.e. when connecting to an external Elasticsearch service). For the
+    # bundled ECK Elasticsearch, the host, the "elastic" user and the operator-
+    # generated password are resolved automatically and these values are ignored.
+    host: external-elasticsearch-host
     initialisation: true
     pass: ""
     port: 9200
@@ -693,48 +696,57 @@ initContainers: []
 
 # dependency charts config
 
-# Settings for the elasticsearch subchart
+# Settings for the elasticsearch subchart (eck-elasticsearch).
+#
+# This deploys an Elasticsearch custom resource that is managed by the ECK operator.
+# The ECK operator (and its CRDs) MUST already be installed in the cluster, see README.
+#
+# ECK names the resource "<release name>-elasticsearch" (nameOverride below), which
+# results in the in-cluster service "<release name>-elasticsearch-es-http" and the
+# auto-generated superuser secret "<release name>-elasticsearch-es-elastic-user"
+# (user "elastic"). The Zammad templates pick these up automatically.
 elasticsearch:
-  # Workaround: switch to bitnami legacy image repository (https://github.com/bitnami/containers/issues/83267)
-  image:
-    repository: bitnamilegacy/elasticsearch
-  sysctlImage:
-    repository: bitnamilegacy/os-shell
-  volumePermissions:
-    image:
-      repository: bitnamilegacy/os-shell
-  metrics:
-    image:
-      repository: bitnamilegacy/elasticsearch-exporter
-  global:
-    security:
-      allowInsecureImages: true
-
-  clusterName: zammad
-  coordinating:
-    replicaCount: 0
-  data:
-    replicaCount: 0
-  ingest:
-    replicaCount: 0
-  master:
-    heapSize: 512m
-    masterOnly: false
-    replicaCount: 1
-    resourcesPreset: medium
-    resources: {}
-      # requests:
-      #   cpu: 50m
-      #   memory: 512Mi
-      # limits:
-      #   cpu: 100m
-      #   memory: 1024Mi
-
-  # To use an existing Kubernetes secret containing the credentials,
-  # remove the comments on the lines below and adjust them accordingly
-  #
-  # security:
-  #   existingSecret: elastic-credentials
+  nameOverride: elasticsearch
+  version: 9.4.2
+  http:
+    tls:
+      selfSignedCertificate:
+        # Disable TLS on the HTTP layer (Variant A): Zammad talks plain http to ES.
+        # Native authentication (user "elastic" + generated password) stays enabled.
+        # Set to false and adjust zammadConfig.elasticsearch.schema to "https" to
+        # use the operator-managed self-signed certificate instead.
+        disabled: true
+  nodeSets:
+    - name: default
+      count: 1
+      config:
+        # Single-node default deployment without raising vm.max_map_count.
+        # For production, raise the kernel setting vm.max_map_count to 1048576 at
+        # the node level and remove this line instead. See:
+        # https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html
+        node.store.allow_mmap: false
+      volumeClaimTemplates:
+        - metadata:
+            name: elasticsearch-data
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 5Gi
+            # storageClassName: ""
+      podTemplate:
+        spec:
+          containers:
+            - name: elasticsearch
+              env:
+                - name: ES_JAVA_OPTS
+                  value: "-Xms512m -Xmx512m"
+              resources:
+                requests:
+                  memory: 1Gi
+                limits:
+                  memory: 1Gi
 
 # settings for the memcached subchart
 memcached: