From b24916d06ae0f72475ff32c49a8d4c4bc5690dd8 Mon Sep 17 00:00:00 2001
From: Sam Crauwels <sam.crauwels@gmail.com>
Date: Fri, 13 Mar 2026 09:56:01 +0100
Subject: [PATCH 1/5] Add kibana_system_password variable for custom password
 management
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When set, the kibana role changes the kibana_system user's password via
the Elasticsearch security API and uses that value for the connection to
Elasticsearch. When empty (default), behavior is unchanged — the auto-
generated password from initial_passwords is used.

The password change runs once on the CA host, authenticates as the elastic
superuser, and updates the kibana_password fact so the template and keystore
tasks pick up the new value transparently.

Closes #102
---
 docs/roles/kibana.md                   |  3 ++
 roles/kibana/defaults/main.yml         |  8 +++++
 roles/kibana/tasks/kibana-security.yml | 42 ++++++++++++++++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/docs/roles/kibana.md b/docs/roles/kibana.md
index 8da2f35..267167a 100644
--- a/docs/roles/kibana.md
+++ b/docs/roles/kibana.md
@@ -51,6 +51,7 @@ kibana_config_backup: false
 ```yaml
 # kibana_elasticsearch_hosts: (undefined by default)
 kibana_security: true
+kibana_system_password: ""
 kibana_sniff_on_start: false
 kibana_sniff_on_connection_fault: false
 ```
@@ -66,6 +67,8 @@ kibana_sniff_on_connection_fault: false
 
 `kibana_security` enables authenticated, encrypted connections to Elasticsearch. When `true`, Kibana connects over HTTPS using the `kibana_system` user and the password from the Elasticsearch security setup. The CA certificate is deployed automatically from the ES CA host.
 
+`kibana_system_password` lets you set a specific password for the `kibana_system` Elasticsearch user. When empty (the default), Kibana uses the auto-generated password from the initial security setup. When set, the role changes the password via the Elasticsearch `/_security/user/kibana_system/_password` API on every run and uses the new value for Kibana's connection to Elasticsearch. This is useful when you need a known password for external monitoring, when you rotate credentials on a schedule, or when multiple Kibana instances need a consistent password that isn't tied to the initial setup file.
+
 `kibana_sniff_on_start` and `kibana_sniff_on_connection_fault` control Elasticsearch node discovery. When enabled, Kibana queries the ES cluster for the full list of nodes at startup or when a connection drops. These settings only apply to Elastic Stack versions prior to 9.x (Kibana 9.x removed sniffing support).
 
 ### TLS for the Kibana Web Interface
diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml
index 548f6a8..9225504 100644
--- a/roles/kibana/defaults/main.yml
+++ b/roles/kibana/defaults/main.yml
@@ -21,6 +21,14 @@ kibana_config_backup: false
 # @var kibana_manage_yaml:description: Let the role manage kibana.yml. Set to false to manage it yourself
 kibana_manage_yaml: true
 
+# @var kibana_system_password:description: >
+#   User-defined password for the kibana_system user. When set, the role
+#   changes the auto-generated password via the Elasticsearch security API
+#   and uses this value for Kibana's connection to Elasticsearch. Leave
+#   empty to keep the auto-generated password from initial_passwords.
+# @end
+kibana_system_password: ""
+
 # @var kibana_security:description: Enable security features (connect to Elasticsearch over HTTPS with authentication)
 kibana_security: true
 # @var kibana_tls:description: Enable TLS on the Kibana web interface itself (serve Kibana over HTTPS)
diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml
index 9c38ca4..794e253 100644
--- a/roles/kibana/tasks/kibana-security.yml
+++ b/roles/kibana/tasks/kibana-security.yml
@@ -416,6 +416,48 @@
     _password_user: kibana_system
     _password_fact: kibana_password
 
+# -- Change kibana_system password if user defined one --
+#
+# When kibana_system_password is set, the role changes the auto-generated
+# password to the user-provided value via the Elasticsearch security API.
+# This runs once on the CA host, authenticating as the elastic superuser.
+# After the API call, kibana_password is updated so the rest of the role
+# (kibana.yml template, keystore) uses the new password transparently.
+- name: kibana-security | Change kibana_system password to user-defined value
+  when:
+    - inventory_hostname == elasticstack_ca_host
+    - kibana_system_password | default('') | length > 0
+    - kibana_password.stdout | default('') != kibana_system_password
+  block:
+    - name: kibana-security | Fetch elastic password for API authentication
+      ansible.builtin.include_tasks:
+        file: "{{ role_path }}/../elasticstack/tasks/fetch_password.yml"
+      vars:
+        _password_user: elastic
+        _password_fact: _kibana_elastic_password
+
+    - name: kibana-security | Change kibana_system password via Elasticsearch API
+      ansible.builtin.uri:
+        url: "{{ _kibana_es_protocol }}://{{ elasticsearch_api_host | default('localhost') }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/kibana_system/_password"
+        method: POST
+        body_format: json
+        body:
+          password: "{{ kibana_system_password }}"
+        user: elastic
+        password: "{{ elasticstack_password.stdout | default(_kibana_elastic_password.stdout) }}"
+        force_basic_auth: true
+        validate_certs: false
+        status_code: 200
+      no_log: "{{ elasticstack_no_log }}"
+      vars:
+        _kibana_es_protocol: "{{ 'https' if kibana_security | bool else 'http' }}"
+
+    - name: kibana-security | Update kibana_password fact to user-defined value  # noqa: var-naming[no-role-prefix]
+      ansible.builtin.set_fact:
+        kibana_password:
+          stdout: "{{ kibana_system_password }}"
+      no_log: "{{ elasticstack_no_log }}"
+
 # -- Distribute CA certificate to Kibana nodes --
 - name: kibana-security | Distribute CA certificate to Kibana nodes
   ansible.builtin.include_tasks:

From 2e1dbfcaf157a3b1c07b05c542af0131abde9e4a Mon Sep 17 00:00:00 2001
From: Sam Crauwels <sam.crauwels@gmail.com>
Date: Fri, 13 Mar 2026 09:56:45 +0100
Subject: [PATCH 2/5] Add elasticsearch_keystore_entries for custom keystore
 management

Adds a dict variable that lets users put arbitrary secrets into the
Elasticsearch keystore without writing custom tasks. Each entry is set
with elasticsearch-keystore add -f -x, values passed via stdin.

The role validates that none of the user-provided keys overlap with the
eight keys managed internally (bootstrap.password, SSL keystore/truststore
passwords, PEM key passphrases, autoconfiguration hash). If there is a
conflict, the playbook fails immediately with an error listing all reserved
keys and explaining why they cannot be set through this variable.

On each run the role reads current values and only writes changed entries,
so Elasticsearch is not restarted unnecessarily.

Closes #103
---
 docs/roles/elasticsearch.md                   | 16 +++++
 roles/elasticsearch/defaults/main.yml         | 23 +++++++
 .../tasks/elasticsearch-keystore.yml          | 66 +++++++++++++++++++
 3 files changed, 105 insertions(+)

diff --git a/docs/roles/elasticsearch.md b/docs/roles/elasticsearch.md
index 6642f77..b72f29f 100644
--- a/docs/roles/elasticsearch.md
+++ b/docs/roles/elasticsearch.md
@@ -382,6 +382,22 @@ elasticsearch_logging_audit: true
 
     JSON logs use `ECSJsonLayout` with `dataset` fields (Elastic Common Schema). Deprecation logs add a `RateLimitingFilter` to prevent log flooding and a `HeaderWarningAppender` for HTTP response warnings. Indexing slow log logger name changed to `index.indexing.slowlog.index`.
 
+### Custom Keystore Entries
+
+```yaml
+elasticsearch_keystore_entries:
+  xpack.notification.slack.account.monitoring.secure_url: "https://hooks.slack.com/services/T00/B00/XXX"
+  xpack.notification.email.account.work.smtp.secure_password: "smtp-password"
+```
+
+`elasticsearch_keystore_entries` is a dictionary of custom entries to add to the Elasticsearch keystore. Each key-value pair is set using `elasticsearch-keystore add -f -x`, with the value passed via stdin so it never appears in process listings or Ansible logs.
+
+Use this for any sensitive Elasticsearch setting that belongs in the keystore rather than in `elasticsearch.yml` — Watcher notification credentials, repository passwords, LDAP bind passwords, custom plugin secrets, etc.
+
+The role manages a fixed set of keystore keys internally (SSL keystore/truststore passwords, bootstrap password). If you try to set any of these via `elasticsearch_keystore_entries`, the playbook fails immediately with an error listing exactly which keys are reserved and why. The reserved keys are: `bootstrap.password`, `autoconfiguration.password_hash`, and the six `xpack.security.*.ssl.*` password entries. Use the dedicated role variables for those instead.
+
+On each run, the role reads the current value of each custom entry and only writes it if the value has changed, so the keystore is not unnecessarily modified and Elasticsearch is only restarted when an entry actually changes.
+
 ### Extra Configuration
 
 ```yaml
diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml
index 8c871c8..67b9af6 100644
--- a/roles/elasticsearch/defaults/main.yml
+++ b/roles/elasticsearch/defaults/main.yml
@@ -171,6 +171,29 @@ elasticsearch_validate_api_certs: false
 
 elasticsearch_unsafe_upgrade_restart: false
 
+# @var elasticsearch_keystore_entries:description: >
+#   Custom entries to add to the Elasticsearch keystore. Each key-value pair
+#   is added with `elasticsearch-keystore add -f -x`. Values are passed via
+#   stdin so they never appear in process listings or Ansible logs.
+#
+#   The role manages these keystore keys internally — do NOT set them here:
+#     bootstrap.password,
+#     xpack.security.http.ssl.keystore.secure_password,
+#     xpack.security.http.ssl.truststore.secure_password,
+#     xpack.security.transport.ssl.keystore.secure_password,
+#     xpack.security.transport.ssl.truststore.secure_password,
+#     xpack.security.http.ssl.secure_key_passphrase,
+#     xpack.security.transport.ssl.secure_key_passphrase,
+#     autoconfiguration.password_hash
+#   The playbook will fail with an error if any of these keys appear in
+#   elasticsearch_keystore_entries.
+# @var elasticsearch_keystore_entries:example: >
+# elasticsearch_keystore_entries:
+#   xpack.notification.slack.account.monitoring.secure_url: "https://hooks.slack.com/services/T00/B00/XXX"
+#   xpack.notification.email.account.work.smtp.secure_password: "smtp-password"
+# @end
+elasticsearch_keystore_entries: {}
+
 # @var elasticsearch_extra_config:description: >
 #   Additional key-value pairs merged into elasticsearch.yml. Keys that
 #   conflict with dedicated role variables (e.g. cluster.name) are
diff --git a/roles/elasticsearch/tasks/elasticsearch-keystore.yml b/roles/elasticsearch/tasks/elasticsearch-keystore.yml
index d9b242b..feb52c6 100644
--- a/roles/elasticsearch/tasks/elasticsearch-keystore.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-keystore.yml
@@ -306,6 +306,72 @@
   notify:
     - Restart Elasticsearch
 
+# ============================================================
+# Custom keystore entries (elasticsearch_keystore_entries)
+# ============================================================
+#
+# These tasks run AFTER the built-in keystore management above.
+# The role owns a fixed set of keystore keys (SSL passwords,
+# bootstrap password, etc.) — those are off-limits. User-provided
+# entries in elasticsearch_keystore_entries must not overlap with
+# them or the playbook fails with a clear error explaining why.
+
+- name: elasticsearch-keystore | Validate custom entries do not conflict with role-managed keys
+  ansible.builtin.assert:
+    that:
+      - item.key not in _elasticsearch_reserved_keystore_keys
+    fail_msg: >-
+      Cannot set '{{ item.key }}' via elasticsearch_keystore_entries because
+      this key is managed automatically by the elasticsearch role. Remove it
+      from elasticsearch_keystore_entries. The role manages these keys
+      internally: {{ _elasticsearch_reserved_keystore_keys | join(', ') }}
+  loop: "{{ elasticsearch_keystore_entries | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  vars:
+    _elasticsearch_reserved_keystore_keys:
+      - bootstrap.password
+      - autoconfiguration.password_hash
+      - xpack.security.http.ssl.keystore.secure_password
+      - xpack.security.http.ssl.truststore.secure_password
+      - xpack.security.transport.ssl.keystore.secure_password
+      - xpack.security.transport.ssl.truststore.secure_password
+      - xpack.security.http.ssl.secure_key_passphrase
+      - xpack.security.transport.ssl.secure_key_passphrase
+  when: elasticsearch_keystore_entries | length > 0
+
+- name: elasticsearch-keystore | Get current value for custom entry '{{ item.key }}'
+  ansible.builtin.command: >
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    show {{ item.key }}
+  loop: "{{ elasticsearch_keystore_entries | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  register: _elasticsearch_custom_keystore_current
+  changed_when: false
+  no_log: true
+  ignore_errors: true
+  when: elasticsearch_keystore_entries | length > 0
+
+- name: elasticsearch-keystore | Set custom entry '{{ item.item.key }}'
+  ansible.builtin.command: >
+    /usr/share/elasticsearch/bin/elasticsearch-keystore
+    add -f -x '{{ item.item.key }}'
+  args:
+    stdin: "{{ item.item.value }}"
+  loop: "{{ _elasticsearch_custom_keystore_current.results | default([]) }}"
+  loop_control:
+    label: "{{ item.item.key }}"
+  changed_when: true
+  no_log: true
+  when:
+    - elasticsearch_keystore_entries | length > 0
+    - item.rc != 0 or item.stdout != item.item.value | string
+  notify:
+    - Restart Elasticsearch
+
+# ============================================================
+
 - name: elasticsearch-keystore | Check keystore exists
   ansible.builtin.stat:
     path: /etc/elasticsearch/elasticsearch.keystore

From 099608d216037b02e01e94d19f487f1f92a5619b Mon Sep 17 00:00:00 2001
From: Sam Crauwels <sam.crauwels@gmail.com>
Date: Thu, 26 Mar 2026 09:57:01 +0100
Subject: [PATCH 3/5] Improve custom keystore entries: error handling, stale
 removal, tests

Three changes to elasticsearch_keystore_entries:

The show command previously used ignore_errors: true which would silently
swallow real errors like permission denied or a corrupt keystore. Now it
uses failed_when to only tolerate a missing key (rc != 0 when the key
isn't in the keystore list), while letting genuine errors propagate.

Stale keystore entries are now removed automatically. On each run the
role lists the keystore, subtracts the declared custom entries and the
role-managed reserved keys, and removes whatever is left. This keeps
the keystore in sync with declared state and makes removal as simple
as deleting the entry from elasticsearch_keystore_entries.

The whole custom keystore section is wrapped in a block with shared
vars so the reserved keys list is defined once.

Added molecule test coverage in the es_kibana scenario for both new
features: elasticsearch_keystore_entries sets a test entry and verify
checks it exists with the right value, kibana_system_password sets a
custom password and verify confirms kibana_system can authenticate
with it via the ES security API.
---
 docs/roles/elasticsearch.md                   |   2 +
 molecule/es_kibana/converge.yml               |   3 +
 molecule/es_kibana/verify.yml                 |  51 ++++++++
 .../tasks/elasticsearch-keystore.yml          | 115 +++++++++++-------
 4 files changed, 130 insertions(+), 41 deletions(-)

diff --git a/docs/roles/elasticsearch.md b/docs/roles/elasticsearch.md
index b72f29f..482c042 100644
--- a/docs/roles/elasticsearch.md
+++ b/docs/roles/elasticsearch.md
@@ -398,6 +398,8 @@ The role manages a fixed set of keystore keys internally (SSL keystore/truststor
 
 On each run, the role reads the current value of each custom entry and only writes it if the value has changed, so the keystore is not unnecessarily modified and Elasticsearch is only restarted when an entry actually changes.
 
+The keystore is kept in sync with the declared state: entries that were previously added via `elasticsearch_keystore_entries` but are no longer present in the dictionary are automatically removed. Role-managed keys and `keystore.seed` are never touched by this cleanup.
+
 ### Extra Configuration
 
 ```yaml
diff --git a/molecule/es_kibana/converge.yml b/molecule/es_kibana/converge.yml
index a6dbaa1..f4ccff3 100644
--- a/molecule/es_kibana/converge.yml
+++ b/molecule/es_kibana/converge.yml
@@ -8,6 +8,9 @@
     elasticstack_full_stack: true
     elasticstack_no_log: false
     elasticsearch_heap: "1"
+    elasticsearch_keystore_entries:
+      xpack.notification.slack.account.test.secure_url: "https://hooks.example.com/test"
+    kibana_system_password: "molecule-kibana-test-pw"
 
   tasks:
     - name: Include Elastic repos
diff --git a/molecule/es_kibana/verify.yml b/molecule/es_kibana/verify.yml
index 13d3307..6522106 100644
--- a/molecule/es_kibana/verify.yml
+++ b/molecule/es_kibana/verify.yml
@@ -70,3 +70,54 @@
           - kibana_status.json.status.overall.level == 'available'
         fail_msg: "Kibana status: {{ kibana_status.json.status.overall.level | default('unknown') }}"
       when: "'kibana' in group_names"
+
+    # -- Custom keystore entries --
+    - name: List Elasticsearch keystore entries
+      ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list
+      register: keystore_list
+      changed_when: false
+      when: "'elasticsearch' in group_names"
+
+    - name: Verify custom keystore entry exists
+      ansible.builtin.assert:
+        that:
+          - "'xpack.notification.slack.account.test.secure_url' in keystore_list.stdout_lines"
+        fail_msg: "Custom keystore entry not found in: {{ keystore_list.stdout_lines }}"
+      when: "'elasticsearch' in group_names"
+
+    - name: Verify custom keystore entry has correct value
+      ansible.builtin.command: >
+        /usr/share/elasticsearch/bin/elasticsearch-keystore
+        show xpack.notification.slack.account.test.secure_url
+      register: keystore_value
+      changed_when: false
+      when: "'elasticsearch' in group_names"
+
+    - name: Assert custom keystore value
+      ansible.builtin.assert:
+        that:
+          - keystore_value.stdout == "https://hooks.example.com/test"
+        fail_msg: "Expected 'https://hooks.example.com/test', got '{{ keystore_value.stdout }}'"
+      when: "'elasticsearch' in group_names"
+
+    # -- kibana_system_password --
+    - name: Verify kibana_system can authenticate with custom password
+      ansible.builtin.uri:
+        url: "https://localhost:9200/_security/_authenticate"
+        method: GET
+        force_basic_auth: true
+        user: kibana_system
+        password: "molecule-kibana-test-pw"
+        validate_certs: false
+        status_code: 200
+      register: kibana_auth
+      when: "'elasticsearch' in group_names"
+      delegate_to: "{{ elasticsearch_ca }}"
+
+    - name: Assert kibana_system authentication succeeded
+      ansible.builtin.assert:
+        that:
+          - kibana_auth.json.username == 'kibana_system'
+        fail_msg: "kibana_system auth failed: {{ kibana_auth.json | default('no response') }}"
+      when: "'elasticsearch' in group_names"
+      delegate_to: "{{ elasticsearch_ca }}"
diff --git a/roles/elasticsearch/tasks/elasticsearch-keystore.yml b/roles/elasticsearch/tasks/elasticsearch-keystore.yml
index feb52c6..f625aed 100644
--- a/roles/elasticsearch/tasks/elasticsearch-keystore.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-keystore.yml
@@ -315,60 +315,93 @@
 # bootstrap password, etc.) — those are off-limits. User-provided
 # entries in elasticsearch_keystore_entries must not overlap with
 # them or the playbook fails with a clear error explaining why.
+#
+# Stale entries (present in the keystore but absent from
+# elasticsearch_keystore_entries and not role-managed) are removed
+# to keep the keystore in sync with declared state.
 
-- name: elasticsearch-keystore | Validate custom entries do not conflict with role-managed keys
-  ansible.builtin.assert:
-    that:
-      - item.key not in _elasticsearch_reserved_keystore_keys
-    fail_msg: >-
-      Cannot set '{{ item.key }}' via elasticsearch_keystore_entries because
-      this key is managed automatically by the elasticsearch role. Remove it
-      from elasticsearch_keystore_entries. The role manages these keys
-      internally: {{ _elasticsearch_reserved_keystore_keys | join(', ') }}
-  loop: "{{ elasticsearch_keystore_entries | dict2items }}"
-  loop_control:
-    label: "{{ item.key }}"
+- name: elasticsearch-keystore | Manage custom entries
+  when: elasticsearch_keystore_entries | length > 0
   vars:
     _elasticsearch_reserved_keystore_keys:
       - bootstrap.password
       - autoconfiguration.password_hash
+      - keystore.seed
       - xpack.security.http.ssl.keystore.secure_password
       - xpack.security.http.ssl.truststore.secure_password
       - xpack.security.transport.ssl.keystore.secure_password
       - xpack.security.transport.ssl.truststore.secure_password
       - xpack.security.http.ssl.secure_key_passphrase
       - xpack.security.transport.ssl.secure_key_passphrase
-  when: elasticsearch_keystore_entries | length > 0
+  block:
+    - name: elasticsearch-keystore | Validate custom entries do not conflict with role-managed keys
+      ansible.builtin.assert:
+        that:
+          - item.key not in _elasticsearch_reserved_keystore_keys
+        fail_msg: >-
+          Cannot set '{{ item.key }}' via elasticsearch_keystore_entries because
+          this key is managed automatically by the elasticsearch role. Remove it
+          from elasticsearch_keystore_entries. The role manages these keys
+          internally: {{ _elasticsearch_reserved_keystore_keys | join(', ') }}
+      loop: "{{ elasticsearch_keystore_entries | dict2items }}"
+      loop_control:
+        label: "{{ item.key }}"
 
-- name: elasticsearch-keystore | Get current value for custom entry '{{ item.key }}'
-  ansible.builtin.command: >
-    /usr/share/elasticsearch/bin/elasticsearch-keystore
-    show {{ item.key }}
-  loop: "{{ elasticsearch_keystore_entries | dict2items }}"
-  loop_control:
-    label: "{{ item.key }}"
-  register: _elasticsearch_custom_keystore_current
-  changed_when: false
-  no_log: true
-  ignore_errors: true
-  when: elasticsearch_keystore_entries | length > 0
+    - name: elasticsearch-keystore | Get current value for custom entry '{{ item.key }}'
+      ansible.builtin.command: >
+        /usr/share/elasticsearch/bin/elasticsearch-keystore
+        show {{ item.key }}
+      loop: "{{ elasticsearch_keystore_entries | dict2items }}"
+      loop_control:
+        label: "{{ item.key }}"
+      register: _elasticsearch_custom_keystore_current
+      changed_when: false
+      no_log: true
+      # Only expect failure when the key doesn't exist yet. If the key IS
+      # listed in the keystore but show still fails, that's a real error
+      # (corrupt keystore, permission denied) — let it propagate.
+      failed_when: >-
+        _elasticsearch_custom_keystore_current.rc != 0 and
+        item.key in elasticsearch_keystore.stdout_lines
 
-- name: elasticsearch-keystore | Set custom entry '{{ item.item.key }}'
-  ansible.builtin.command: >
-    /usr/share/elasticsearch/bin/elasticsearch-keystore
-    add -f -x '{{ item.item.key }}'
-  args:
-    stdin: "{{ item.item.value }}"
-  loop: "{{ _elasticsearch_custom_keystore_current.results | default([]) }}"
-  loop_control:
-    label: "{{ item.item.key }}"
-  changed_when: true
-  no_log: true
-  when:
-    - elasticsearch_keystore_entries | length > 0
-    - item.rc != 0 or item.stdout != item.item.value | string
-  notify:
-    - Restart Elasticsearch
+    - name: elasticsearch-keystore | Set custom entry '{{ item.item.key }}'
+      ansible.builtin.command: >
+        /usr/share/elasticsearch/bin/elasticsearch-keystore
+        add -f -x '{{ item.item.key }}'
+      args:
+        stdin: "{{ item.item.value }}"
+      loop: "{{ _elasticsearch_custom_keystore_current.results | default([]) }}"
+      loop_control:
+        label: "{{ item.item.key }}"
+      changed_when: true
+      no_log: true
+      when: item.rc != 0 or item.stdout != item.item.value | string
+      notify:
+        - Restart Elasticsearch
+
+    # --- Remove stale custom keystore entries ---
+    #
+    # Re-list the keystore, then subtract: declared custom entries,
+    # all role-managed keys. Whatever remains is stale and gets removed.
+
+    - name: elasticsearch-keystore | Refresh keystore listing
+      ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list
+      changed_when: false
+      register: _elasticsearch_keystore_after_custom
+
+    - name: elasticsearch-keystore | Remove stale custom entries
+      ansible.builtin.command: >
+        /usr/share/elasticsearch/bin/elasticsearch-keystore
+        remove {{ item }}
+      loop: >-
+        {{ _elasticsearch_keystore_after_custom.stdout_lines
+           | difference(elasticsearch_keystore_entries.keys() | list)
+           | difference(_elasticsearch_reserved_keystore_keys) }}
+      loop_control:
+        label: "{{ item }}"
+      changed_when: true
+      notify:
+        - Restart Elasticsearch
 
 # ============================================================
 

From 2e2189f26858d6b39d6767a2307ba7331dbdaa56 Mon Sep 17 00:00:00 2001
From: Sam Crauwels <sam.crauwels@gmail.com>
Date: Thu, 26 Mar 2026 19:11:11 +0100
Subject: [PATCH 4/5] Fix kibana_system_password verify: run on first ES node
 only

The verify assertion was delegating to elasticsearch_ca which is not
set during the verify context (only during role execution). Also
running the auth check on all ES nodes was redundant. Now it runs
directly on the first elasticsearch group member only.
---
 molecule/es_kibana/verify.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/molecule/es_kibana/verify.yml b/molecule/es_kibana/verify.yml
index 6522106..6b7087e 100644
--- a/molecule/es_kibana/verify.yml
+++ b/molecule/es_kibana/verify.yml
@@ -111,13 +111,15 @@
         validate_certs: false
         status_code: 200
       register: kibana_auth
-      when: "'elasticsearch' in group_names"
-      delegate_to: "{{ elasticsearch_ca }}"
+      when:
+        - "'elasticsearch' in group_names"
+        - inventory_hostname == groups['elasticsearch'][0]
 
     - name: Assert kibana_system authentication succeeded
       ansible.builtin.assert:
         that:
           - kibana_auth.json.username == 'kibana_system'
         fail_msg: "kibana_system auth failed: {{ kibana_auth.json | default('no response') }}"
-      when: "'elasticsearch' in group_names"
-      delegate_to: "{{ elasticsearch_ca }}"
+      when:
+        - "'elasticsearch' in group_names"
+        - inventory_hostname == groups['elasticsearch'][0]

From 0e50e3f2ad830a5aa9a1ba8e2eaa7f050437b60a Mon Sep 17 00:00:00 2001
From: Sam Crauwels <sam.crauwels@gmail.com>
Date: Thu, 26 Mar 2026 20:44:31 +0100
Subject: [PATCH 5/5] Fix kibana_system_password: run from Kibana node, target
 CA host
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The password change block required inventory_hostname == elasticstack_ca_host,
but the Kibana role only runs on Kibana group hosts — never on the CA host.
The API call now targets elasticstack_ca_host directly and uses run_once
so it executes once regardless of how many Kibana nodes exist.
---
 roles/kibana/tasks/kibana-security.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml
index 794e253..f050ca8 100644
--- a/roles/kibana/tasks/kibana-security.yml
+++ b/roles/kibana/tasks/kibana-security.yml
@@ -425,7 +425,6 @@
 # (kibana.yml template, keystore) uses the new password transparently.
 - name: kibana-security | Change kibana_system password to user-defined value
   when:
-    - inventory_hostname == elasticstack_ca_host
     - kibana_system_password | default('') | length > 0
     - kibana_password.stdout | default('') != kibana_system_password
   block:
@@ -438,7 +437,7 @@
 
     - name: kibana-security | Change kibana_system password via Elasticsearch API
       ansible.builtin.uri:
-        url: "{{ _kibana_es_protocol }}://{{ elasticsearch_api_host | default('localhost') }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/kibana_system/_password"
+        url: "{{ _kibana_es_protocol }}://{{ elasticsearch_api_host | default(elasticstack_ca_host) }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/kibana_system/_password"
         method: POST
         body_format: json
         body:
@@ -451,6 +450,7 @@
       no_log: "{{ elasticstack_no_log }}"
       vars:
         _kibana_es_protocol: "{{ 'https' if kibana_security | bool else 'http' }}"
+      run_once: true
 
     - name: kibana-security | Update kibana_password fact to user-defined value  # noqa: var-naming[no-role-prefix]
       ansible.builtin.set_fact: