Fix password timing on fresh install and cluster settings

molecule/shared/set_ci_watermarks.yml

@@ -4,6 +4,8 @@
 # scenarios where disk space is constrained. Runs once per cluster.
 #
 # Requires: Elasticsearch running with security enabled.
+# Uses elasticstack_password (set by the elasticsearch role) when
+# available, falls back to reading from the initial_passwords file.
 - name: Read elastic password for watermark setup
   ansible.builtin.shell: |
     set -o pipefail
@@ -15,6 +17,7 @@
   changed_when: false
   no_log: true
   run_once: true
+  when: elasticstack_password is not defined or elasticstack_password.stdout | default('') | length == 0
 
 - name: Set lenient disk watermarks (CI)
   ansible.builtin.uri:
@@ -28,7 +31,7 @@
         cluster.routing.allocation.disk.watermark.flood_stage.frozen: "99%"
     body_format: json
     user: elastic
-    password: "{{ _ci_elastic_pass.stdout }}"
+    password: "{{ elasticstack_password.stdout | default(_ci_elastic_pass.stdout | default('')) }}"
     validate_certs: false
     force_basic_auth: true
   register: _ci_watermark_result

molecule/shared/verify_fetch_password.yml

@@ -7,13 +7,6 @@
 #   _verify_delegate_to: host to delegate to (default: omitted, runs locally)
 #   _verify_run_once: whether to run once (default: true)
 
-- name: Use user-defined elastic password
-  ansible.builtin.set_fact:
-    elastic_pass:
-      stdout: "{{ _verify_elastic_password }}"
-  when: _verify_elastic_password | default('') | length > 0
-  run_once: "{{ _verify_run_once | default(true) }}"  # noqa: run-once[task]
-
 - name: Fetch Elastic password from file
   ansible.builtin.shell: |
     set -o pipefail
@@ -26,3 +19,13 @@
   run_once: "{{ _verify_run_once | default(true) }}"  # noqa: run-once[task]
   delegate_to: "{{ _verify_delegate_to | default(omit) }}"
   when: _verify_elastic_password | default('') | length == 0
+
+# set_fact MUST come after the shell task: Ansible's `register` runs
+# even on skipped tasks, overwriting the variable with a dict that
+# lacks `.stdout`.  Putting set_fact second ensures it wins.
+- name: Use user-defined elastic password
+  ansible.builtin.set_fact:
+    elastic_pass:
+      stdout: "{{ _verify_elastic_password }}"
+  when: _verify_elastic_password | default('') | length > 0
+  run_once: "{{ _verify_run_once | default(true) }}"  # noqa: run-once[task]

roles/elasticsearch/tasks/elasticsearch-security.yml

@@ -256,7 +256,7 @@
 
     # Extract CA chain from the already-deployed transport cert on the node.
     # Uses copy+content for idempotency (only writes when content changes).
     - name: Read CA chain from PEM bundle (transport cert)
       ansible.builtin.shell:
         cmd: >-
           awk '/-----BEGIN CERTIFICATE-----/{n++} n>1'
@@ -289,7 +289,7 @@
 
     # -- Remove stale auto-generated P12 files from a previous elasticsearch_ca deployment --
 
     - name: Find stale auto-generated P12 certificate files
       ansible.builtin.find:
         paths: /etc/elasticsearch/certs
         patterns: "*.p12"
@@ -355,7 +355,7 @@
     # The CA backup removes the entire elasticstack_ca_dir, which also
     # contains Kibana encryption key files. If those are regenerated,
     # Kibana can't decrypt its existing saved objects and enters 503.
     - name: Check for Kibana encryption keys before CA backup
       ansible.builtin.stat:
         path: "{{ elasticstack_ca_dir }}/{{ item }}"
       loop:
@@ -676,7 +676,7 @@
         - elasticsearch_api_status_bootstrap is failed
         - inventory_hostname == elasticstack_ca_host
       block:
         - name: Reset elastic user password via elasticsearch-reset-password
           ansible.builtin.command: >
             /usr/share/elasticsearch/bin/elasticsearch-reset-password
             -u elastic -b -s
@@ -684,7 +684,7 @@
           changed_when: true
           no_log: "{{ elasticstack_no_log }}"
 
         - name: Verify API access with recovered password
           ansible.builtin.uri:
             url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}"
             user: elastic
@@ -747,23 +747,29 @@
       retries: 30
       delay: 10
 
-    - name: Use user-defined elastic password
-      ansible.builtin.set_fact:
-        elasticstack_password:
-          stdout: "{{ elasticsearch_elastic_password }}"
-      no_log: "{{ elasticstack_no_log }}"
-      when: elasticsearch_elastic_password | default('') | length > 0
-
     - name: Fetch Elastic password from file
       ansible.builtin.include_tasks:
         file: "{{ role_path }}/../elasticstack/tasks/fetch_password.yml"
       vars:
         _password_user: elastic
         _password_fact: elasticstack_password
       when:
-        - elasticsearch_elastic_password | default('') | length == 0
         - elasticsearch_passwords_file.stat.exists | bool
 
+    # On fresh install the passwords file doesn't exist yet, so the fetch
+    # above is skipped.  But the elasticstack shared role may have already
+    # set elasticstack_password to the user-defined value — which hasn't
+    # actually been applied to ES yet.  Clear it so the API check below
+    # is safely skipped (the bootstrap-password check already confirmed
+    # the cluster is reachable).
+    - name: Clear premature user-defined password on fresh install
+      ansible.builtin.set_fact:
+        elasticstack_password:
+          stdout: ""
+      when:
+        - not elasticsearch_passwords_file.stat.exists | bool
+        - elasticsearch_elastic_password | default('') | length > 0
+
     - name: Check for API availability with elastic password
       ansible.builtin.uri:
         url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}"
@@ -773,14 +779,47 @@
         force_basic_auth: true
       register: elasticsearch_api_status
       changed_when: false
+      failed_when: false
       no_log: "{{ elasticstack_no_log }}"
       when:
         - elasticstack_password is defined
         - elasticstack_password.stdout | default('') | length > 0
-      until: (elasticsearch_api_status.json | default({})).cluster_name is defined
+      until: >-
+        (elasticsearch_api_status.json | default({})).cluster_name is defined
+        or (elasticsearch_api_status.status | default(0)) == 401
       retries: 20
       delay: 10
 
+    # If the auto-generated password returned 401, the user-defined
+    # password was already applied in a previous run.  Switch to it.
+    - name: Switch to user-defined password after prior change
+      ansible.builtin.set_fact:
+        elasticstack_password:
+          stdout: "{{ elasticsearch_elastic_password }}"
+      no_log: "{{ elasticstack_no_log }}"
+      when:
+        - elasticsearch_elastic_password | default('') | length > 0
+        - elasticsearch_api_status is defined
+        - (elasticsearch_api_status.status | default(0)) == 401
+
+    - name: Verify API availability with user-defined password
+      ansible.builtin.uri:
+        url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}"
+        user: elastic
+        password: "{{ elasticstack_password.stdout }}"
+        validate_certs: "{{ elasticsearch_validate_api_certs }}"
+        force_basic_auth: true
+      register: elasticsearch_api_status
+      changed_when: false
+      no_log: "{{ elasticstack_no_log }}"
+      when:
+        - elasticsearch_elastic_password | default('') | length > 0
+        - elasticsearch_api_status is defined
+        - (elasticsearch_api_status.status | default(0)) == 401
+      until: (elasticsearch_api_status.json | default({})).cluster_name is defined
+      retries: 10
+      delay: 5
+
     - name: Work around low resources on CI/CD nodes
       when: ansible_facts.virtualization_type in ["container", "docker", "lxc"]
       block:
@@ -802,7 +841,7 @@
                 cluster.routing.allocation.disk.watermark.flood_stage.frozen: "99%"
             body_format: json
             user: elastic
-            password: "{{ elasticstack_password.stdout | default(elasticsearch_bootstrap_pw) }}"
+            password: "{{ (elasticstack_password.stdout | default('')) or elasticsearch_bootstrap_pw }}"
             validate_certs: "{{ elasticsearch_validate_api_certs }}"
             force_basic_auth: true
           register: elasticsearch_watermark_response
@@ -846,7 +885,7 @@
 
     # ES 8.x: elasticsearch-setup-passwords creates all built-in user passwords
     # ES 9.x: setup-passwords is removed; use elasticsearch-reset-password per user
     - name: Create initial passwords (ES 8.x)
       ansible.builtin.shell: >
         set -o pipefail;
         /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto -b >
@@ -915,7 +954,7 @@
       when:
         - inventory_hostname == elasticstack_ca_host
         - elasticsearch_elastic_password | default('') | length > 0
-        - elasticsearch_freshstart_security.changed | bool
+        - elasticstack_password.stdout | default('') != elasticsearch_elastic_password
       block:
         - name: Fetch auto-generated elastic password
           ansible.builtin.include_tasks:
@@ -944,6 +983,18 @@
               stdout: "{{ elasticsearch_elastic_password }}"
           no_log: "{{ elasticstack_no_log }}"
 
+    # On fresh install the passwords file was just created but elasticstack_password
+    # may still be unset (the shared role couldn't fetch it because the file didn't
+    # exist yet).  Fetch now so downstream tasks (cluster settings, watermarks) work.
+    - name: Fetch elastic password after initial setup
+      ansible.builtin.include_tasks:
+        file: "{{ role_path }}/../elasticstack/tasks/fetch_password.yml"
+      vars:
+        _password_user: elastic
+        _password_fact: elasticstack_password
+      when:
+        - elasticstack_password is not defined or (elasticstack_password.stdout | default('') | length == 0)
+
 # Maybe make sure that Elasticsearch is using the right protocol http(s) to connect, even in newly setup clusters
 
 # -- Certificate expiry warnings --

roles/elasticsearch/tasks/main.yml

@@ -531,9 +531,11 @@
   ansible.builtin.debug:
     msg: >-
       Using {{ elasticsearch_heap | int * 1024 }} of
-      {{ (_es_cgroup_memory.content is defined and (_es_cgroup_memory.content | b64decode | trim) != 'max')
-         | ternary((_es_cgroup_memory.content | b64decode | trim) | int // 1048576 ~ ' MB (cgroup limit)',
-                   ansible_facts.memtotal_mb ~ ' MB') }}
+      {% if _es_cgroup_memory.content is defined and (_es_cgroup_memory.content | b64decode | trim) != 'max' %}
+      {{ (_es_cgroup_memory.content | b64decode | trim) | int // 1048576 }} MB (cgroup limit)
+      {% else %}
+      {{ ansible_facts.memtotal_mb }} MB
+      {% endif %}
       as heap for Elasticsearch
   when: elasticsearch_heap is defined and (elasticsearch_heap | string | length > 0)
 
@@ -580,7 +582,7 @@
 
     - name: Check if settings already match
       ansible.builtin.set_fact:
-        _es_cluster_settings_changed: "{{ _needs_update }}"
+        _es_cluster_settings_changed: "{{ _needs_update | trim }}"
       vars:
         _current: "{{ _es_current_cluster_settings.json.persistent }}"
         _needs_update: >-

roles/elasticsearch/templates/log4j2.properties.j2

@@ -1,4 +1,4 @@
-{{ ansible_managed | comment('#') }}
+{{ ansible_managed | comment }}
 
 status = error
 

roles/kibana/tasks/kibana-security.yml

@@ -67,6 +67,15 @@
             _validate_ca_extracted_fact: _kibana_ca_extracted
           when: not (_kibana_content_mode | bool)
 
+        # -- Ensure cert directory exists before deploying --
+        - name: Create certificate directory
+          ansible.builtin.file:
+            path: /etc/kibana/certs
+            state: directory
+            owner: root
+            group: kibana
+            mode: "0750"
+
         # -- Deploy certificates --
 
         - name: Write Kibana certificate (from content)