From 79a29887557205cf0e4a92059fd68b26c6bfdf6e Mon Sep 17 00:00:00 2001 From: Sam Crauwels Date: Sun, 8 Mar 2026 08:27:25 +0100 Subject: [PATCH 1/2] Bump logstash and beats standalone containers from 1024 to 2048 MB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1024 MB limit was too tight — dnf/yum with rpm scriptlets spikes memory during package install, causing OOM kills (rc=137) on Rocky 9. --- molecule/beats_advanced/molecule.yml | 2 +- molecule/beats_peculiar/molecule.yml | 2 +- molecule/beats_security/molecule.yml | 2 +- molecule/logstash_advanced/molecule.yml | 2 +- molecule/logstash_centralized_pipelines/molecule.yml | 2 +- molecule/logstash_custom_pipeline/molecule.yml | 2 +- molecule/logstash_default/molecule.yml | 2 +- molecule/logstash_ssl/molecule.yml | 2 +- molecule/logstash_standalone_certs/molecule.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/molecule/beats_advanced/molecule.yml b/molecule/beats_advanced/molecule.yml index 0af0d27b..4ced7c1d 100644 --- a/molecule/beats_advanced/molecule.yml +++ b/molecule/beats_advanced/molecule.yml @@ -9,7 +9,7 @@ driver: platforms: - name: "beats-adv-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/beats_peculiar/molecule.yml b/molecule/beats_peculiar/molecule.yml index 6a332f22..1e8e7fff 100644 --- a/molecule/beats_peculiar/molecule.yml +++ b/molecule/beats_peculiar/molecule.yml @@ -9,7 +9,7 @@ driver: platforms: - name: beats-pec-${MOLECULE_DISTRO:-debian11}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX} distro: "${MOLECULE_DISTRO:-debian11}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/beats_security/molecule.yml b/molecule/beats_security/molecule.yml index 658e3f97..f60fbb55 100644 --- a/molecule/beats_security/molecule.yml +++ b/molecule/beats_security/molecule.yml @@ -16,7 +16,7 @@ platforms: groups: - beats distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/logstash_advanced/molecule.yml b/molecule/logstash_advanced/molecule.yml index bba6be60..14bc9776 100644 --- a/molecule/logstash_advanced/molecule.yml +++ b/molecule/logstash_advanced/molecule.yml @@ -11,7 +11,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/logstash_centralized_pipelines/molecule.yml b/molecule/logstash_centralized_pipelines/molecule.yml index 86b4b355..2bf3cfd7 100644 --- a/molecule/logstash_centralized_pipelines/molecule.yml +++ b/molecule/logstash_centralized_pipelines/molecule.yml @@ -11,7 +11,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/logstash_custom_pipeline/molecule.yml b/molecule/logstash_custom_pipeline/molecule.yml index 34c78088..e3993952 100644 --- a/molecule/logstash_custom_pipeline/molecule.yml +++ b/molecule/logstash_custom_pipeline/molecule.yml @@ -11,7 +11,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/logstash_default/molecule.yml b/molecule/logstash_default/molecule.yml index 57c1cc39..a1b09ddf 100644 --- a/molecule/logstash_default/molecule.yml +++ b/molecule/logstash_default/molecule.yml @@ -11,7 +11,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/logstash_ssl/molecule.yml b/molecule/logstash_ssl/molecule.yml index 2d4b0524..7717684f 100644 --- a/molecule/logstash_ssl/molecule.yml +++ b/molecule/logstash_ssl/molecule.yml @@ -11,7 +11,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: diff --git a/molecule/logstash_standalone_certs/molecule.yml b/molecule/logstash_standalone_certs/molecule.yml index bb69010b..6d5d1496 100644 --- a/molecule/logstash_standalone_certs/molecule.yml +++ b/molecule/logstash_standalone_certs/molecule.yml @@ -11,7 +11,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 1024 + memory_mb: 2048 provisioner: name: ansible env: From 9d8a879a470e9372abfb8b1e2b276a5a8880a699 Mon Sep 17 00:00:00 2001 From: Sam Crauwels Date: Sun, 8 Mar 2026 08:35:53 +0100 Subject: [PATCH 2/2] Fix transient CI failures in verify and diagnostics scenarios Add retries to _cat/nodes checks in no-security, upgrade, and elasticstack_default verify playbooks. ES can return HTTP 500 (array_index_out_of_bounds_exception) briefly after startup when node info isn't fully initialized. Fix diagnostics scenario restart failure on debian13 r9 by clearing systemd failure state and waiting for node lock release before restarting ES after the intentional bad-config crash test. --- molecule/elasticsearch_diagnostics/converge.yml | 11 +++++++++++ molecule/elasticsearch_no-security/verify.yml | 3 +++ molecule/elasticsearch_upgrade_8to9/verify.yml | 3 +++ molecule/elasticstack_default/verify.yml | 3 +++ 4 files changed, 20 insertions(+) diff --git a/molecule/elasticsearch_diagnostics/converge.yml b/molecule/elasticsearch_diagnostics/converge.yml index e9e4586f..ae0013e7 100644 --- a/molecule/elasticsearch_diagnostics/converge.yml +++ b/molecule/elasticsearch_diagnostics/converge.yml @@ -60,6 +60,17 @@ remote_src: true mode: "0644" + # systemd may refuse to restart a recently-failed service; clear the + # failure state and give ES a moment to release node locks. + - name: Reset systemd failure state before recovery restart + ansible.builtin.command: + cmd: systemctl reset-failed elasticsearch.service + changed_when: false + + - name: Wait for node lock to be released + ansible.builtin.pause: + seconds: 5 + - name: Restart Elasticsearch with restored config ansible.builtin.include_tasks: file: "{{ lookup('env', 'ANSIBLE_COLLECTIONS_PATH') | default(lookup('env', 'HOME') + '/.ansible/collections', true) }}/ansible_collections/oddly/elasticstack/roles/elasticsearch/tasks/restart_and_verify_elasticsearch.yml" diff --git a/molecule/elasticsearch_no-security/verify.yml b/molecule/elasticsearch_no-security/verify.yml index 8cb8097e..2a2665b9 100644 --- a/molecule/elasticsearch_no-security/verify.yml +++ b/molecule/elasticsearch_no-security/verify.yml @@ -47,6 +47,9 @@ status_code: 200 validate_certs: false register: nodes + until: nodes.status == 200 + retries: 3 + delay: 5 - name: Check if all Nodes see each other ansible.builtin.assert: diff --git a/molecule/elasticsearch_upgrade_8to9/verify.yml b/molecule/elasticsearch_upgrade_8to9/verify.yml index 41e37503..329da2cd 100644 --- a/molecule/elasticsearch_upgrade_8to9/verify.yml +++ b/molecule/elasticsearch_upgrade_8to9/verify.yml @@ -83,6 +83,9 @@ return_content: true validate_certs: false register: nodes_info + until: nodes_info.status | default(0) == 200 + retries: 3 + delay: 5 when: groups[elasticstack_elasticsearch_group_name] | length > 1 - name: Display nodes diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index 726fe577..da02a36a 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -169,6 +169,9 @@ status_code: 200 validate_certs: false register: nodes + until: nodes.status | default(0) == 200 + retries: 3 + delay: 5 when: groups[elasticstack_elasticsearch_group_name] | length > 1 - name: Check if all Nodes see each other