CI fixes: venv isolation, python 3.12, logstash memory

.github/workflows/molecule.yml

@@ -64,7 +64,11 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Install dependencies
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install -r requirements-test.txt
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 

.github/workflows/test_linting.yml

@@ -33,7 +33,11 @@ jobs:
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
       - name: Install test dependencies.
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install -r requirements-test.txt
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 

.github/workflows/test_plugins.yml

@@ -39,8 +39,12 @@ jobs:
       - name: Check out the codebase.
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
-      - name: Install test dependencies.
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 pycodestyle
+      - name: Create venv and install dependencies
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install pycodestyle
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 
@@ -66,8 +70,21 @@ jobs:
       - name: Check out code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
-      - name: Install ansible-core
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible-core~=${{ matrix.ansible_core }}.0
+      - name: Select Python for ansible-core version
+        id: pyver
+        run: |
+          PYTHON=/usr/bin/python3
+          if [[ "${{ matrix.ansible_core }}" == "2.20" ]] && command -v python3.12 &>/dev/null; then
+            PYTHON=$(command -v python3.12)
+          fi
+          echo "python=$PYTHON" >> "$GITHUB_OUTPUT"
+
+      - name: Create venv and install ansible-core
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python ${{ steps.pyver.outputs.python }}
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install ansible-core~=${{ matrix.ansible_core }}.0
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 
@@ -109,8 +126,12 @@ jobs:
       - name: Check out code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
-      - name: Install dependencies
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible
+      - name: Create venv and install dependencies
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install ansible
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 
@@ -154,8 +175,12 @@ jobs:
       - name: Check out code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
-      - name: Install dependencies
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible
+      - name: Create venv and install dependencies
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install ansible
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 
@@ -197,8 +222,21 @@ jobs:
       - name: Check out code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
-      - name: Install dependencies
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible-core==${{ matrix.ansible_core_version }}
+      - name: Select Python for ansible-core version
+        id: pyver
+        run: |
+          PYTHON=/usr/bin/python3
+          if [[ "${{ matrix.ansible_core_version }}" == 2.20* ]] && command -v python3.12 &>/dev/null; then
+            PYTHON=$(command -v python3.12)
+          fi
+          echo "python=$PYTHON" >> "$GITHUB_OUTPUT"
+
+      - name: Create venv and install dependencies
+        run: |
+          uv venv "$RUNNER_TEMP/venv" --python ${{ steps.pyver.outputs.python }}
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install ansible-core==${{ matrix.ansible_core_version }}
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 
@@ -240,10 +278,13 @@ jobs:
       - name: Check out code
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
 
-      - name: Install dependencies
+      - name: Create venv and install dependencies
         run: |
-          uv pip install --system --break-system-packages --python /usr/bin/python3 cryptography==${{ matrix.python_cryptography_version }}
-          uv pip install --system --break-system-packages --python /usr/bin/python3 ansible
+          uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3
+          source "$RUNNER_TEMP/venv/bin/activate"
+          uv pip install cryptography==${{ matrix.python_cryptography_version }}
+          uv pip install ansible
+          echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
         env:
           SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
 

docs/reference/elasticstack.md

@@ -36,6 +36,14 @@ elasticstack_no_log: true
 `elasticstack_release`
 :   Major version of the Elastic Stack to install (`8` or `9`). Controls which package repository is configured and which configuration format is generated by templates. Set to `9` for new deployments; keep at `8` for existing clusters until you're ready to upgrade.
 
+`elasticstack_version`
+:   Pin a specific Elastic Stack version (e.g. `"9.1.2"`). When set, the package manager installs exactly this version. When unset (the default), the latest available version within the major release is installed.
+
+    Set to `"latest"` to always upgrade to the newest version in the repository. This means every playbook run where a newer version is available will trigger a **rolling restart** of Elasticsearch nodes one at a time. This is safe (shard allocation is managed), but be aware that routine playbook runs may cause node restarts if a new minor or patch release was published since the last run.
+
+    !!! warning
+        When `elasticstack_version` is set to `"latest"`, every new minor or patch release triggers a rolling restart of your Elasticsearch cluster. If you prefer controlled upgrades, leave this unset and pin specific versions when you're ready to upgrade.
+
 `elasticstack_full_stack`
 :   When `true`, roles auto-discover each other through inventory groups and share TLS certificates via the central CA. When `false`, each role operates standalone — you must provide explicit host lists (e.g. `beats_target_hosts`, `logstash_elasticsearch_hosts`) instead of relying on group lookups.
 

molecule/elasticsearch_upgrade_8to9/converge.yml

@@ -118,14 +118,19 @@
       delay: 5
       until: _refresh_result.status | default(0) == 200
 
-- name: Upgrade to Elasticsearch 9.x (with config change to exercise handler suppression)
+- name: Upgrade to Elasticsearch 9.x via release change (rolling upgrade)
   hosts: all
   serial: 1
   vars:
     elasticstack_full_stack: false
+    # Only change the release — don't pin elasticstack_version.
+    # This exercises the real-world upgrade path: the role detects the
+    # major version mismatch and triggers a rolling upgrade automatically.
     elasticstack_release: 9
     elasticsearch_heap: "1"
     elasticstack_no_log: false
+    # Skip the countdown pause in CI
+    elasticsearch_upgrade_countdown: 0
     # Add a JVM custom parameter during upgrade to trigger a config change.
     # This exercises the fix: the rolling upgrade restarts ES, and the config
     # change notifies the handler, which should be suppressed to prevent an
@@ -146,43 +151,51 @@
       ansible.builtin.set_fact:
         _elasticstack_role_imported: false
 
-    - name: Gather package facts for version comparison
-      ansible.builtin.package_facts:
-        manager: auto
-
-    - name: Detect latest available Elasticsearch 9.x version (Debian)
-      ansible.builtin.shell: >
-        set -o pipefail &&
-        apt-cache policy elasticsearch | grep 'Candidate:' | awk '{print $2}'
-      args:
-        executable: /bin/bash
-      register: _es_candidate_version_deb
-      changed_when: false
-      when: ansible_facts.os_family == "Debian"
-
-    - name: Detect latest available Elasticsearch 9.x version (RHEL)
-      ansible.builtin.shell: >
-        set -o pipefail &&
-        dnf info --available elasticsearch 2>/dev/null | grep '^Version' | awk '{print $3}' | head -1
-      args:
-        executable: /bin/bash
-      register: _es_candidate_version_rhel
-      changed_when: false
-      when: ansible_facts.os_family == "RedHat"
-
-    - name: Combine candidate version
-      ansible.builtin.set_fact:
-        _es_candidate_version:
-          stdout: "{{ _es_candidate_version_deb.stdout | default(_es_candidate_version_rhel.stdout | default('')) }}"
+    - name: Include Elasticsearch role (upgrade to 9.x)
+      ansible.builtin.include_role:
+        name: oddly.elasticstack.elasticsearch
 
-    - name: Set target upgrade version
+# Re-run with elasticstack_version: "latest" to exercise the post-install
+# detection path. Since we're already on the latest 9.x, the package should
+# NOT change and the rolling restart should NOT trigger. This verifies that
+# the "latest" mode doesn't cause unnecessary restarts.
+- name: Re-run with elasticstack_version latest (no-op expected)
+  hosts: all
+  vars:
+    elasticstack_full_stack: false
+    elasticstack_release: 9
+    elasticstack_version: "latest"
+    elasticsearch_heap: "1"
+    elasticstack_no_log: false
+    elasticsearch_upgrade_countdown: 0
+    elasticsearch_jvm_custom_parameters:
+      - "-Des.upgrade.test.marker=true"
+  tasks:
+    - name: Reset shared role guard
       ansible.builtin.set_fact:
-        elasticstack_version: "{{ _es_candidate_version.stdout }}"
+        _elasticstack_role_imported: false
 
-    - name: Display upgrade target
-      ansible.builtin.debug:
-        msg: "Upgrading from {{ ansible_facts.packages['elasticsearch'][0].version }} to {{ elasticstack_version }}"
+    - name: Record ES PID before re-run
+      ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch'
+      register: _es_pid_before
+      changed_when: false
 
-    - name: Include Elasticsearch role (upgrade to 9.x)
+    - name: Include Elasticsearch role (latest, should be no-op)
       ansible.builtin.include_role:
         name: oddly.elasticstack.elasticsearch
+
+    - name: Record ES PID after re-run
+      ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch'
+      register: _es_pid_after
+      changed_when: false
+
+    - name: Verify ES was NOT restarted (PID unchanged)
+      ansible.builtin.assert:
+        that:
+          - _es_pid_before.stdout == _es_pid_after.stdout
+        fail_msg: >-
+          ES was restarted unnecessarily! PID changed from
+          {{ _es_pid_before.stdout }} to {{ _es_pid_after.stdout }}.
+          The 'latest' mode should not trigger a restart when already
+          at the latest version.
+        success_msg: "ES PID unchanged ({{ _es_pid_after.stdout }}) — no unnecessary restart"

molecule/elasticsearch_upgrade_8to9_single/converge.yml

@@ -174,15 +174,19 @@
         fail_msg: "Must be on 8.19.x before upgrading to 9.x. Current: {{ es_info.json.version.number }}"
         success_msg: "On 8.19.x ({{ es_info.json.version.number }}), ready for 9.x upgrade"
 
-- name: Upgrade to Elasticsearch 9.x (via role)
+- name: Upgrade to Elasticsearch 9.x via release change (rolling upgrade)
   hosts: all
   vars:
     elasticstack_full_stack: false
+    # Only change the release — don't pin elasticstack_version.
+    # This exercises the real-world upgrade path.
     elasticstack_release: 9
     elasticsearch_heap: "1"
     elasticstack_no_log: false
     elasticsearch_seed_hosts: []
     elasticsearch_initial_master_nodes: []
+    # Skip the countdown pause in CI
+    elasticsearch_upgrade_countdown: 0
   tasks:
     - name: Include Elastic repos role (9.x)
       ansible.builtin.include_role:
@@ -192,38 +196,6 @@
       ansible.builtin.set_fact:
         _elasticstack_role_imported: false
 
-    - name: Gather package facts for version comparison
-      ansible.builtin.package_facts:
-        manager: auto
-
-    - name: Detect latest available Elasticsearch 9.x version (Debian)
-      ansible.builtin.shell: >
-        set -o pipefail &&
-        apt-cache policy elasticsearch | grep 'Candidate:' | awk '{print $2}'
-      args:
-        executable: /bin/bash
-      register: _es_candidate_version_deb
-      changed_when: false
-      when: ansible_facts.os_family == "Debian"
-
-    - name: Detect latest available Elasticsearch 9.x version (RHEL)
-      ansible.builtin.shell: >
-        set -o pipefail &&
-        dnf info --available elasticsearch 2>/dev/null | grep '^Version' | awk '{print $3}' | head -1
-      args:
-        executable: /bin/bash
-      register: _es_candidate_version_rhel
-      changed_when: false
-      when: ansible_facts.os_family == "RedHat"
-
-    - name: Set target upgrade version
-      ansible.builtin.set_fact:
-        elasticstack_version: "{{ _es_candidate_version_deb.stdout | default(_es_candidate_version_rhel.stdout | default('')) }}"
-
-    - name: Display upgrade target
-      ansible.builtin.debug:
-        msg: "Upgrading from {{ ansible_facts.packages['elasticsearch'][0].version }} to {{ elasticstack_version }}"
-
     - name: Include Elasticsearch role (upgrade to 9.x)
       ansible.builtin.include_role:
         name: oddly.elasticstack.elasticsearch

molecule/logstash_elasticsearch/molecule.yml

@@ -16,7 +16,7 @@ platforms:
     groups:
       - logstash
     distro: "${MOLECULE_DISTRO:-debian12}"
-    memory_mb: 2048
+    memory_mb: 3072
 provisioner:
   name: ansible
   env:

roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml

@@ -28,6 +28,7 @@
     - name: elasticsearch-rolling-upgrade | Update stopped Elasticsearch - rpm with managed repositories
       ansible.builtin.package:
         name: "{{ elasticsearch_package }}"
+        state: latest  # noqa: package-latest
         enablerepo:
           - 'elastic-{{ elasticstack_release }}.x'
       when:
@@ -37,6 +38,7 @@
     - name: elasticsearch-rolling-upgrade | Update stopped Elasticsearch - deb or unmanaged repositories rpm
       ansible.builtin.package:
         name: "{{ elasticsearch_package }}"
+        state: latest  # noqa: package-latest
       when:
         - ansible_facts.os_family == "Debian" or
           not elasticstack_full_stack | bool
@@ -48,6 +50,7 @@
     - name: elasticsearch-rolling-upgrade | Update single instances without extra caution - deb or unmanaged repositories rpm
       ansible.builtin.package:
         name: "{{ elasticsearch_package }}"
+        state: latest  # noqa: package-latest
       when:
         - ansible_facts.os_family == "Debian" or
           not elasticstack_full_stack | bool
@@ -57,6 +60,7 @@
     - name: elasticsearch-rolling-upgrade | Update single instances without extra caution - rpm with managed repositories
       ansible.builtin.package:
         name: "{{ elasticsearch_package }}"
+        state: latest  # noqa: package-latest
         enablerepo:
           - 'elastic-{{ elasticstack_release }}.x'
       when:
@@ -215,6 +219,7 @@
     - name: elasticsearch-rolling-upgrade | Update Elasticsearch - rpm with managed repositories
       ansible.builtin.package:
         name: "{{ elasticsearch_package }}"
+        state: latest  # noqa: package-latest
         enablerepo:
           - 'elastic-{{ elasticstack_release }}.x'
       when:
@@ -224,6 +229,7 @@
     - name: elasticsearch-rolling-upgrade | Update Elasticsearch - deb or unmanaged repositories rpm
       ansible.builtin.package:
         name: "{{ elasticsearch_package }}"
+        state: latest  # noqa: package-latest
       when:
         - ansible_facts.os_family == "Debian" or
           not elasticstack_full_stack | bool