diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 891c365..9985193 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -64,7 +64,11 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt + run: | + uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3 + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install -r requirements-test.txt + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt diff --git a/.github/workflows/test_linting.yml b/.github/workflows/test_linting.yml index 88f7283..766e16e 100644 --- a/.github/workflows/test_linting.yml +++ b/.github/workflows/test_linting.yml @@ -33,7 +33,11 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install test dependencies. - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt + run: | + uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3 + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install -r requirements-test.txt + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index 6e7b718..965bf95 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -39,8 +39,12 @@ jobs: - name: Check out the codebase. uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install test dependencies. - run: uv pip install --system --break-system-packages --python /usr/bin/python3 pycodestyle + - name: Create venv and install dependencies + run: | + uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3 + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install pycodestyle + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt @@ -66,8 +70,21 @@ jobs: - name: Check out code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install ansible-core - run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible-core~=${{ matrix.ansible_core }}.0 + - name: Select Python for ansible-core version + id: pyver + run: | + PYTHON=/usr/bin/python3 + if [[ "${{ matrix.ansible_core }}" == "2.20" ]] && command -v python3.12 &>/dev/null; then + PYTHON=$(command -v python3.12) + fi + echo "python=$PYTHON" >> "$GITHUB_OUTPUT" + + - name: Create venv and install ansible-core + run: | + uv venv "$RUNNER_TEMP/venv" --python ${{ steps.pyver.outputs.python }} + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install ansible-core~=${{ matrix.ansible_core }}.0 + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt @@ -109,8 +126,12 @@ jobs: - name: Check out code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible + - name: Create venv and install dependencies + run: | + uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3 + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install ansible + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt @@ -154,8 +175,12 @@ jobs: - name: Check out code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible + - name: Create venv and install dependencies + run: | + uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3 + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install ansible + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt @@ -197,8 +222,21 @@ jobs: - name: Check out code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 ansible-core==${{ matrix.ansible_core_version }} + - name: Select Python for ansible-core version + id: pyver + run: | + PYTHON=/usr/bin/python3 + if [[ "${{ matrix.ansible_core_version }}" == 2.20* ]] && command -v python3.12 &>/dev/null; then + PYTHON=$(command -v python3.12) + fi + echo "python=$PYTHON" >> "$GITHUB_OUTPUT" + + - name: Create venv and install dependencies + run: | + uv venv "$RUNNER_TEMP/venv" --python ${{ steps.pyver.outputs.python }} + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install ansible-core==${{ matrix.ansible_core_version }} + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt @@ -240,10 +278,13 @@ jobs: - name: Check out code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Install dependencies + - name: Create venv and install dependencies run: | - uv pip install --system --break-system-packages --python /usr/bin/python3 cryptography==${{ matrix.python_cryptography_version }} - uv pip install --system --break-system-packages --python /usr/bin/python3 ansible + uv venv "$RUNNER_TEMP/venv" --python /usr/bin/python3 + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install cryptography==${{ matrix.python_cryptography_version }} + uv pip install ansible + echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" env: SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt diff --git a/docs/reference/elasticstack.md b/docs/reference/elasticstack.md index 74896bf..4be2475 100644 --- a/docs/reference/elasticstack.md +++ b/docs/reference/elasticstack.md @@ -36,6 +36,14 @@ elasticstack_no_log: true `elasticstack_release` : Major version of the Elastic Stack to install (`8` or `9`). Controls which package repository is configured and which configuration format is generated by templates. Set to `9` for new deployments; keep at `8` for existing clusters until you're ready to upgrade. +`elasticstack_version` +: Pin a specific Elastic Stack version (e.g. `"9.1.2"`). When set, the package manager installs exactly this version. When unset (the default), the latest available version within the major release is installed. + + Set to `"latest"` to always upgrade to the newest version in the repository. This means every playbook run where a newer version is available will trigger a **rolling restart** of Elasticsearch nodes one at a time. This is safe (shard allocation is managed), but be aware that routine playbook runs may cause node restarts if a new minor or patch release was published since the last run. + + !!! warning + When `elasticstack_version` is set to `"latest"`, every new minor or patch release triggers a rolling restart of your Elasticsearch cluster. If you prefer controlled upgrades, leave this unset and pin specific versions when you're ready to upgrade. + `elasticstack_full_stack` : When `true`, roles auto-discover each other through inventory groups and share TLS certificates via the central CA. When `false`, each role operates standalone — you must provide explicit host lists (e.g. `beats_target_hosts`, `logstash_elasticsearch_hosts`) instead of relying on group lookups. diff --git a/molecule/elasticsearch_upgrade_8to9/converge.yml b/molecule/elasticsearch_upgrade_8to9/converge.yml index f05924e..956628a 100644 --- a/molecule/elasticsearch_upgrade_8to9/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9/converge.yml @@ -118,14 +118,19 @@ delay: 5 until: _refresh_result.status | default(0) == 200 -- name: Upgrade to Elasticsearch 9.x (with config change to exercise handler suppression) +- name: Upgrade to Elasticsearch 9.x via release change (rolling upgrade) hosts: all serial: 1 vars: elasticstack_full_stack: false + # Only change the release — don't pin elasticstack_version. + # This exercises the real-world upgrade path: the role detects the + # major version mismatch and triggers a rolling upgrade automatically. elasticstack_release: 9 elasticsearch_heap: "1" elasticstack_no_log: false + # Skip the countdown pause in CI + elasticsearch_upgrade_countdown: 0 # Add a JVM custom parameter during upgrade to trigger a config change. # This exercises the fix: the rolling upgrade restarts ES, and the config # change notifies the handler, which should be suppressed to prevent an @@ -146,43 +151,51 @@ ansible.builtin.set_fact: _elasticstack_role_imported: false - - name: Gather package facts for version comparison - ansible.builtin.package_facts: - manager: auto - - - name: Detect latest available Elasticsearch 9.x version (Debian) - ansible.builtin.shell: > - set -o pipefail && - apt-cache policy elasticsearch | grep 'Candidate:' | awk '{print $2}' - args: - executable: /bin/bash - register: _es_candidate_version_deb - changed_when: false - when: ansible_facts.os_family == "Debian" - - - name: Detect latest available Elasticsearch 9.x version (RHEL) - ansible.builtin.shell: > - set -o pipefail && - dnf info --available elasticsearch 2>/dev/null | grep '^Version' | awk '{print $3}' | head -1 - args: - executable: /bin/bash - register: _es_candidate_version_rhel - changed_when: false - when: ansible_facts.os_family == "RedHat" - - - name: Combine candidate version - ansible.builtin.set_fact: - _es_candidate_version: - stdout: "{{ _es_candidate_version_deb.stdout | default(_es_candidate_version_rhel.stdout | default('')) }}" + - name: Include Elasticsearch role (upgrade to 9.x) + ansible.builtin.include_role: + name: oddly.elasticstack.elasticsearch - - name: Set target upgrade version +# Re-run with elasticstack_version: "latest" to exercise the post-install +# detection path. Since we're already on the latest 9.x, the package should +# NOT change and the rolling restart should NOT trigger. This verifies that +# the "latest" mode doesn't cause unnecessary restarts. +- name: Re-run with elasticstack_version latest (no-op expected) + hosts: all + vars: + elasticstack_full_stack: false + elasticstack_release: 9 + elasticstack_version: "latest" + elasticsearch_heap: "1" + elasticstack_no_log: false + elasticsearch_upgrade_countdown: 0 + elasticsearch_jvm_custom_parameters: + - "-Des.upgrade.test.marker=true" + tasks: + - name: Reset shared role guard ansible.builtin.set_fact: - elasticstack_version: "{{ _es_candidate_version.stdout }}" + _elasticstack_role_imported: false - - name: Display upgrade target - ansible.builtin.debug: - msg: "Upgrading from {{ ansible_facts.packages['elasticsearch'][0].version }} to {{ elasticstack_version }}" + - name: Record ES PID before re-run + ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch' + register: _es_pid_before + changed_when: false - - name: Include Elasticsearch role (upgrade to 9.x) + - name: Include Elasticsearch role (latest, should be no-op) ansible.builtin.include_role: name: oddly.elasticstack.elasticsearch + + - name: Record ES PID after re-run + ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch' + register: _es_pid_after + changed_when: false + + - name: Verify ES was NOT restarted (PID unchanged) + ansible.builtin.assert: + that: + - _es_pid_before.stdout == _es_pid_after.stdout + fail_msg: >- + ES was restarted unnecessarily! PID changed from + {{ _es_pid_before.stdout }} to {{ _es_pid_after.stdout }}. + The 'latest' mode should not trigger a restart when already + at the latest version. + success_msg: "ES PID unchanged ({{ _es_pid_after.stdout }}) — no unnecessary restart" diff --git a/molecule/elasticsearch_upgrade_8to9_single/converge.yml b/molecule/elasticsearch_upgrade_8to9_single/converge.yml index c0f2784..9eba0fe 100644 --- a/molecule/elasticsearch_upgrade_8to9_single/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9_single/converge.yml @@ -174,15 +174,19 @@ fail_msg: "Must be on 8.19.x before upgrading to 9.x. Current: {{ es_info.json.version.number }}" success_msg: "On 8.19.x ({{ es_info.json.version.number }}), ready for 9.x upgrade" -- name: Upgrade to Elasticsearch 9.x (via role) +- name: Upgrade to Elasticsearch 9.x via release change (rolling upgrade) hosts: all vars: elasticstack_full_stack: false + # Only change the release — don't pin elasticstack_version. + # This exercises the real-world upgrade path. elasticstack_release: 9 elasticsearch_heap: "1" elasticstack_no_log: false elasticsearch_seed_hosts: [] elasticsearch_initial_master_nodes: [] + # Skip the countdown pause in CI + elasticsearch_upgrade_countdown: 0 tasks: - name: Include Elastic repos role (9.x) ansible.builtin.include_role: @@ -192,38 +196,6 @@ ansible.builtin.set_fact: _elasticstack_role_imported: false - - name: Gather package facts for version comparison - ansible.builtin.package_facts: - manager: auto - - - name: Detect latest available Elasticsearch 9.x version (Debian) - ansible.builtin.shell: > - set -o pipefail && - apt-cache policy elasticsearch | grep 'Candidate:' | awk '{print $2}' - args: - executable: /bin/bash - register: _es_candidate_version_deb - changed_when: false - when: ansible_facts.os_family == "Debian" - - - name: Detect latest available Elasticsearch 9.x version (RHEL) - ansible.builtin.shell: > - set -o pipefail && - dnf info --available elasticsearch 2>/dev/null | grep '^Version' | awk '{print $3}' | head -1 - args: - executable: /bin/bash - register: _es_candidate_version_rhel - changed_when: false - when: ansible_facts.os_family == "RedHat" - - - name: Set target upgrade version - ansible.builtin.set_fact: - elasticstack_version: "{{ _es_candidate_version_deb.stdout | default(_es_candidate_version_rhel.stdout | default('')) }}" - - - name: Display upgrade target - ansible.builtin.debug: - msg: "Upgrading from {{ ansible_facts.packages['elasticsearch'][0].version }} to {{ elasticstack_version }}" - - name: Include Elasticsearch role (upgrade to 9.x) ansible.builtin.include_role: name: oddly.elasticstack.elasticsearch diff --git a/molecule/logstash_elasticsearch/molecule.yml b/molecule/logstash_elasticsearch/molecule.yml index 4e6f9a1..75797af 100644 --- a/molecule/logstash_elasticsearch/molecule.yml +++ b/molecule/logstash_elasticsearch/molecule.yml @@ -16,7 +16,7 @@ platforms: groups: - logstash distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 2048 + memory_mb: 3072 provisioner: name: ansible env: diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index fe7690c..fa4fba6 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -28,6 +28,7 @@ - name: elasticsearch-rolling-upgrade | Update stopped Elasticsearch - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" + state: latest # noqa: package-latest enablerepo: - 'elastic-{{ elasticstack_release }}.x' when: @@ -37,6 +38,7 @@ - name: elasticsearch-rolling-upgrade | Update stopped Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" + state: latest # noqa: package-latest when: - ansible_facts.os_family == "Debian" or not elasticstack_full_stack | bool @@ -48,6 +50,7 @@ - name: elasticsearch-rolling-upgrade | Update single instances without extra caution - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" + state: latest # noqa: package-latest when: - ansible_facts.os_family == "Debian" or not elasticstack_full_stack | bool @@ -57,6 +60,7 @@ - name: elasticsearch-rolling-upgrade | Update single instances without extra caution - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" + state: latest # noqa: package-latest enablerepo: - 'elastic-{{ elasticstack_release }}.x' when: @@ -215,6 +219,7 @@ - name: elasticsearch-rolling-upgrade | Update Elasticsearch - rpm with managed repositories ansible.builtin.package: name: "{{ elasticsearch_package }}" + state: latest # noqa: package-latest enablerepo: - 'elastic-{{ elasticstack_release }}.x' when: @@ -224,6 +229,7 @@ - name: elasticsearch-rolling-upgrade | Update Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" + state: latest # noqa: package-latest when: - ansible_facts.os_family == "Debian" or not elasticstack_full_stack | bool diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index eabf7ac..0960d76 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -4,6 +4,28 @@ ansible.builtin.package_facts: manager: auto +# Detect whether a rolling upgrade is needed. Any version change — major, +# minor, or patch — should restart nodes one at a time with shard allocation +# management rather than restarting all nodes simultaneously. +# +# Pre-install detection covers cases where we know the target version: +# 1. Pinned version higher than installed (elasticstack_version: "9.2.0") +# 2. Major version change (elasticstack_release differs from installed) +# +# For "latest" mode, we can't know pre-install whether a newer version is +# available. The normal package tasks handle installation with state: latest, +# and if the package changed, a rolling restart is triggered post-install. +- name: Detect if rolling upgrade is needed (pre-install) + ansible.builtin.set_fact: + _elasticsearch_needs_rolling_upgrade: >- + {{ ansible_facts.packages['elasticsearch'] is defined and + ansible_facts.packages['elasticsearch'][0].version is defined and + ((elasticstack_version is defined and + elasticstack_version != 'latest' and + elasticstack_version is version(ansible_facts.packages['elasticsearch'][0].version, '>')) + or + (elasticstack_release | int != (ansible_facts.packages['elasticsearch'][0].version.split('.')[0] | int))) }} + - name: Check upgrade path requirement for ES 9.x ansible.builtin.fail: msg: | @@ -187,17 +209,25 @@ replace(' ', '') }} +- name: "WARNING: Rolling upgrade will be performed" + ansible.builtin.pause: + seconds: "{{ elasticsearch_upgrade_countdown | default(10) }}" + prompt: >- + Elasticsearch {{ ansible_facts.packages['elasticsearch'][0].version }} + will be upgraded to {{ elasticstack_version | default('latest ' ~ elasticstack_release ~ '.x') }}. + Nodes will be restarted one at a time with shard allocation management. + Press Ctrl+C to abort, or wait {{ elasticsearch_upgrade_countdown | default(10) }} seconds to continue. + when: + - _elasticsearch_needs_rolling_upgrade | bool + - ansible_facts.packages['elasticsearch'] is defined + run_once: true + # Write config files BEFORE the rolling upgrade so the upgrade restart picks # up new config in a single restart. Without this, the rolling upgrade restarts # with old config, then config tasks change files and notify the handler, # causing a second uncontrolled restart. - name: Write config before rolling upgrade - when: - - elasticstack_version is defined - - elasticstack_version != 'latest' - - ansible_facts.packages['elasticsearch'] is defined - - ansible_facts.packages['elasticsearch'][0].version is defined - - elasticstack_version is version(ansible_facts.packages['elasticsearch'][0].version, '>') + when: _elasticsearch_needs_rolling_upgrade | bool block: - name: Configure Elasticsearch (pre-upgrade) ansible.builtin.template: @@ -243,13 +273,14 @@ loop: "{{ groups[elasticstack_elasticsearch_group_name] }}" when: - "hostvars[item].inventory_hostname == inventory_hostname" - - elasticstack_version is defined - - elasticstack_version != 'latest' - - ansible_facts.packages['elasticsearch'] is defined - - ansible_facts.packages['elasticsearch'][0].version is defined + - _elasticsearch_needs_rolling_upgrade | bool - elasticstack_password.stdout is defined - - elasticstack_version is version( ansible_facts.packages['elasticsearch'][0].version, '>') +# NOTE: When elasticstack_version is "latest", state: latest means the package +# manager installs the newest available version. If a newer version is installed, +# the post-install detection below triggers a rolling restart (node-by-node with +# shard allocation management). This is intentional — even minor upgrades should +# not restart all nodes at once. - name: Install Elasticsearch - rpm - full stack ansible.builtin.package: name: "{{ elasticsearch_package }}" @@ -287,6 +318,27 @@ when: - ansible_facts.os_family == "Debian" +# If a normal package install (not the rolling upgrade path) changed the +# package, we need a rolling restart to avoid bringing all nodes down at +# once. This covers the "latest" case where we didn't know pre-install +# whether a newer version was available. +- name: Detect if package was upgraded (post-install) + ansible.builtin.set_fact: + _elasticsearch_package_upgraded: >- + {{ (_elasticsearch_install_rpm_full.changed | default(false)) or + (_elasticsearch_install_rpm.changed | default(false)) or + (_elasticsearch_install_deb.changed | default(false)) }} + when: not (_elasticsearch_needs_rolling_upgrade | bool) + +- name: Rolling restart after package upgrade + ansible.builtin.include_tasks: elasticsearch-rolling-upgrade.yml + loop: "{{ groups[elasticstack_elasticsearch_group_name] }}" + when: + - "hostvars[item].inventory_hostname == inventory_hostname" + - not (_elasticsearch_needs_rolling_upgrade | bool) + - _elasticsearch_package_upgraded | default(false) | bool + - elasticstack_password.stdout is defined + # Pre-detect external cert state so the template renders consistently # across first and subsequent runs (idempotency). - name: Detect existing external CA for template rendering