diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3208dc78..8504346b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,5 +4,6 @@ repos: rev: v25.4.0 hooks: - id: ansible-lint + pass_filenames: false additional_dependencies: - ansible diff --git a/docs/roles/elasticsearch.md b/docs/roles/elasticsearch.md index 4b63649b..59ee995a 100644 --- a/docs/roles/elasticsearch.md +++ b/docs/roles/elasticsearch.md @@ -226,6 +226,15 @@ elasticsearch_cluster_settings: ``` Any setting supported by the [cluster settings API](https://www.elastic.co/docs/reference/elasticsearch/rest-api/cluster/update-cluster-settings) can be used. Values can be strings, numbers, booleans, or nested objects — the YAML dict is serialized to JSON directly. + +### LogsDB + +```yaml +elasticsearch_logsdb: true # default: true for 9.x, false for 8.x +``` + +`elasticsearch_logsdb` enables the LogsDB index mode for `logs-*-*` data streams by setting `cluster.logsdb.enabled: true` as a persistent cluster setting. LogsDB uses synthetic `_source` reconstruction and optimized doc_values compression for up to 4x storage savings with under 5% indexing overhead. Fresh 9.x installs enable this by default, but 8.x-to-9.x upgrades do not — this variable ensures consistent behavior. Requires ES 8.17+ (LogsDB GA). Existing backing indices remain in standard mode until ILM deletes them; new backing indices pick up LogsDB on their next rollover. + ### Temperature Attribute ```yaml @@ -248,6 +257,7 @@ Any setting supported by the [cluster settings API](https://www.elastic.co/docs/ elasticsearch_security: true elasticsearch_http_security: true elasticsearch_bootstrap_pw: PleaseChangeMe +elasticsearch_elastic_password: "" elasticsearch_ssl_verification_mode: full elasticsearch_tls_key_passphrase: PleaseChangeMeIndividually elasticsearch_validate_api_certs: false @@ -256,6 +266,8 @@ elasticsearch_validate_api_certs: false !!! warning Both `elasticsearch_bootstrap_pw` and `elasticsearch_tls_key_passphrase` ship with placeholder defaults. Change them before deploying to any environment. The bootstrap password is only used once during initial security setup, but the TLS key passphrase protects every node's private key for the life of the cluster. Use Ansible Vault or a secrets manager. +`elasticsearch_elastic_password` sets a user-defined password for the `elastic` superuser. When set, the role changes the auto-generated password to this value after initial security setup and uses it for all subsequent API calls. Leave empty to keep using the auto-generated password from the `initial_passwords` file. The `initial_passwords` file is preserved for other built-in users (kibana_system, beats_system, etc.). + `elasticsearch_security` is the main security toggle. When enabled, the role generates a certificate authority, creates per-node TLS certificates, configures transport and HTTP encryption, initializes the `elastic` superuser password, and enables RBAC. Elasticsearch 8.x and later require security -- the role fails the play if you try to disable it on 8.x+. `elasticsearch_http_security` controls TLS on the HTTP interface (port 9200) independently of transport encryption. Only relevant when `elasticsearch_security` is also `true`. Disabling HTTP security while keeping transport encryption is unusual but sometimes done behind a TLS-terminating reverse proxy. diff --git a/examples/full-stack.yml b/examples/full-stack.yml index e3805653..f97129fc 100644 --- a/examples/full-stack.yml +++ b/examples/full-stack.yml @@ -45,6 +45,9 @@ # Elasticsearch bootstrap password (used for initial setup, built-in users) elasticsearch_bootstrap_pw: "CHANGE-ME-bootstrap-password" + # User-defined elastic superuser password (optional — replaces auto-generated) + # elasticsearch_elastic_password: "CHANGE-ME-elastic-password" + # TLS key passphrases per service elasticsearch_tls_key_passphrase: "CHANGE-ME-es-key-passphrase" # kibana_tls_key_passphrase: "CHANGE-ME-kibana-key-passphrase" diff --git a/molecule/elasticsearch_default/converge.yml b/molecule/elasticsearch_default/converge.yml index ee6dce45..aa0b1b48 100644 --- a/molecule/elasticsearch_default/converge.yml +++ b/molecule/elasticsearch_default/converge.yml @@ -9,6 +9,7 @@ elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int }}" elasticsearch_heap: "1" elasticstack_no_log: false + elasticsearch_elastic_password: "TestPassword123!" elasticsearch_cluster_settings: action.destructive_requires_name: "true" tasks: diff --git a/molecule/elasticsearch_default/verify.yml b/molecule/elasticsearch_default/verify.yml index 9cd684ee..8775a256 100644 --- a/molecule/elasticsearch_default/verify.yml +++ b/molecule/elasticsearch_default/verify.yml @@ -4,6 +4,8 @@ tasks: - name: Fetch elastic password ansible.builtin.include_tasks: ../shared/verify_fetch_password.yml + vars: + _verify_elastic_password: "TestPassword123!" - name: Check cluster health ansible.builtin.include_tasks: ../shared/verify_es_health.yml @@ -17,6 +19,30 @@ fail_msg: "Expected {{ groups['elasticsearch'] | length }} nodes, got {{ health.json.number_of_nodes }}" run_once: true # noqa: run-once[task] + - name: Verify auto-generated password no longer works # noqa: run-once[task] + block: + - name: Read auto-generated password from file # noqa: run-once[task] + ansible.builtin.shell: | + set -o pipefail + grep "PASSWORD elastic " /usr/share/elasticsearch/initial_passwords | + awk {' print $4 '} + args: + executable: /bin/bash + register: _old_pass + changed_when: false + run_once: true + + - name: Try API with old password (should fail) # noqa: run-once[task] + ansible.builtin.uri: + url: "https://localhost:9200/" + user: elastic + password: "{{ _old_pass.stdout }}" + force_basic_auth: true + validate_certs: false + status_code: 401 + run_once: true + when: _old_pass.stdout != "TestPassword123!" + - name: Read cluster settings # noqa: run-once[task] ansible.builtin.uri: url: "https://localhost:9200/_cluster/settings?flat_settings=true" @@ -36,3 +62,12 @@ elasticsearch_cluster_settings not applied. Got: {{ cluster_settings.json.persistent }} run_once: true + + - name: Verify LogsDB is enabled (9.x default) # noqa: run-once[task] + ansible.builtin.assert: + that: + - cluster_settings.json.persistent['cluster.logsdb.enabled'] == 'true' + fail_msg: >- + LogsDB not enabled. + Got: {{ cluster_settings.json.persistent }} + run_once: true diff --git a/molecule/shared/verify_fetch_password.yml b/molecule/shared/verify_fetch_password.yml index ba9e62b0..bbf6e0de 100644 --- a/molecule/shared/verify_fetch_password.yml +++ b/molecule/shared/verify_fetch_password.yml @@ -1,9 +1,20 @@ --- -# Fetch the elastic user password from the initial_passwords file. +# Fetch the elastic user password for verify tasks. +# If _verify_elastic_password is set, use it directly. +# Otherwise, read from the initial_passwords file. # Optional variables: +# _verify_elastic_password: user-defined elastic password (skips file read) # _verify_delegate_to: host to delegate to (default: omitted, runs locally) # _verify_run_once: whether to run once (default: true) -- name: Fetch Elastic password + +- name: Use user-defined elastic password + ansible.builtin.set_fact: + elastic_pass: + stdout: "{{ _verify_elastic_password }}" + when: _verify_elastic_password | default('') | length > 0 + run_once: "{{ _verify_run_once | default(true) }}" # noqa: run-once[task] + +- name: Fetch Elastic password from file ansible.builtin.shell: | set -o pipefail grep "PASSWORD elastic " /usr/share/elasticsearch/initial_passwords | @@ -14,3 +25,4 @@ changed_when: false run_once: "{{ _verify_run_once | default(true) }}" # noqa: run-once[task] delegate_to: "{{ _verify_delegate_to | default(omit) }}" + when: _verify_elastic_password | default('') | length == 0 diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index 67c67cfe..f27e39af 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -39,6 +39,13 @@ elasticsearch_logging_audit: true elasticsearch_security: true # @var elasticsearch_bootstrap_pw:description: Bootstrap password for the elastic superuser during initial cluster setup elasticsearch_bootstrap_pw: PleaseChangeMe +# @var elasticsearch_elastic_password:description: > +# User-defined password for the elastic superuser. When set, the role +# changes the auto-generated password after initial security setup and +# uses this value for all subsequent API calls. Leave empty to keep the +# auto-generated password from the initial_passwords file. +# @end +elasticsearch_elastic_password: "" # @var elasticsearch_http_security:description: Enable TLS on the Elasticsearch HTTP interface (port 9200) elasticsearch_http_security: true # @var elasticsearch_http_protocol:description: Protocol for Elasticsearch HTTP API. Automatically set to https when security is enabled @@ -84,13 +91,20 @@ elasticsearch_memory_lock: false # @var elasticsearch_systemd_override_type_exec:description: Override systemd service Type to exec. Enable for Docker-in-Docker environments where sd_notify is non-functional elasticsearch_systemd_override_type_exec: false +# @var elasticsearch_logsdb:description: > +# Enable LogsDB index mode for logs-*-* data streams. Uses synthetic _source +# and optimized compression for ~4x storage savings. Fresh 9.x installs enable +# this by default; 8.x→9.x upgrades do not. Set to true to match 9.x default +# behavior. Requires ES 8.17+ (LogsDB GA). Applied as a persistent cluster setting. +# @end +elasticsearch_logsdb: "{{ elasticstack_release | int >= 9 }}" + # @var elasticsearch_cluster_settings:description: > # Persistent cluster settings applied via PUT _cluster/settings after the # cluster is healthy. Accepts any setting the cluster settings API supports. # Only applied when non-empty. Runs once per play on the CA host. # Example: # elasticsearch_cluster_settings: -# cluster.logsdb.enabled: true # indices.recovery.max_bytes_per_sec: "100mb" elasticsearch_cluster_settings: {} diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index 222397fb..ddb8c0f2 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -747,13 +747,22 @@ retries: 30 delay: 10 - - name: Fetch Elastic password + - name: Use user-defined elastic password + ansible.builtin.set_fact: + elasticstack_password: + stdout: "{{ elasticsearch_elastic_password }}" + no_log: "{{ elasticstack_no_log }}" + when: elasticsearch_elastic_password | default('') | length > 0 + + - name: Fetch Elastic password from file ansible.builtin.include_tasks: file: "{{ role_path }}/../elasticstack/tasks/fetch_password.yml" vars: _password_user: elastic _password_fact: elasticstack_password - when: elasticsearch_passwords_file.stat.exists | bool + when: + - elasticsearch_elastic_password | default('') | length == 0 + - elasticsearch_passwords_file.stat.exists | bool - name: Check for API availability with elastic password ansible.builtin.uri: @@ -766,7 +775,8 @@ changed_when: false no_log: "{{ elasticstack_no_log }}" when: - - elasticsearch_passwords_file.stat.exists | bool + - elasticstack_password is defined + - elasticstack_password.stdout | default('') | length > 0 until: (elasticsearch_api_status.json | default({})).cluster_name is defined retries: 20 delay: 10 @@ -816,7 +826,8 @@ changed_when: false no_log: "{{ elasticstack_no_log }}" when: - - elasticsearch_passwords_file.stat.exists | bool + - elasticstack_password is defined + - elasticstack_password.stdout | default('') | length > 0 until: ((elasticsearch_cluster_status.json | default({})).status | default('')) in ['green', 'yellow'] retries: 20 delay: 10 @@ -900,6 +911,39 @@ mode: "0600" when: inventory_hostname == elasticstack_ca_host + - name: Change elastic password to user-defined value + when: + - inventory_hostname == elasticstack_ca_host + - elasticsearch_elastic_password | default('') | length > 0 + - elasticsearch_freshstart_security.changed | bool + block: + - name: Fetch auto-generated elastic password + ansible.builtin.include_tasks: + file: "{{ role_path }}/../elasticstack/tasks/fetch_password.yml" + vars: + _password_user: elastic + _password_fact: _elasticsearch_initial_password + + - name: Change elastic user password via API + ansible.builtin.uri: + url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/elastic/_password" + method: POST + body_format: json + body: + password: "{{ elasticsearch_elastic_password }}" + user: elastic + password: "{{ _elasticsearch_initial_password.stdout }}" + force_basic_auth: true + validate_certs: "{{ elasticsearch_validate_api_certs }}" + status_code: 200 + no_log: "{{ elasticstack_no_log }}" + + - name: Set elastic password fact to user-defined value + ansible.builtin.set_fact: + elasticstack_password: + stdout: "{{ elasticsearch_elastic_password }}" + no_log: "{{ elasticstack_no_log }}" + # Maybe make sure that Elasticsearch is using the right protocol http(s) to connect, even in newly setup clusters # -- Certificate expiry warnings -- diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 1628e20d..f0102647 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -539,16 +539,29 @@ - name: Show hint about passwords ansible.builtin.debug: - msg: "Remember, your temporary passwords can be found on {{ elasticstack_ca_host }} in {{ elasticstack_initial_passwords }}" + msg: >- + {{ 'Elastic password set to user-defined value. Other service passwords are in ' + ~ elasticstack_initial_passwords + if elasticsearch_elastic_password | default('') | length > 0 + else 'Remember, your temporary passwords can be found on ' + ~ elasticstack_ca_host ~ ' in ' ~ elasticstack_initial_passwords }} when: - elasticsearch_security | bool - inventory_hostname == elasticstack_ca_host # -- Persistent cluster settings via _cluster/settings API -- +- name: Build effective cluster settings + ansible.builtin.set_fact: + _es_effective_cluster_settings: >- + {{ (elasticsearch_logsdb | bool) + | ternary({'cluster.logsdb.enabled': 'true'}, {}) + | combine(elasticsearch_cluster_settings | default({})) }} + - name: Apply persistent cluster settings # noqa: run-once[task] when: - - elasticsearch_cluster_settings | default({}) | length > 0 + - _es_effective_cluster_settings | length > 0 + - elasticsearch_security | bool | ternary(elasticstack_password is defined and (elasticstack_password.stdout | default('') | length > 0), true) - not ansible_check_mode run_once: true delegate_to: "{{ elasticstack_ca_host | default(inventory_hostname) }}" @@ -572,7 +585,7 @@ _current: "{{ _es_current_cluster_settings.json.persistent }}" _needs_update: >- {% set ns = namespace(changed=false) %} - {% for key, value in elasticsearch_cluster_settings.items() %} + {% for key, value in _es_effective_cluster_settings.items() %} {% if _current.get(key) is none or _current[key] | string != value | string %} {% set ns.changed = true %} {% endif %} @@ -585,7 +598,7 @@ method: PUT body_format: json body: - persistent: "{{ elasticsearch_cluster_settings }}" + persistent: "{{ _es_effective_cluster_settings }}" user: "{{ 'elastic' if elasticsearch_security | bool else omit }}" password: "{{ elasticstack_password.stdout if elasticsearch_security | bool else omit }}" force_basic_auth: "{{ elasticsearch_security | bool }}" diff --git a/roles/elasticstack/tasks/elasticstack-passwords.yml b/roles/elasticstack/tasks/elasticstack-passwords.yml index 74fdc7a5..ea44b3b5 100644 --- a/roles/elasticstack/tasks/elasticstack-passwords.yml +++ b/roles/elasticstack/tasks/elasticstack-passwords.yml @@ -1,17 +1,28 @@ --- -- name: Check for passwords being set - ansible.builtin.stat: - path: "{{ elasticstack_initial_passwords }}" - delegate_to: "{{ elasticstack_ca_host }}" - register: elasticstack_passwords_file - when: groups[elasticstack_elasticsearch_group_name] is defined +- name: Use user-defined elastic password + ansible.builtin.set_fact: + elasticstack_password: + stdout: "{{ elasticsearch_elastic_password }}" + no_log: "{{ elasticstack_no_log }}" + when: + - elasticsearch_elastic_password | default('') | length > 0 -- name: Fetch Elastic password - ansible.builtin.include_tasks: fetch_password.yml - vars: - _password_user: elastic - _password_fact: elasticstack_password +- name: Fetch elastic password from file when: + - elasticsearch_elastic_password | default('') | length == 0 - groups[elasticstack_elasticsearch_group_name] is defined - - elasticstack_passwords_file.stat.exists | default(false) | bool + block: + - name: Check for passwords being set + ansible.builtin.stat: + path: "{{ elasticstack_initial_passwords }}" + delegate_to: "{{ elasticstack_ca_host }}" + register: elasticstack_passwords_file + + - name: Fetch Elastic password + ansible.builtin.include_tasks: fetch_password.yml + vars: + _password_user: elastic + _password_fact: elasticstack_password + when: + - elasticstack_passwords_file.stat.exists | default(false) | bool