diff --git a/docs/roles/elasticsearch.md b/docs/roles/elasticsearch.md index 6642f77..482c042 100644 --- a/docs/roles/elasticsearch.md +++ b/docs/roles/elasticsearch.md @@ -382,6 +382,24 @@ elasticsearch_logging_audit: true JSON logs use `ECSJsonLayout` with `dataset` fields (Elastic Common Schema). Deprecation logs add a `RateLimitingFilter` to prevent log flooding and a `HeaderWarningAppender` for HTTP response warnings. Indexing slow log logger name changed to `index.indexing.slowlog.index`. +### Custom Keystore Entries + +```yaml +elasticsearch_keystore_entries: + xpack.notification.slack.account.monitoring.secure_url: "https://hooks.slack.com/services/T00/B00/XXX" + xpack.notification.email.account.work.smtp.secure_password: "smtp-password" +``` + +`elasticsearch_keystore_entries` is a dictionary of custom entries to add to the Elasticsearch keystore. Each key-value pair is set using `elasticsearch-keystore add -f -x`, with the value passed via stdin so it never appears in process listings or Ansible logs. + +Use this for any sensitive Elasticsearch setting that belongs in the keystore rather than in `elasticsearch.yml` — Watcher notification credentials, repository passwords, LDAP bind passwords, custom plugin secrets, etc. + +The role manages a fixed set of keystore keys internally (SSL keystore/truststore passwords, bootstrap password). If you try to set any of these via `elasticsearch_keystore_entries`, the playbook fails immediately with an error listing exactly which keys are reserved and why. The reserved keys are: `bootstrap.password`, `autoconfiguration.password_hash`, and the six `xpack.security.*.ssl.*` password entries. Use the dedicated role variables for those instead. + +On each run, the role reads the current value of each custom entry and only writes it if the value has changed, so the keystore is not unnecessarily modified and Elasticsearch is only restarted when an entry actually changes. + +The keystore is kept in sync with the declared state: entries that were previously added via `elasticsearch_keystore_entries` but are no longer present in the dictionary are automatically removed. Role-managed keys and `keystore.seed` are never touched by this cleanup. + ### Extra Configuration ```yaml diff --git a/docs/roles/kibana.md b/docs/roles/kibana.md index 8da2f35..267167a 100644 --- a/docs/roles/kibana.md +++ b/docs/roles/kibana.md @@ -51,6 +51,7 @@ kibana_config_backup: false ```yaml # kibana_elasticsearch_hosts: (undefined by default) kibana_security: true +kibana_system_password: "" kibana_sniff_on_start: false kibana_sniff_on_connection_fault: false ``` @@ -66,6 +67,8 @@ kibana_sniff_on_connection_fault: false `kibana_security` enables authenticated, encrypted connections to Elasticsearch. When `true`, Kibana connects over HTTPS using the `kibana_system` user and the password from the Elasticsearch security setup. The CA certificate is deployed automatically from the ES CA host. +`kibana_system_password` lets you set a specific password for the `kibana_system` Elasticsearch user. When empty (the default), Kibana uses the auto-generated password from the initial security setup. When set, the role changes the password via the Elasticsearch `/_security/user/kibana_system/_password` API on every run and uses the new value for Kibana's connection to Elasticsearch. This is useful when you need a known password for external monitoring, when you rotate credentials on a schedule, or when multiple Kibana instances need a consistent password that isn't tied to the initial setup file. + `kibana_sniff_on_start` and `kibana_sniff_on_connection_fault` control Elasticsearch node discovery. When enabled, Kibana queries the ES cluster for the full list of nodes at startup or when a connection drops. These settings only apply to Elastic Stack versions prior to 9.x (Kibana 9.x removed sniffing support). ### TLS for the Kibana Web Interface diff --git a/molecule/es_kibana/converge.yml b/molecule/es_kibana/converge.yml index a6dbaa1..f4ccff3 100644 --- a/molecule/es_kibana/converge.yml +++ b/molecule/es_kibana/converge.yml @@ -8,6 +8,9 @@ elasticstack_full_stack: true elasticstack_no_log: false elasticsearch_heap: "1" + elasticsearch_keystore_entries: + xpack.notification.slack.account.test.secure_url: "https://hooks.example.com/test" + kibana_system_password: "molecule-kibana-test-pw" tasks: - name: Include Elastic repos diff --git a/molecule/es_kibana/verify.yml b/molecule/es_kibana/verify.yml index 13d3307..6b7087e 100644 --- a/molecule/es_kibana/verify.yml +++ b/molecule/es_kibana/verify.yml @@ -70,3 +70,56 @@ - kibana_status.json.status.overall.level == 'available' fail_msg: "Kibana status: {{ kibana_status.json.status.overall.level | default('unknown') }}" when: "'kibana' in group_names" + + # -- Custom keystore entries -- + - name: List Elasticsearch keystore entries + ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list + register: keystore_list + changed_when: false + when: "'elasticsearch' in group_names" + + - name: Verify custom keystore entry exists + ansible.builtin.assert: + that: + - "'xpack.notification.slack.account.test.secure_url' in keystore_list.stdout_lines" + fail_msg: "Custom keystore entry not found in: {{ keystore_list.stdout_lines }}" + when: "'elasticsearch' in group_names" + + - name: Verify custom keystore entry has correct value + ansible.builtin.command: > + /usr/share/elasticsearch/bin/elasticsearch-keystore + show xpack.notification.slack.account.test.secure_url + register: keystore_value + changed_when: false + when: "'elasticsearch' in group_names" + + - name: Assert custom keystore value + ansible.builtin.assert: + that: + - keystore_value.stdout == "https://hooks.example.com/test" + fail_msg: "Expected 'https://hooks.example.com/test', got '{{ keystore_value.stdout }}'" + when: "'elasticsearch' in group_names" + + # -- kibana_system_password -- + - name: Verify kibana_system can authenticate with custom password + ansible.builtin.uri: + url: "https://localhost:9200/_security/_authenticate" + method: GET + force_basic_auth: true + user: kibana_system + password: "molecule-kibana-test-pw" + validate_certs: false + status_code: 200 + register: kibana_auth + when: + - "'elasticsearch' in group_names" + - inventory_hostname == groups['elasticsearch'][0] + + - name: Assert kibana_system authentication succeeded + ansible.builtin.assert: + that: + - kibana_auth.json.username == 'kibana_system' + fail_msg: "kibana_system auth failed: {{ kibana_auth.json | default('no response') }}" + when: + - "'elasticsearch' in group_names" + - inventory_hostname == groups['elasticsearch'][0] diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index 8c871c8..67b9af6 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -171,6 +171,29 @@ elasticsearch_validate_api_certs: false elasticsearch_unsafe_upgrade_restart: false +# @var elasticsearch_keystore_entries:description: > +# Custom entries to add to the Elasticsearch keystore. Each key-value pair +# is added with `elasticsearch-keystore add -f -x`. Values are passed via +# stdin so they never appear in process listings or Ansible logs. +# +# The role manages these keystore keys internally — do NOT set them here: +# bootstrap.password, +# xpack.security.http.ssl.keystore.secure_password, +# xpack.security.http.ssl.truststore.secure_password, +# xpack.security.transport.ssl.keystore.secure_password, +# xpack.security.transport.ssl.truststore.secure_password, +# xpack.security.http.ssl.secure_key_passphrase, +# xpack.security.transport.ssl.secure_key_passphrase, +# autoconfiguration.password_hash +# The playbook will fail with an error if any of these keys appear in +# elasticsearch_keystore_entries. +# @var elasticsearch_keystore_entries:example: > +# elasticsearch_keystore_entries: +# xpack.notification.slack.account.monitoring.secure_url: "https://hooks.slack.com/services/T00/B00/XXX" +# xpack.notification.email.account.work.smtp.secure_password: "smtp-password" +# @end +elasticsearch_keystore_entries: {} + # @var elasticsearch_extra_config:description: > # Additional key-value pairs merged into elasticsearch.yml. Keys that # conflict with dedicated role variables (e.g. cluster.name) are diff --git a/roles/elasticsearch/tasks/elasticsearch-keystore.yml b/roles/elasticsearch/tasks/elasticsearch-keystore.yml index d9b242b..f625aed 100644 --- a/roles/elasticsearch/tasks/elasticsearch-keystore.yml +++ b/roles/elasticsearch/tasks/elasticsearch-keystore.yml @@ -306,6 +306,105 @@ notify: - Restart Elasticsearch +# ============================================================ +# Custom keystore entries (elasticsearch_keystore_entries) +# ============================================================ +# +# These tasks run AFTER the built-in keystore management above. +# The role owns a fixed set of keystore keys (SSL passwords, +# bootstrap password, etc.) — those are off-limits. User-provided +# entries in elasticsearch_keystore_entries must not overlap with +# them or the playbook fails with a clear error explaining why. +# +# Stale entries (present in the keystore but absent from +# elasticsearch_keystore_entries and not role-managed) are removed +# to keep the keystore in sync with declared state. + +- name: elasticsearch-keystore | Manage custom entries + when: elasticsearch_keystore_entries | length > 0 + vars: + _elasticsearch_reserved_keystore_keys: + - bootstrap.password + - autoconfiguration.password_hash + - keystore.seed + - xpack.security.http.ssl.keystore.secure_password + - xpack.security.http.ssl.truststore.secure_password + - xpack.security.transport.ssl.keystore.secure_password + - xpack.security.transport.ssl.truststore.secure_password + - xpack.security.http.ssl.secure_key_passphrase + - xpack.security.transport.ssl.secure_key_passphrase + block: + - name: elasticsearch-keystore | Validate custom entries do not conflict with role-managed keys + ansible.builtin.assert: + that: + - item.key not in _elasticsearch_reserved_keystore_keys + fail_msg: >- + Cannot set '{{ item.key }}' via elasticsearch_keystore_entries because + this key is managed automatically by the elasticsearch role. Remove it + from elasticsearch_keystore_entries. The role manages these keys + internally: {{ _elasticsearch_reserved_keystore_keys | join(', ') }} + loop: "{{ elasticsearch_keystore_entries | dict2items }}" + loop_control: + label: "{{ item.key }}" + + - name: elasticsearch-keystore | Get current value for custom entry '{{ item.key }}' + ansible.builtin.command: > + /usr/share/elasticsearch/bin/elasticsearch-keystore + show {{ item.key }} + loop: "{{ elasticsearch_keystore_entries | dict2items }}" + loop_control: + label: "{{ item.key }}" + register: _elasticsearch_custom_keystore_current + changed_when: false + no_log: true + # Only expect failure when the key doesn't exist yet. If the key IS + # listed in the keystore but show still fails, that's a real error + # (corrupt keystore, permission denied) — let it propagate. + failed_when: >- + _elasticsearch_custom_keystore_current.rc != 0 and + item.key in elasticsearch_keystore.stdout_lines + + - name: elasticsearch-keystore | Set custom entry '{{ item.item.key }}' + ansible.builtin.command: > + /usr/share/elasticsearch/bin/elasticsearch-keystore + add -f -x '{{ item.item.key }}' + args: + stdin: "{{ item.item.value }}" + loop: "{{ _elasticsearch_custom_keystore_current.results | default([]) }}" + loop_control: + label: "{{ item.item.key }}" + changed_when: true + no_log: true + when: item.rc != 0 or item.stdout != item.item.value | string + notify: + - Restart Elasticsearch + + # --- Remove stale custom keystore entries --- + # + # Re-list the keystore, then subtract: declared custom entries, + # all role-managed keys. Whatever remains is stale and gets removed. + + - name: elasticsearch-keystore | Refresh keystore listing + ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore list + changed_when: false + register: _elasticsearch_keystore_after_custom + + - name: elasticsearch-keystore | Remove stale custom entries + ansible.builtin.command: > + /usr/share/elasticsearch/bin/elasticsearch-keystore + remove {{ item }} + loop: >- + {{ _elasticsearch_keystore_after_custom.stdout_lines + | difference(elasticsearch_keystore_entries.keys() | list) + | difference(_elasticsearch_reserved_keystore_keys) }} + loop_control: + label: "{{ item }}" + changed_when: true + notify: + - Restart Elasticsearch + +# ============================================================ + - name: elasticsearch-keystore | Check keystore exists ansible.builtin.stat: path: /etc/elasticsearch/elasticsearch.keystore diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml index 548f6a8..9225504 100644 --- a/roles/kibana/defaults/main.yml +++ b/roles/kibana/defaults/main.yml @@ -21,6 +21,14 @@ kibana_config_backup: false # @var kibana_manage_yaml:description: Let the role manage kibana.yml. Set to false to manage it yourself kibana_manage_yaml: true +# @var kibana_system_password:description: > +# User-defined password for the kibana_system user. When set, the role +# changes the auto-generated password via the Elasticsearch security API +# and uses this value for Kibana's connection to Elasticsearch. Leave +# empty to keep the auto-generated password from initial_passwords. +# @end +kibana_system_password: "" + # @var kibana_security:description: Enable security features (connect to Elasticsearch over HTTPS with authentication) kibana_security: true # @var kibana_tls:description: Enable TLS on the Kibana web interface itself (serve Kibana over HTTPS) diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml index 9c38ca4..f050ca8 100644 --- a/roles/kibana/tasks/kibana-security.yml +++ b/roles/kibana/tasks/kibana-security.yml @@ -416,6 +416,48 @@ _password_user: kibana_system _password_fact: kibana_password +# -- Change kibana_system password if user defined one -- +# +# When kibana_system_password is set, the role changes the auto-generated +# password to the user-provided value via the Elasticsearch security API. +# This runs once on the CA host, authenticating as the elastic superuser. +# After the API call, kibana_password is updated so the rest of the role +# (kibana.yml template, keystore) uses the new password transparently. +- name: kibana-security | Change kibana_system password to user-defined value + when: + - kibana_system_password | default('') | length > 0 + - kibana_password.stdout | default('') != kibana_system_password + block: + - name: kibana-security | Fetch elastic password for API authentication + ansible.builtin.include_tasks: + file: "{{ role_path }}/../elasticstack/tasks/fetch_password.yml" + vars: + _password_user: elastic + _password_fact: _kibana_elastic_password + + - name: kibana-security | Change kibana_system password via Elasticsearch API + ansible.builtin.uri: + url: "{{ _kibana_es_protocol }}://{{ elasticsearch_api_host | default(elasticstack_ca_host) }}:{{ elasticstack_elasticsearch_http_port }}/_security/user/kibana_system/_password" + method: POST + body_format: json + body: + password: "{{ kibana_system_password }}" + user: elastic + password: "{{ elasticstack_password.stdout | default(_kibana_elastic_password.stdout) }}" + force_basic_auth: true + validate_certs: false + status_code: 200 + no_log: "{{ elasticstack_no_log }}" + vars: + _kibana_es_protocol: "{{ 'https' if kibana_security | bool else 'http' }}" + run_once: true + + - name: kibana-security | Update kibana_password fact to user-defined value # noqa: var-naming[no-role-prefix] + ansible.builtin.set_fact: + kibana_password: + stdout: "{{ kibana_system_password }}" + no_log: "{{ elasticstack_no_log }}" + # -- Distribute CA certificate to Kibana nodes -- - name: kibana-security | Distribute CA certificate to Kibana nodes ansible.builtin.include_tasks: