diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml new file mode 100644 index 0000000..891c365 --- /dev/null +++ b/.github/workflows/molecule.yml @@ -0,0 +1,157 @@ +--- +name: Molecule Test +on: + workflow_call: + inputs: + scenarios: + required: true + type: string + description: JSON array of scenario names + distros: + required: true + type: string + description: JSON array of distro identifiers + releases: + type: string + default: '["8","9"]' + description: JSON array of Elastic major versions + timeout: + type: number + default: 45 + max-parallel: + type: number + default: 10 + skip-idempotence: + type: boolean + default: false + +permissions: + contents: read + +jobs: + molecule: + runs-on: self-hosted + timeout-minutes: ${{ inputs.timeout }} + + env: + COLLECTION_NAMESPACE: oddly + COLLECTION_NAME: elasticstack + ANSIBLE_PIPELINING: 'true' + ANSIBLE_GATHERING: smart + ANSIBLE_ANY_ERRORS_FATAL: 'true' + ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' + ANSIBLE_TRANSFER_METHOD: sftp + ANSIBLE_SSH_RETRIES: '5' + ANSIBLE_HOST_KEY_CHECKING: 'false' + ANSIBLE_INJECT_FACTS_AS_VARS: 'false' + ANSIBLE_TIMEOUT: '60' + ANSIBLE_FORKS: '10' + ANSIBLE_SSH_ARGS: >- + -o ControlMaster=auto + -o ControlPersist=60s + -o PreferredAuthentications=publickey + + strategy: + fail-fast: false + max-parallel: ${{ inputs.max-parallel }} + matrix: + distro: ${{ fromJSON(inputs.distros) }} + scenario: ${{ fromJSON(inputs.scenarios) }} + release: ${{ fromJSON(inputs.releases) }} + + steps: + - name: Check out code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Install dependencies + run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt + env: + SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt + + - name: Set up collections path + run: | + echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" + echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" + echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" + echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/ssh-cp" + + - name: Install collection + run: | + mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE + cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME + ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz + + - name: Set up SSH key for molecule + run: | + echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 + chmod 600 ${{ runner.temp }}/molecule_id_ed25519 + mkdir -p ~/.ssh + ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Converge + run: molecule converge -s ${{ matrix.scenario }} + env: + MOLECULE_DISTRO: ${{ matrix.distro }} + ELASTIC_RELEASE: ${{ matrix.release }} + ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + ANSIBLE_DIFF_ALWAYS: 'true' + ANSIBLE_CALLBACKS_ENABLED: profile_tasks + INCUS_HOST: ${{ secrets.INCUS_HOST }} + MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 + DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + + - name: Verify + run: molecule verify -s ${{ matrix.scenario }} + env: + MOLECULE_DISTRO: ${{ matrix.distro }} + ELASTIC_RELEASE: ${{ matrix.release }} + INCUS_HOST: ${{ secrets.INCUS_HOST }} + MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 + DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + + - name: Idempotence check + if: ${{ !inputs.skip-idempotence }} + run: molecule idempotence -s ${{ matrix.scenario }} + env: + MOLECULE_DISTRO: ${{ matrix.distro }} + ELASTIC_RELEASE: ${{ matrix.release }} + ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + ANSIBLE_DIFF_ALWAYS: 'true' + ANSIBLE_CALLBACKS_ENABLED: profile_tasks + INCUS_HOST: ${{ secrets.INCUS_HOST }} + MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 + DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + + - name: Collect and upload diagnostics + if: failure() + uses: ./.github/actions/collect-diagnostics + with: + artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} + env: + INCUS_HOST: ${{ secrets.INCUS_HOST }} + MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 + + - name: Cleanup molecule + if: always() + run: molecule destroy -s ${{ matrix.scenario }} + continue-on-error: true + env: + MOLECULE_DISTRO: ${{ matrix.distro }} + ELASTIC_RELEASE: ${{ matrix.release }} + INCUS_HOST: ${{ secrets.INCUS_HOST }} + MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 + DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + + - name: Clean up SSH state + if: always() + run: | + # Kill orphaned SSH processes from this job + pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true + # Remove ControlMaster sockets and SSH key + rm -rf "${{ runner.temp }}/ssh-cp" + rm -f "${{ runner.temp }}/molecule_id_ed25519" diff --git a/.github/workflows/test_elasticsearch_custom_certs.yml b/.github/workflows/test_elasticsearch_custom_certs.yml index d1d1baf..7dcc9e4 100644 --- a/.github/workflows/test_elasticsearch_custom_certs.yml +++ b/.github/workflows/test_elasticsearch_custom_certs.yml @@ -15,11 +15,13 @@ on: pull_request: paths: - 'roles/elasticsearch/**' - - 'roles/elasticstack/tasks/certs/**' - - '.github/workflows/test_elasticsearch_custom_certs.yml' + - 'roles/elasticstack/**' - 'molecule/elasticsearch_custom_certs*/**' + - 'molecule/shared/**' + - '.github/workflows/test_elasticsearch_custom_certs.yml' + - '.github/workflows/molecule.yml' schedule: - - cron: "30 2 * * *" + - cron: "30 2 * * 2,4,6" # Tue/Thu/Sat — alternate with ES role permissions: contents: read @@ -37,125 +39,8 @@ jobs: molecule_elasticsearch_custom_certs: needs: lint_elasticsearch - runs-on: self-hosted - timeout-minutes: 45 - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: - - elasticsearch_custom_certs - - elasticsearch_custom_certs_minimal - release: - - 8 - - 9 - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection and dependencies - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge and verify - run: | - molecule converge -s ${{ matrix.scenario }} - molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - timeout-minutes: 20 - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["elasticsearch_custom_certs","elasticsearch_custom_certs_minimal"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + secrets: inherit diff --git a/.github/workflows/test_elasticsearch_modules.yml b/.github/workflows/test_elasticsearch_modules.yml index a1febeb..4bf4984 100644 --- a/.github/workflows/test_elasticsearch_modules.yml +++ b/.github/workflows/test_elasticsearch_modules.yml @@ -15,8 +15,11 @@ on: pull_request: paths: - 'roles/elasticsearch/**' + - 'roles/elasticstack/**' + - 'molecule/elasticsearch_test_modules/**' + - 'molecule/shared/**' - '.github/workflows/test_elasticsearch_modules.yml' - - 'molecule/elasticsearch_test_modules/*' + - '.github/workflows/molecule.yml' schedule: - cron: "0 3 * * *" @@ -36,124 +39,9 @@ jobs: molecule_elasticsearch_modules: needs: lint_elasticsearch - runs-on: self-hosted - timeout-minutes: 20 - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: - - elasticsearch_test_modules - release: - - 8 - - 9 - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection and dependencies - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge and verify - run: | - molecule converge -s ${{ matrix.scenario }} - molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - timeout-minutes: 20 - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["elasticsearch_test_modules"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + timeout: 20 + secrets: inherit diff --git a/.github/workflows/test_elasticsearch_upgrade.yml b/.github/workflows/test_elasticsearch_upgrade.yml index 3366d7f..8d4679b 100644 --- a/.github/workflows/test_elasticsearch_upgrade.yml +++ b/.github/workflows/test_elasticsearch_upgrade.yml @@ -15,8 +15,10 @@ on: pull_request: paths: - 'roles/elasticsearch/**' - - '.github/workflows/test_elasticsearch_upgrade.yml' + - 'roles/elasticstack/**' - 'molecule/elasticsearch_upgrade*/**' + - 'molecule/shared/**' + - '.github/workflows/test_elasticsearch_upgrade.yml' schedule: - cron: "0 2 * * *" # Daily at 02:00 UTC diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml index e1f22ae..871157a 100644 --- a/.github/workflows/test_full_stack.yml +++ b/.github/workflows/test_full_stack.yml @@ -15,7 +15,7 @@ on: pull_request: merge_group: schedule: - - cron: "0 4 * * *" + - cron: "0 4 * * 2,4,6" # Tue/Thu/Sat — 48 jobs, alternate with ES/logstash permissions: contents: read diff --git a/.github/workflows/test_role_beats.yml b/.github/workflows/test_role_beats.yml index f1aeee6..c2ffa9f 100644 --- a/.github/workflows/test_role_beats.yml +++ b/.github/workflows/test_role_beats.yml @@ -13,16 +13,15 @@ on: - warning - debug pull_request: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' paths: - 'roles/beats/**' - - '.github/workflows/test_role_beats.yml' + - 'roles/elasticstack/**' - 'molecule/beats_**' + - 'molecule/shared/**' + - '.github/workflows/test_role_beats.yml' + - '.github/workflows/molecule.yml' schedule: - - cron: "30 5 * * *" + - cron: "30 5 * * 2,4,6" # Tue/Thu/Sat — 48 jobs, alternate with logstash permissions: contents: read @@ -40,127 +39,8 @@ jobs: molecule_beats: needs: lint_beats - runs-on: self-hosted - timeout-minutes: 45 - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: [beats_default, beats_peculiar, beats_advanced, beats_security] - release: [8, 9] - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge - run: molecule converge -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Verify - run: molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["beats_default","beats_peculiar","beats_advanced","beats_security"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + secrets: inherit diff --git a/.github/workflows/test_role_elasticsearch.yml b/.github/workflows/test_role_elasticsearch.yml index dded38c..7f4655e 100644 --- a/.github/workflows/test_role_elasticsearch.yml +++ b/.github/workflows/test_role_elasticsearch.yml @@ -13,16 +13,15 @@ on: - warning - debug pull_request: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' paths: - 'roles/elasticsearch/**' - - '.github/workflows/test_role_elasticsearch.yml' + - 'roles/elasticstack/**' - 'molecule/elasticsearch_**' + - 'molecule/shared/**' + - '.github/workflows/test_role_elasticsearch.yml' + - '.github/workflows/molecule.yml' schedule: - - cron: "0 2 * * *" + - cron: "0 2 * * 1,3,5" # Mon/Wed/Fri — 60 jobs, stagger with full_stack permissions: contents: read @@ -40,138 +39,19 @@ jobs: molecule_elasticsearch: needs: lint_elasticsearch - runs-on: self-hosted - timeout-minutes: 45 - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: - - elasticsearch_default - - elasticsearch_roles_calculation - - elasticsearch_custom - - elasticsearch_no-security - - elasticsearch_cert_content - release: - - 8 - - 9 - include: - - distro: debian13 - scenario: elasticsearch_diagnostics - release: 9 - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge - run: molecule converge -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Verify - run: molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["elasticsearch_default","elasticsearch_roles_calculation","elasticsearch_custom","elasticsearch_no-security","elasticsearch_cert_content"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + secrets: inherit - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + molecule_elasticsearch_diagnostics: + needs: lint_elasticsearch + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["elasticsearch_diagnostics"]' + distros: '["debian13"]' + releases: '["9"]' + timeout: 30 + skip-idempotence: true + secrets: inherit diff --git a/.github/workflows/test_role_kibana.yml b/.github/workflows/test_role_kibana.yml index bce3792..825b60b 100644 --- a/.github/workflows/test_role_kibana.yml +++ b/.github/workflows/test_role_kibana.yml @@ -13,14 +13,13 @@ on: - warning - debug pull_request: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' paths: - 'roles/kibana/**' - - '.github/workflows/test_role_kibana.yml' + - 'roles/elasticstack/**' - 'molecule/kibana_**' + - 'molecule/shared/**' + - '.github/workflows/test_role_kibana.yml' + - '.github/workflows/molecule.yml' schedule: - cron: "30 3 * * *" @@ -40,127 +39,8 @@ jobs: molecule_kibana: needs: lint_kibana - runs-on: self-hosted - timeout-minutes: 45 - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: [kibana_default, kibana_custom, kibana_custom_certs] - release: [8, 9] - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge - run: molecule converge -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Verify - run: molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["kibana_default","kibana_custom","kibana_custom_certs"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + secrets: inherit diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index 7b20793..7769728 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -13,16 +13,15 @@ on: - warning - debug pull_request: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' paths: - 'roles/logstash/**' - - '.github/workflows/test_role_logstash.yml' + - 'roles/elasticstack/**' - 'molecule/logstash_**' + - 'molecule/shared/**' + - '.github/workflows/test_role_logstash.yml' + - '.github/workflows/molecule.yml' schedule: - - cron: "0 5 * * *" + - cron: "0 5 * * 1,3,5" # Mon/Wed/Fri — 72 jobs, alternate with full_stack permissions: contents: read @@ -39,136 +38,9 @@ jobs: secrets: inherit molecule_logstash: - runs-on: self-hosted - timeout-minutes: 45 needs: lint_logstash - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: - - logstash_default - - logstash_ssl - - logstash_custom_pipeline - - logstash_advanced - - logstash_standalone_certs - - logstash_centralized_pipelines - release: - - 8 - - 9 - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge - run: molecule converge -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Verify - run: molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_standalone_certs","logstash_centralized_pipelines"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + secrets: inherit diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml index 9f3a6c3..6e44756 100644 --- a/.github/workflows/test_role_repos.yml +++ b/.github/workflows/test_role_repos.yml @@ -13,14 +13,13 @@ on: - warning - debug pull_request: - branches: - - 'feature/**' - - 'fix/**' - - '!doc/**' paths: - 'roles/repos/**' - - '.github/workflows/test_role_repos.yml' + - 'roles/elasticstack/**' - 'molecule/repos_**' + - 'molecule/shared/**' + - '.github/workflows/test_role_repos.yml' + - '.github/workflows/molecule.yml' schedule: - cron: "0 1 * * *" @@ -40,131 +39,9 @@ jobs: molecule_repos: needs: lint_repos - runs-on: self-hosted - timeout-minutes: 30 - - env: - COLLECTION_NAMESPACE: oddly - COLLECTION_NAME: elasticstack - ANSIBLE_PIPELINING: 'true' - ANSIBLE_GATHERING: smart - ANSIBLE_ANY_ERRORS_FATAL: 'true' - ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false' - ANSIBLE_TRANSFER_METHOD: sftp - ANSIBLE_SSH_RETRIES: '5' - ANSIBLE_HOST_KEY_CHECKING: 'false' - ANSIBLE_INJECT_FACTS_AS_VARS: 'false' - ANSIBLE_TIMEOUT: '60' - ANSIBLE_FORKS: '10' - ANSIBLE_SSH_ARGS: >- - -o ControlMaster=auto - -o ControlPersist=60s - -o PreferredAuthentications=publickey - - strategy: - fail-fast: false - max-parallel: 10 - matrix: - distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }} - scenario: - - repos_default - release: - - 8 - - 9 - - steps: - - name: Check out code - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Install dependencies - run: | - uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt - ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz - env: - SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt - - - name: Set up collections path - run: | - echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV" - echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV" - echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV" - echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/ssh-cp" - - - name: Install collection - run: | - mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE - cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME - - - name: Set up SSH key for molecule - run: | - echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519 - chmod 600 ${{ runner.temp }}/molecule_id_ed25519 - mkdir -p ~/.ssh - ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Converge - run: molecule converge -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Verify - run: molecule verify -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Idempotence check - run: molecule idempotence -s ${{ matrix.scenario }} - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080 - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - ANSIBLE_DIFF_ALWAYS: 'true' - ANSIBLE_CALLBACKS_ENABLED: profile_tasks - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Collect and upload diagnostics - if: failure() - uses: ./.github/actions/collect-diagnostics - with: - artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }} - env: - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - - - name: Cleanup molecule - if: always() - run: molecule destroy -s ${{ matrix.scenario }} - continue-on-error: true - env: - MOLECULE_DISTRO: ${{ matrix.distro }} - ELASTIC_RELEASE: ${{ matrix.release }} - INCUS_HOST: ${{ secrets.INCUS_HOST }} - MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 - DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 - - - name: Clean up SSH state - if: always() - run: | - # Kill orphaned SSH processes from this job - pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true - # Remove ControlMaster sockets and SSH key - rm -rf "${{ runner.temp }}/ssh-cp" - rm -f "${{ runner.temp }}/molecule_id_ed25519" + uses: ./.github/workflows/molecule.yml + with: + scenarios: '["repos_default"]' + distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }} + timeout: 30 + secrets: inherit diff --git a/molecule/elasticsearch_default/verify.yml b/molecule/elasticsearch_default/verify.yml index 8775a25..b457b95 100644 --- a/molecule/elasticsearch_default/verify.yml +++ b/molecule/elasticsearch_default/verify.yml @@ -1,6 +1,8 @@ --- - name: Verify Elasticsearch default cluster hosts: elasticsearch + vars: + elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int }}" tasks: - name: Fetch elastic password ansible.builtin.include_tasks: ../shared/verify_fetch_password.yml @@ -71,3 +73,4 @@ LogsDB not enabled. Got: {{ cluster_settings.json.persistent }} run_once: true + when: elasticstack_release | int >= 9 diff --git a/molecule/logstash_ssl/converge.yml b/molecule/logstash_ssl/converge.yml index 4c1227f..924345f 100644 --- a/molecule/logstash_ssl/converge.yml +++ b/molecule/logstash_ssl/converge.yml @@ -24,82 +24,8 @@ logstash_extra_outputs: | stdout { codec => rubydebug } tasks: - - name: Install openssl and cryptography for certificate generation - ansible.builtin.package: - name: - - openssl - - python3-cryptography - state: present - - - name: Create test certificate directory - ansible.builtin.file: - path: /tmp/test-certs - state: directory - mode: "0755" - - - name: Generate CA private key - community.crypto.openssl_privatekey: - path: /tmp/test-certs/ca.key - size: 2048 - - - name: Generate CA CSR - community.crypto.openssl_csr: - path: /tmp/test-certs/ca.csr - privatekey_path: /tmp/test-certs/ca.key - common_name: Test CA - organization_name: Test - basic_constraints: - - "CA:TRUE" - basic_constraints_critical: true - key_usage: - - keyCertSign - - cRLSign - key_usage_critical: true - - - name: Generate CA certificate - community.crypto.x509_certificate: - path: /tmp/test-certs/ca.crt - privatekey_path: /tmp/test-certs/ca.key - csr_path: /tmp/test-certs/ca.csr - provider: selfsigned - selfsigned_not_after: "+365d" - - - name: Generate server private key - community.crypto.openssl_privatekey: - path: /tmp/test-certs/server.key - size: 2048 - - - name: Generate server CSR - community.crypto.openssl_csr: - path: /tmp/test-certs/server.csr - privatekey_path: /tmp/test-certs/server.key - common_name: "{{ ansible_facts.hostname }}" - subject_alt_name: - - "DNS:{{ ansible_facts.hostname }}" - - "DNS:localhost" - - "IP:127.0.0.1" - - - name: Sign server certificate with CA - community.crypto.x509_certificate: - path: /tmp/test-certs/server.crt - csr_path: /tmp/test-certs/server.csr - ownca_path: /tmp/test-certs/ca.crt - ownca_privatekey_path: /tmp/test-certs/ca.key - provider: ownca - ownca_not_after: "+365d" - - - name: Create unencrypted PKCS8 key - ansible.builtin.command: >- - openssl pkcs8 -topk8 -nocrypt - -in /tmp/test-certs/server.key - -out /tmp/test-certs/server-pkcs8.key - args: - creates: /tmp/test-certs/server-pkcs8.key - - - name: Set key permissions - ansible.builtin.file: - path: /tmp/test-certs/server-pkcs8.key - mode: "0640" + - name: Generate test certificates + ansible.builtin.include_tasks: ../shared/generate_test_certs.yml - name: Include Elastic repos role ansible.builtin.include_role: diff --git a/molecule/logstash_standalone_certs/converge.yml b/molecule/logstash_standalone_certs/converge.yml index 72973c7..c00121d 100644 --- a/molecule/logstash_standalone_certs/converge.yml +++ b/molecule/logstash_standalone_certs/converge.yml @@ -26,82 +26,8 @@ logstash_extra_outputs: | stdout { codec => rubydebug } tasks: - - name: Install openssl and cryptography for certificate generation - ansible.builtin.package: - name: - - openssl - - python3-cryptography - state: present - - - name: Create test certificate directory - ansible.builtin.file: - path: /tmp/test-certs - state: directory - mode: "0755" - - - name: Generate CA private key - community.crypto.openssl_privatekey: - path: /tmp/test-certs/ca.key - size: 2048 - - - name: Generate CA CSR - community.crypto.openssl_csr: - path: /tmp/test-certs/ca.csr - privatekey_path: /tmp/test-certs/ca.key - common_name: Test CA - organization_name: Test - basic_constraints: - - "CA:TRUE" - basic_constraints_critical: true - key_usage: - - keyCertSign - - cRLSign - key_usage_critical: true - - - name: Generate CA certificate - community.crypto.x509_certificate: - path: /tmp/test-certs/ca.crt - privatekey_path: /tmp/test-certs/ca.key - csr_path: /tmp/test-certs/ca.csr - provider: selfsigned - selfsigned_not_after: "+365d" - - - name: Generate server private key - community.crypto.openssl_privatekey: - path: /tmp/test-certs/server.key - size: 2048 - - - name: Generate server CSR - community.crypto.openssl_csr: - path: /tmp/test-certs/server.csr - privatekey_path: /tmp/test-certs/server.key - common_name: "{{ ansible_facts.hostname }}" - subject_alt_name: - - "DNS:{{ ansible_facts.hostname }}" - - "DNS:localhost" - - "IP:127.0.0.1" - - - name: Sign server certificate with CA - community.crypto.x509_certificate: - path: /tmp/test-certs/server.crt - csr_path: /tmp/test-certs/server.csr - ownca_path: /tmp/test-certs/ca.crt - ownca_privatekey_path: /tmp/test-certs/ca.key - provider: ownca - ownca_not_after: "+365d" - - - name: Create unencrypted PKCS8 key - ansible.builtin.command: >- - openssl pkcs8 -topk8 -nocrypt - -in /tmp/test-certs/server.key - -out /tmp/test-certs/server-pkcs8.key - args: - creates: /tmp/test-certs/server-pkcs8.key - - - name: Set key permissions - ansible.builtin.file: - path: /tmp/test-certs/server-pkcs8.key - mode: "0640" + - name: Generate test certificates + ansible.builtin.include_tasks: ../shared/generate_test_certs.yml - name: Include Elastic repos role ansible.builtin.include_role: diff --git a/molecule/shared/generate_test_certs.yml b/molecule/shared/generate_test_certs.yml new file mode 100644 index 0000000..98cde05 --- /dev/null +++ b/molecule/shared/generate_test_certs.yml @@ -0,0 +1,79 @@ +--- +# Generate a self-signed CA and server certificate for testing TLS. +# Outputs to /tmp/test-certs/: ca.crt, server.crt, server-pkcs8.key +- name: Install openssl and cryptography for certificate generation + ansible.builtin.package: + name: + - openssl + - python3-cryptography + state: present + +- name: Create test certificate directory + ansible.builtin.file: + path: /tmp/test-certs + state: directory + mode: "0755" + +- name: Generate CA private key + community.crypto.openssl_privatekey: + path: /tmp/test-certs/ca.key + size: 2048 + +- name: Generate CA CSR + community.crypto.openssl_csr: + path: /tmp/test-certs/ca.csr + privatekey_path: /tmp/test-certs/ca.key + common_name: Test CA + organization_name: Test + basic_constraints: + - "CA:TRUE" + basic_constraints_critical: true + key_usage: + - keyCertSign + - cRLSign + key_usage_critical: true + +- name: Generate CA certificate + community.crypto.x509_certificate: + path: /tmp/test-certs/ca.crt + privatekey_path: /tmp/test-certs/ca.key + csr_path: /tmp/test-certs/ca.csr + provider: selfsigned + selfsigned_not_after: "+365d" + +- name: Generate server private key + community.crypto.openssl_privatekey: + path: /tmp/test-certs/server.key + size: 2048 + +- name: Generate server CSR + community.crypto.openssl_csr: + path: /tmp/test-certs/server.csr + privatekey_path: /tmp/test-certs/server.key + common_name: "{{ ansible_facts.hostname }}" + subject_alt_name: + - "DNS:{{ ansible_facts.hostname }}" + - "DNS:localhost" + - "IP:127.0.0.1" + +- name: Sign server certificate with CA + community.crypto.x509_certificate: + path: /tmp/test-certs/server.crt + csr_path: /tmp/test-certs/server.csr + ownca_path: /tmp/test-certs/ca.crt + ownca_privatekey_path: /tmp/test-certs/ca.key + provider: ownca + ownca_not_after: "+365d" + +- name: Create unencrypted PKCS8 key + ansible.builtin.command: >- + openssl pkcs8 -topk8 -nocrypt + -in /tmp/test-certs/server.key + -out /tmp/test-certs/server-pkcs8.key + args: + creates: /tmp/test-certs/server-pkcs8.key + +- name: Set key permissions + ansible.builtin.file: + path: /tmp/test-certs/server-pkcs8.key + mode: "0640" diff --git a/molecule/shared/prepare_common.yml b/molecule/shared/prepare_common.yml index 13a6b0a..682c369 100644 --- a/molecule/shared/prepare_common.yml +++ b/molecule/shared/prepare_common.yml @@ -99,3 +99,5 @@ - openssl - unzip - systemd + - python3-cryptography + - python3-packaging