Consolidate CI workflows and fix test gaps

.github/workflows/molecule.yml

@@ -0,0 +1,157 @@
+---
+name: Molecule Test
+on:
+  workflow_call:
+    inputs:
+      scenarios:
+        required: true
+        type: string
+        description: JSON array of scenario names
+      distros:
+        required: true
+        type: string
+        description: JSON array of distro identifiers
+      releases:
+        type: string
+        default: '["8","9"]'
+        description: JSON array of Elastic major versions
+      timeout:
+        type: number
+        default: 45
+      max-parallel:
+        type: number
+        default: 10
+      skip-idempotence:
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  molecule:
+    runs-on: self-hosted
+    timeout-minutes: ${{ inputs.timeout }}
+
+    env:
+      COLLECTION_NAMESPACE: oddly
+      COLLECTION_NAME: elasticstack
+      ANSIBLE_PIPELINING: 'true'
+      ANSIBLE_GATHERING: smart
+      ANSIBLE_ANY_ERRORS_FATAL: 'true'
+      ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false'
+      ANSIBLE_TRANSFER_METHOD: sftp
+      ANSIBLE_SSH_RETRIES: '5'
+      ANSIBLE_HOST_KEY_CHECKING: 'false'
+      ANSIBLE_INJECT_FACTS_AS_VARS: 'false'
+      ANSIBLE_TIMEOUT: '60'
+      ANSIBLE_FORKS: '10'
+      ANSIBLE_SSH_ARGS: >-
+        -o ControlMaster=auto
+        -o ControlPersist=60s
+        -o PreferredAuthentications=publickey
+
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ inputs.max-parallel }}
+      matrix:
+        distro: ${{ fromJSON(inputs.distros) }}
+        scenario: ${{ fromJSON(inputs.scenarios) }}
+        release: ${{ fromJSON(inputs.releases) }}
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+      - name: Install dependencies
+        run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt
+        env:
+          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
+
+      - name: Set up collections path
+        run: |
+          echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV"
+          echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV"
+          echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV"
+          echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV"
+          mkdir -p "$RUNNER_TEMP/ssh-cp"
+
+      - name: Install collection
+        run: |
+          mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE
+          cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
+          ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz
+
+      - name: Set up SSH key for molecule
+        run: |
+          echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519
+          chmod 600 ${{ runner.temp }}/molecule_id_ed25519
+          mkdir -p ~/.ssh
+          ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
+
+      - name: Converge
+        run: molecule converge -s ${{ matrix.scenario }}
+        env:
+          MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}
+          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+          ANSIBLE_DIFF_ALWAYS: 'true'
+          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
+          INCUS_HOST: ${{ secrets.INCUS_HOST }}
+          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
+          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
+
+      - name: Verify
+        run: molecule verify -s ${{ matrix.scenario }}
+        env:
+          MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}
+          INCUS_HOST: ${{ secrets.INCUS_HOST }}
+          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
+          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
+
+      - name: Idempotence check
+        if: ${{ !inputs.skip-idempotence }}
+        run: molecule idempotence -s ${{ matrix.scenario }}
+        env:
+          MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}
+          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+          ANSIBLE_DIFF_ALWAYS: 'true'
+          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
+          INCUS_HOST: ${{ secrets.INCUS_HOST }}
+          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
+          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
+
+      - name: Collect and upload diagnostics
+        if: failure()
+        uses: ./.github/actions/collect-diagnostics
+        with:
+          artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }}
+        env:
+          INCUS_HOST: ${{ secrets.INCUS_HOST }}
+          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
+
+      - name: Cleanup molecule
+        if: always()
+        run: molecule destroy -s ${{ matrix.scenario }}
+        continue-on-error: true
+        env:
+          MOLECULE_DISTRO: ${{ matrix.distro }}
+          ELASTIC_RELEASE: ${{ matrix.release }}
+          INCUS_HOST: ${{ secrets.INCUS_HOST }}
+          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
+          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
+
+      - name: Clean up SSH state
+        if: always()
+        run: |
+          # Kill orphaned SSH processes from this job
+          pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true
+          # Remove ControlMaster sockets and SSH key
+          rm -rf "${{ runner.temp }}/ssh-cp"
+          rm -f "${{ runner.temp }}/molecule_id_ed25519"

.github/workflows/test_elasticsearch_custom_certs.yml

@@ -15,11 +15,13 @@ on:
   pull_request:
     paths:
       - 'roles/elasticsearch/**'
-      - 'roles/elasticstack/tasks/certs/**'
-      - '.github/workflows/test_elasticsearch_custom_certs.yml'
+      - 'roles/elasticstack/**'
       - 'molecule/elasticsearch_custom_certs*/**'
+      - 'molecule/shared/**'
+      - '.github/workflows/test_elasticsearch_custom_certs.yml'
+      - '.github/workflows/molecule.yml'
   schedule:
-    - cron: "30 2 * * *"
+    - cron: "30 2 * * 2,4,6"  # Tue/Thu/Sat — alternate with ES role
 
 permissions:
   contents: read
@@ -37,125 +39,8 @@ jobs:
 
   molecule_elasticsearch_custom_certs:
     needs: lint_elasticsearch
-    runs-on: self-hosted
-    timeout-minutes: 45
-
-    env:
-      COLLECTION_NAMESPACE: oddly
-      COLLECTION_NAME: elasticstack
-      ANSIBLE_PIPELINING: 'true'
-      ANSIBLE_GATHERING: smart
-      ANSIBLE_ANY_ERRORS_FATAL: 'true'
-      ANSIBLE_DISPLAY_SKIPPED_HOSTS: 'false'
-      ANSIBLE_TRANSFER_METHOD: sftp
-      ANSIBLE_SSH_RETRIES: '5'
-      ANSIBLE_HOST_KEY_CHECKING: 'false'
-      ANSIBLE_INJECT_FACTS_AS_VARS: 'false'
-      ANSIBLE_TIMEOUT: '60'
-      ANSIBLE_FORKS: '10'
-      ANSIBLE_SSH_ARGS: >-
-        -o ControlMaster=auto
-        -o ControlPersist=60s
-        -o PreferredAuthentications=publickey
-
-    strategy:
-      fail-fast: false
-      max-parallel: 10
-      matrix:
-        distro: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && fromJSON('["rockylinux10","debian13"]') || fromJSON('["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]') }}
-        scenario:
-          - elasticsearch_custom_certs
-          - elasticsearch_custom_certs_minimal
-        release:
-          - 8
-          - 9
-
-    steps:
-      - name: Check out code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-
-      - name: Install dependencies
-        run: uv pip install --system --break-system-packages --python /usr/bin/python3 -r requirements-test.txt
-        env:
-          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
-
-      - name: Set up collections path
-        run: |
-          echo "ANSIBLE_COLLECTIONS_PATH=$RUNNER_TEMP/collections" >> "$GITHUB_ENV"
-          echo "MOLECULE_EPHEMERAL_DIRECTORY=$RUNNER_TEMP/molecule" >> "$GITHUB_ENV"
-          echo "ANSIBLE_SSH_CONTROL_PATH_DIR=$RUNNER_TEMP/ssh-cp" >> "$GITHUB_ENV"
-          echo "MOLECULE_RUN_SUFFIX=-${GITHUB_RUN_ID: -6}" >> "$GITHUB_ENV"
-          mkdir -p "$RUNNER_TEMP/ssh-cp"
-
-      - name: Install collection and dependencies
-        run: |
-          mkdir -p $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE
-          cp -a "$GITHUB_WORKSPACE" $ANSIBLE_COLLECTIONS_PATH/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
-          ansible-galaxy collection install http://${{ secrets.INCUS_HOST }}:8082/collections/community-general-12.3.0.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/community-crypto-3.1.1.tar.gz http://${{ secrets.INCUS_HOST }}:8082/collections/ansible-posix-2.1.0.tar.gz
-
-      - name: Set up SSH key for molecule
-        run: |
-          echo "${{ secrets.MOLECULE_SSH_PRIVATE_KEY }}" > ${{ runner.temp }}/molecule_id_ed25519
-          chmod 600 ${{ runner.temp }}/molecule_id_ed25519
-          mkdir -p ~/.ssh
-          ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
-
-      - name: Converge and verify
-        run: |
-          molecule converge -s ${{ matrix.scenario }}
-          molecule verify -s ${{ matrix.scenario }}
-        env:
-          MOLECULE_DISTRO: ${{ matrix.distro }}
-          ELASTIC_RELEASE: ${{ matrix.release }}
-          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
-          PY_COLORS: '1'
-          ANSIBLE_FORCE_COLOR: '1'
-          ANSIBLE_DIFF_ALWAYS: 'true'
-          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
-          INCUS_HOST: ${{ secrets.INCUS_HOST }}
-          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
-          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
-
-      - name: Idempotence check
-        timeout-minutes: 20
-        run: molecule idempotence -s ${{ matrix.scenario }}
-        env:
-          MOLECULE_DISTRO: ${{ matrix.distro }}
-          ELASTIC_RELEASE: ${{ matrix.release }}
-          ELASTICSTACK_REPO_BASE_URL: http://elastic-cache.chiark.dev:8080
-          PY_COLORS: '1'
-          ANSIBLE_FORCE_COLOR: '1'
-          ANSIBLE_DIFF_ALWAYS: 'true'
-          ANSIBLE_CALLBACKS_ENABLED: profile_tasks
-          INCUS_HOST: ${{ secrets.INCUS_HOST }}
-          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
-          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
-
-      - name: Collect and upload diagnostics
-        if: failure()
-        uses: ./.github/actions/collect-diagnostics
-        with:
-          artifact-name: diag-${{ matrix.scenario }}-${{ matrix.distro }}-r${{ matrix.release }}
-        env:
-          INCUS_HOST: ${{ secrets.INCUS_HOST }}
-          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
-
-      - name: Cleanup molecule
-        if: always()
-        run: molecule destroy -s ${{ matrix.scenario }}
-        continue-on-error: true
-        env:
-          MOLECULE_DISTRO: ${{ matrix.distro }}
-          ELASTIC_RELEASE: ${{ matrix.release }}
-          INCUS_HOST: ${{ secrets.INCUS_HOST }}
-          MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519
-          DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081
-
-      - name: Clean up SSH state
-        if: always()
-        run: |
-          # Kill orphaned SSH processes from this job
-          pkill -f "${{ runner.temp }}/molecule_id_ed25519" 2>/dev/null || true
-          # Remove ControlMaster sockets and SSH key
-          rm -rf "${{ runner.temp }}/ssh-cp"
-          rm -f "${{ runner.temp }}/molecule_id_ed25519"
+    uses: ./.github/workflows/molecule.yml
+    with:
+      scenarios: '["elasticsearch_custom_certs","elasticsearch_custom_certs_minimal"]'
+      distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","debian12","debian13"]' }}
+    secrets: inherit