diff --git a/.github/workflows/kubescape-cli-e2e-tests.yaml b/.github/workflows/kubescape-cli-e2e-tests.yaml index a5ed639..cc8cd9a 100644 --- a/.github/workflows/kubescape-cli-e2e-tests.yaml +++ b/.github/workflows/kubescape-cli-e2e-tests.yaml @@ -6,6 +6,11 @@ on: description: "Configuration file for Kind setup" required: false type: string + SYSTEM_TESTS_BRANCH: + required: false + default: master + type: string + description: "system tests branch" DOWNLOAD_ARTIFACT_PATH: description: "Download artifact path" required: true @@ -31,7 +36,6 @@ on: "scan_git_repository_and_submit_to_backend", "scan_with_exception_to_backend", "scan_customer_configuration", - "host_scanner", "scan_compliance_score" ]' jobs: @@ -78,77 +82,194 @@ jobs: input: ${{ inputs.BINARY_TESTS }} run-tests: - strategy: - fail-fast: false - matrix: - TEST: ${{ fromJson(needs.wf-preparation.outputs.TEST_NAMES) }} needs: [wf-preparation] - # Down here we have the previous if statement that contains the "is-secret-set" validation. - # if: ${{ (needs.wf-preparation.outputs.is-secret-set == 'true') && (always() && contains(needs.*.result, 'success') && !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled'))) }} if: ${{ (always() && contains(needs.*.result, 'success') && !(contains(needs.*.result, 'failure')) && !(contains(needs.*.result, 'cancelled'))) }} runs-on: ubuntu-latest # This cannot change + permissions: + actions: read + contents: read steps: - - name: Checkout systests repo - uses: actions/checkout@v4 - with: - repository: armosec/system-tests - path: . + - name: Set dispatch info + id: dispatch-info + run: | + # Correlation ID WITHOUT attempt - so re-runs can find the original run + CORRELATION_ID="${GITHUB_REPOSITORY##*/}-${{ github.run_id }}" + echo "correlation_id=${CORRELATION_ID}" >> "$GITHUB_OUTPUT" + echo "Correlation ID: ${CORRELATION_ID}, Attempt: ${{ github.run_attempt }}" - - uses: actions/setup-python@v5 + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 with: - python-version: "3.9" - cache: "pip" + app-id: ${{ secrets.E2E_DISPATCH_APP_ID }} + private-key: ${{ secrets.E2E_DISPATCH_APP_PRIVATE_KEY }} + owner: armosec + repositories: shared-workflows - - name: create env - run: ./create_env.sh + - name: Dispatch system tests to private repo + if: ${{ github.run_attempt == 1 }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + CORRELATION_ID: ${{ steps.dispatch-info.outputs.correlation_id }} + REQUIRED_TESTS: ${{ needs.wf-preparation.outputs.TEST_NAMES }} + run: | + ADDITIONAL_TESTS=$(python3 - <<'PY' + import json, os + raw = os.environ.get("REQUIRED_TESTS", "").strip() + if not raw: + print("") + else: + tests = json.loads(raw) + print(" ".join(tests)) + PY + ) + TESTS_GROUP="CUSTOM_ONLY" - - name: Generate uuid - id: uuid + echo "Dispatching E2E tests with correlation_id: ${CORRELATION_ID}" + echo "Using tests group: ${TESTS_GROUP}" + + gh api "repos/armosec/shared-workflows/dispatches" \ + -f event_type="e2e-test-trigger" \ + -f "client_payload[correlation_id]=${CORRELATION_ID}" \ + -f "client_payload[github_repository]=${GITHUB_REPOSITORY}" \ + -f "client_payload[environment]=production" \ + -f "client_payload[tests_groups]=${TESTS_GROUP}" \ + -f "client_payload[additional_tests]=${ADDITIONAL_TESTS}" \ + -f "client_payload[systests_branch]=${{ inputs.SYSTEM_TESTS_BRANCH }}" \ + -f "client_payload[ks_branch]=release" \ + -f "client_payload[source_artifact_repo]=${GITHUB_REPOSITORY}" \ + -f "client_payload[source_artifact_run_id]=${{ github.run_id }}" \ + -f "client_payload[source_artifact_name]=${{ inputs.DOWNLOAD_ARTIFACT_KEY_NAME }}" \ + -f "client_payload[source_artifact_path]=${{ inputs.DOWNLOAD_ARTIFACT_PATH }}" \ + -f "client_payload[use_artifacts_file]=${{ inputs.USE_ARTIFACTS_FILE }}" + + echo "Dispatch completed" + + - name: Find E2E workflow run + id: find-run + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + CORRELATION_ID: ${{ steps.dispatch-info.outputs.correlation_id }} run: | - echo "RANDOM_UUID=$(uuidgen)" >> $GITHUB_OUTPUT + for i in {1..15}; do + run_id=$(gh api "repos/armosec/shared-workflows/actions/runs?event=repository_dispatch&per_page=30" \ + --jq '.workflow_runs | map(select(.name | contains("'"$CORRELATION_ID"'"))) | first | .id // empty') - - name: Create k8s Kind Cluster - id: kind-cluster-install - uses: helm/kind-action@v1 - with: - cluster_name: ${{ steps.uuid.outputs.RANDOM_UUID }} - config: ${{ inputs.KIND_CONFIG_FILE }} + if [ -n "$run_id" ]; then + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" + gh api "repos/armosec/shared-workflows/actions/runs/${run_id}" --jq '"url=" + .html_url' >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "Attempt $i: waiting for run..." + sleep $((i < 5 ? 10 : 30)) + done + echo "::error::Could not find workflow run" + exit 1 - - uses: actions/download-artifact@v4 - id: download-artifact - with: - name: ${{ inputs.DOWNLOAD_ARTIFACT_KEY_NAME }} - path: ${{ inputs.DOWNLOAD_ARTIFACT_PATH }} + - name: Re-run failed jobs in private repo + id: rerun + if: ${{ github.run_attempt > 1 }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RUN_ID: ${{ steps.find-run.outputs.run_id }} + run: | + conclusion=$(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" --jq '.conclusion') + echo "Previous conclusion: $conclusion" + + if [ "$conclusion" = "success" ]; then + echo "Previous run passed. Nothing to re-run." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi - - name: run-tests-on-latest-release-of-kubescape + # Full rerun if cancelled, partial if failed + if [ "$conclusion" = "cancelled" ]; then + echo "Run was cancelled - triggering full re-run" + gh api --method POST "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/rerun" + else + echo "Re-running failed jobs only" + gh api --method POST "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/rerun-failed-jobs" + fi + + # Wait for status to flip from 'completed' + for i in {1..30}; do + [ "$(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" --jq '.status')" != "completed" ] && break + sleep 2 + done + + - name: Wait for E2E tests to complete + if: ${{ steps.rerun.outputs.skip != 'true' }} env: - CUSTOMER: ${{ secrets.CUSTOMER }} - USERNAME: ${{ secrets.USERNAME }} - PASSWORD: ${{ secrets.PASSWORD }} - CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }} - SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RUN_ID: ${{ steps.find-run.outputs.run_id }} + URL: ${{ steps.find-run.outputs.url }} run: | - echo "Test history:" - echo " ${{ matrix.TEST }} " >/tmp/testhistory - cat /tmp/testhistory - source systests_python_env/bin/activate - - python3 systest-cli.py \ - -t ${{ matrix.TEST }} \ - -b production \ - -c CyberArmorTests \ - --duration 3 \ - --logger DEBUG \ - --kwargs ks_branch=release \ - use_artifacts=${{steps.download-artifact.outputs.download-path}}/${{ inputs.USE_ARTIFACTS_FILE }} - - deactivate - - - name: Test Report - uses: mikepenz/action-junit-report@v5 - if: always() # always run even if the previous step fails + echo "Monitoring: ${URL}" + + for i in {1..60}; do # 60 iterations × 60s = 1 hour max + read status conclusion < <(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" \ + --jq '[.status, .conclusion // "null"] | @tsv') + + echo "Status: ${status} | Conclusion: ${conclusion}" + + if [ "$status" = "completed" ]; then + if [ "$conclusion" = "success" ]; then + echo "E2E tests passed!" + exit 0 + fi + + echo "::error::E2E tests failed: ${conclusion}" + echo "" + + # Get failed job IDs to a file first + gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/jobs" \ + --jq '.jobs[] | select(.conclusion == "failure") | [.id, .name, (.steps[] | select(.conclusion == "failure") | .name)] | @tsv' > /tmp/failed_jobs.txt + + # Process each failed job + while IFS=$'\t' read -r job_id job_name step_name; do + # Extract test name: "run-helm-e2e / ST (relevancy_python)" → "relevancy_python" + test_name=$(echo "$job_name" | sed 's/.*(\(.*\))/\1/') + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "${job_name}" + echo " Step: ${step_name}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + # Fetch logs to temp file + gh api "repos/armosec/shared-workflows/actions/jobs/${job_id}/logs" 2>/dev/null > /tmp/job_logs.txt + + # Show summary in console + grep -E "(ERROR|FAILURE)" /tmp/job_logs.txt | tail -10 + echo "" + + # Save to separate file per test + log_file="failed_${test_name}.txt" + echo "════════════════════════════════════════" > "$log_file" + echo "${job_name}" >> "$log_file" + echo " Step: ${step_name}" >> "$log_file" + echo "════════════════════════════════════════" >> "$log_file" + last_endgroup=$(grep -n "##\\[endgroup\\]" /tmp/job_logs.txt | tail -1 | cut -d: -f1) + if [ -n "$last_endgroup" ]; then + tail -n +$((last_endgroup + 1)) /tmp/job_logs.txt >> "$log_file" + else + tail -500 /tmp/job_logs.txt >> "$log_file" + fi + done < /tmp/failed_jobs.txt + + echo "View full logs: ${URL}" + exit 1 + fi + + sleep 60 + done + + echo "::error::Timeout waiting for tests" + exit 1 + + - name: Upload failed step logs + if: failure() + uses: actions/upload-artifact@v4 with: - report_paths: "**/results_xml_format/**.xml" - commit: ${{github.event.workflow_run.head_sha}} + name: failed-e2e-logs-attempt-${{ github.run_attempt }} + path: failed_*.txt + retention-days: 7