Skip to content

run system test from private repo #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bvolovat merged 1 commit into from
Feb 2, 2026
Merged

run system test from private repo #75

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 175 additions & 56 deletions .github/workflows/incluster-comp-pr-merged.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,76 +152,195 @@ jobs:
input: ${{ inputs.REQUIRED_TESTS }}

run-tests:
strategy:
fail-fast: false
matrix:
TEST: ${{ fromJson(needs.docker-build.outputs.TEST_NAMES) }}
needs: docker-build
if: ${{ needs.docker-build.outputs.TEST_NAMES != '' && toJson(fromJson(needs.docker-build.outputs.TEST_NAMES)) != '[]' }}
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
steps:
- name: Checkout systests repo
uses: actions/checkout@v4
with:
repository: armosec/system-tests
ref: ${{ inputs.SYSTEM_TESTS_BRANCH }}
path: .
- name: Set dispatch info
id: dispatch-info
run: |
# Correlation ID WITHOUT attempt - so re-runs can find the original run
CORRELATION_ID="${GITHUB_REPOSITORY##*/}-${{ github.run_id }}"
echo "correlation_id=${CORRELATION_ID}" >> "$GITHUB_OUTPUT"
echo "Correlation ID: ${CORRELATION_ID}, Attempt: ${{ github.run_attempt }}"

- uses: actions/setup-python@v5
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@v1
with:
python-version: "3.9"
cache: "pip"
app-id: ${{ secrets.E2E_DISPATCH_APP_ID }}
private-key: ${{ secrets.E2E_DISPATCH_APP_PRIVATE_KEY }}
owner: armosec
repositories: shared-workflows

- name: Dispatch system tests to private repo
if: ${{ github.run_attempt == 1 }}
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
CORRELATION_ID: ${{ steps.dispatch-info.outputs.correlation_id }}
REQUIRED_TESTS: ${{ needs.docker-build.outputs.TEST_NAMES }}
run: |
ADDITIONAL_TESTS=$(python3 - <<'PY'
import json, os
raw = os.environ.get("REQUIRED_TESTS", "").strip()
if not raw:
print("")
else:
tests = json.loads(raw)
print(" ".join(tests))
PY
)
TESTS_GROUP="OPERATOR_E2E"

echo "Dispatching E2E tests with correlation_id: ${CORRELATION_ID}"
echo "Using tests group: ${TESTS_GROUP}"

gh api "repos/armosec/shared-workflows/dispatches" \
-f event_type="e2e-test-trigger" \
-f "client_payload[correlation_id]=${CORRELATION_ID}" \
-f "client_payload[github_repository]=${GITHUB_REPOSITORY}" \
-f "client_payload[environment]=production" \
-f "client_payload[tests_groups]=${TESTS_GROUP}" \
-f "client_payload[additional_tests]=${ADDITIONAL_TESTS}" \
-f "client_payload[systests_branch]=${{ inputs.SYSTEM_TESTS_BRANCH }}" \
-f "client_payload[in_cluster_chart_branch]=${{ inputs.HELM_BRANCH }}" \
-f "client_payload[ks_branch]=release" \
-f "client_payload[charts_name]=kubescape-operator" \
-f "client_payload[charts_repo]=kubescape/helm-charts"

- name: create env
run: ./create_env.sh
echo "Dispatch completed"

- name: Generate uuid
id: uuid
- name: Find E2E workflow run
id: find-run
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
CORRELATION_ID: ${{ steps.dispatch-info.outputs.correlation_id }}
run: |
echo "RANDOM_UUID=systets-$(uuidgen)" >> $GITHUB_OUTPUT
for i in {1..15}; do
run_id=$(gh api "repos/armosec/shared-workflows/actions/runs?event=repository_dispatch&per_page=30" \
--jq '.workflow_runs | map(select(.name | contains("'"$CORRELATION_ID"'"))) | first | .id // empty')

- name: Create k8s Kind Cluster
id: kind-cluster-install
uses: helm/kind-action@v1
with:
cluster_name: ${{ steps.uuid.outputs.RANDOM_UUID }}
# kubectl_version: v1.23.12
if [ -n "$run_id" ]; then
echo "run_id=${run_id}" >> "$GITHUB_OUTPUT"
gh api "repos/armosec/shared-workflows/actions/runs/${run_id}" --jq '"url=" + .html_url' >> "$GITHUB_OUTPUT"
exit 0
fi
echo "Attempt $i: waiting for run..."
sleep $((i < 5 ? 10 : 30))
done
echo "::error::Could not find workflow run"
exit 1

- name: Re-run failed jobs in private repo
id: rerun
if: ${{ github.run_attempt > 1 }}
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
RUN_ID: ${{ steps.find-run.outputs.run_id }}
run: |
conclusion=$(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" --jq '.conclusion')
echo "Previous conclusion: $conclusion"

if [ "$conclusion" = "success" ]; then
echo "Previous run passed. Nothing to re-run."
echo "skip=true" >> "$GITHUB_OUTPUT"
exit 0
fi

# Full rerun if cancelled, partial if failed
if [ "$conclusion" = "cancelled" ]; then
echo "Run was cancelled - triggering full re-run"
gh api --method POST "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/rerun"
else
echo "Re-running failed jobs only"
gh api --method POST "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/rerun-failed-jobs"
fi

# Wait for status to flip from 'completed'
for i in {1..30}; do
[ "$(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" --jq '.status')" != "completed" ] && break
sleep 2
done

- name: run-tests
- name: Wait for E2E tests to complete
if: ${{ steps.rerun.outputs.skip != 'true' }}
env:
CUSTOMER: ${{ secrets.CUSTOMER }}
USERNAME: ${{ secrets.USERNAME }}
PASSWORD: ${{ secrets.PASSWORD }}
CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }}
SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
QUAY_REGISTRY_ACCESS_TOKEN: "${{ secrets.QUAY_REGISTRY_ACCESS_TOKEN }}"
AZURE_REGISTRY_ACCESS_TOKEN: "${{ secrets.AZURE_REGISTRY_ACCESS_TOKEN }}"
AWS_REGISTRY_SECRET_KEY: "${{ secrets.AWS_REGISTRY_SECRET_KEY }}"
GOOGLE_REGISTRY_KEY: "${{ secrets.GOOGLE_REGISTRY_KEY }}"
GH_TOKEN: ${{ steps.app-token.outputs.token }}
RUN_ID: ${{ steps.find-run.outputs.run_id }}
URL: ${{ steps.find-run.outputs.url }}
run: |
echo "Test history:"
echo " ${{ matrix.TEST }} " >/tmp/testhistory
cat /tmp/testhistory
source systests_python_env/bin/activate

python3 systest-cli.py \
-t ${{ matrix.TEST }} \
-b production \
-c CyberArmorTests \
--logger DEBUG \
--kwargs helm_branch=${{ inputs.HELM_BRANCH }} \
${{ inputs.COMPONENT_NAME }}-tag=${{ needs.docker-build.outputs.IMAGE_TAG_PRERELEASE }}

deactivate

- name: Test Report
uses: mikepenz/action-junit-report@v5
if: always() # always run even if the previous step fails
echo "Monitoring: ${URL}"

for i in {1..60}; do # 60 iterations × 60s = 1 hour max
read status conclusion < <(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" \
--jq '[.status, .conclusion // "null"] | @tsv')

echo "Status: ${status} | Conclusion: ${conclusion}"

if [ "$status" = "completed" ]; then
if [ "$conclusion" = "success" ]; then
echo "E2E tests passed!"
exit 0
fi

echo "::error::E2E tests failed: ${conclusion}"
echo ""

# Get failed job IDs to a file first
gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/jobs" \
--jq '.jobs[] | select(.conclusion == "failure") | [.id, .name, (.steps[] | select(.conclusion == "failure") | .name)] | @tsv' > /tmp/failed_jobs.txt

# Process each failed job
while IFS=$'\t' read -r job_id job_name step_name; do
# Extract test name: "run-helm-e2e / ST (relevancy_python)" → "relevancy_python"
test_name=$(echo "$job_name" | sed 's/.*(\(.*\))/\1/')

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "${job_name}"
echo " Step: ${step_name}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Fetch logs to temp file
gh api "repos/armosec/shared-workflows/actions/jobs/${job_id}/logs" 2>/dev/null > /tmp/job_logs.txt

# Show summary in console
grep -E "(ERROR|FAILURE)" /tmp/job_logs.txt | tail -10
echo ""

# Save to separate file per test
log_file="failed_${test_name}.txt"
echo "════════════════════════════════════════" > "$log_file"
echo "${job_name}" >> "$log_file"
echo " Step: ${step_name}" >> "$log_file"
echo "════════════════════════════════════════" >> "$log_file"
last_endgroup=$(grep -n "##\\[endgroup\\]" /tmp/job_logs.txt | tail -1 | cut -d: -f1)
if [ -n "$last_endgroup" ]; then
tail -n +$((last_endgroup + 1)) /tmp/job_logs.txt >> "$log_file"
else
tail -500 /tmp/job_logs.txt >> "$log_file"
fi
done < /tmp/failed_jobs.txt

echo "View full logs: ${URL}"
exit 1
fi

sleep 60
done

echo "::error::Timeout waiting for tests"
exit 1

- name: Upload failed step logs
if: failure()
uses: actions/upload-artifact@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
report_paths: "**/results_xml_format/**.xml"
name: failed-e2e-logs-attempt-${{ github.run_attempt }}
path: failed_*.txt
retention-days: 7

create-release-and-retag:
if: ${{ contains(github.event.pull_request.labels.*.name, 'release') && always() && contains(needs.*.result, 'success') && !(contains(needs.*.result, 'failure')) && !(contains (needs.*.result,'cancelled')) || inputs.FORCE }}
Expand Down
Loading