From dbabdffd7801279bf4053a98026fa496f9505f9f Mon Sep 17 00:00:00 2001 From: mrjf Date: Wed, 27 May 2026 22:36:38 -0700 Subject: [PATCH] ci: ensure migration benchmark runs --- .github/workflows/migration-ci.yml | 68 +++++++++++++++++++-------- scripts/ci/migration_cli_benchmark.py | 18 +++++-- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/.github/workflows/migration-ci.yml b/.github/workflows/migration-ci.yml index 37e305c8..9554c988 100644 --- a/.github/workflows/migration-ci.yml +++ b/.github/workflows/migration-ci.yml @@ -3,23 +3,12 @@ name: Migration Parity and Benchmarks on: pull_request: branches: [main] - paths: - - ".crane/**" - - ".github/workflows/migration-ci.yml" - - "cmd/**" - - "internal/**" - - "pkg/**" - - "go.mod" - - "go.sum" - - "pyproject.toml" - - "scripts/ci/**" - - "src/**" - - "tests/benchmarks/**" - - "tests/unit/test_crane_score.py" workflow_dispatch: permissions: contents: read + issues: write + pull-requests: write env: PYTHON_VERSION: "3.12" @@ -37,6 +26,8 @@ jobs: - name: Check changed paths id: filter + env: + HEAD_REF: ${{ github.event.pull_request.head.ref }} shell: bash run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then @@ -44,12 +35,17 @@ jobs: exit 0 fi + if [ "${{ github.event_name }}" = "pull_request" ] && [[ "$HEAD_REF" == crane/* ]]; then + echo "should-run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + git diff --name-only \ "${{ github.event.pull_request.base.sha }}" \ "${{ github.event.pull_request.head.sha }}" \ | tee "$RUNNER_TEMP/changed-files.txt" - if grep -Eq '^(\.crane/|cmd/|internal/|pkg/|go\.mod$|go\.sum$|pyproject\.toml$|src/|tests/benchmarks/|tests/unit/test_crane_score\.py$)' "$RUNNER_TEMP/changed-files.txt"; then + if grep -Eq '^(\.crane/|\.github/workflows/migration-ci\.yml$|cmd/|internal/|pkg/|go\.mod$|go\.sum$|pyproject\.toml$|scripts/ci/|src/|tests/benchmarks/|tests/unit/test_crane_score\.py$)' "$RUNNER_TEMP/changed-files.txt"; then echo "should-run=true" >> "$GITHUB_OUTPUT" else echo "should-run=false" >> "$GITHUB_OUTPUT" @@ -137,11 +133,6 @@ jobs: - name: Build Go CLI run: go build -o "$RUNNER_TEMP/apm-go" ./cmd/apm - - name: Run Python performance guards - run: | - uv run pytest tests/benchmarks/test_scaling_guards.py -v - uv run pytest tests/benchmarks -v --tb=short -m benchmark - - name: Run Python-vs-Go CLI benchmark run: | python scripts/ci/migration_cli_benchmark.py \ @@ -151,6 +142,9 @@ jobs: --markdown-out "$RUNNER_TEMP/migration-cli-benchmark.md" \ --max-ratio 5.0 + - name: Run Python scaling guards + run: uv run pytest tests/benchmarks/test_scaling_guards.py -v + - name: Add benchmark summary if: always() run: | @@ -158,6 +152,42 @@ jobs: cat "$RUNNER_TEMP/migration-cli-benchmark.md" >> "$GITHUB_STEP_SUMMARY" fi + - name: Post benchmark PR comment + if: always() && github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ ! -f "$RUNNER_TEMP/migration-cli-benchmark.md" ]; then + echo "No migration benchmark markdown found; skipping PR comment." + exit 0 + fi + + marker="" + { + echo "$marker" + echo "## Migration Benchmark Results" + echo + echo "- **Commit**: \`${HEAD_SHA}\`" + echo "- **Run**: ${RUN_URL}" + echo + cat "$RUNNER_TEMP/migration-cli-benchmark.md" + } > "$RUNNER_TEMP/migration-benchmark-pr-comment.md" + + comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq ".[] | select(.body | contains(\"${marker}\")) | .id" | tail -n 1) + + if [ -n "$comment_id" ]; then + gh api \ + --method PATCH \ + "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" \ + --field body@"$RUNNER_TEMP/migration-benchmark-pr-comment.md" + else + gh pr comment "$PR_NUMBER" --body-file "$RUNNER_TEMP/migration-benchmark-pr-comment.md" + fi + - name: Upload benchmark evidence if: always() uses: actions/upload-artifact@v4 diff --git a/scripts/ci/migration_cli_benchmark.py b/scripts/ci/migration_cli_benchmark.py index b8cf7cf3..e1fc970c 100644 --- a/scripts/ci/migration_cli_benchmark.py +++ b/scripts/ci/migration_cli_benchmark.py @@ -60,6 +60,7 @@ def _measure( label: str, env: dict[str, str], ) -> dict[str, object]: + base.mkdir(parents=True, exist_ok=True) samples: list[dict[str, object]] = [] for index in range(repeats): cwd = _workspace(base, label, index) if mutates_workspace else base @@ -75,22 +76,33 @@ def _measure( } +def _speed_label(ratio: float) -> str: + if ratio == 0: + return "n/a" + if ratio < 1: + return f"{1 / ratio:.2f}x faster" + if ratio > 1: + return f"{ratio:.2f}x slower" + return "same" + + def _markdown(results: list[dict[str, object]], max_ratio: float) -> str: lines = [ "## Migration CLI Benchmark", "", f"Max allowed Go/Python median ratio: `{max_ratio:.2f}`", "", - "| Command | Python median | Go median | Go/Python | Return codes |", - "|---|---:|---:|---:|---|", + "| Command | Python median | Go median | Go/Python | Result | Return codes |", + "|---|---:|---:|---:|---|---|", ] for row in results: lines.append( - "| {command} | {python:.4f}s | {go:.4f}s | {ratio:.2f}x | {codes} |".format( + "| {command} | {python:.4f}s | {go:.4f}s | {ratio:.2f}x | {result} | {codes} |".format( command=row["command"], python=row["python_median_seconds"], go=row["go_median_seconds"], ratio=row["ratio"], + result=_speed_label(float(row["ratio"])), codes=row["returncodes"], ) )