From e306e79d279b3de365e6ddc8138b58c1999ece96 Mon Sep 17 00:00:00 2001 From: Derek Misler Date: Tue, 12 May 2026 16:13:58 -0400 Subject: [PATCH 1/4] feat(e2e): add reviewer e2e testing infrastructure Signed-off-by: Derek Misler --- .github/workflows/self-review-pr.yml | 2 +- .github/workflows/test-e2e-reviewer.yml | 275 ++++++++++++++++++++ .github/workflows/test-e2e.yml | 323 ++++++++++++++++++++++++ 3 files changed, 599 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-e2e-reviewer.yml diff --git a/.github/workflows/self-review-pr.yml b/.github/workflows/self-review-pr.yml index 961621e..3724ad6 100644 --- a/.github/workflows/self-review-pr.yml +++ b/.github/workflows/self-review-pr.yml @@ -14,7 +14,7 @@ jobs: if: | github.event_name == 'issue_comment' || github.event.workflow_run.conclusion == 'success' - uses: docker/cagent-action/.github/workflows/review-pr.yml@f208610469d69f20983cad64c577949a132caa33 # v1.5.3 + uses: ./.github/workflows/review-pr.yml permissions: contents: read # Read repository files and PR diffs pull-requests: write # Post review comments diff --git a/.github/workflows/test-e2e-reviewer.yml b/.github/workflows/test-e2e-reviewer.yml new file mode 100644 index 0000000..647677b --- /dev/null +++ b/.github/workflows/test-e2e-reviewer.yml @@ -0,0 +1,275 @@ +name: Test Reviewer E2E (Manual) + +on: + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to run the scenario against' + required: true + default: '207' + scenario: + description: 'Scenario to test' + required: true + type: choice + options: + - full-review + - top-level-mention + - inline-mention + default: full-review + +permissions: + contents: read + +jobs: + full-review: + name: Full Review E2E + if: inputs.scenario == 'full-review' + uses: ./.github/workflows/review-pr.yml + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + checks: write + actions: read + with: + pr-number: ${{ inputs.pr_number }} + secrets: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + NEBIUS_API_KEY: ${{ secrets.NEBIUS_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + + top-level-mention: + name: Top-Level Mention E2E + if: inputs.scenario == 'top-level-mention' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Check if fork PR + id: fork-check + run: | + HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}" + if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then + echo "⏭️ Skipping - fork PR (secrets not available)" + echo "is_fork=true" >> $GITHUB_OUTPUT + else + echo "is_fork=false" >> $GITHUB_OUTPUT + fi + + - name: Checkout code + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + if: steps.fork-check.outputs.is_fork != 'true' + uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + with: + run_install: false + + - name: Setup Node.js + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + + - name: Build action + if: steps.fork-check.outputs.is_fork != 'true' + run: pnpm install --frozen-lockfile && pnpm build + + - name: Setup credentials + if: steps.fork-check.outputs.is_fork != 'true' + uses: docker/cagent-action/setup-credentials@2a43a3882401f45e3114df7f6d66eca184993a90 # v1.5.2 + + - name: Write synthetic issue_comment event + if: steps.fork-check.outputs.is_fork != 'true' + run: | + jq -n \ + --arg actor "${{ github.actor }}" \ + --argjson pr_number "${{ inputs.pr_number }}" \ + '{ + "action": "created", + "issue": { + "number": $pr_number, + "pull_request": { "url": ("https://api.github.com/repos/docker/cagent-action/pulls/" + ($pr_number | tostring)) } + }, + "comment": { + "id": 9999999901, + "body": "@docker-agent this is a manual e2e test — please reply with a brief acknowledgement.", + "user": { "login": $actor, "type": "User" } + }, + "repository": { + "owner": { "login": "docker" }, + "name": "cagent-action" + }, + "sender": { "login": $actor, "type": "User" } + }' > /tmp/test-event-toplevel.json + + - name: Run mention-reply handler + if: steps.fork-check.outputs.is_fork != 'true' + id: mention-handler + uses: ./.github/actions/mention-reply + env: + GITHUB_EVENT_PATH: /tmp/test-event-toplevel.json + GITHUB_EVENT_NAME: issue_comment + with: + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + org-membership-token: ${{ env.ORG_MEMBERSHIP_TOKEN || github.token }} + + - name: Run mention reply + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + id: run-reply + uses: ./review-pr/mention-reply + with: + mention-context: ${{ steps.mention-handler.outputs.prompt }} + owner: ${{ steps.mention-handler.outputs.owner }} + repo: ${{ steps.mention-handler.outputs.repo }} + pr-number: ${{ steps.mention-handler.outputs.pr-number }} + is-inline: ${{ steps.mention-handler.outputs.is-inline }} + in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }} + anthropic-api-key: ${{ env.ANTHROPIC_API_KEY_FROM_SSM || secrets.ANTHROPIC_API_KEY }} + openai-api-key: ${{ env.OPENAI_API_KEY_FROM_SSM || secrets.OPENAI_API_KEY }} + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + skip-auth: "true" + + - name: Report outcome + if: steps.fork-check.outputs.is_fork != 'true' + run: | + echo "should-reply: ${{ steps.mention-handler.outputs.should-reply }}" + echo "owner: ${{ steps.mention-handler.outputs.owner }}" + echo "repo: ${{ steps.mention-handler.outputs.repo }}" + echo "pr-number: ${{ steps.mention-handler.outputs.pr-number }}" + echo "is-inline: ${{ steps.mention-handler.outputs.is-inline }}" + echo "✅ Top-level mention scenario completed (manual run — no assertion)" + + inline-mention: + name: Inline Mention E2E + if: inputs.scenario == 'inline-mention' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Check if fork PR + id: fork-check + run: | + HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}" + if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then + echo "⏭️ Skipping - fork PR (secrets not available)" + echo "is_fork=true" >> $GITHUB_OUTPUT + else + echo "is_fork=false" >> $GITHUB_OUTPUT + fi + + - name: Checkout code + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + if: steps.fork-check.outputs.is_fork != 'true' + uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + with: + run_install: false + + - name: Setup Node.js + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + + - name: Build action + if: steps.fork-check.outputs.is_fork != 'true' + run: pnpm install --frozen-lockfile && pnpm build + + - name: Setup credentials + if: steps.fork-check.outputs.is_fork != 'true' + uses: docker/cagent-action/setup-credentials@2a43a3882401f45e3114df7f6d66eca184993a90 # v1.5.2 + + - name: Create anchor review comment + if: steps.fork-check.outputs.is_fork != 'true' + id: create-anchor + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + HEAD_SHA=$(gh api "repos/docker/cagent-action/pulls/$PR_NUMBER" --jq '.head.sha') + DIFF_FILE=$(gh api "repos/docker/cagent-action/pulls/$PR_NUMBER/files" --jq '.[0].filename') + echo "Using diff file: $DIFF_FILE" + COMMENT_ID=$(gh api "repos/docker/cagent-action/pulls/$PR_NUMBER/comments" \ + -X POST \ + --input - <<< $(jq -n \ + --arg sha "$HEAD_SHA" \ + --arg path "$DIFF_FILE" \ + '{"body": "manual e2e test anchor comment — safe to delete", "commit_id": $sha, "path": $path, "side": "RIGHT", "position": 1}') \ + --jq '.id') + echo "Created anchor comment ID: $COMMENT_ID" + echo "test_comment_id=$COMMENT_ID" >> $GITHUB_OUTPUT + + - name: Write synthetic pull_request_review_comment event + if: steps.fork-check.outputs.is_fork != 'true' + run: | + COMMENT_ID="${{ steps.create-anchor.outputs.test_comment_id }}" + jq -n \ + --arg actor "${{ github.actor }}" \ + --argjson comment_id "$COMMENT_ID" \ + --argjson pr_number "${{ inputs.pr_number }}" \ + '{ + "action": "created", + "pull_request": { "number": $pr_number }, + "comment": { + "id": $comment_id, + "body": "@docker-agent this is a manual e2e test of the inline mention path.", + "path": "README.md", + "line": 1, + "original_line": 1, + "diff_hunk": "@@ -1,1 +1,1 @@\n-old\n+new", + "user": { "login": $actor, "type": "User" } + }, + "repository": { + "owner": { "login": "docker" }, + "name": "cagent-action" + }, + "sender": { "login": $actor, "type": "User" } + }' > /tmp/test-event-inline.json + + - name: Run mention-reply handler + if: steps.fork-check.outputs.is_fork != 'true' + id: mention-handler + uses: ./.github/actions/mention-reply + env: + GITHUB_EVENT_PATH: /tmp/test-event-inline.json + GITHUB_EVENT_NAME: pull_request_review_comment + with: + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + org-membership-token: ${{ env.ORG_MEMBERSHIP_TOKEN || github.token }} + + - name: Run mention reply + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + id: run-reply + uses: ./review-pr/mention-reply + with: + mention-context: ${{ steps.mention-handler.outputs.prompt }} + owner: ${{ steps.mention-handler.outputs.owner }} + repo: ${{ steps.mention-handler.outputs.repo }} + pr-number: ${{ steps.mention-handler.outputs.pr-number }} + is-inline: ${{ steps.mention-handler.outputs.is-inline }} + in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }} + anthropic-api-key: ${{ env.ANTHROPIC_API_KEY_FROM_SSM || secrets.ANTHROPIC_API_KEY }} + openai-api-key: ${{ env.OPENAI_API_KEY_FROM_SSM || secrets.OPENAI_API_KEY }} + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + skip-auth: "true" + + - name: Report outcome + if: steps.fork-check.outputs.is_fork != 'true' + run: | + echo "should-reply: ${{ steps.mention-handler.outputs.should-reply }}" + echo "is-inline: ${{ steps.mention-handler.outputs.is-inline }}" + echo "in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }}" + echo "✅ Inline mention scenario completed (manual run — no assertion)" diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index a6e69a3..67f3259 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -184,3 +184,326 @@ jobs: else echo "✅ Invalid agent correctly failed (non-zero exit code)" fi + + test-mention-reply-toplevel: + name: Mention Reply (Top-Level) E2E Test + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Check if fork PR + id: fork-check + run: | + # Use default empty string to handle edge cases (deleted branches, malformed events) + HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}" + if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then + echo "⏭️ Skipping - fork PR (secrets not available)" + echo "is_fork=true" >> $GITHUB_OUTPUT + else + echo "is_fork=false" >> $GITHUB_OUTPUT + fi + + - name: Checkout code + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + if: steps.fork-check.outputs.is_fork != 'true' + uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + with: + run_install: false + + - name: Setup Node.js + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + + - name: Build action + if: steps.fork-check.outputs.is_fork != 'true' + run: pnpm install --frozen-lockfile && pnpm build + + - name: Setup credentials + if: steps.fork-check.outputs.is_fork != 'true' + uses: docker/cagent-action/setup-credentials@2a43a3882401f45e3114df7f6d66eca184993a90 # v1.5.2 + + - name: Write synthetic issue_comment event + if: steps.fork-check.outputs.is_fork != 'true' + run: | + jq -n \ + --arg actor "${{ github.actor }}" \ + '{ + "action": "created", + "issue": { + "number": 207, + "pull_request": { "url": "https://api.github.com/repos/docker/cagent-action/pulls/207" } + }, + "comment": { + "id": 9999999901, + "body": "@docker-agent this is an automated e2e test — please reply with a brief acknowledgement.", + "user": { "login": $actor, "type": "User" } + }, + "repository": { + "owner": { "login": "docker" }, + "name": "cagent-action" + }, + "sender": { "login": $actor, "type": "User" } + }' > /tmp/test-event-toplevel.json + + - name: Run mention-reply handler + if: steps.fork-check.outputs.is_fork != 'true' + id: mention-handler + uses: ./.github/actions/mention-reply + env: + GITHUB_EVENT_PATH: /tmp/test-event-toplevel.json + GITHUB_EVENT_NAME: issue_comment + with: + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + org-membership-token: ${{ env.ORG_MEMBERSHIP_TOKEN || github.token }} + + - name: Assert should-reply output + if: steps.fork-check.outputs.is_fork != 'true' + run: | + SHOULD_REPLY="${{ steps.mention-handler.outputs.should-reply }}" + echo "should-reply=$SHOULD_REPLY" + echo "owner=${{ steps.mention-handler.outputs.owner }}" + echo "repo=${{ steps.mention-handler.outputs.repo }}" + echo "pr-number=${{ steps.mention-handler.outputs.pr-number }}" + echo "is-inline=${{ steps.mention-handler.outputs.is-inline }}" + if [ "$SHOULD_REPLY" == 'false' ]; then + echo "⚠️ Warning: should-reply=false — ${{ github.actor }} may not be a docker org member (fork runners and external contributors are expected to see this). Skipping reply steps." + exit 0 + fi + + - name: Run mention reply + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + id: run-reply + uses: ./review-pr/mention-reply + with: + mention-context: ${{ steps.mention-handler.outputs.prompt }} + owner: ${{ steps.mention-handler.outputs.owner }} + repo: ${{ steps.mention-handler.outputs.repo }} + pr-number: ${{ steps.mention-handler.outputs.pr-number }} + is-inline: ${{ steps.mention-handler.outputs.is-inline }} + in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }} + anthropic-api-key: ${{ env.ANTHROPIC_API_KEY_FROM_SSM || secrets.ANTHROPIC_API_KEY }} + openai-api-key: ${{ env.OPENAI_API_KEY_FROM_SSM || secrets.OPENAI_API_KEY }} + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + skip-auth: "true" + + - name: Verify reply was posted + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + run: | + FOUND=$(gh api repos/docker/cagent-action/issues/207/comments \ + --jq '[.[] | select(.body | contains("")) | select(.created_at > (now - 300 | todate))] | length') + if [ "$FOUND" -eq 0 ]; then + echo "❌ No reply comment found within the last 5 minutes" + exit 1 + fi + echo "✅ Reply posted successfully ($FOUND comment(s) found)" + + - name: Cleanup test comments + if: always() && steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + run: | + # Delete any test reply comments posted in the last 5 minutes + gh api repos/docker/cagent-action/issues/207/comments \ + --jq '.[] | select(.body | contains("")) | select(.created_at > (now - 300 | todate)) | .id' | \ + while read -r comment_id; do + gh api "repos/docker/cagent-action/issues/comments/$comment_id" -X DELETE || true + echo "Deleted comment $comment_id" + done + + test-mention-reply-inline: + name: Mention Reply (Inline) E2E Test + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Check if fork PR + id: fork-check + run: | + HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}" + if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then + echo "⏭️ Skipping - fork PR (secrets not available)" + echo "is_fork=true" >> $GITHUB_OUTPUT + else + echo "is_fork=false" >> $GITHUB_OUTPUT + fi + + - name: Checkout code + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + if: steps.fork-check.outputs.is_fork != 'true' + uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 + with: + run_install: false + + - name: Setup Node.js + if: steps.fork-check.outputs.is_fork != 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + + - name: Build action + if: steps.fork-check.outputs.is_fork != 'true' + run: pnpm install --frozen-lockfile && pnpm build + + - name: Setup credentials + if: steps.fork-check.outputs.is_fork != 'true' + uses: docker/cagent-action/setup-credentials@2a43a3882401f45e3114df7f6d66eca184993a90 # v1.5.2 + + - name: Create anchor review comment on PR #207 + if: steps.fork-check.outputs.is_fork != 'true' + id: create-anchor + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + run: | + # Get the PR head SHA + HEAD_SHA=$(gh api repos/docker/cagent-action/pulls/207 --jq '.head.sha') + echo "PR head SHA: $HEAD_SHA" + + # Get first file in the diff to use as a safe anchor + DIFF_FILE=$(gh api repos/docker/cagent-action/pulls/207/files --jq '.[0].filename') + echo "Using diff file: $DIFF_FILE" + + # Post a test inline comment to get a real comment ID + COMMENT_ID=$(gh api repos/docker/cagent-action/pulls/207/comments \ + -X POST \ + --input - <<< $(jq -n \ + --arg sha "$HEAD_SHA" \ + --arg path "$DIFF_FILE" \ + '{"body": "e2e test anchor comment — safe to delete", "commit_id": $sha, "path": $path, "side": "RIGHT", "position": 1}') \ + --jq '.id') + echo "Created test anchor comment ID: $COMMENT_ID" + echo "test_comment_id=$COMMENT_ID" >> $GITHUB_OUTPUT + + - name: Write synthetic pull_request_review_comment event + if: steps.fork-check.outputs.is_fork != 'true' + run: | + COMMENT_ID="${{ steps.create-anchor.outputs.test_comment_id }}" + jq -n \ + --arg actor "${{ github.actor }}" \ + --argjson comment_id "$COMMENT_ID" \ + '{ + "action": "created", + "pull_request": { "number": 207 }, + "comment": { + "id": $comment_id, + "body": "@docker-agent this is an automated e2e test of the inline mention path.", + "path": "README.md", + "line": 1, + "original_line": 1, + "diff_hunk": "@@ -1,1 +1,1 @@\n-old\n+new", + "user": { "login": $actor, "type": "User" } + }, + "repository": { + "owner": { "login": "docker" }, + "name": "cagent-action" + }, + "sender": { "login": $actor, "type": "User" } + }' > /tmp/test-event-inline.json + + - name: Run mention-reply handler + if: steps.fork-check.outputs.is_fork != 'true' + id: mention-handler + uses: ./.github/actions/mention-reply + env: + GITHUB_EVENT_PATH: /tmp/test-event-inline.json + GITHUB_EVENT_NAME: pull_request_review_comment + with: + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + org-membership-token: ${{ env.ORG_MEMBERSHIP_TOKEN || github.token }} + + - name: Assert should-reply output + if: steps.fork-check.outputs.is_fork != 'true' + run: | + SHOULD_REPLY="${{ steps.mention-handler.outputs.should-reply }}" + echo "should-reply=$SHOULD_REPLY" + echo "is-inline=${{ steps.mention-handler.outputs.is-inline }}" + echo "in-reply-to-id=${{ steps.mention-handler.outputs.in-reply-to-id }}" + if [ "$SHOULD_REPLY" == 'false' ]; then + echo "⚠️ Warning: should-reply=false — ${{ github.actor }} may not be a docker org member. Skipping reply steps." + exit 0 + fi + + - name: Assert inline outputs + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + run: | + IS_INLINE="${{ steps.mention-handler.outputs.is-inline }}" + IN_REPLY_TO_ID="${{ steps.mention-handler.outputs.in-reply-to-id }}" + EXPECTED_ID="${{ steps.create-anchor.outputs.test_comment_id }}" + if [ "$IS_INLINE" != 'true' ]; then + echo "❌ Expected is-inline=true, got: $IS_INLINE" + exit 1 + fi + echo "✅ is-inline=true" + if [ "$IN_REPLY_TO_ID" != "$EXPECTED_ID" ]; then + echo "❌ Expected in-reply-to-id=$EXPECTED_ID, got: $IN_REPLY_TO_ID" + exit 1 + fi + echo "✅ in-reply-to-id=$IN_REPLY_TO_ID (matches anchor comment)" + + - name: Run mention reply + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + id: run-reply + uses: ./review-pr/mention-reply + with: + mention-context: ${{ steps.mention-handler.outputs.prompt }} + owner: ${{ steps.mention-handler.outputs.owner }} + repo: ${{ steps.mention-handler.outputs.repo }} + pr-number: ${{ steps.mention-handler.outputs.pr-number }} + is-inline: ${{ steps.mention-handler.outputs.is-inline }} + in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }} + anthropic-api-key: ${{ env.ANTHROPIC_API_KEY_FROM_SSM || secrets.ANTHROPIC_API_KEY }} + openai-api-key: ${{ env.OPENAI_API_KEY_FROM_SSM || secrets.OPENAI_API_KEY }} + github-token: ${{ env.GITHUB_APP_TOKEN || github.token }} + skip-auth: "true" + + - name: Verify inline reply was posted in thread + if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true' + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }} + run: | + FOUND=$(gh api repos/docker/cagent-action/pulls/207/comments \ + --jq --argjson id "$ANCHOR_ID" \ + '[.[] | select(.in_reply_to_id == $id and (.body | contains(""))) ] | length') + if [ "$FOUND" -eq 0 ]; then + echo "❌ No inline reply found in thread $ANCHOR_ID" + exit 1 + fi + echo "✅ Inline reply posted successfully" + + - name: Cleanup anchor comment and thread replies + if: always() && steps.fork-check.outputs.is_fork != 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }} + run: | + if [ -z "$ANCHOR_ID" ]; then + echo "No anchor comment ID — nothing to clean up" + exit 0 + fi + # Delete any replies in the thread first + gh api repos/docker/cagent-action/pulls/207/comments \ + --jq --argjson id "$ANCHOR_ID" \ + '[.[] | select(.in_reply_to_id == $id)] | .[].id' | \ + while read -r reply_id; do + gh api "repos/docker/cagent-action/pulls/comments/$reply_id" -X DELETE || true + echo "Deleted reply comment $reply_id" + done + # Delete the anchor comment itself + gh api "repos/docker/cagent-action/pulls/comments/$ANCHOR_ID" -X DELETE || true + echo "Deleted anchor comment $ANCHOR_ID" From 4afb06d6c6629e272217ed4820978e5b2e03ee56 Mon Sep 17 00:00:00 2001 From: Derek Misler Date: Tue, 12 May 2026 16:50:26 -0400 Subject: [PATCH 2/4] fix: four review issues on feat/e2e-reviewer-testing Fix 1 & 2: Replace invalid 'gh api --jq --argjson' with pipe to jq - In test-mention-reply-inline 'Verify inline reply' step: pipe gh api output to 'jq --argjson' instead of using invalid '--jq --argjson' flag combo - Same fix in 'Cleanup anchor comment and thread replies' step Fix 3: Replace hardcoded PR #207 with dynamic TEST_PR_NUMBER - Add job-level 'if: github.event_name == pull_request' guard to both test-mention-reply-toplevel and test-mention-reply-inline jobs (prevents running on push-to-main where no PR number exists) - Add job-level 'env: TEST_PR_NUMBER: ${{ github.event.pull_request.number }}' to both jobs - Update all synthetic event JSON, gh api endpoint URLs, and assertions to use $TEST_PR_NUMBER - Remove hardcoded default '207' from test-e2e-reviewer.yml workflow_dispatch input Fix 4: Revert self-review-pr.yml local ref back to pinned SHA - Restore 'uses: docker/cagent-action/.github/workflows/review-pr.yml@f208610469d69f20983cad64c577949a132caa33 # v1.5.3' - Local './.github/workflows/review-pr.yml' ref is resolved from default branch (main), not the PR branch, so it doesn't achieve dogfooding --- .github/workflows/self-review-pr.yml | 2 +- .github/workflows/test-e2e-reviewer.yml | 1 - .github/workflows/test-e2e.yml | 34 +++++++++++++++---------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/self-review-pr.yml b/.github/workflows/self-review-pr.yml index 3724ad6..961621e 100644 --- a/.github/workflows/self-review-pr.yml +++ b/.github/workflows/self-review-pr.yml @@ -14,7 +14,7 @@ jobs: if: | github.event_name == 'issue_comment' || github.event.workflow_run.conclusion == 'success' - uses: ./.github/workflows/review-pr.yml + uses: docker/cagent-action/.github/workflows/review-pr.yml@f208610469d69f20983cad64c577949a132caa33 # v1.5.3 permissions: contents: read # Read repository files and PR diffs pull-requests: write # Post review comments diff --git a/.github/workflows/test-e2e-reviewer.yml b/.github/workflows/test-e2e-reviewer.yml index 647677b..10e577a 100644 --- a/.github/workflows/test-e2e-reviewer.yml +++ b/.github/workflows/test-e2e-reviewer.yml @@ -6,7 +6,6 @@ on: pr_number: description: 'PR number to run the scenario against' required: true - default: '207' scenario: description: 'Scenario to test' required: true diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 67f3259..569c817 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -188,9 +188,12 @@ jobs: test-mention-reply-toplevel: name: Mention Reply (Top-Level) E2E Test runs-on: ubuntu-latest + if: github.event_name == 'pull_request' permissions: contents: read id-token: write + env: + TEST_PR_NUMBER: ${{ github.event.pull_request.number }} steps: - name: Check if fork PR id: fork-check @@ -234,11 +237,12 @@ jobs: run: | jq -n \ --arg actor "${{ github.actor }}" \ + --argjson pr_number "$TEST_PR_NUMBER" \ '{ "action": "created", "issue": { - "number": 207, - "pull_request": { "url": "https://api.github.com/repos/docker/cagent-action/pulls/207" } + "number": $pr_number, + "pull_request": { "url": ("https://api.github.com/repos/docker/cagent-action/pulls/" + ($pr_number | tostring)) } }, "comment": { "id": 9999999901, @@ -298,7 +302,7 @@ jobs: env: GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} run: | - FOUND=$(gh api repos/docker/cagent-action/issues/207/comments \ + FOUND=$(gh api repos/docker/cagent-action/issues/$TEST_PR_NUMBER/comments \ --jq '[.[] | select(.body | contains("")) | select(.created_at > (now - 300 | todate))] | length') if [ "$FOUND" -eq 0 ]; then echo "❌ No reply comment found within the last 5 minutes" @@ -313,7 +317,7 @@ jobs: GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} run: | # Delete any test reply comments posted in the last 5 minutes - gh api repos/docker/cagent-action/issues/207/comments \ + gh api repos/docker/cagent-action/issues/$TEST_PR_NUMBER/comments \ --jq '.[] | select(.body | contains("")) | select(.created_at > (now - 300 | todate)) | .id' | \ while read -r comment_id; do gh api "repos/docker/cagent-action/issues/comments/$comment_id" -X DELETE || true @@ -323,9 +327,12 @@ jobs: test-mention-reply-inline: name: Mention Reply (Inline) E2E Test runs-on: ubuntu-latest + if: github.event_name == 'pull_request' permissions: contents: read id-token: write + env: + TEST_PR_NUMBER: ${{ github.event.pull_request.number }} steps: - name: Check if fork PR id: fork-check @@ -363,22 +370,22 @@ jobs: if: steps.fork-check.outputs.is_fork != 'true' uses: docker/cagent-action/setup-credentials@2a43a3882401f45e3114df7f6d66eca184993a90 # v1.5.2 - - name: Create anchor review comment on PR #207 + - name: Create anchor review comment on current PR if: steps.fork-check.outputs.is_fork != 'true' id: create-anchor env: GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} run: | # Get the PR head SHA - HEAD_SHA=$(gh api repos/docker/cagent-action/pulls/207 --jq '.head.sha') + HEAD_SHA=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER --jq '.head.sha') echo "PR head SHA: $HEAD_SHA" # Get first file in the diff to use as a safe anchor - DIFF_FILE=$(gh api repos/docker/cagent-action/pulls/207/files --jq '.[0].filename') + DIFF_FILE=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/files --jq '.[0].filename') echo "Using diff file: $DIFF_FILE" # Post a test inline comment to get a real comment ID - COMMENT_ID=$(gh api repos/docker/cagent-action/pulls/207/comments \ + COMMENT_ID=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/comments \ -X POST \ --input - <<< $(jq -n \ --arg sha "$HEAD_SHA" \ @@ -395,9 +402,10 @@ jobs: jq -n \ --arg actor "${{ github.actor }}" \ --argjson comment_id "$COMMENT_ID" \ + --argjson pr_number "$TEST_PR_NUMBER" \ '{ "action": "created", - "pull_request": { "number": 207 }, + "pull_request": { "number": $pr_number }, "comment": { "id": $comment_id, "body": "@docker-agent this is an automated e2e test of the inline mention path.", @@ -476,8 +484,8 @@ jobs: GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }} run: | - FOUND=$(gh api repos/docker/cagent-action/pulls/207/comments \ - --jq --argjson id "$ANCHOR_ID" \ + FOUND=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/comments \ + | jq --argjson id "$ANCHOR_ID" \ '[.[] | select(.in_reply_to_id == $id and (.body | contains(""))) ] | length') if [ "$FOUND" -eq 0 ]; then echo "❌ No inline reply found in thread $ANCHOR_ID" @@ -497,8 +505,8 @@ jobs: exit 0 fi # Delete any replies in the thread first - gh api repos/docker/cagent-action/pulls/207/comments \ - --jq --argjson id "$ANCHOR_ID" \ + gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/comments \ + | jq --argjson id "$ANCHOR_ID" \ '[.[] | select(.in_reply_to_id == $id)] | .[].id' | \ while read -r reply_id; do gh api "repos/docker/cagent-action/pulls/comments/$reply_id" -X DELETE || true From bc8b1a95cf130d00b9ba69fda85199af24464edb Mon Sep 17 00:00:00 2001 From: Derek Misler Date: Tue, 12 May 2026 16:51:54 -0400 Subject: [PATCH 3/4] revert: restore local path ref in self-review-pr.yml (undo Fix 4) --- .github/workflows/self-review-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-review-pr.yml b/.github/workflows/self-review-pr.yml index 961621e..3724ad6 100644 --- a/.github/workflows/self-review-pr.yml +++ b/.github/workflows/self-review-pr.yml @@ -14,7 +14,7 @@ jobs: if: | github.event_name == 'issue_comment' || github.event.workflow_run.conclusion == 'success' - uses: docker/cagent-action/.github/workflows/review-pr.yml@f208610469d69f20983cad64c577949a132caa33 # v1.5.3 + uses: ./.github/workflows/review-pr.yml permissions: contents: read # Read repository files and PR diffs pull-requests: write # Post review comments From 90a10c72da68d4d6c9434ef58082e4dbd6aa0248 Mon Sep 17 00:00:00 2001 From: Derek Misler Date: Tue, 12 May 2026 17:04:59 -0400 Subject: [PATCH 4/4] fix: add missing permissions and cleanup step to e2e jobs - Add `issues: write` to test-mention-reply-toplevel job (test-e2e.yml) - Add `pull-requests: write` to test-mention-reply-inline job (test-e2e.yml) - Add cleanup step to inline-mention job (test-e2e-reviewer.yml) to delete the anchor PR review comment and its thread replies on every manual run, preventing orphan comments from accumulating --- .github/workflows/test-e2e-reviewer.yml | 17 +++++++++++++++++ .github/workflows/test-e2e.yml | 2 ++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/test-e2e-reviewer.yml b/.github/workflows/test-e2e-reviewer.yml index 10e577a..8894151 100644 --- a/.github/workflows/test-e2e-reviewer.yml +++ b/.github/workflows/test-e2e-reviewer.yml @@ -272,3 +272,20 @@ jobs: echo "is-inline: ${{ steps.mention-handler.outputs.is-inline }}" echo "in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }}" echo "✅ Inline mention scenario completed (manual run — no assertion)" + + - name: Cleanup anchor and replies + if: always() && steps.fork-check.outputs.is_fork != 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }} + ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }} + run: | + if [ -z "$ANCHOR_ID" ]; then exit 0; fi + # Delete thread replies first + gh api "repos/docker/cagent-action/pulls/${{ inputs.pr_number }}/comments" \ + | jq --argjson id "$ANCHOR_ID" '[.[] | select(.in_reply_to_id == $id)] | .[].id' \ + | while read -r reply_id; do + gh api "repos/docker/cagent-action/pulls/comments/$reply_id" -X DELETE || true + done + # Delete anchor + gh api "repos/docker/cagent-action/pulls/comments/$ANCHOR_ID" -X DELETE || true diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 569c817..1b25776 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -192,6 +192,7 @@ jobs: permissions: contents: read id-token: write + issues: write env: TEST_PR_NUMBER: ${{ github.event.pull_request.number }} steps: @@ -331,6 +332,7 @@ jobs: permissions: contents: read id-token: write + pull-requests: write env: TEST_PR_NUMBER: ${{ github.event.pull_request.number }} steps: