From 757306866a6d2f41d7944dbed013a6fb163aaeb5 Mon Sep 17 00:00:00 2001 From: henrrypg Date: Thu, 18 Jun 2026 11:12:18 -0500 Subject: [PATCH 1/4] chore: move comment trigger to separate file --- .../workflows/integration-test-trigger.yml | 55 ++++++++++++++++++ .github/workflows/integration-tests.yml | 58 ++++++------------- backend/test_settings.py | 2 +- 3 files changed, 75 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/integration-test-trigger.yml diff --git a/.github/workflows/integration-test-trigger.yml b/.github/workflows/integration-test-trigger.yml new file mode 100644 index 00000000..fa3b3da0 --- /dev/null +++ b/.github/workflows/integration-test-trigger.yml @@ -0,0 +1,55 @@ +name: Trigger Integration Tests + +on: + issue_comment: + types: + - created + +jobs: + trigger: + if: | + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/integration-test') && + ( + github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'OWNER' + ) + + runs-on: ubuntu-latest + + permissions: + actions: write + contents: read + pull-requests: read + + steps: + - name: Get PR details + id: pr + uses: actions/github-script@v9 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + + core.setOutput('head_sha', pr.data.head.sha); + core.setOutput('head_ref', pr.data.head.ref); + core.setOutput('head_repo', pr.data.head.repo.full_name); + + - name: Dispatch integration workflow + uses: actions/github-script@v9 + with: + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'integration-tests.yml', + ref: 'main', + inputs: { + sha: '${{ steps.pr.outputs.head_sha }}', + ref: '${{ steps.pr.outputs.head_ref }}', + repository: '${{ steps.pr.outputs.head_repo }}' + } + }); diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9a369d02..8973c4f4 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -6,28 +6,23 @@ on: - main workflow_dispatch: + inputs: + sha: + required: false + type: string - issue_comment: - types: - - created + ref: + required: false + type: string + + repository: + required: false + type: string jobs: integration: name: Live LLM Integration Tests - if: | - github.event_name == 'push' || - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'issue_comment' && - github.event.issue.pull_request && - startsWith(github.event.comment.body, '/integration-test') && - ( - github.event.comment.author_association == 'MEMBER' || - github.event.comment.author_association == 'OWNER' - ) - ) - runs-on: ubuntu-latest defaults: @@ -35,34 +30,19 @@ jobs: working-directory: ./backend steps: - - name: Get PR details - if: github.event_name == 'issue_comment' - id: pr - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const pr = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number - }); - - core.setOutput('head_ref', pr.data.head.ref); - core.setOutput('head_repo', pr.data.head.repo.full_name); - - - name: Checkout PR branch - if: github.event_name == 'issue_comment' - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 #v4.3.1 + - name: Checkout PR commit + if: inputs.sha != '' + uses: actions/checkout@v4 with: - repository: ${{ steps.pr.outputs.head_repo }} - ref: ${{ steps.pr.outputs.head_ref }} + repository: ${{ inputs.repository }} + ref: ${{ inputs.sha }} - name: Checkout repository - if: github.event_name != 'issue_comment' - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 #v4.3.1 + if: inputs.sha == '' + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@v5 with: python-version: "3.12" diff --git a/backend/test_settings.py b/backend/test_settings.py index b9789f57..dce0671d 100644 --- a/backend/test_settings.py +++ b/backend/test_settings.py @@ -114,7 +114,7 @@ def root(*args): }, "root": { "handlers": ["console"], - "level": "DEBUG", + "level": "INFO", }, } From bdcad9ec39cd0d22142dc3da669b6a070050aff4 Mon Sep 17 00:00:00 2001 From: henrrypg Date: Thu, 18 Jun 2026 11:39:28 -0500 Subject: [PATCH 2/4] chore: pin version sha --- .github/workflows/integration-test-trigger.yml | 4 ++-- .github/workflows/integration-tests.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-test-trigger.yml b/.github/workflows/integration-test-trigger.yml index fa3b3da0..c3adb5ec 100644 --- a/.github/workflows/integration-test-trigger.yml +++ b/.github/workflows/integration-test-trigger.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Get PR details id: pr - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const pr = await github.rest.pulls.get({ @@ -39,7 +39,7 @@ jobs: core.setOutput('head_repo', pr.data.head.repo.full_name); - name: Dispatch integration workflow - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | await github.rest.actions.createWorkflowDispatch({ diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8973c4f4..c38c1ddb 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -32,17 +32,17 @@ jobs: steps: - name: Checkout PR commit if: inputs.sha != '' - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 #v4.3.1 with: repository: ${{ inputs.repository }} ref: ${{ inputs.sha }} - name: Checkout repository if: inputs.sha == '' - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 #v4.3.1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" From 9b0acc07f33f50eb57e676fe44e9f1d96a6e0d70 Mon Sep 17 00:00:00 2001 From: henrrypg Date: Thu, 18 Jun 2026 12:01:15 -0500 Subject: [PATCH 3/4] chore: show check --- .github/workflows/integration-tests.yml | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c38c1ddb..dc7f832d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -25,11 +25,31 @@ jobs: runs-on: ubuntu-latest + permissions: + checks: write + contents: read + defaults: run: working-directory: ./backend steps: + - name: Create check run + if: inputs.sha != '' + id: check + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { data: check } = await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'Live LLM Integration Tests', + head_sha: '${{ inputs.sha }}', + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + }); + core.setOutput('check_id', check.id); + - name: Checkout PR commit if: inputs.sha != '' uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 #v4.3.1 @@ -52,7 +72,21 @@ jobs: pip install -e . - name: Run integration tests + id: tests env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: make test-integration + + - name: Complete check run + if: always() && inputs.sha != '' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + await github.rest.checks.update({ + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: ${{ steps.check.outputs.check_id }}, + status: 'completed', + conclusion: '${{ steps.tests.outcome }}' === 'success' ? 'success' : 'failure' + }); From e1e2f899534739bd5d0ee94501a61381e99f710a Mon Sep 17 00:00:00 2001 From: henrrypg Date: Thu, 18 Jun 2026 12:06:20 -0500 Subject: [PATCH 4/4] chore: show check --- .github/workflows/integration-tests.yml | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index dc7f832d..9585add1 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest permissions: - checks: write + statuses: write contents: read defaults: @@ -34,21 +34,19 @@ jobs: working-directory: ./backend steps: - - name: Create check run + - name: Set pending status if: inputs.sha != '' - id: check uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { data: check } = await github.rest.checks.create({ + await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - name: 'Live LLM Integration Tests', - head_sha: '${{ inputs.sha }}', - status: 'in_progress', - details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + sha: '${{ inputs.sha }}', + state: 'pending', + context: 'Live LLM Integration Tests', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` }); - core.setOutput('check_id', check.id); - name: Checkout PR commit if: inputs.sha != '' @@ -78,15 +76,16 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: make test-integration - - name: Complete check run + - name: Set final status if: always() && inputs.sha != '' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - await github.rest.checks.update({ + await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - check_run_id: ${{ steps.check.outputs.check_id }}, - status: 'completed', - conclusion: '${{ steps.tests.outcome }}' === 'success' ? 'success' : 'failure' + sha: '${{ inputs.sha }}', + state: '${{ steps.tests.outcome }}' === 'success' ? 'success' : 'failure', + context: 'Live LLM Integration Tests', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` });