conformance-weekly #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: conformance-weekly | |
| # Runs the MCP conformance suite weekly against the latest | |
| # @modelcontextprotocol/conformance release. The on:pull_request pipeline | |
| # pins to whatever version is available at PR time; this schedule catches | |
| # upstream releases that add scenarios between PRs. | |
| # | |
| # It also scores each run and publishes the client/server pass-rate as | |
| # shields.io endpoint JSON to the orphan `badges` branch (consumed by the | |
| # README); that branch is created on the first run. | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' # Mondays 06:00 UTC | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| issues: write | |
| jobs: | |
| server: | |
| name: conformance / server (latest) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: '22' | |
| - run: composer install --prefer-dist --no-progress --no-interaction | |
| - name: Start conformance server | |
| run: | | |
| mkdir -p tests/Conformance/sessions tests/Conformance/logs | |
| chmod -R 777 tests/Conformance/sessions tests/Conformance/logs | |
| docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d | |
| sleep 5 | |
| - name: Run conformance tests | |
| working-directory: ./tests/Conformance | |
| run: npx --yes @modelcontextprotocol/conformance@latest server --url http://localhost:8000/ --expected-failures conformance-baseline.yml --output-dir results | |
| - name: Generate score badge | |
| if: always() | |
| run: php tests/Conformance/score.php server | |
| - name: Show docker logs on failure | |
| if: failure() | |
| run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs | |
| - name: Upload conformance results | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-server-results | |
| path: | | |
| tests/Conformance/logs | |
| tests/Conformance/results | |
| - name: Upload score badge | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: server-badge | |
| path: tests/Conformance/server-conformance.json | |
| client: | |
| name: conformance / client (latest) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: '8.4' | |
| coverage: none | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: '22' | |
| - run: composer install --prefer-dist --no-progress --no-interaction | |
| - run: mkdir -p tests/Conformance/logs | |
| - name: Run conformance tests | |
| working-directory: ./tests/Conformance | |
| run: npx --yes @modelcontextprotocol/conformance@latest client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml --output-dir results | |
| - name: Generate score badge | |
| if: always() | |
| run: php tests/Conformance/score.php client | |
| - name: Upload conformance results | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-client-results | |
| path: | | |
| tests/Conformance/logs | |
| tests/Conformance/results | |
| - name: Upload score badge | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: client-badge | |
| path: tests/Conformance/client-conformance.json | |
| notify: | |
| name: Open issue on failure | |
| runs-on: ubuntu-latest | |
| needs: [server, client] | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GH_REPO: ${{ github.repository }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - name: File or comment tracking issue | |
| run: | | |
| existing=$(gh issue list --label conformance-weekly --state open --json number --jq '.[0].number // empty') | |
| if [ -n "$existing" ]; then | |
| gh issue comment "$existing" --body "New failure on $(date -u +%FT%TZ): $RUN_URL" | |
| else | |
| gh issue create \ | |
| --title '[conformance] Weekly conformance run failed' \ | |
| --label conformance-weekly \ | |
| --body "Weekly conformance against \`@modelcontextprotocol/conformance@latest\` failed. | |
| - Run: $RUN_URL | |
| - Triggered: $(date -u +%FT%TZ) | |
| Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline.yml\`." | |
| fi | |
| publish: | |
| name: Publish conformance badges | |
| runs-on: ubuntu-latest | |
| needs: [server, client] | |
| # Publish even when the suite regressed (the badge should reflect reality); | |
| # skip on forks, which cannot push the `badges` branch. | |
| if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: server-badge | |
| path: badges-in | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: client-badge | |
| path: badges-in | |
| - name: Publish to badges branch | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| if git ls-remote --exit-code --heads origin badges >/dev/null 2>&1; then | |
| git fetch origin badges | |
| git worktree add badges-wt badges | |
| else | |
| git worktree add --detach badges-wt | |
| git -C badges-wt checkout --orphan badges | |
| git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true | |
| fi | |
| cp badges-in/server-conformance.json badges-in/client-conformance.json badges-wt/ | |
| cd badges-wt | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "Conformance scores unchanged." | |
| else | |
| git commit -m "Update conformance score badges" | |
| git push origin badges | |
| fi |