From 1f8f8059220d39c04406a295cce7275985f0fae3 Mon Sep 17 00:00:00 2001 From: Anubhav Dhawan Date: Mon, 20 Apr 2026 21:20:38 +0530 Subject: [PATCH 1/4] ci(core): Adds GitHub Action workflow, test harness, and baseline config --- .github/workflows/conformance.yml | 81 +++++++++++++++++++ .../toolbox-core/conformance-baseline.yml | 32 ++++++++ .../toolbox-core/tests/conformance/client.py | 80 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 .github/workflows/conformance.yml create mode 100644 packages/toolbox-core/conformance-baseline.yml create mode 100644 packages/toolbox-core/tests/conformance/client.py diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..f97f49fd7 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,81 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: conformance-core +on: + pull_request: + paths: + - 'packages/toolbox-core/**' + - '!packages/toolbox-core/**/*.md' + pull_request_target: + types: [labeled] + +permissions: read-all + +jobs: + conformance: + if: "${{ github.event.action != 'labeled' || github.event.label.name == 'tests: run' }}" + name: conformance + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + defaults: + run: + working-directory: ./packages/toolbox-core + permissions: + contents: 'read' + issues: 'write' + pull-requests: 'write' + steps: + - name: Remove PR Label + if: "${{ github.event.action == 'labeled' && github.event.label.name == 'tests: run' }}" + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + try { + await github.rest.issues.removeLabel({ + name: 'tests: run', + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number + }); + } catch (e) { + console.log('Failed to remove label. Another job may have already removed it!'); + } + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install test requirements + run: pip install -e .[test] + + - name: Install library requirements + run: pip install -r requirements.txt + + - name: Run Conformance Tests + uses: modelcontextprotocol/conformance@main + with: + mode: client + command: 'python packages/toolbox-core/tests/conformance/client.py' + suite: core + expected-failures: packages/toolbox-core/conformance-baseline.yml diff --git a/packages/toolbox-core/conformance-baseline.yml b/packages/toolbox-core/conformance-baseline.yml new file mode 100644 index 000000000..0b09ceef6 --- /dev/null +++ b/packages/toolbox-core/conformance-baseline.yml @@ -0,0 +1,32 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +client: + - tools_call + - elicitation-sep1034-client-defaults + - sse-retry + - auth/metadata-default + - auth/metadata-var1 + - auth/metadata-var2 + - auth/metadata-var3 + - auth/basic-cimd + - auth/scope-from-www-authenticate + - auth/scope-from-scopes-supported + - auth/scope-omitted-when-undefined + - auth/scope-step-up + - auth/scope-retry-limit + - auth/token-endpoint-auth-basic + - auth/token-endpoint-auth-post + - auth/token-endpoint-auth-none + - auth/pre-registration diff --git a/packages/toolbox-core/tests/conformance/client.py b/packages/toolbox-core/tests/conformance/client.py new file mode 100644 index 000000000..0608523f7 --- /dev/null +++ b/packages/toolbox-core/tests/conformance/client.py @@ -0,0 +1,80 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import os +import sys + +from toolbox_core.client import ToolboxClient + + +async def main(): + if len(sys.argv) < 2: + print("Usage: client.py ", file=sys.stderr) + sys.exit(1) + + server_url = sys.argv[-1] + # Prevent requests going to /mcp/mcp/ + if server_url.endswith("/mcp"): + server_url = server_url[:-4] + elif server_url.endswith("/mcp/"): + server_url = server_url[:-5] + scenario = os.environ.get("MCP_CONFORMANCE_SCENARIO", "") + context_json = os.environ.get("MCP_CONFORMANCE_CONTEXT", "{}") + context = json.loads(context_json) + + print(f"Running scenario: {scenario}", file=sys.stderr) + print(f"Server URL: {server_url}", file=sys.stderr) + print(f"Context: {context_json}", file=sys.stderr) + + client_headers = {"Accept": "application/json, text/event-stream"} + + async with ToolboxClient(server_url, client_headers=client_headers) as client: + if scenario == "initialize": + print( + "Client initialized, loading toolset to trigger tools/list", + file=sys.stderr, + ) + try: + await client.load_toolset() + print("Client initialization test completed", file=sys.stderr) + except Exception as e: + print(f"Failed to load toolset: {e}", file=sys.stderr) + + elif scenario == "tools_call": + print("Loading tool 'add_numbers'", file=sys.stderr) + try: + add_numbers = await client.load_tool("add_numbers") + print("Invoking add_numbers(a=1, b=2)", file=sys.stderr) + result = await add_numbers(a=1, b=2) + print(f"Result: {result}", file=sys.stderr) + except Exception as e: + print(f"Failed to call tool: {e}", file=sys.stderr) + + else: + print(f"Unknown or unsupported scenario: {scenario}", file=sys.stderr) + # Default behavior: load default toolset + try: + await client.load_toolset() + except Exception as e: + print(f"Default interaction failed: {e}", file=sys.stderr) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except Exception as e: + print(f"Fatal error in client: {e}", file=sys.stderr) + sys.exit(1) From 6af5419ca7534d0fa94bb5a92a5ebbe4222b109b Mon Sep 17 00:00:00 2001 From: Anubhav Dhawan Date: Mon, 20 Apr 2026 21:46:22 +0530 Subject: [PATCH 2/4] ci: Setup nightly reporter for Tier assessment --- .github/workflows/nightly_tier_report.yml | 112 ++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 .github/workflows/nightly_tier_report.yml diff --git a/.github/workflows/nightly_tier_report.yml b/.github/workflows/nightly_tier_report.yml new file mode 100644 index 000000000..75b1a98c3 --- /dev/null +++ b/.github/workflows/nightly_tier_report.yml @@ -0,0 +1,112 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Nightly SDK Tier Assessment + +on: + schedule: + - cron: '0 6 * * *' # Runs at 6 AM every morning + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + tier-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install test requirements + working-directory: ./packages/toolbox-core + run: pip install -e .[test] + + - name: Install library requirements + working-directory: ./packages/toolbox-core + run: pip install -r requirements.txt + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Run SDK Tier Assessment + id: assessment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npx --yes @modelcontextprotocol/conformance@latest tier-check --repo ${{ github.repository }} --client-cmd "python packages/toolbox-core/tests/conformance/client.py" > tier_report.txt 2>&1 + continue-on-error: true + + - name: Report Tier Status + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const report = fs.readFileSync('tier_report.txt', 'utf8'); + const title = 'Nightly SDK Tier Assessment Failure'; + const periodicLabel = 'periodic-failure'; + + const prevIssues = await github.rest.issues.listForRepo({ + ...context.repo, + state: 'open', + creator: 'github-actions[bot]', + labels: [periodicLabel] + }); + + let existingIssue = prevIssues.data.find(issue => issue.title === title); + + if (report.includes('Tier Assessment: Tier 1')) { + console.log('Tier 1 achieved! Closing open issue if it exists.'); + if (existingIssue) { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: existingIssue.number, + body: "🎉 The nightly SDK Tier Assessment confirms we are now **Tier 1**! Closing this issue." + }); + await github.rest.issues.update({ + ...context.repo, + issue_number: existingIssue.number, + state: 'closed' + }); + } + return; + } + + const body = `The nightly SDK Tier Assessment found gaps preventing Tier 1 status:\n\n\`\`\`text\n${report}\n\`\`\`\n\nPlease fix these operational gaps.`; + + if (existingIssue) { + console.log(`Found previous issue ${existingIssue.html_url}, updating body`); + await github.rest.issues.update({ + ...context.repo, + issue_number: existingIssue.number, + body: body + }); + } else { + console.log('No previous issue found, creating one'); + await github.rest.issues.create({ + ...context.repo, + title: title, + body: body, + labels: [periodicLabel] + }); + } From 6641a244b06067779f05c1a0dff62ae5ad3b5616 Mon Sep 17 00:00:00 2001 From: Anubhav Dhawan Date: Tue, 21 Apr 2026 03:42:33 +0530 Subject: [PATCH 3/4] chore: beautify issue content format --- .github/workflows/nightly_tier_report.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly_tier_report.yml b/.github/workflows/nightly_tier_report.yml index 75b1a98c3..e6b79e039 100644 --- a/.github/workflows/nightly_tier_report.yml +++ b/.github/workflows/nightly_tier_report.yml @@ -53,7 +53,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - npx --yes @modelcontextprotocol/conformance@latest tier-check --repo ${{ github.repository }} --client-cmd "python packages/toolbox-core/tests/conformance/client.py" > tier_report.txt 2>&1 + npx --yes @modelcontextprotocol/conformance@latest tier-check --repo ${{ github.repository }} --output markdown --client-cmd "python packages/toolbox-core/tests/conformance/client.py" > tier_report.txt continue-on-error: true - name: Report Tier Status @@ -92,7 +92,7 @@ jobs: return; } - const body = `The nightly SDK Tier Assessment found gaps preventing Tier 1 status:\n\n\`\`\`text\n${report}\n\`\`\`\n\nPlease fix these operational gaps.`; + const body = `The nightly SDK Tier Assessment found gaps preventing Tier 1 status:\n\n${report}\n\nPlease fix these operational gaps.`; if (existingIssue) { console.log(`Found previous issue ${existingIssue.html_url}, updating body`); From d63e0c91367c3f4625062ec5cbfec772bd235214 Mon Sep 17 00:00:00 2001 From: Anubhav Dhawan Date: Tue, 21 Apr 2026 15:24:52 +0530 Subject: [PATCH 4/4] chore: move conformance baseline file --- .github/workflows/conformance.yml | 2 +- packages/toolbox-core/{ => tests}/conformance-baseline.yml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packages/toolbox-core/{ => tests}/conformance-baseline.yml (100%) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f97f49fd7..92c68ff4b 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -78,4 +78,4 @@ jobs: mode: client command: 'python packages/toolbox-core/tests/conformance/client.py' suite: core - expected-failures: packages/toolbox-core/conformance-baseline.yml + expected-failures: packages/toolbox-core/tests/conformance-baseline.yml diff --git a/packages/toolbox-core/conformance-baseline.yml b/packages/toolbox-core/tests/conformance-baseline.yml similarity index 100% rename from packages/toolbox-core/conformance-baseline.yml rename to packages/toolbox-core/tests/conformance-baseline.yml