From f448e87e2aebe6fa79fe821088a086ffccac2bc2 Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Thu, 19 Mar 2026 18:35:54 -0700 Subject: [PATCH 1/3] Add conformance-tier-audit skill and update skill output paths Add a new conformance-tier-audit skill that orchestrates MCP SDK tier audits by starting the conformance server, pre-building the client, cloning the conformance repo, and delegating to its mcp-sdk-tier-audit skill. Update both conformance-tier-audit and issue-triage skills to write output files to artifacts/skill-output/ (already gitignored via artifacts/). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../skills/conformance-tier-audit/SKILL.md | 173 ++++++++++++++++++ .github/skills/issue-triage/SKILL.md | 2 +- 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 .github/skills/conformance-tier-audit/SKILL.md diff --git a/.github/skills/conformance-tier-audit/SKILL.md b/.github/skills/conformance-tier-audit/SKILL.md new file mode 100644 index 000000000..b51feaff5 --- /dev/null +++ b/.github/skills/conformance-tier-audit/SKILL.md @@ -0,0 +1,173 @@ +--- +name: conformance-tier-audit +description: >- + Run an MCP SDK conformance tier audit for the C# MCP SDK. Starts the conformance server, + pre-builds the conformance client, clones the conformance repo, and delegates + to its mcp-sdk-tier-audit skill for all evaluation and reporting. +argument-hint: '[--port ] [--framework ] [--branch ]' +compatibility: >- + Requires: Node.js >= 20, .NET SDK (net9.0+), + and internet access to clone the github.com/modelcontextprotocol/conformance repo. +--- + +# Conformance Tier Audit — C# MCP SDK + +This skill orchestrates a tier audit by preparing the C# SDK's conformance server and client, then delegating to the `mcp-sdk-tier-audit` skill from the `modelcontextprotocol/conformance` repo for all tier evaluation, scoring, and report generation. + +## Step 0: Pre-flight Checks + +### 0a. Parse arguments + +Extract optional overrides from the user's input (all have defaults): + +- **port** (default: `3001`): Port for the conformance server +- **framework** (default: `net9.0`): Target framework for `dotnet run`. Available: `net8.0`, `net9.0`, `net10.0` +- **branch** (default: current branch): Git branch for GitHub API checks. Derive from: `git rev-parse --abbrev-ref HEAD` + +## Step 1: Start the Conformance Server + +Start the C# SDK's conformance server as a detached background process from the SDK root (the cwd): + +``` +dotnet run --project tests/ModelContextProtocol.ConformanceServer --framework -p:NuGetAudit=false -- --urls http://localhost: +``` + +Use `mode: async, detach: true` so the server persists. + +Wait a few seconds, then verify it's reachable: + +**PowerShell** (Windows): +```powershell +curl -sf http://localhost: -o NUL -w '%{http_code}' +``` + +**Bash** (Linux/macOS): +```bash +curl -sf http://localhost: -o /dev/null -w '%{http_code}' +``` + +A `400` response is expected and means the server is running (it rejects plain GET requests). + +If the server fails to start, check stderr for build errors. Common issues: +- **NU1903 (NuGet vulnerability)**: The `-p:NuGetAudit=false` flag should suppress this. +- **Multiple TFMs**: The `--framework` flag is required because the project multi-targets. + +## Step 2: Pre-build the Conformance Client + +**CRITICAL**: Pre-build the conformance client before the audit runs tests. The conformance runner executes 26 scenarios in parallel — without pre-building, each `dotnet run` invocation triggers a full compilation, causing massive CPU contention and 30-second timeouts. + +**PowerShell** (Windows): +```powershell +dotnet build tests\ModelContextProtocol.ConformanceClient --framework -p:NuGetAudit=false --nologo -v q +``` + +**Bash** (Linux/macOS): +```bash +dotnet build tests/ModelContextProtocol.ConformanceClient --framework -p:NuGetAudit=false --nologo -v q +``` + +## Step 3: Clone the Conformance Repo + +Clone the conformance repo to a temporary directory and build it: + +**PowerShell** (Windows): +```powershell +$conformanceDir = Join-Path $env:TEMP "mcp-conformance-$(Get-Date -Format 'yyyyMMdd-HHmmss')" +git clone --depth 1 https://github.com/modelcontextprotocol/conformance.git $conformanceDir +cd $conformanceDir +npm install --silent +npm run build +``` + +**Bash** (Linux/macOS): +```bash +conformanceDir=$(mktemp -d) +git clone --depth 1 https://github.com/modelcontextprotocol/conformance.git "$conformanceDir" +cd "$conformanceDir" +npm install --silent +npm run build +``` + +Store the conformance directory path for cleanup later. + +## Step 4: Delegate to the Conformance Repo's Audit Skill + +Read the `mcp-sdk-tier-audit` skill from the cloned conformance repo: + +``` +$conformanceDir/.claude/skills/mcp-sdk-tier-audit/SKILL.md +``` + +Follow that skill's instructions end-to-end, providing these inputs: + +| Input | Value | +|-------|-------| +| `--repo` | `modelcontextprotocol/csharp-sdk` | +| `--branch` | `` | +| `--conformance-server-url` | `http://localhost:` | +| `--client-cmd` | See platform-specific commands below | + +Where `` is the absolute path to the SDK checkout (the original cwd, not the conformance temp dir). + +**PowerShell** (Windows) — use backslashes and `%MCP_CONFORMANCE_SCENARIO%`: +``` +dotnet run --project \tests\ModelContextProtocol.ConformanceClient --framework -p:NuGetAudit=false --no-build -- %MCP_CONFORMANCE_SCENARIO% +``` + +**Bash** (Linux/macOS) — use forward slashes and `$MCP_CONFORMANCE_SCENARIO`: +``` +dotnet run --project /tests/ModelContextProtocol.ConformanceClient --framework -p:NuGetAudit=false --no-build -- $MCP_CONFORMANCE_SCENARIO +``` + +### Windows Quoting Note + +The Windows `--client-cmd` uses `%MCP_CONFORMANCE_SCENARIO%` — the conformance runner sets this as an environment variable and spawns the client with `shell: true`, so the Windows shell expands it. If the tier-check CLI reports 0/N client scenarios with 0 checks passed AND 0 checks failed, the command is being parsed incorrectly due to the CLI wrapping it in single quotes (which don't work on Windows cmd.exe). In that case, run the client suite directly: + +**PowerShell** (Windows): +```powershell +cd $conformanceDir +node dist/index.js client ` + --command "dotnet run --project \tests\ModelContextProtocol.ConformanceClient --framework -p:NuGetAudit=false --no-build -- %MCP_CONFORMANCE_SCENARIO%" ` + --suite all ` + -o +``` + +**Bash** (Linux/macOS): +```bash +cd "$conformanceDir" +node dist/index.js client \ + --command "dotnet run --project /tests/ModelContextProtocol.ConformanceClient --framework -p:NuGetAudit=false --no-build -- $MCP_CONFORMANCE_SCENARIO" \ + --suite all \ + -o +``` + +### Output Location Override + +The conformance skill may specify its own output location. Override it: write all output files to `artifacts/skill-output/` at the SDK repo root. Create the directory if it doesn't exist. The `artifacts/` directory is already gitignored. + +## Step 5: Cleanup + +Stop the conformance server process. Remove the temporary conformance repo directory: + +**PowerShell** (Windows): +```powershell +Remove-Item -Recurse -Force $conformanceDir +``` + +**Bash** (Linux/macOS): +```bash +rm -rf "$conformanceDir" +``` + +## Usage Examples + +``` +# Default settings (port 3001, net9.0, current branch) +/conformance-tier-audit + +# Custom port and framework +/conformance-tier-audit --port 3003 --framework net10.0 + +# Specific branch for GitHub API checks +/conformance-tier-audit --branch main +``` diff --git a/.github/skills/issue-triage/SKILL.md b/.github/skills/issue-triage/SKILL.md index 635c6307a..017293ba4 100644 --- a/.github/skills/issue-triage/SKILL.md +++ b/.github/skills/issue-triage/SKILL.md @@ -140,7 +140,7 @@ This step adds significant value but also significant API calls. If the user ask Produce the triage report following the template in [references/report-format.md](references/report-format.md). The report must follow the BLUF structure with urgency-descending ordering. **Output destination:** -- **Default (local file):** Save as `{YYYY-MM-DD}-mcp-issue-triage.md` in the current working directory. If a file with that name already exists, suffix with `-2`, `-3`, etc. +- **Default (local file):** Save as `artifacts/skill-output/{YYYY-MM-DD}-mcp-issue-triage.md` at the SDK repo root. Create the `artifacts/skill-output/` directory if it doesn't exist (the `artifacts/` directory is already gitignored). If a file with that name already exists, suffix with `-2`, `-3`, etc. - **Gist (if requested):** If the user asked to save as a gist, create a **secret** gist using `gh gist create` with a `--desc` describing the report. No confirmation is needed — create the gist, then notify the user with a clickable link to it. The user may request a gist with phrases like "save as a gist", "create a gist", "gist it", "post to gist", etc. From 97934ab4151f425270c26bd7b46934fdf9b7fad6 Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Thu, 19 Mar 2026 21:32:37 -0700 Subject: [PATCH 2/3] Add conformance-tier-audit agentic workflow Create a GitHub Agentic Workflow that runs the MCP SDK conformance tier audit every Thursday at 14:00 UTC (6:00 AM PST). The workflow uses the Copilot CLI engine to follow the conformance-tier-audit skill and writes the assessment and remediation reports to the GitHub Actions step summary. Scheduled runs are blocked on forks; workflow_dispatch works on both upstream and forks. - conformance-tier-audit.md: Source workflow with frontmatter (schedule, runtimes, network, tools) and natural-language instructions for the agent - conformance-tier-audit.lock.yml: Compiled workflow (gh-aw v0.62.2) with SHA-pinned actions and firewall configuration - actions-lock.json: SHA-pinned action references Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitattributes | 2 + .github/aw/actions-lock.json | 14 + .../workflows/conformance-tier-audit.lock.yml | 530 ++++++++++++++++++ .github/workflows/conformance-tier-audit.md | 87 +++ 4 files changed, 633 insertions(+) create mode 100644 .github/aw/actions-lock.json create mode 100644 .github/workflows/conformance-tier-audit.lock.yml create mode 100644 .github/workflows/conformance-tier-audit.md diff --git a/.gitattributes b/.gitattributes index 956f7739c..0fa3a553c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -62,3 +62,5 @@ *.fsproj text *.dbproj text *.sln text eol=crlf + +.github/workflows/*.lock.yml linguist-generated=true merge=ours diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 000000000..78f5feb7f --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v8": { + "repo": "actions/github-script", + "version": "v8", + "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" + }, + "github/gh-aw-actions/setup@v0.62.2": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.62.2", + "sha": "20045bbd5ad2632b9809856c389708eab1bd16ef" + } + } +} diff --git a/.github/workflows/conformance-tier-audit.lock.yml b/.github/workflows/conformance-tier-audit.lock.yml new file mode 100644 index 000000000..26a640e62 --- /dev/null +++ b/.github/workflows/conformance-tier-audit.lock.yml @@ -0,0 +1,530 @@ +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.62.2). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# +# gh-aw-metadata: {"schema_version":"v2","frontmatter_hash":"0dc01f773705e447f0ab8c72439f7e6c346d43cdbb828a172c7fe717dbdfd4d1","compiler_version":"v0.62.2","strict":true} + +name: "Weekly Conformance Tier Audit" +"on": + schedule: + - cron: "0 14 * * 4" + workflow_dispatch: + inputs: + conformance-branch: + default: main + description: Conformance repo branch to clone + required: false + type: string + conformance-repo: + default: modelcontextprotocol/conformance + description: Conformance repo to clone (org/repo) + required: false + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Weekly Conformance Tier Audit" + +jobs: + activation: + if: github.repository == 'modelcontextprotocol/csharp-sdk' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw-actions/setup@20045bbd5ad2632b9809856c389708eab1bd16ef # v0.62.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_INFO_VERSION: "" + GH_AW_INFO_AGENT_VERSION: "latest" + GH_AW_INFO_CLI_VERSION: "v0.62.2" + GH_AW_INFO_WORKFLOW_NAME: "Weekly Conformance Tier Audit" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","node","dotnet"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.24.3" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: ${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "conformance-tier-audit.lock.yml" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_7CF528D1: ${{ github.event.inputs.conformance-repo || 'modelcontextprotocol/conformance' }} + GH_AW_EXPR_9411329A: ${{ github.event.inputs.conformance-branch || 'main' }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash ${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh + { + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" + cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" + cat << 'GH_AW_PROMPT_EOF' + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_prompt.md" + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + {{#runtime-import .github/workflows/conformance-tier-audit.md}} + GH_AW_PROMPT_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_9411329A: ${{ github.event.inputs.conformance-branch || 'main' }} + GH_AW_EXPR_7CF528D1: ${{ github.event.inputs.conformance-repo || 'modelcontextprotocol/conformance' }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_7CF528D1: ${{ github.event.inputs.conformance-repo || 'modelcontextprotocol/conformance' }} + GH_AW_EXPR_9411329A: ${{ github.event.inputs.conformance-branch || 'main' }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + + const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_7CF528D1: process.env.GH_AW_EXPR_7CF528D1, + GH_AW_EXPR_9411329A: process.env.GH_AW_EXPR_9411329A, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash ${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash ${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: activation + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + retention-days: 1 + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + GH_AW_WORKFLOW_ID_SANITIZED: conformancetieraudit + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + inference_access_error: ${{ steps.detect-inference-error.outputs.inference_access_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + steps: + - name: Setup Scripts + uses: github/gh-aw-actions/setup@20045bbd5ad2632b9809856c389708eab1bd16ef # v0.62.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup .NET + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 + with: + dotnet-version: '10.0' + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: '24' + package-manager-cache: false + - name: Create gh-aw temp directory + run: bash ${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh + - name: Configure gh CLI for GitHub Enterprise + run: bash ${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh + env: + GH_TOKEN: ${{ github.token }} + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + (github.event.pull_request) || (github.event.issue.pull_request) + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Install GitHub Copilot CLI + run: ${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh latest + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash ${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh v0.24.3 + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download container images + run: bash ${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.24.3 ghcr.io/github/gh-aw-firewall/api-proxy:0.24.3 ghcr.io/github/gh-aw-firewall/squid:0.24.3 ghcr.io/github/gh-aw-mcpg:v0.1.19 ghcr.io/github/github-mcp-server:v0.32.0 + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} + GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.19' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.32.0", + "env": { + "GITHUB_HOST": "\${GITHUB_SERVER_URL}", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", + "repos": "$GITHUB_MCP_GUARD_REPOS" + } + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Clean git credentials + continue-on-error: true + run: bash ${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + touch /tmp/gh-aw/agent-step-summary.md + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --allow-domains "*.jsr.io,*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.npms.io,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,bun.sh,cdn.jsdelivr.net,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,deb.nodesource.com,deno.land,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,esm.sh,get.pnpm.io,github.com,googleapis.deno.dev,googlechromelabs.github.io,host.docker.internal,json-schema.org,json.schemastore.org,jsr.io,keyserver.ubuntu.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,storage.googleapis.com,telemetry.enterprise.githubcopilot.com,telemetry.vercel.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com,www.npmjs.com,www.npmjs.org,yarnpkg.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.24.3 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: v0.62.2 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + XDG_CONFIG_HOME: /home/runner + - name: Detect inference access error + id: detect-inference-error + if: always() + continue-on-error: true + run: bash ${RUNNER_TEMP}/gh-aw/actions/detect_inference_access_error.sh + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash ${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Append agent step summary + if: always() + run: bash ${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) + if command -v awf &> /dev/null; then + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + else + echo 'AWF binary not installed, skipping firewall log summary' + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + if-no-files-found: ignore + diff --git a/.github/workflows/conformance-tier-audit.md b/.github/workflows/conformance-tier-audit.md new file mode 100644 index 000000000..cf436c5fc --- /dev/null +++ b/.github/workflows/conformance-tier-audit.md @@ -0,0 +1,87 @@ +--- +on: + schedule: + - cron: "0 14 * * 4" + workflow_dispatch: + inputs: + conformance-repo: + description: 'Conformance repo to clone (org/repo)' + required: false + type: string + default: 'modelcontextprotocol/conformance' + conformance-branch: + description: 'Conformance repo branch to clone' + required: false + type: string + default: 'main' + +if: github.repository == 'modelcontextprotocol/csharp-sdk' || github.event_name == 'workflow_dispatch' + +permissions: + contents: read + issues: read + pull-requests: read + +runtimes: + dotnet: + version: "10.0" + action-version: "v5" + node: + version: "24" + +network: + allowed: + - defaults + - node + - dotnet + +tools: + bash: true + github: true +--- + +# Weekly Conformance Tier Audit + +Run the MCP SDK conformance tier audit every Thursday and publish the assessment and remediation reports to the workflow summary. + +## Inputs + +These values are configurable via `workflow_dispatch` inputs. On scheduled runs, the defaults are used. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `conformance-repo` | `modelcontextprotocol/conformance` | The conformance repo to clone | +| `conformance-branch` | `main` | The conformance repo branch to clone | + +The workflow always uses `modelcontextprotocol/csharp-sdk` with branch `main` for issue triage, labels, and policy checks — regardless of which fork or branch the workflow runs on. The SDK source code is taken from the current repository and branch (selectable via workflow_dispatch's native branch picker). + +## Instructions + +Read and follow the conformance-tier-audit skill at `.github/skills/conformance-tier-audit/SKILL.md`. Use these parameter overrides: + +- **`--repo`**: `modelcontextprotocol/csharp-sdk` (always — for issue triage, labels, policy signals) +- **`--branch`**: `main` (always — for GitHub API checks against the upstream repo) +- **`--framework net9.0`** for the conformance server and client +- When cloning the conformance repo, use `https://github.com/${{ github.event.inputs.conformance-repo || 'modelcontextprotocol/conformance' }}.git` and checkout branch `${{ github.event.inputs.conformance-branch || 'main' }}` + +**Important**: The `--repo` and `--branch` values above are for GitHub API checks (issue triage, labels, policy signals) and must always target the upstream `modelcontextprotocol/csharp-sdk` repo on `main`. The SDK source code being audited (conformance server/client) comes from the current repository checkout. + +### Output to Workflow Summary + +Instead of writing files to `artifacts/skill-output/`, write **all** reports to the GitHub Actions step summary (`$GITHUB_STEP_SUMMARY`). This makes the reports visible directly in the workflow run summary page. + +Format the step summary as: + +1. **Executive summary** at the top — final tier result, conformance matrix table, repository health table +2. **Full assessment report** in a collapsible `
` block with a 📋 prefix +3. **Full remediation report** in a separate collapsible `
` block with a 🔧 prefix + +Write the content to `$GITHUB_STEP_SUMMARY` using bash, for example: + + echo "# Conformance Tier Audit — C# MCP SDK" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "## Executive Summary" >> "$GITHUB_STEP_SUMMARY" + echo "...tables and tier result..." >> "$GITHUB_STEP_SUMMARY" + echo "
📋 Full Assessment Report" >> "$GITHUB_STEP_SUMMARY" + echo "...assessment content..." >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" From 439085870db9bfa2425caaa781dce2e85722c7a3 Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Thu, 2 Apr 2026 18:26:31 -0700 Subject: [PATCH 3/3] Add Copilot PAT pool support for agentic workflows - Add select-copilot-pat composite action with README - Update agentic-workflows agent with mandatory PAT pool instructions - Add PAT pool frontmatter blocks to conformance-tier-audit workflow - Use fork detection instead of repo name check in workflow condition - Change schedule to fuzzy 'weekly on thursday around 9am utc-7' - Move release.md out of .github/workflows/ to avoid treatment as agentic workflow - Recompile conformance-tier-audit lock file Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/actions/select-copilot-pat/README.md | 115 ++++++++ .github/actions/select-copilot-pat/action.yml | 53 ++++ .github/agents/agentic-workflows.agent.md | 261 ++++++++++++++++++ .github/aw/actions-lock.json | 5 + .../release.md => release-workflow.md} | 0 .../workflows/conformance-tier-audit.lock.yml | 200 ++++++++++---- .github/workflows/conformance-tier-audit.md | 66 ++++- .vscode/settings.json | 5 + 8 files changed, 648 insertions(+), 57 deletions(-) create mode 100644 .github/actions/select-copilot-pat/README.md create mode 100644 .github/actions/select-copilot-pat/action.yml create mode 100644 .github/agents/agentic-workflows.agent.md rename .github/{workflows/release.md => release-workflow.md} (100%) create mode 100644 .vscode/settings.json diff --git a/.github/actions/select-copilot-pat/README.md b/.github/actions/select-copilot-pat/README.md new file mode 100644 index 000000000..e125a7e3d --- /dev/null +++ b/.github/actions/select-copilot-pat/README.md @@ -0,0 +1,115 @@ +# Select Copilot PAT + +Selects a random Copilot PAT from a numbered pool of secrets. This addresses limitations that arise from having a single PAT shared across all agentic workflows, such as rate-limiting. + +**This is a stop-gap workaround.** As soon as organization/enterprise billing is offered for agentic workflows, this approach will be removed from our workflows. + +## Repository Onboarding + +To use this action in a repository: + +1. Copy this `select-copilot-pat` folder into the repository under `.github/actions/select-copilot-pat`, including both the `README.md` and `action.yml`. +2. Merge those additions into the repository and then follow the instructions for PAT Creation and Usage below. + +## PAT Management + +Team members provide PATs into the pools for the repository by adding them as repository secrets with secret names matching the pattern of `_<0-9>`, such as `COPILOT_PAT_0`. + +[Use this link to prefill the PAT creation form with the required settings][create-pat]: + +1. **Resource owner** is your **user account**, not an organization. +2. **Copilot Requests (Read)** must be the only permission granted. +3. **8-day expiration** must be used, which enforces a weekly renewal. +4. **Repository access** set to **Public repositories** only. + +The **Token Name** _does not_ need to match the secret name and is only visible to the owner of the PAT. It's recommended to use a token name indicating the PAT is used for modelcontextprotocol org agentic workflows. The **Description** is also only used for your own reference. + +Team members providing PATs for workflows should set weekly recurring reminders to regenerate and update their PATs in the repository secrets. With an 8-day expiration, renewal can be done on the same day each week. + +PATs are added to repositories through the **Settings > Secrets and variables > Actions** UI, saved as **Repository secrets** and matching the `_<0-9>` naming convention. This can also be done using the GitHub CLI. + +```sh +gh aw secrets set "_<0-9>" --value "" --repo modelcontextprotocol/ +``` + +## Workflow Output Attribution + +Team members' PATs are _only_ used for the Copilot requests from within the agentic portion of the workflow. All outputs from the workflow use the `github-actions[bot]` account token. Issues, PRs, comments, and all other content generated by the workflow will be attributed to `github-actions[bot]`--not the team member's account or token. + +## Usage + +Add the following frontmatter at the top-level of an agentic workflow. These elements are not supported through [imports][imports], so they must be copied into all workflows. + +Up to 10 `SECRET_#` environment variables can be passed to the action, numbered 0-9. Different workflows can use different pools of PATs if desired. Change the `secrets.COPILOT_PAT_0` through `secrets.COPILOT_PAT_9` secret names in both the `select-copilot-pat` step `env` values and in the `case` expression under the `engine: env` configuration. + +```yml +on: + # Add the pre-activation step of selecting a random PAT from the supplied secrets + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + name: Checkout the select-copilot-pat action folder + with: + persist-credentials: false + sparse-checkout: .github/actions/select-copilot-pat + sparse-checkout-cone-mode: true + fetch-depth: 1 + + - id: select-copilot-pat + name: Select Copilot token from pool + uses: ./.github/actions/select-copilot-pat + env: + # If the secret names are changed here, they must also be changed + # in the `engine: env` case expression + SECRET_0: ${{ secrets.COPILOT_PAT_0 }} + SECRET_1: ${{ secrets.COPILOT_PAT_1 }} + SECRET_2: ${{ secrets.COPILOT_PAT_2 }} + SECRET_3: ${{ secrets.COPILOT_PAT_3 }} + SECRET_4: ${{ secrets.COPILOT_PAT_4 }} + SECRET_5: ${{ secrets.COPILOT_PAT_5 }} + SECRET_6: ${{ secrets.COPILOT_PAT_6 }} + SECRET_7: ${{ secrets.COPILOT_PAT_7 }} + SECRET_8: ${{ secrets.COPILOT_PAT_8 }} + SECRET_9: ${{ secrets.COPILOT_PAT_9 }} + +# Add the pre-activation output of the randomly selected PAT +jobs: + pre-activation: + outputs: + copilot_pat_number: ${{ steps.select-copilot-pat.outputs.copilot_pat_number }} + +# Override the COPILOT_GITHUB_TOKEN expression used in the activation job +# Consume the PAT number from the pre-activation step and select the corresponding secret +engine: + id: copilot + env: + # We cannot use line breaks in this expression as it leads to a syntax error in the compiled workflow + # If none of the `COPILOT_PAT_#` secrets were selected, then the default COPILOT_GITHUB_TOKEN is used + COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} +``` + +## References + +- [Agentic Workflows CLI Extension][cli-setup] +- [Agentic Authoring][configure-repo] +- [Authentication][authentication] +- [Agentic Workflow Imports][imports] +- [Custom Steps][steps] +- [Custom Jobs][jobs] +- [Job Outputs][job-outputs] +- [Engine Configuration][engine] +- [Engine Environment Variables][engine-vars] +- [Case Function in Workflow Expressions][case-expression] +- [Update agentic engine token handling to use user-provided secrets (github/gh-aw#18017)][secret-override] + +[cli-setup]: https://github.github.com/gh-aw/setup/cli/ +[configure-repo]: https://github.github.com/gh-aw/guides/agentic-authoring/#configuring-your-repository +[authentication]: https://github.github.com/gh-aw/reference/auth/ +[create-pat]: https://github.com/settings/personal-access-tokens/new?name=modelcontextprotocol%20org%20agentic%20workflows&description=GitHub+Agentic+Workflows+-+Copilot+engine+authentication.++Used+for+modelcontextprotocol+org+workflows.+MUST+be+configured+with+only+Copilot+Requests+permissions+and+user+account+as+resource+owner.+Weekly+expiration+and+required+renewal.&user_copilot_requests=read&expires_in=8 +[imports]: https://github.github.com/gh-aw/reference/imports/ +[steps]: https://github.github.com/gh-aw/reference/frontmatter/#custom-steps-steps +[jobs]: https://github.github.com/gh-aw/reference/frontmatter/#custom-jobs-jobs +[job-outputs]: https://github.github.com/gh-aw/reference/frontmatter/#job-outputs +[engine]: https://github.github.com/gh-aw/reference/frontmatter/#ai-engine-engine +[engine-vars]: https://github.github.com/gh-aw/reference/engines/#engine-environment-variables +[case-expression]: https://docs.github.com/en/actions/reference/workflows-and-actions/expressions#case +[secret-override]: https://github.com/github/gh-aw/pull/18017 diff --git a/.github/actions/select-copilot-pat/action.yml b/.github/actions/select-copilot-pat/action.yml new file mode 100644 index 000000000..198eb0393 --- /dev/null +++ b/.github/actions/select-copilot-pat/action.yml @@ -0,0 +1,53 @@ +name: 'Select Copilot PAT from Pool' +description: > + Selects a random Copilot PAT from a numbered pool of secrets. Secrets + are passed as environment variables SECRET_0 through SECRET_9 + by the calling workflow step. + +inputs: + random-seed: + description: 'A seed number to use for the random PAT selection, for deterministic selection if needed.' + required: false + default: '' + +outputs: + copilot_pat_number: + description: 'The 0-9 secret number selected from the pool of specified secrets' + value: ${{ steps.select-pat-number.outputs.copilot_pat_number }} + +runs: + using: composite + steps: + - id: select-pat-number + shell: bash + env: + RANDOM_SEED: ${{ inputs.random-seed }} + run: | + # Collect all secret numbers with non-empty values from SECRET_0..SECRET_9 + PAT_NUMBERS=() + for i in $(seq 0 9); do + var="SECRET_${i}" + val="${!var}" + if [ -n "$val" ]; then + PAT_NUMBERS+=(${i}) + fi + done + + # If none of the secrets in the pool have values, then emit a warning and do not + # set an output value. The consumer can then fall back to using COPILOT_GITHUB_TOKEN. + if [ ${#PAT_NUMBERS[@]} -eq 0 ]; then + echo "::warning::None of the specified secrets had values (checked SECRET_0 through SECRET_9)" + exit 0 + fi + + # Select a random index using the seed if specified + if [ -n "$RANDOM_SEED" ]; then + RANDOM=$RANDOM_SEED + fi + + PAT_INDEX=$(( RANDOM % ${#PAT_NUMBERS[@]} )) + PAT_NUMBER="${PAT_NUMBERS[$PAT_INDEX]}" + echo "Selected token ${PAT_NUMBER} (index: ${PAT_INDEX}; pool size: ${#PAT_NUMBERS[@]})" + + # Set the PAT number as the output + echo "copilot_pat_number=${PAT_NUMBER}" >> "$GITHUB_OUTPUT" diff --git a/.github/agents/agentic-workflows.agent.md b/.github/agents/agentic-workflows.agent.md new file mode 100644 index 000000000..a1ecd9fea --- /dev/null +++ b/.github/agents/agentic-workflows.agent.md @@ -0,0 +1,261 @@ +--- +description: GitHub Agentic Workflows (gh-aw) - Create, debug, and upgrade AI-powered workflows with intelligent prompt routing +disable-model-invocation: true +--- + +# GitHub Agentic Workflows Agent + +This agent helps you work with **GitHub Agentic Workflows (gh-aw)**, a CLI extension for creating AI-powered workflows in natural language using markdown files. + +## What This Agent Does + +This is a **dispatcher agent** that routes your request to the appropriate specialized prompt based on your task: + +- **Creating new workflows**: Routes to `create` prompt +- **Updating existing workflows**: Routes to `update` prompt +- **Debugging workflows**: Routes to `debug` prompt +- **Upgrading workflows**: Routes to `upgrade-agentic-workflows` prompt +- **Creating report-generating workflows**: Routes to `report` prompt — consult this whenever the workflow posts status updates, audits, analyses, or any structured output as issues, discussions, or comments +- **Creating shared components**: Routes to `create-shared-agentic-workflow` prompt +- **Fixing Dependabot PRs**: Routes to `dependabot` prompt — use this when Dependabot opens PRs that modify generated manifest files (`.github/workflows/package.json`, `.github/workflows/requirements.txt`, `.github/workflows/go.mod`). Never merge those PRs directly; instead update the source `.md` files and rerun `gh aw compile --dependabot` to bundle all fixes +- **Analyzing test coverage**: Routes to `test-coverage` prompt — consult this whenever the workflow reads, analyzes, or reports on test coverage data from PRs or CI runs + +Workflows may optionally include: + +- **Project tracking / monitoring** (GitHub Projects updates, status reporting) +- **Orchestration / coordination** (one workflow assigning agents or dispatching and coordinating other workflows) + +## Files This Applies To + +- Workflow files: `.github/workflows/*.md` and `.github/workflows/**/*.md` +- Workflow lock files: `.github/workflows/*.lock.yml` +- Shared components: `.github/workflows/shared/*.md` +- Configuration: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/github-agentic-workflows.md + +## Problems This Solves + +- **Workflow Creation**: Design secure, validated agentic workflows with proper triggers, tools, and permissions +- **Workflow Debugging**: Analyze logs, identify missing tools, investigate failures, and fix configuration issues +- **Version Upgrades**: Migrate workflows to new gh-aw versions, apply codemods, fix breaking changes +- **Component Design**: Create reusable shared workflow components that wrap MCP servers + +## How to Use + +When you interact with this agent, it will: + +1. **Understand your intent** - Determine what kind of task you're trying to accomplish +2. **Route to the right prompt** - Load the specialized prompt file for your task +3. **Execute the task** - Follow the detailed instructions in the loaded prompt + +## Available Prompts + +### Create New Workflow +**Load when**: User wants to create a new workflow from scratch, add automation, or design a workflow that doesn't exist yet + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/create-agentic-workflow.md + +**Use cases**: +- "Create a workflow that triages issues" +- "I need a workflow to label pull requests" +- "Design a weekly research automation" + +### Update Existing Workflow +**Load when**: User wants to modify, improve, or refactor an existing workflow + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/update-agentic-workflow.md + +**Use cases**: +- "Add web-fetch tool to the issue-classifier workflow" +- "Update the PR reviewer to use discussions instead of issues" +- "Improve the prompt for the weekly-research workflow" + +### Debug Workflow +**Load when**: User needs to investigate, audit, debug, or understand a workflow, troubleshoot issues, analyze logs, or fix errors + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/debug-agentic-workflow.md + +**Use cases**: +- "Why is this workflow failing?" +- "Analyze the logs for workflow X" +- "Investigate missing tool calls in run #12345" + +### Upgrade Agentic Workflows +**Load when**: User wants to upgrade workflows to a new gh-aw version or fix deprecations + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/upgrade-agentic-workflows.md + +**Use cases**: +- "Upgrade all workflows to the latest version" +- "Fix deprecated fields in workflows" +- "Apply breaking changes from the new release" + +### Create a Report-Generating Workflow +**Load when**: The workflow being created or updated produces reports — recurring status updates, audit summaries, analyses, or any structured output posted as a GitHub issue, discussion, or comment + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/report.md + +**Use cases**: +- "Create a weekly CI health report" +- "Post a daily security audit to Discussions" +- "Add a status update comment to open PRs" + +### Create Shared Agentic Workflow +**Load when**: User wants to create a reusable workflow component or wrap an MCP server + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/create-shared-agentic-workflow.md + +**Use cases**: +- "Create a shared component for Notion integration" +- "Wrap the Slack MCP server as a reusable component" +- "Design a shared workflow for database queries" + +### Fix Dependabot PRs +**Load when**: User needs to close or fix open Dependabot PRs that update dependencies in generated manifest files (`.github/workflows/package.json`, `.github/workflows/requirements.txt`, `.github/workflows/go.mod`) + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/dependabot.md + +**Use cases**: +- "Fix the open Dependabot PRs for npm dependencies" +- "Bundle and close the Dependabot PRs for workflow dependencies" +- "Update @playwright/test to fix the Dependabot PR" + +### Analyze Test Coverage +**Load when**: The workflow reads, analyzes, or reports test coverage — whether triggered by a PR, a schedule, or a slash command. Always consult this prompt before designing the coverage data strategy. + +**Prompt file**: https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/test-coverage.md + +**Use cases**: +- "Create a workflow that comments coverage on PRs" +- "Analyze coverage trends over time" +- "Add a coverage gate that blocks PRs below a threshold" + +## Instructions + +When a user interacts with you: + +1. **Identify the task type** from the user's request +2. **Load the appropriate prompt** from the GitHub repository URLs listed above +3. **Follow the loaded prompt's instructions** exactly +4. **If uncertain**, ask clarifying questions to determine the right prompt + +## Quick Reference + +```bash +# Initialize repository for agentic workflows +gh aw init + +# Generate the lock file for a workflow +gh aw compile [workflow-name] + +# Debug workflow runs +gh aw logs [workflow-name] +gh aw audit + +# Upgrade workflows +gh aw fix --write +gh aw compile --validate +``` + +## Key Features of gh-aw + +- **Natural Language Workflows**: Write workflows in markdown with YAML frontmatter +- **AI Engine Support**: Copilot, Claude, Codex, or custom engines +- **MCP Server Integration**: Connect to Model Context Protocol servers for tools +- **Safe Outputs**: Structured communication between AI and GitHub API +- **Strict Mode**: Security-first validation and sandboxing +- **Shared Components**: Reusable workflow building blocks +- **Repo Memory**: Persistent git-backed storage for agents +- **Sandboxed Execution**: All workflows run in the Agent Workflow Firewall (AWF) sandbox, enabling full `bash` and `edit` tools by default + +## Important Notes + +- Always reference the instructions file at https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/github-agentic-workflows.md for complete documentation +- Use the MCP tool `agentic-workflows` when running in GitHub Copilot Cloud +- Workflows must be compiled to `.lock.yml` files before running in GitHub Actions +- **Bash tools are enabled by default** - Don't restrict bash commands unnecessarily since workflows are sandboxed by the AWF +- Follow security best practices: minimal permissions, explicit network access, no template injection +- **Network configuration**: Use ecosystem identifiers (`node`, `python`, `go`, etc.) or explicit FQDNs in `network.allowed`. Bare shorthands like `npm` or `pypi` are **not** valid. See https://github.com/github/gh-aw/blob/v0.65.6/.github/aw/network.md for the full list of valid ecosystem identifiers and domain patterns. +- **Single-file output**: When creating a workflow, produce exactly **one** workflow `.md` file. Do not create separate documentation files (architecture docs, runbooks, usage guides, etc.). If documentation is needed, add a brief `## Usage` section inside the workflow file itself. + +## Repository-Specific Requirements: Copilot PAT Pool + +**⚠️ MANDATORY**: Every agentic workflow in this repository **must** include the `select-copilot-pat` integration. This selects a random Copilot PAT from a numbered pool of secrets to avoid rate-limiting from a single shared PAT. This is a stop-gap workaround until organization-level billing is offered for agentic workflows. + +See `.github/actions/select-copilot-pat/README.md` for full documentation and `.github/workflows/conformance-tier-audit.md` for a working example. + +### How it works + +When **creating** or **updating** an agentic workflow, always include the following three frontmatter blocks. These elements are **not supported through imports**, so they must be copied into every workflow's frontmatter. See `.github/workflows/conformance-tier-audit.md` for a working example. + +### Comment block (placed before the steps, inside the `on:` block) + +Always include this comment block before the steps to explain the PAT pool override: + +```yaml +# ############################################################### +# Override the COPILOT_GITHUB_TOKEN secret usage for the workflow +# with a randomly-selected token from a pool of secrets. +# +# As soon as organization-level billing is offered for Agentic +# Workflows, this stop-gap approach will be removed. +# +# See: /.github/actions/select-copilot-pat/README.md +# ############################################################### +``` + +### Block 1: Pre-activation steps (nested under `on:`) + +Add this under the `on:` key, at the same level as the trigger configuration, immediately after the comment block above: + +```yaml + # Add the pre-activation step of selecting a random PAT from the supplied secrets + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + name: Checkout the select-copilot-pat action folder + with: + persist-credentials: false + sparse-checkout: .github/actions/select-copilot-pat + sparse-checkout-cone-mode: true + fetch-depth: 1 + + - id: select-copilot-pat + name: Select Copilot token from pool + uses: ./.github/actions/select-copilot-pat + env: + SECRET_0: ${{ secrets.COPILOT_PAT_0 }} + SECRET_1: ${{ secrets.COPILOT_PAT_1 }} + SECRET_2: ${{ secrets.COPILOT_PAT_2 }} + SECRET_3: ${{ secrets.COPILOT_PAT_3 }} + SECRET_4: ${{ secrets.COPILOT_PAT_4 }} + SECRET_5: ${{ secrets.COPILOT_PAT_5 }} + SECRET_6: ${{ secrets.COPILOT_PAT_6 }} + SECRET_7: ${{ secrets.COPILOT_PAT_7 }} + SECRET_8: ${{ secrets.COPILOT_PAT_8 }} + SECRET_9: ${{ secrets.COPILOT_PAT_9 }} +``` + +### Block 2: Pre-activation job outputs (top-level `jobs:`) + +```yaml +jobs: + pre-activation: + outputs: + copilot_pat_number: ${{ steps.select-copilot-pat.outputs.copilot_pat_number }} +``` + +### Block 3: Engine configuration with PAT override (top-level `engine:`) + +```yaml +engine: + id: copilot + env: + # We cannot use line breaks in this expression as it leads to a syntax error in the compiled workflow + # If none of the `COPILOT_PAT_#` secrets were selected, then the default COPILOT_GITHUB_TOKEN is used + COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} +``` + +**Important notes:** +- The `COPILOT_GITHUB_TOKEN` `case()` expression **must** remain on a single line — line breaks cause syntax errors in the compiled workflow. +- If no `COPILOT_PAT_#` secrets are configured, the expression falls back to the default `COPILOT_GITHUB_TOKEN` secret. +- Do **not** specify `engine: copilot` as a simple string — use the object form so the `env:` override can be included. +- Different workflows can use different pool name prefixes if desired. diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 78f5feb7f..f5d8ada9d 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -9,6 +9,11 @@ "repo": "github/gh-aw-actions/setup", "version": "v0.62.2", "sha": "20045bbd5ad2632b9809856c389708eab1bd16ef" + }, + "github/gh-aw-actions/setup@v0.65.6": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.65.6", + "sha": "31130b20a8fd3ef263acbe2091267c0aace07e09" } } } diff --git a/.github/workflows/release.md b/.github/release-workflow.md similarity index 100% rename from .github/workflows/release.md rename to .github/release-workflow.md diff --git a/.github/workflows/conformance-tier-audit.lock.yml b/.github/workflows/conformance-tier-audit.lock.yml index 26a640e62..718b31765 100644 --- a/.github/workflows/conformance-tier-audit.lock.yml +++ b/.github/workflows/conformance-tier-audit.lock.yml @@ -12,7 +12,7 @@ # \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ # \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ # -# This file was automatically generated by gh-aw (v0.62.2). DO NOT EDIT. +# This file was automatically generated by gh-aw (v0.65.6). DO NOT EDIT. # # To update this file, edit the corresponding .md file and run: # gh aw compile @@ -21,14 +21,42 @@ # For more information: https://github.github.com/gh-aw/introduction/overview/ # # -# gh-aw-metadata: {"schema_version":"v2","frontmatter_hash":"0dc01f773705e447f0ab8c72439f7e6c346d43cdbb828a172c7fe717dbdfd4d1","compiler_version":"v0.62.2","strict":true} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"f40005fcf19ed91cc9c1a2f6eea8fa28519fa75894aae4564cf47e5c8f549287","compiler_version":"v0.65.6","strict":true,"agent_id":"copilot"} name: "Weekly Conformance Tier Audit" "on": schedule: - - cron: "0 14 * * 4" + - cron: "6 15 * * 4" + # Friendly format: weekly on thursday around 9am utc-7 (scattered) + # steps: # Steps injected into pre-activation job + # - name: Checkout the select-copilot-pat action folder + # uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + # with: + # fetch-depth: 1 + # persist-credentials: false + # sparse-checkout: .github/actions/select-copilot-pat + # sparse-checkout-cone-mode: true + # - env: + # SECRET_0: ${{ secrets.COPILOT_PAT_0 }} + # SECRET_1: ${{ secrets.COPILOT_PAT_1 }} + # SECRET_2: ${{ secrets.COPILOT_PAT_2 }} + # SECRET_3: ${{ secrets.COPILOT_PAT_3 }} + # SECRET_4: ${{ secrets.COPILOT_PAT_4 }} + # SECRET_5: ${{ secrets.COPILOT_PAT_5 }} + # SECRET_6: ${{ secrets.COPILOT_PAT_6 }} + # SECRET_7: ${{ secrets.COPILOT_PAT_7 }} + # SECRET_8: ${{ secrets.COPILOT_PAT_8 }} + # SECRET_9: ${{ secrets.COPILOT_PAT_9 }} + # id: select-copilot-pat + # name: Select Copilot token from pool + # uses: ./.github/actions/select-copilot-pat workflow_dispatch: inputs: + aw_context: + default: "" + description: Agent caller context (used internally by Agentic Workflows). + required: false + type: string conformance-branch: default: main description: Conformance repo branch to clone @@ -49,7 +77,9 @@ run-name: "Weekly Conformance Tier Audit" jobs: activation: - if: github.repository == 'modelcontextprotocol/csharp-sdk' || github.event_name == 'workflow_dispatch' + needs: pre_activation + if: > + needs.pre_activation.outputs.activated == 'true' && ((!github.event.repository.fork) || github.event_name == 'workflow_dispatch') runs-on: ubuntu-slim permissions: contents: read @@ -61,7 +91,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw-actions/setup@20045bbd5ad2632b9809856c389708eab1bd16ef # v0.62.2 + uses: github/gh-aw-actions/setup@31130b20a8fd3ef263acbe2091267c0aace07e09 # v0.65.6 with: destination: ${{ runner.temp }}/gh-aw/actions - name: Generate agentic run info @@ -69,17 +99,17 @@ jobs: env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} - GH_AW_INFO_VERSION: "" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'auto' }} + GH_AW_INFO_VERSION: "latest" GH_AW_INFO_AGENT_VERSION: "latest" - GH_AW_INFO_CLI_VERSION: "v0.62.2" + GH_AW_INFO_CLI_VERSION: "v0.65.6" GH_AW_INFO_WORKFLOW_NAME: "Weekly Conformance Tier Audit" GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","node","dotnet"]' GH_AW_INFO_FIREWALL_ENABLED: "true" - GH_AW_INFO_AWF_VERSION: "v0.24.3" + GH_AW_INFO_AWF_VERSION: "v0.25.11" GH_AW_INFO_AWMG_VERSION: "" GH_AW_INFO_FIREWALL_TYPE: "squid" GH_AW_COMPILED_STRICT: "true" @@ -94,7 +124,7 @@ jobs: id: validate-secret run: ${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} - name: Checkout .github and .agents folders uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -114,6 +144,16 @@ jobs: setupGlobals(core, github, context, exec, io); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); await main(); + - name: Check compile-agentic version + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_COMPILED_VERSION: "v0.65.6" + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + await main(); - name: Create prompt with built-in context env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt @@ -127,16 +167,17 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + # poutine:ignore untrusted_checkout_exec run: | bash ${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh { - cat << 'GH_AW_PROMPT_EOF' + cat << 'GH_AW_PROMPT_9124d618aea251e4_EOF' - GH_AW_PROMPT_EOF + GH_AW_PROMPT_9124d618aea251e4_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" - cat << 'GH_AW_PROMPT_EOF' + cat << 'GH_AW_PROMPT_9124d618aea251e4_EOF' The following GitHub context information is available for this workflow: {{#if __GH_AW_GITHUB_ACTOR__ }} @@ -165,14 +206,12 @@ jobs: {{/if}} - GH_AW_PROMPT_EOF + GH_AW_PROMPT_9124d618aea251e4_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_prompt.md" - cat << 'GH_AW_PROMPT_EOF' + cat << 'GH_AW_PROMPT_9124d618aea251e4_EOF' - GH_AW_PROMPT_EOF - cat << 'GH_AW_PROMPT_EOF' {{#runtime-import .github/workflows/conformance-tier-audit.md}} - GH_AW_PROMPT_EOF + GH_AW_PROMPT_9124d618aea251e4_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -200,6 +239,7 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -220,20 +260,23 @@ jobs: GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, - GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED } }); - name: Validate prompt placeholders env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec run: bash ${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh - name: Print prompt env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + # poutine:ignore untrusted_checkout_exec run: bash ${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh - name: Upload activation artifact if: success() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: activation path: | @@ -254,11 +297,12 @@ jobs: GH_AW_WORKFLOW_ID_SANITIZED: conformancetieraudit outputs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} inference_access_error: ${{ steps.detect-inference-error.outputs.inference_access_error || 'false' }} model: ${{ needs.activation.outputs.model }} steps: - name: Setup Scripts - uses: github/gh-aw-actions/setup@20045bbd5ad2632b9809856c389708eab1bd16ef # v0.62.2 + uses: github/gh-aw-actions/setup@31130b20a8fd3ef263acbe2091267c0aace07e09 # v0.65.6 with: destination: ${{ runner.temp }}/gh-aw/actions - name: Checkout repository @@ -295,7 +339,7 @@ jobs: - name: Checkout PR branch id: checkout-pr if: | - (github.event.pull_request) || (github.event.issue.pull_request) + github.event.pull_request || github.event.issue.pull_request uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} @@ -308,10 +352,8 @@ jobs: await main(); - name: Install GitHub Copilot CLI run: ${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh latest - env: - GH_HOST: github.com - name: Install AWF binary - run: bash ${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh v0.24.3 + run: bash ${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh v0.25.11 - name: Determine automatic lockdown mode for GitHub MCP Server id: determine-automatic-lockdown uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -323,7 +365,7 @@ jobs: const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs'); await determineAutomaticLockdown(github, context, core); - name: Download container images - run: bash ${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.24.3 ghcr.io/github/gh-aw-firewall/api-proxy:0.24.3 ghcr.io/github/gh-aw-firewall/squid:0.24.3 ghcr.io/github/gh-aw-mcpg:v0.1.19 ghcr.io/github/github-mcp-server:v0.32.0 + run: bash ${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.25.11 ghcr.io/github/gh-aw-firewall/api-proxy:0.25.11 ghcr.io/github/gh-aw-firewall/squid:0.25.11 ghcr.io/github/gh-aw-mcpg:v0.2.11 ghcr.io/github/github-mcp-server:v0.32.0 - name: Start MCP Gateway id: start-mcp-gateway env: @@ -346,10 +388,10 @@ jobs: export DEBUG="*" export GH_AW_ENGINE="copilot" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.19' + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.11' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh + cat << GH_AW_MCP_CONFIG_b03f9bc659383e57_EOF | bash ${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh { "mcpServers": { "github": { @@ -376,7 +418,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_EOF + GH_AW_MCP_CONFIG_b03f9bc659383e57_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -393,16 +435,16 @@ jobs: set -o pipefail touch /tmp/gh-aw/agent-step-summary.md # shellcheck disable=SC1003 - sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --allow-domains "*.jsr.io,*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.npms.io,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,bun.sh,cdn.jsdelivr.net,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,deb.nodesource.com,deno.land,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,esm.sh,get.pnpm.io,github.com,googleapis.deno.dev,googlechromelabs.github.io,host.docker.internal,json-schema.org,json.schemastore.org,jsr.io,keyserver.ubuntu.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,storage.googleapis.com,telemetry.enterprise.githubcopilot.com,telemetry.vercel.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com,www.npmjs.com,www.npmjs.org,yarnpkg.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.24.3 --skip-pull --enable-api-proxy \ - -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + sudo -E awf --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --allow-domains '*.jsr.io,*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.npms.io,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,bun.sh,cdn.jsdelivr.net,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,deb.nodesource.com,deno.land,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,esm.sh,get.pnpm.io,github.com,googleapis.deno.dev,googlechromelabs.github.io,host.docker.internal,json-schema.org,json.schemastore.org,jsr.io,keyserver.ubuntu.com,nodejs.org,npm.pkg.github.com,npmjs.com,npmjs.org,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.bower.io,registry.npmjs.com,registry.npmjs.org,registry.yarnpkg.com,repo.yarnpkg.com,s.symcb.com,s.symcd.com,security.ubuntu.com,skimdb.npmjs.com,storage.googleapis.com,telemetry.enterprise.githubcopilot.com,telemetry.vercel.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com,www.npmjs.com,www.npmjs.org,yarnpkg.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --image-tag 0.25.11 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_VERSION: v0.62.2 + GH_AW_VERSION: v0.65.6 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_HEAD_REF: ${{ github.head_ref }} @@ -436,20 +478,7 @@ jobs: - name: Copy Copilot session state files to logs if: always() continue-on-error: true - run: | - # Copy Copilot session state files to logs folder for artifact collection - # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them - SESSION_STATE_DIR="$HOME/.copilot/session-state" - LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" - - if [ -d "$SESSION_STATE_DIR" ]; then - echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" - mkdir -p "$LOGS_DIR" - cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true - echo "Session state files copied successfully" - else - echo "No session-state directory found at $SESSION_STATE_DIR" - fi + run: bash ${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh - name: Stop MCP Gateway if: always() continue-on-error: true @@ -469,8 +498,18 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); await main(); env: - GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,COPILOT_PAT_0,COPILOT_PAT_1,COPILOT_PAT_2,COPILOT_PAT_3,COPILOT_PAT_4,COPILOT_PAT_5,COPILOT_PAT_6,COPILOT_PAT_7,COPILOT_PAT_8,COPILOT_PAT_9,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_COPILOT_PAT_0: ${{ secrets.COPILOT_PAT_0 }} + SECRET_COPILOT_PAT_1: ${{ secrets.COPILOT_PAT_1 }} + SECRET_COPILOT_PAT_2: ${{ secrets.COPILOT_PAT_2 }} + SECRET_COPILOT_PAT_3: ${{ secrets.COPILOT_PAT_3 }} + SECRET_COPILOT_PAT_4: ${{ secrets.COPILOT_PAT_4 }} + SECRET_COPILOT_PAT_5: ${{ secrets.COPILOT_PAT_5 }} + SECRET_COPILOT_PAT_6: ${{ secrets.COPILOT_PAT_6 }} + SECRET_COPILOT_PAT_7: ${{ secrets.COPILOT_PAT_7 }} + SECRET_COPILOT_PAT_8: ${{ secrets.COPILOT_PAT_8 }} + SECRET_COPILOT_PAT_9: ${{ secrets.COPILOT_PAT_9 }} SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -490,6 +529,7 @@ jobs: await main(); - name: Parse MCP Gateway logs for step summary if: always() + id: parse-mcp-gateway uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: | @@ -512,10 +552,14 @@ jobs: else echo 'AWF binary not installed, skipping firewall log summary' fi + - name: Parse token usage for step summary + if: always() + continue-on-error: true + run: bash ${RUNNER_TEMP}/gh-aw/actions/parse_token_usage.sh - name: Upload agent artifacts if: always() continue-on-error: true - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: agent path: | @@ -523,8 +567,64 @@ jobs: /tmp/gh-aw/sandbox/agent/logs/ /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ - /tmp/gh-aw/sandbox/firewall/logs/ /tmp/gh-aw/agent-stdio.log /tmp/gh-aw/agent/ if-no-files-found: ignore + - name: Upload firewall audit logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: firewall-audit-logs + path: | + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + if-no-files-found: ignore + + pre_activation: + if: (!github.event.repository.fork) || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-slim + outputs: + activated: ${{ steps.check_membership.outputs.is_team_member == 'true' }} + copilot_pat_number: ${{ steps.select-copilot-pat.outputs.copilot_pat_number }} + matched_command: '' + select-copilot-pat_result: ${{ steps.select-copilot-pat.outcome }} + steps: + - name: Setup Scripts + uses: github/gh-aw-actions/setup@31130b20a8fd3ef263acbe2091267c0aace07e09 # v0.65.6 + with: + destination: ${{ runner.temp }}/gh-aw/actions + - name: Check team membership for workflow + id: check_membership + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_REQUIRED_ROLES: "admin,maintainer,write" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_membership.cjs'); + await main(); + - name: Checkout the select-copilot-pat action folder + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + persist-credentials: false + sparse-checkout: .github/actions/select-copilot-pat + sparse-checkout-cone-mode: true + - name: Select Copilot token from pool + id: select-copilot-pat + uses: ./.github/actions/select-copilot-pat + env: + SECRET_0: ${{ secrets.COPILOT_PAT_0 }} + SECRET_1: ${{ secrets.COPILOT_PAT_1 }} + SECRET_2: ${{ secrets.COPILOT_PAT_2 }} + SECRET_3: ${{ secrets.COPILOT_PAT_3 }} + SECRET_4: ${{ secrets.COPILOT_PAT_4 }} + SECRET_5: ${{ secrets.COPILOT_PAT_5 }} + SECRET_6: ${{ secrets.COPILOT_PAT_6 }} + SECRET_7: ${{ secrets.COPILOT_PAT_7 }} + SECRET_8: ${{ secrets.COPILOT_PAT_8 }} + SECRET_9: ${{ secrets.COPILOT_PAT_9 }} diff --git a/.github/workflows/conformance-tier-audit.md b/.github/workflows/conformance-tier-audit.md index cf436c5fc..171111e3b 100644 --- a/.github/workflows/conformance-tier-audit.md +++ b/.github/workflows/conformance-tier-audit.md @@ -1,7 +1,14 @@ --- +if: (!github.event.repository.fork) || github.event_name == 'workflow_dispatch' + +permissions: + contents: read + issues: read + pull-requests: read + on: schedule: - - cron: "0 14 * * 4" + - cron: "weekly on thursday around 9am utc-7" workflow_dispatch: inputs: conformance-repo: @@ -15,12 +22,48 @@ on: type: string default: 'main' -if: github.repository == 'modelcontextprotocol/csharp-sdk' || github.event_name == 'workflow_dispatch' - -permissions: - contents: read - issues: read - pull-requests: read + # ############################################################### + # Override the COPILOT_GITHUB_TOKEN secret usage for the workflow + # with a randomly-selected token from a pool of secrets. + # + # As soon as organization-level billing is offered for Agentic + # Workflows, this stop-gap approach will be removed. + # + # See: /.github/actions/select-copilot-pat/README.md + # ############################################################### + + # Add the pre-activation step of selecting a random PAT from the supplied secrets + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + name: Checkout the select-copilot-pat action folder + with: + persist-credentials: false + sparse-checkout: .github/actions/select-copilot-pat + sparse-checkout-cone-mode: true + fetch-depth: 1 + + - id: select-copilot-pat + name: Select Copilot token from pool + uses: ./.github/actions/select-copilot-pat + env: + # If the secret names are changed here, they must also be changed + # in the `engine: env` case expression + SECRET_0: ${{ secrets.COPILOT_PAT_0 }} + SECRET_1: ${{ secrets.COPILOT_PAT_1 }} + SECRET_2: ${{ secrets.COPILOT_PAT_2 }} + SECRET_3: ${{ secrets.COPILOT_PAT_3 }} + SECRET_4: ${{ secrets.COPILOT_PAT_4 }} + SECRET_5: ${{ secrets.COPILOT_PAT_5 }} + SECRET_6: ${{ secrets.COPILOT_PAT_6 }} + SECRET_7: ${{ secrets.COPILOT_PAT_7 }} + SECRET_8: ${{ secrets.COPILOT_PAT_8 }} + SECRET_9: ${{ secrets.COPILOT_PAT_9 }} + +# Add the pre-activation output of the randomly selected PAT +jobs: + pre-activation: + outputs: + copilot_pat_number: ${{ steps.select-copilot-pat.outputs.copilot_pat_number }} runtimes: dotnet: @@ -38,6 +81,15 @@ network: tools: bash: true github: true + +# Override the COPILOT_GITHUB_TOKEN expression used in the activation job +# Consume the PAT number from the pre-activation step and select the corresponding secret +engine: + id: copilot + env: + # We cannot use line breaks in this expression as it leads to a syntax error in the compiled workflow + # If none of the `COPILOT_PAT_#` secrets were selected, then the default COPILOT_GITHUB_TOKEN is used + COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} --- # Weekly Conformance Tier Audit diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..dbd4bd793 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "github.copilot.enable": { + "markdown": true + } +} \ No newline at end of file