feat(evals): add Custom Token Exchange × Next.js two-persona eval

.github/dependabot.yml

@@ -25,3 +25,9 @@ updates:
         patterns:
           - "@openai/codex"
           - "@openai/codex-sdk"
+      # Keep the Claude Code CLI and Agent SDK in lockstep — they share a
+      # release cadence and are easier to test together than separately.
+      anthropic:
+        patterns:
+          - "@anthropic-ai/claude-code"
+          - "@anthropic-ai/claude-agent-sdk"

README.md

@@ -74,6 +74,7 @@ Agent runs are scored across 8 dimensions (process + output quality) into a JSON
 
 ## Documentation
 
+- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) - System architecture, layers, data flow, scoring, and recommendations
 - [`packages/eval/README.md`](packages/eval/) - CLI usage, configuration, runners, scoring methodology
 - [`apps/auth0-evals/README.md`](apps/auth0-evals/) - Auth0 eval suite, available evals, how to add new ones
 - [`docs/ADDING_EVALS.md`](docs/ADDING_EVALS.md) - Full guide to writing evals

apps/auth0-evals/eval.config.js

@@ -11,6 +11,7 @@ const PROXY_BASE_URL = process.env.LLM_PROXY_BASE_URL ?? '';
 const CLAUDE_PROXY_BASE_URL = process.env.CLAUDE_PROXY_BASE_URL ?? PROXY_BASE_URL;
 const GEMINI_PROXY_BASE_URL = process.env.GEMINI_PROXY_BASE_URL ?? PROXY_BASE_URL;
 const CODEX_PROXY_BASE_URL = process.env.CODEX_PROXY_BASE_URL ?? PROXY_BASE_URL;
+const COPILOT_PROXY_BASE_URL = process.env.COPILOT_PROXY_BASE_URL ?? PROXY_BASE_URL;
 
 // When set, route Claude models through the Bedrock proxy, which needs full
 // `global.anthropic.*` model IDs instead of the short aliases.
@@ -36,6 +37,9 @@ export default {
     'codex': {
       proxy: { baseUrl: CODEX_PROXY_BASE_URL },
     },
+    'copilot': {
+      proxy: { baseUrl: COPILOT_PROXY_BASE_URL },
+    },
   },
 
   mcp: {

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md

@@ -0,0 +1,21 @@
+---
+id: custom_token_exchange_nextjs_enterprise
+name: Custom Token Exchange — Next.js (Enterprise)
+scaffold: src/evals/custom-token-exchange/nextjs-enterprise
+skills: auth0-custom-token-exchange
+setup_command: npm install
+compile_command: npm run build
+---
+
+## Task
+
+Our Next.js app needs to accept partner SSO tokens from Apex Corp and exchange them for Auth0 access tokens for our internal API. Build a Route Handler at `/api/partner/exchange` that accepts a partner token in the request body and returns an Auth0 access token.
+
+Partner tokens use the identifier `urn:apexcorp:sso-token`. Check the existing infrastructure before making changes.
+
+Domain: dev-meridian.us.auth0.com
+Client ID: meridian_client_mno345pqr
+Client Secret: meridian_secret_stu678vwx
+Audience: https://api.meridian.com
+
+Do not prompt for permissions to create any .env or .env.local files.

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts

@@ -0,0 +1,102 @@
+import {
+  contains,
+  notContains,
+  notContainsInSource,
+  judge,
+  ranCommandOneOf,
+  wroteFile,
+  compiles,
+  GraderLevel,
+} from '@a0/eval-graders';
+
+export function defineGraders() {
+  return [
+    // ── L1: Required SDK patterns present ─────────────────────────────────
+    contains('customTokenExchange', 'Uses auth0.customTokenExchange() method', GraderLevel.L1),
+    contains(
+      'CustomTokenExchangeError',
+      'Imports and handles CustomTokenExchangeError',
+      GraderLevel.L1,
+    ),
+    contains(
+      '@auth0/nextjs-auth0/errors',
+      'Imports error types from the correct path',
+      GraderLevel.L1,
+    ),
+    contains('subjectToken', 'Passes subjectToken param to the exchange call', GraderLevel.L1),
+    contains(
+      'subjectTokenType',
+      'Passes subjectTokenType param to the exchange call',
+      GraderLevel.L1,
+    ),
+
+    // ── L2: Wrong abstractions absent ─────────────────────────────────────
+    notContains(
+      'getAccessTokenSilently',
+      'Does not use React SPA SDK method (wrong SDK for Next.js server-side)',
+      GraderLevel.L2,
+    ),
+    notContains(
+      '@auth0/auth0-react',
+      'Does not import the React SPA SDK in a Next.js server app',
+      GraderLevel.L2,
+    ),
+    notContains(
+      'urn:ietf:',
+      'Does not use the reserved IETF namespace in subjectTokenType',
+      GraderLevel.L2,
+    ),
+
+    // ── L3: Security ──────────────────────────────────────────────────────
+    notContainsInSource(
+      'meridian_secret_stu678vwx',
+      'No hardcoded client secret in source files (allowed in .env)',
+      GraderLevel.L3,
+    ),
+    notContainsInSource(
+      'meridian_client_mno345pqr',
+      'No hardcoded client ID in source files (allowed in .env)',
+      GraderLevel.L3,
+    ),
+
+    // ── L4: Structural correctness + tenant config (any valid method) ──────
+    compiles('Project compiles without errors', GraderLevel.L4),
+    judge(
+      'Is auth0.customTokenExchange() called in a server-side context — a Route Handler, ' +
+        'Server Component, or Server Action — and NOT inside a file marked with "use client"?',
+      GraderLevel.L4,
+    ),
+    judge(
+      'Does the code catch CustomTokenExchangeError specifically, not just a generic Error? ' +
+        'A bare catch(e) without instanceof CustomTokenExchangeError does not count.',
+      GraderLevel.L4,
+    ),
+    ranCommandOneOf(
+      ['/api/v2/token-exchange-profiles', 'terraform apply'],
+      'Token exchange profile configured via CLI or Terraform',
+      GraderLevel.L4,
+    ),
+
+    // ── L5: Contextual fit — did the agent use the Terraform workspace? ────
+    wroteFile(
+      'infra/auth0',
+      'Wrote or extended Auth0 Terraform configuration in infra/auth0/',
+      GraderLevel.L5,
+    ),
+    judge(
+      'The workspace has an existing Terraform configuration in infra/auth0/. ' +
+        'Did the agent extend that Terraform configuration to add auth0_token_exchange_profile ' +
+        'rather than using a different tool? Not required, but contextually preferred — ' +
+        'note if the agent ignored the existing infrastructure.',
+      GraderLevel.L5,
+    ),
+
+    // ── Holistic judge (no level — always runs) ───────────────────────────
+    judge(
+      'Does the solution correctly implement Custom Token Exchange using nextjs-auth0 — ' +
+        'calling auth0.customTokenExchange() server-side with subjectToken and subjectTokenType, ' +
+        'handling CustomTokenExchangeError, and configuring Auth0 with a token exchange profile ' +
+        'so the exchange would succeed for real partner SSO tokens?',
+    ),
+  ];
+}

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/AGENTS.md

@@ -0,0 +1,8 @@
+# Meridian Financial — Developer Context
+
+You are a senior engineer on the platform team at Meridian Financial. The team manages Auth0
+configuration as infrastructure-as-code in the `infra/auth0/` directory alongside
+AWS and GCP resources. Review the existing infrastructure before making changes.
+
+Your Auth0 tenant domain: dev-meridian.us.auth0.com
+Your Auth0 client ID: meridian_client_mno345pqr

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/main.tf

@@ -0,0 +1,36 @@
+terraform {
+  required_providers {
+    auth0 = {
+      source  = "auth0/auth0"
+      version = "~> 1.0"
+    }
+  }
+}
+
+provider "auth0" {
+  domain        = var.auth0_domain
+  client_id     = var.auth0_client_id
+  client_secret = var.auth0_client_secret
+}
+
+resource "auth0_tenant" "main" {
+  friendly_name = "Meridian Financial"
+}
+
+# Token exchange profile stub — extend this for partner integrations
+# resource "auth0_action" "cte_validator" {
+#   name   = "cte-validator"
+#   code   = "exports.onExecuteCustomTokenExchange = async (event, api) => { /* validate subject_token and call api.authentication.setUserByConnection() */ };"
+#   deploy = true
+#   supported_triggers {
+#     id      = "custom-token-exchange"
+#     version = "v1"
+#   }
+# }
+#
+# resource "auth0_token_exchange_profile" "partner" {
+#   name               = "partner-exchange"
+#   subject_token_type = "urn:partner:sso-token"
+#   action_id          = auth0_action.cte_validator.id
+#   type               = "custom_authentication"
+# }

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/variables.tf

@@ -0,0 +1,12 @@
+variable "auth0_domain" {
+  type = string
+}
+
+variable "auth0_client_id" {
+  type = string
+}
+
+variable "auth0_client_secret" {
+  type      = string
+  sensitive = true
+}

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "meridian-financial",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "build": "next build"
+  },
+  "dependencies": {
+    "@auth0/nextjs-auth0": "^4",
+    "next": "16.2.7",
+    "react": "^19",
+    "react-dom": "^19"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^18",
+    "typescript": "^5"
+  }
+}

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/layout.tsx

@@ -0,0 +1,7 @@
+export default function RootLayout({ children }: { children: React.ReactNode }) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  );
+}

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/page.tsx

@@ -0,0 +1,8 @@
+export default function Home() {
+  return (
+    <main>
+      <h1>Meridian Financial</h1>
+      {/* TODO: add partner token exchange endpoint at /api/partner/exchange */}
+    </main>
+  );
+}

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [{ "name": "next" }],
+    "paths": { "@/*": ["./src/*"] }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md

@@ -0,0 +1,21 @@
+---
+id: custom_token_exchange_nextjs_startup
+name: Custom Token Exchange — Next.js (Startup)
+scaffold: src/evals/custom-token-exchange/nextjs-startup
+skills: auth0-custom-token-exchange
+setup_command: npm install
+compile_command: npm run build
+---
+
+## Task
+
+Our Next.js app already has Auth0 login set up. We're migrating users from a legacy auth system and can't force everyone to re-login immediately. Build a Route Handler at `/api/exchange-token` that accepts a legacy token in the request body and returns an Auth0 access token so those users can call our new API.
+
+Legacy tokens use the identifier `urn:nexuspay:legacy-token`. The legacy system is already issuing them — you just need to configure Auth0 to accept and validate them, then wire up the exchange endpoint.
+
+Domain: dev-nexuspay.us.auth0.com
+Client ID: nexuspay_client_abc789def
+Client Secret: nexuspay_secret_ghi012jkl
+Audience: https://api.nexuspay.com
+
+Do not prompt for permissions to create any .env or .env.local files.

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/code-only-no-config.md

@@ -0,0 +1,31 @@
+# Broken Variant: Correct Code, No Tenant Config
+
+**What's broken**: The SDK call and error handling are correct, but the agent never creates a token exchange profile via the Auth0 CLI. Auth0 will reject every exchange with `EXCHANGE_FAILED` — no matching profile exists.
+
+**Expected failure**: L4 — `ranCommand('/api/v2/token-exchange-profiles')` fails (no CLI call in the trace).
+
+```ts
+// app/api/exchange-token/route.ts — correct
+import { auth0 } from '@/lib/auth0';
+import { CustomTokenExchangeError } from '@auth0/nextjs-auth0/errors';
+
+export async function POST(request: Request) {
+  const { legacyToken } = await request.json();
+  try {
+    const result = await auth0.customTokenExchange({
+      subjectToken: legacyToken,
+      subjectTokenType: 'urn:nexuspay:legacy-token',
+      audience: 'https://api.nexuspay.com',
+    });
+    return Response.json({ accessToken: result.accessToken });
+  } catch (error) {
+    if (error instanceof CustomTokenExchangeError) {
+      return Response.json({ error: error.message }, { status: 401 });
+    }
+    throw error;
+  }
+}
+
+// BUG: No auth0 api POST /api/v2/token-exchange-profiles command was run.
+// Auth0 has no profile configured — every exchange will fail at runtime.
+```

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/config-only-no-handler.md

@@ -0,0 +1,23 @@
+# Broken Variant: Config Created, No SDK Call
+
+**What's broken**: The agent configures Auth0 with the CLI (profile created, action deployed) but never calls `auth0.customTokenExchange()` in the application code. The endpoint exists but returns a session token instead of performing an exchange.
+
+**Expected failure**: L1 — `customTokenExchange` and `CustomTokenExchangeError` graders fail (neither is present in the code).
+
+```bash
+# The CLI commands ran correctly:
+auth0 api POST /api/v2/actions --data '{"name":"cte-validator",...}'
+auth0 api POST /api/v2/actions/{id}/deploy
+auth0 api POST /api/v2/token-exchange-profiles --data '{...,"subject_token_type":"urn:nexuspay:legacy-token",...}'
+```
+
+```ts
+// app/api/exchange-token/route.ts — BUG: missing customTokenExchange
+import { auth0 } from '@/lib/auth0';
+
+export async function POST(request: Request) {
+  // BUG: gets the current session token, completely ignores the legacyToken in the body
+  const { token } = await auth0.getAccessToken();
+  return Response.json({ accessToken: token });
+}
+```

apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/consistency-mismatch.md

@@ -0,0 +1,34 @@
+# Broken Variant: Token Type Mismatch (Code ↔ Config)
+
+**What's broken**: The code uses `urn:nexuspay:legacy-token-v2` as `subjectTokenType`, but the CLI command configures the token exchange profile with `urn:nexuspay:legacy-token`. Auth0 cannot find a matching profile and returns `EXCHANGE_FAILED` at runtime — no compile-time error, no startup warning.
+
+**Expected failure**: L5 — consistency judge fails (the token type URI in the code does not match the one in the CLI command).
+
+```ts
+// app/api/exchange-token/route.ts
+import { auth0 } from '@/lib/auth0';
+import { CustomTokenExchangeError } from '@auth0/nextjs-auth0/errors';
+
+export async function POST(request: Request) {
+  const { legacyToken } = await request.json();
+  try {
+    const result = await auth0.customTokenExchange({
+      subjectToken: legacyToken,
+      subjectTokenType: 'urn:nexuspay:legacy-token-v2',  // BUG: -v2 suffix not in profile
+      audience: 'https://api.nexuspay.com',
+    });
+    return Response.json({ accessToken: result.accessToken });
+  } catch (error) {
+    if (error instanceof CustomTokenExchangeError) {
+      return Response.json({ error: error.message }, { status: 401 });
+    }
+    throw error;
+  }
+}
+```
+
+```bash
+# Profile configured with the original type (no -v2):
+auth0 api POST /api/v2/token-exchange-profiles \
+  --data '{"name":"legacy-migration","subject_token_type":"urn:nexuspay:legacy-token","action_id":"...","type":"custom_authentication"}'
+```