From b26c7508bfcdaef133b1cb4248dccc59e6f25950 Mon Sep 17 00:00:00 2001 From: Bharath Natarajan Date: Mon, 22 Jun 2026 14:12:58 +0530 Subject: [PATCH 01/23] =?UTF-8?q?feat(evals):=20add=20Custom=20Token=20Exc?= =?UTF-8?q?hange=20=C3=97=20Next.js=20two-persona=20eval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-leg graders covering SDK integration (auth0.customTokenExchange, CustomTokenExchangeError) and tenant configuration (token exchange profile via CLI or Terraform). Startup persona uses CLI; enterprise persona uses Terraform with a pre-seeded infra/auth0/ workspace. Graders: L1 SDK method + error import, L2 wrong-SDK anti-patterns, L3 credential hygiene, L4 structural + ranCommand/ranCommandOneOf config, L5 code↔config consistency + Terraform contextual fit. Validated against 4 known-broken variants (meta-eval.md). --- .../nextjs-enterprise/PROMPT.md | 20 ++++ .../nextjs-enterprise/graders.ts | 100 ++++++++++++++++++ .../nextjs-enterprise/scaffold/AGENTS.md | 8 ++ .../scaffold/infra/auth0/main.tf | 36 +++++++ .../scaffold/infra/auth0/variables.tf | 12 +++ .../nextjs-enterprise/scaffold/package.json | 19 ++++ .../scaffold/src/app/layout.tsx | 7 ++ .../scaffold/src/app/page.tsx | 8 ++ .../nextjs-enterprise/scaffold/tsconfig.json | 20 ++++ .../nextjs-startup/PROMPT.md | 20 ++++ .../broken-variants/code-only-no-config.md | 31 ++++++ .../broken-variants/config-only-no-handler.md | 23 ++++ .../broken-variants/consistency-mismatch.md | 34 ++++++ .../broken-variants/meta-eval.md | 22 ++++ .../broken-variants/wrong-abstraction.md | 23 ++++ .../nextjs-startup/graders.ts | 94 ++++++++++++++++ .../nextjs-startup/scaffold/AGENTS.md | 7 ++ .../nextjs-startup/scaffold/package.json | 19 ++++ .../scaffold/src/app/layout.tsx | 7 ++ .../nextjs-startup/scaffold/src/app/page.tsx | 8 ++ .../nextjs-startup/scaffold/tsconfig.json | 20 ++++ 21 files changed, 538 insertions(+) create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/AGENTS.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/main.tf create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/variables.tf create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/package.json create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/layout.tsx create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/page.tsx create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/tsconfig.json create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/code-only-no-config.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/config-only-no-handler.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/consistency-mismatch.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/meta-eval.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/wrong-abstraction.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/AGENTS.md create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/package.json create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/layout.tsx create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/page.tsx create mode 100644 apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/tsconfig.json diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md new file mode 100644 index 00000000..59fdb279 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md @@ -0,0 +1,20 @@ +--- +id: custom_token_exchange_nextjs_enterprise +name: Custom Token Exchange — Next.js (Enterprise) +scaffold: src/evals/custom-token-exchange/nextjs-enterprise +skills: auth0-custom-token-exchange +setup_command: npm install +--- + +## Task + +Our Next.js app needs to accept partner SSO tokens from Apex Corp and exchange them for Auth0 access tokens for our internal API. Build a Route Handler at `/api/partner/exchange` that accepts a partner token in the request body and returns an Auth0 access token. + +Partner tokens use the identifier `urn:apexcorp:sso-token`. Check the existing infrastructure before making changes. + +Domain: dev-meridian.us.auth0.com +Client ID: meridian_client_mno345pqr +Client Secret: meridian_secret_stu678vwx +Audience: https://api.meridian.com + +Do not prompt for permissions to create any .env or .env.local files. diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts new file mode 100644 index 00000000..e3eb5bdb --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts @@ -0,0 +1,100 @@ +import { + contains, + notContains, + notContainsInSource, + judge, + ranCommandOneOf, + wroteFile, + GraderLevel, +} from '@a0/eval-graders'; + +export function defineGraders() { + return [ + // ── L1: Required SDK patterns present ───────────────────────────────── + contains('customTokenExchange', 'Uses auth0.customTokenExchange() method', GraderLevel.L1), + contains( + 'CustomTokenExchangeError', + 'Imports and handles CustomTokenExchangeError', + GraderLevel.L1, + ), + contains( + '@auth0/nextjs-auth0/errors', + 'Imports error types from the correct path', + GraderLevel.L1, + ), + contains('subjectToken', 'Passes subjectToken param to the exchange call', GraderLevel.L1), + contains( + 'subjectTokenType', + 'Passes subjectTokenType param to the exchange call', + GraderLevel.L1, + ), + + // ── L2: Wrong abstractions absent ───────────────────────────────────── + notContains( + 'getAccessTokenSilently', + 'Does not use React SPA SDK method (wrong SDK for Next.js server-side)', + GraderLevel.L2, + ), + notContains( + '@auth0/auth0-react', + 'Does not import the React SPA SDK in a Next.js server app', + GraderLevel.L2, + ), + notContains( + 'urn:ietf:', + 'Does not use the reserved IETF namespace in subjectTokenType', + GraderLevel.L2, + ), + + // ── L3: Security ────────────────────────────────────────────────────── + notContainsInSource( + 'meridian_secret_stu678vwx', + 'No hardcoded client secret in source files (allowed in .env)', + GraderLevel.L3, + ), + notContainsInSource( + 'meridian_client_mno345pqr', + 'No hardcoded client ID in source files (allowed in .env)', + GraderLevel.L3, + ), + + // ── L4: Structural correctness + tenant config (any valid method) ────── + judge( + 'Is auth0.customTokenExchange() called in a server-side context — a Route Handler, ' + + 'Server Component, or Server Action — and NOT inside a file marked with "use client"?', + GraderLevel.L4, + ), + judge( + 'Does the code catch CustomTokenExchangeError specifically, not just a generic Error? ' + + 'A bare catch(e) without instanceof CustomTokenExchangeError does not count.', + GraderLevel.L4, + ), + ranCommandOneOf( + ['/api/v2/token-exchange-profiles', 'terraform apply'], + 'Token exchange profile configured via CLI or Terraform', + GraderLevel.L4, + ), + + // ── L5: Contextual fit — did the agent use the Terraform workspace? ──── + wroteFile( + 'infra/auth0', + 'Wrote or extended Auth0 Terraform configuration in infra/auth0/', + GraderLevel.L5, + ), + judge( + 'The workspace has an existing Terraform configuration in infra/auth0/. ' + + 'Did the agent extend that Terraform configuration to add auth0_token_exchange_profile ' + + 'rather than using a different tool? Not required, but contextually preferred — ' + + 'note if the agent ignored the existing infrastructure.', + GraderLevel.L5, + ), + + // ── Holistic judge (no level — always runs) ─────────────────────────── + judge( + 'Does the solution correctly implement Custom Token Exchange using nextjs-auth0 — ' + + 'calling auth0.customTokenExchange() server-side with subjectToken and subjectTokenType, ' + + 'handling CustomTokenExchangeError, and configuring Auth0 with a token exchange profile ' + + 'so the exchange would succeed for real partner SSO tokens?', + ), + ]; +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/AGENTS.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/AGENTS.md new file mode 100644 index 00000000..05b91bf3 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/AGENTS.md @@ -0,0 +1,8 @@ +# Meridian Financial — Developer Context + +You are a senior engineer on the platform team at Meridian Financial. The team manages Auth0 +configuration as infrastructure-as-code in the `infra/auth0/` directory alongside +AWS and GCP resources. Review the existing infrastructure before making changes. + +Your Auth0 tenant domain: dev-meridian.us.auth0.com +Your Auth0 client ID: meridian_client_mno345pqr diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/main.tf b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/main.tf new file mode 100644 index 00000000..9f98b86b --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/main.tf @@ -0,0 +1,36 @@ +terraform { + required_providers { + auth0 = { + source = "auth0/auth0" + version = "~> 1.0" + } + } +} + +provider "auth0" { + domain = var.auth0_domain + client_id = var.auth0_client_id + client_secret = var.auth0_client_secret +} + +resource "auth0_tenant" "main" { + friendly_name = "Meridian Financial" +} + +# Token exchange profile stub — extend this for partner integrations +# resource "auth0_action" "cte_validator" { +# name = "cte-validator" +# code = "exports.onExecuteCustomTokenExchange = async (event, api) => { /* validate subject_token and call api.authentication.setUserByConnection() */ };" +# deploy = true +# supported_triggers { +# id = "custom-token-exchange" +# version = "v1" +# } +# } +# +# resource "auth0_token_exchange_profile" "partner" { +# name = "partner-exchange" +# subject_token_type = "urn:partner:sso-token" +# action_id = auth0_action.cte_validator.id +# type = "custom_authentication" +# } diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/variables.tf b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/variables.tf new file mode 100644 index 00000000..69ba70ac --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/infra/auth0/variables.tf @@ -0,0 +1,12 @@ +variable "auth0_domain" { + type = string +} + +variable "auth0_client_id" { + type = string +} + +variable "auth0_client_secret" { + type = string + sensitive = true +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/package.json b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/package.json new file mode 100644 index 00000000..f22b77e6 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/package.json @@ -0,0 +1,19 @@ +{ + "name": "meridian-financial", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "next build" + }, + "dependencies": { + "@auth0/nextjs-auth0": "^4", + "next": "16.2.7", + "react": "^19", + "react-dom": "^19" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^18", + "typescript": "^5" + } +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/layout.tsx b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/layout.tsx new file mode 100644 index 00000000..f3ef34cd --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/layout.tsx @@ -0,0 +1,7 @@ +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/page.tsx b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/page.tsx new file mode 100644 index 00000000..2ca726d8 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/src/app/page.tsx @@ -0,0 +1,8 @@ +export default function Home() { + return ( +
+

Meridian Financial

+ {/* TODO: add partner token exchange endpoint at /api/partner/exchange */} +
+ ); +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/tsconfig.json b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/tsconfig.json new file mode 100644 index 00000000..49e4cf3b --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/scaffold/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./src/*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md new file mode 100644 index 00000000..9f29de6e --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md @@ -0,0 +1,20 @@ +--- +id: custom_token_exchange_nextjs_startup +name: Custom Token Exchange — Next.js (Startup) +scaffold: src/evals/custom-token-exchange/nextjs-startup +skills: auth0-custom-token-exchange +setup_command: npm install +--- + +## Task + +Our Next.js app already has Auth0 login set up. We're migrating users from a legacy auth system and can't force everyone to re-login immediately. Build a Route Handler at `/api/exchange-token` that accepts a legacy token in the request body and returns an Auth0 access token so those users can call our new API. + +Legacy tokens use the identifier `urn:nexuspay:legacy-token`. The legacy system is already issuing them — you just need to configure Auth0 to accept and validate them, then wire up the exchange endpoint. + +Domain: dev-nexuspay.us.auth0.com +Client ID: nexuspay_client_abc789def +Client Secret: nexuspay_secret_ghi012jkl +Audience: https://api.nexuspay.com + +Do not prompt for permissions to create any .env or .env.local files. diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/code-only-no-config.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/code-only-no-config.md new file mode 100644 index 00000000..3731dfda --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/code-only-no-config.md @@ -0,0 +1,31 @@ +# Broken Variant: Correct Code, No Tenant Config + +**What's broken**: The SDK call and error handling are correct, but the agent never creates a token exchange profile via the Auth0 CLI. Auth0 will reject every exchange with `EXCHANGE_FAILED` — no matching profile exists. + +**Expected failure**: L4 — `ranCommand('/api/v2/token-exchange-profiles')` fails (no CLI call in the trace). + +```ts +// app/api/exchange-token/route.ts — correct +import { auth0 } from '@/lib/auth0'; +import { CustomTokenExchangeError } from '@auth0/nextjs-auth0/errors'; + +export async function POST(request: Request) { + const { legacyToken } = await request.json(); + try { + const result = await auth0.customTokenExchange({ + subjectToken: legacyToken, + subjectTokenType: 'urn:nexuspay:legacy-token', + audience: 'https://api.nexuspay.com', + }); + return Response.json({ accessToken: result.accessToken }); + } catch (error) { + if (error instanceof CustomTokenExchangeError) { + return Response.json({ error: error.message }, { status: 401 }); + } + throw error; + } +} + +// BUG: No auth0 api POST /api/v2/token-exchange-profiles command was run. +// Auth0 has no profile configured — every exchange will fail at runtime. +``` diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/config-only-no-handler.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/config-only-no-handler.md new file mode 100644 index 00000000..8b4bd924 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/config-only-no-handler.md @@ -0,0 +1,23 @@ +# Broken Variant: Config Created, No SDK Call + +**What's broken**: The agent configures Auth0 with the CLI (profile created, action deployed) but never calls `auth0.customTokenExchange()` in the application code. The endpoint exists but returns a session token instead of performing an exchange. + +**Expected failure**: L1 — `customTokenExchange` and `CustomTokenExchangeError` graders fail (neither is present in the code). + +```bash +# The CLI commands ran correctly: +auth0 api POST /api/v2/actions --data '{"name":"cte-validator",...}' +auth0 api POST /api/v2/actions/{id}/deploy +auth0 api POST /api/v2/token-exchange-profiles --data '{...,"subject_token_type":"urn:nexuspay:legacy-token",...}' +``` + +```ts +// app/api/exchange-token/route.ts — BUG: missing customTokenExchange +import { auth0 } from '@/lib/auth0'; + +export async function POST(request: Request) { + // BUG: gets the current session token, completely ignores the legacyToken in the body + const { token } = await auth0.getAccessToken(); + return Response.json({ accessToken: token }); +} +``` diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/consistency-mismatch.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/consistency-mismatch.md new file mode 100644 index 00000000..41bf3f5e --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/consistency-mismatch.md @@ -0,0 +1,34 @@ +# Broken Variant: Token Type Mismatch (Code ↔ Config) + +**What's broken**: The code uses `urn:nexuspay:legacy-token-v2` as `subjectTokenType`, but the CLI command configures the token exchange profile with `urn:nexuspay:legacy-token`. Auth0 cannot find a matching profile and returns `EXCHANGE_FAILED` at runtime — no compile-time error, no startup warning. + +**Expected failure**: L5 — consistency judge fails (the token type URI in the code does not match the one in the CLI command). + +```ts +// app/api/exchange-token/route.ts +import { auth0 } from '@/lib/auth0'; +import { CustomTokenExchangeError } from '@auth0/nextjs-auth0/errors'; + +export async function POST(request: Request) { + const { legacyToken } = await request.json(); + try { + const result = await auth0.customTokenExchange({ + subjectToken: legacyToken, + subjectTokenType: 'urn:nexuspay:legacy-token-v2', // BUG: -v2 suffix not in profile + audience: 'https://api.nexuspay.com', + }); + return Response.json({ accessToken: result.accessToken }); + } catch (error) { + if (error instanceof CustomTokenExchangeError) { + return Response.json({ error: error.message }, { status: 401 }); + } + throw error; + } +} +``` + +```bash +# Profile configured with the original type (no -v2): +auth0 api POST /api/v2/token-exchange-profiles \ + --data '{"name":"legacy-migration","subject_token_type":"urn:nexuspay:legacy-token","action_id":"...","type":"custom_authentication"}' +``` diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/meta-eval.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/meta-eval.md new file mode 100644 index 00000000..3edf2400 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/meta-eval.md @@ -0,0 +1,22 @@ +# Meta-eval: Custom Token Exchange × Next.js (Startup) + +Verifies that graders correctly discriminate against broken implementations. +Each variant must fail at the indicated level — if any pass, tighten the grader. + +| Variant | What's broken | Expected failure | +|---------|--------------|-----------------| +| `wrong-abstraction.md` | Uses `auth0.getAccessToken()` instead of `auth0.customTokenExchange()`; no `CustomTokenExchangeError` | L1: `customTokenExchange` and `CustomTokenExchangeError` contains() both fail | +| `code-only-no-config.md` | Correct SDK code; no token exchange profile created via CLI | L4: `ranCommand('/api/v2/token-exchange-profiles')` fails | +| `config-only-no-handler.md` | CLI configures profile correctly; code never calls `customTokenExchange()` | L1: `customTokenExchange` contains() fails | +| `consistency-mismatch.md` | Code uses `urn:nexuspay:legacy-token-v2`; CLI uses `urn:nexuspay:legacy-token` | L5: consistency judge fails | + +## META-EVAL GATE: custom-token-exchange × nextjs-startup + +``` +✅ wrong-abstraction: L1 rejects (customTokenExchange absent) +✅ code-only-no-config: L4 rejects (ranCommand absent) +✅ config-only-no-handler: L1 rejects (customTokenExchange absent) +✅ consistency-mismatch: L5 rejects (token type mismatch) + +STATUS: READY TO MERGE +``` diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/wrong-abstraction.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/wrong-abstraction.md new file mode 100644 index 00000000..91b32a22 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/broken-variants/wrong-abstraction.md @@ -0,0 +1,23 @@ +# Broken Variant: Wrong Abstraction + +**What's broken**: Uses `auth0.getAccessToken()` to fetch the current session's access token rather than calling `auth0.customTokenExchange()` to exchange a legacy token. The code also omits `CustomTokenExchangeError` entirely. + +**Expected failure**: L1 — `customTokenExchange` and `CustomTokenExchangeError` graders fail (neither is present). + +```ts +// app/api/exchange-token/route.ts +import { auth0 } from '@/lib/auth0'; + +export async function POST(request: Request) { + const { legacyToken } = await request.json(); + + // BUG: getAccessToken() returns the current session's access token — + // it does NOT exchange the legacyToken for a new one. + try { + const { token } = await auth0.getAccessToken(); + return Response.json({ accessToken: token }); + } catch (error) { + return Response.json({ error: 'Failed' }, { status: 401 }); + } +} +``` diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts new file mode 100644 index 00000000..352341df --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts @@ -0,0 +1,94 @@ +import { + contains, + notContains, + notContainsInSource, + judge, + ranCommand, + GraderLevel, +} from '@a0/eval-graders'; + +export function defineGraders() { + return [ + // ── L1: Required SDK patterns present ───────────────────────────────── + contains('customTokenExchange', 'Uses auth0.customTokenExchange() method', GraderLevel.L1), + contains( + 'CustomTokenExchangeError', + 'Imports and handles CustomTokenExchangeError', + GraderLevel.L1, + ), + contains( + '@auth0/nextjs-auth0/errors', + 'Imports error types from the correct path', + GraderLevel.L1, + ), + contains('subjectToken', 'Passes subjectToken param to the exchange call', GraderLevel.L1), + contains( + 'subjectTokenType', + 'Passes subjectTokenType param to the exchange call', + GraderLevel.L1, + ), + + // ── L2: Wrong abstractions absent ───────────────────────────────────── + notContains( + 'getAccessTokenSilently', + 'Does not use React SPA SDK method (wrong SDK for Next.js server-side)', + GraderLevel.L2, + ), + notContains( + '@auth0/auth0-react', + 'Does not import the React SPA SDK in a Next.js server app', + GraderLevel.L2, + ), + notContains( + 'urn:ietf:', + 'Does not use the reserved IETF namespace in subjectTokenType', + GraderLevel.L2, + ), + + // ── L3: Security ────────────────────────────────────────────────────── + notContainsInSource( + 'nexuspay_secret_ghi012jkl', + 'No hardcoded client secret in source files (allowed in .env)', + GraderLevel.L3, + ), + notContainsInSource( + 'nexuspay_client_abc789def', + 'No hardcoded client ID in source files (allowed in .env)', + GraderLevel.L3, + ), + + // ── L4: Structural correctness + tenant config ───────────────────────── + judge( + 'Is auth0.customTokenExchange() called in a server-side context — a Route Handler, ' + + 'Server Component, or Server Action — and NOT inside a file marked with "use client"?', + GraderLevel.L4, + ), + judge( + 'Does the code catch CustomTokenExchangeError specifically, not just a generic Error? ' + + 'A bare catch(e) without instanceof CustomTokenExchangeError does not count.', + GraderLevel.L4, + ), + ranCommand( + 'auth0', + ['/api/v2/token-exchange-profiles'], + 'Created a token exchange profile via Auth0 CLI', + GraderLevel.L4, + ), + + // ── L5: Consistency (code ↔ config agree) ───────────────────────────── + judge( + 'Does the subjectTokenType value passed to auth0.customTokenExchange() match the ' + + 'subject_token_type used in the auth0 CLI command that creates the token exchange profile? ' + + 'Both should use the same token type URI — a mismatch causes EXCHANGE_FAILED at runtime.', + GraderLevel.L5, + ), + + // ── Holistic judge (no level — always runs) ─────────────────────────── + judge( + 'Does the solution correctly implement Custom Token Exchange using nextjs-auth0 — ' + + 'calling auth0.customTokenExchange() server-side with subjectToken and subjectTokenType, ' + + 'handling CustomTokenExchangeError, and configuring Auth0 with a token exchange profile ' + + 'so the exchange would succeed for real legacy tokens?', + ), + ]; +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/AGENTS.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/AGENTS.md new file mode 100644 index 00000000..0fafb18f --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/AGENTS.md @@ -0,0 +1,7 @@ +# NexusPay — Developer Context + +You are the founding engineer at a Series A fintech startup. The team configures +the Auth0 tenant directly using the Auth0 CLI and Management API. + +Your Auth0 tenant domain: dev-nexuspay.us.auth0.com +Your Auth0 client ID: nexuspay_client_abc789def diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/package.json b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/package.json new file mode 100644 index 00000000..290d3fe6 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/package.json @@ -0,0 +1,19 @@ +{ + "name": "nexuspay", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "next build" + }, + "dependencies": { + "@auth0/nextjs-auth0": "^4", + "next": "16.2.7", + "react": "^19", + "react-dom": "^19" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^18", + "typescript": "^5" + } +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/layout.tsx b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/layout.tsx new file mode 100644 index 00000000..f3ef34cd --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/layout.tsx @@ -0,0 +1,7 @@ +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/page.tsx b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/page.tsx new file mode 100644 index 00000000..2cf6c5b0 --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/src/app/page.tsx @@ -0,0 +1,8 @@ +export default function Home() { + return ( +
+

NexusPay

+ {/* TODO: add token exchange endpoint at /api/exchange-token */} +
+ ); +} diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/tsconfig.json b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/tsconfig.json new file mode 100644 index 00000000..49e4cf3b --- /dev/null +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/scaffold/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./src/*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} From de1ca2f49ed2d613b3f5a3a5da22e6cef597a36e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:46:46 +0000 Subject: [PATCH 02/23] chore(deps): bump @google/gemini-cli from 0.45.2 to 0.46.0 Bumps [@google/gemini-cli](https://github.com/google-gemini/gemini-cli) from 0.45.2 to 0.46.0. - [Release notes](https://github.com/google-gemini/gemini-cli/releases) - [Changelog](https://github.com/google-gemini/gemini-cli/blob/main/docs/releases.md) - [Commits](https://github.com/google-gemini/gemini-cli/compare/v0.45.2...v0.46.0) --- updated-dependencies: - dependency-name: "@google/gemini-cli" dependency-version: 0.46.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- package-lock.json | 8 ++++---- packages/eval/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index d7bb4961..94e47f93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1196,8 +1196,8 @@ } }, "node_modules/@google/gemini-cli": { - "version": "0.45.2", - "integrity": "sha512-Gm9UrodMu8Tw0hk2ncz1b8po5zhjA8tyED6SpDbkuJ91NjwzoYJ0s7sHlnJ1RpFUYuK6I+zMQMgtRqCcd+FbnA==", + "version": "0.46.0", + "integrity": "sha512-+HZtuuDKsL8mvOUWgK08GkrL1BQM4IplaSzxIjAM262FmJFp3Jo/zlHUq9ulRKcvx0agZ4KAzuj7jG9yIAxFBw==", "license": "Apache-2.0", "bin": { "gemini": "bundle/gemini.js" @@ -6099,9 +6099,9 @@ "@a0/eval-reporter": "*", "@ai-sdk/openai": "^3.0.69", "@anthropic-ai/claude-agent-sdk": "^0.3.173", - "@anthropic-ai/claude-code": "^2.1.173", + "@anthropic-ai/claude-code": "2.1.173", "@github/copilot-sdk": "^0.3.0", - "@google/gemini-cli": "^0.45.2", + "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.139.0", "@openai/codex-sdk": "^0.139.0", "ai": "^6.0.201", diff --git a/packages/eval/package.json b/packages/eval/package.json index bcf0b23c..71953e87 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -20,7 +20,7 @@ "@anthropic-ai/claude-agent-sdk": "^0.3.173", "@anthropic-ai/claude-code": "2.1.173", "@github/copilot-sdk": "^0.3.0", - "@google/gemini-cli": "^0.45.2", + "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.139.0", "@openai/codex-sdk": "^0.139.0", "ai": "^6.0.201", From 57fa0221ad42f260852eebe9f3eefe859cf67e88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:37:13 +0000 Subject: [PATCH 03/23] chore(deps): bump braintrust from 3.10.0 to 3.17.0 Bumps [braintrust](https://github.com/braintrustdata/braintrust-sdk-javascript/tree/HEAD/js) from 3.10.0 to 3.17.0. - [Release notes](https://github.com/braintrustdata/braintrust-sdk-javascript/releases) - [Changelog](https://github.com/braintrustdata/braintrust-sdk-javascript/blob/main/js/CHANGELOG.md) - [Commits](https://github.com/braintrustdata/braintrust-sdk-javascript/commits/braintrust@3.17.0/js) --- updated-dependencies: - dependency-name: braintrust dependency-version: 3.17.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- package-lock.json | 865 +++++++++---------------------------- packages/eval/package.json | 2 +- 2 files changed, 198 insertions(+), 669 deletions(-) diff --git a/package-lock.json b/package-lock.json index 94e47f93..c884bcb6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -137,17 +137,6 @@ "node": ">=18" } }, - "node_modules/@ai-sdk/provider": { - "version": "1.1.3", - "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/@ai-sdk/provider-utils": { "version": "4.0.27", "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", @@ -517,6 +506,90 @@ "node": ">=18" } }, + "node_modules/@braintrust/bt-darwin-arm64": { + "version": "0.11.1", + "integrity": "sha512-jhL/X24ss4e4qMMdlXtxO8rA957Z77wJA59XXFHqpuaaVCd4pXE5JPUQBkms9dHcs4efJhY3Lx9zNQqoaeCqWg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@braintrust/bt-darwin-x64": { + "version": "0.11.1", + "integrity": "sha512-f4l25gVUpCJ99mFS9y+zmnYOcevtI7KLLvlLF/xOil2YCIx/KCM6AqS96jj6CJW74B7hYhd74dLnFKMFFL429w==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@braintrust/bt-linux-arm64": { + "version": "0.11.1", + "integrity": "sha512-01GWsP/p17I3yy0kxkp+Yt8w5L28j3nFL+7iLCkQWtDdfCGkuRsnrIbrY8dbnqGo/wvgz7uPXBkCXoIuNsfoxw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@braintrust/bt-linux-x64": { + "version": "0.11.1", + "integrity": "sha512-E/XwRuhPrZxD+IZgSbPCuj4gywBrim/g+tU0MjWxFohpMMkJJW2CxMCIO+6RVk8gTxlVh/ajCIjv2/Ph1Yugeg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@braintrust/bt-linux-x64-musl": { + "version": "0.11.1", + "integrity": "sha512-QLqlFsF6HKON5Vc0c8JfpQ4vrl4KjmInGF1Vsqy+ecVkgXVk8pwCVibb8Ea/udVpBQqctAaNMn7ZsmvWR2vO8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@braintrust/bt-win32-arm64": { + "version": "0.11.1", + "integrity": "sha512-0LSVXZ/tE79VVcpRjaTE+iTMQR4EKNC6tteAS01uKT34eID7OqJ7xJijZOthe11kGqMcX8nnNbdDrWHqLczDow==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@braintrust/bt-win32-x64": { + "version": "0.11.1", + "integrity": "sha512-JPo3xffJvW0OKowqpbh+XtlafpoZq8VXzhOcW2yQmHcN56Me2u9plPiXi4gzrpgz2cnFx6/EiJ1bTa3F25F0RA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@colors/colors": { "version": "1.5.0", "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", @@ -558,8 +631,8 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.7", - "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "version": "0.28.0", + "integrity": "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==", "cpu": [ "ppc64" ], @@ -573,8 +646,8 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.7", - "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "version": "0.28.0", + "integrity": "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==", "cpu": [ "arm" ], @@ -588,8 +661,8 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.7", - "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "version": "0.28.0", + "integrity": "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==", "cpu": [ "arm64" ], @@ -603,8 +676,8 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.7", - "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "version": "0.28.0", + "integrity": "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==", "cpu": [ "x64" ], @@ -618,8 +691,8 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.7", - "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "version": "0.28.0", + "integrity": "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==", "cpu": [ "arm64" ], @@ -633,8 +706,8 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.7", - "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "version": "0.28.0", + "integrity": "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==", "cpu": [ "x64" ], @@ -648,8 +721,8 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.7", - "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "version": "0.28.0", + "integrity": "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==", "cpu": [ "arm64" ], @@ -663,8 +736,8 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.7", - "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "version": "0.28.0", + "integrity": "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==", "cpu": [ "x64" ], @@ -678,8 +751,8 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.7", - "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "version": "0.28.0", + "integrity": "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==", "cpu": [ "arm" ], @@ -693,8 +766,8 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.7", - "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "version": "0.28.0", + "integrity": "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==", "cpu": [ "arm64" ], @@ -708,8 +781,8 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.7", - "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "version": "0.28.0", + "integrity": "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==", "cpu": [ "ia32" ], @@ -723,8 +796,8 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.7", - "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "version": "0.28.0", + "integrity": "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==", "cpu": [ "loong64" ], @@ -738,8 +811,8 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.7", - "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "version": "0.28.0", + "integrity": "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==", "cpu": [ "mips64el" ], @@ -753,8 +826,8 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.7", - "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "version": "0.28.0", + "integrity": "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==", "cpu": [ "ppc64" ], @@ -768,8 +841,8 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.7", - "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "version": "0.28.0", + "integrity": "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==", "cpu": [ "riscv64" ], @@ -783,8 +856,8 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.7", - "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "version": "0.28.0", + "integrity": "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==", "cpu": [ "s390x" ], @@ -798,8 +871,8 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.7", - "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "version": "0.28.0", + "integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==", "cpu": [ "x64" ], @@ -813,8 +886,8 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.7", - "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "version": "0.28.0", + "integrity": "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==", "cpu": [ "arm64" ], @@ -828,8 +901,8 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.7", - "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "version": "0.28.0", + "integrity": "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==", "cpu": [ "x64" ], @@ -843,8 +916,8 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.7", - "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "version": "0.28.0", + "integrity": "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==", "cpu": [ "arm64" ], @@ -858,8 +931,8 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.7", - "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "version": "0.28.0", + "integrity": "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==", "cpu": [ "x64" ], @@ -873,8 +946,8 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.7", - "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "version": "0.28.0", + "integrity": "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==", "cpu": [ "arm64" ], @@ -888,8 +961,8 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.7", - "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "version": "0.28.0", + "integrity": "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==", "cpu": [ "x64" ], @@ -903,8 +976,8 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.7", - "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "version": "0.28.0", + "integrity": "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==", "cpu": [ "arm64" ], @@ -918,8 +991,8 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.7", - "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "version": "0.28.0", + "integrity": "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==", "cpu": [ "ia32" ], @@ -933,8 +1006,8 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.7", - "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "version": "0.28.0", + "integrity": "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==", "cpu": [ "x64" ], @@ -2377,59 +2450,6 @@ } } }, - "node_modules/ansi-align": { - "version": "3.0.1", - "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", - "license": "ISC", - "dependencies": { - "string-width": "^4.1.0" - } - }, - "node_modules/ansi-align/node_modules/ansi-regex": { - "version": "5.0.1", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-align/node_modules/emoji-regex": { - "version": "8.0.0", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/ansi-align/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-align/node_modules/string-width": { - "version": "4.2.3", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-align/node_modules/strip-ansi": { - "version": "6.0.1", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/ansi-escapes": { "version": "7.3.0", "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", @@ -2448,6 +2468,7 @@ "node_modules/ansi-regex": { "version": "6.2.2", "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -2459,6 +2480,7 @@ "node_modules/ansi-styles": { "version": "6.2.3", "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -2472,11 +2494,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, - "node_modules/array-flatten": { - "version": "1.1.1", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, "node_modules/asap": { "version": "2.0.6", "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", @@ -2517,7 +2534,6 @@ "node_modules/balanced-match": { "version": "4.0.4", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, "license": "MIT", "engines": { "node": "18 || 20 || >=22" @@ -2547,58 +2563,9 @@ "url": "https://opencollective.com/express" } }, - "node_modules/boxen": { - "version": "8.0.1", - "integrity": "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw==", - "license": "MIT", - "dependencies": { - "ansi-align": "^3.0.1", - "camelcase": "^8.0.0", - "chalk": "^5.3.0", - "cli-boxes": "^3.0.0", - "string-width": "^7.2.0", - "type-fest": "^4.21.0", - "widest-line": "^5.0.0", - "wrap-ansi": "^9.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/boxen/node_modules/chalk": { - "version": "5.6.2", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/boxen/node_modules/string-width": { - "version": "7.2.0", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/brace-expansion": { "version": "5.0.6", "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -2608,123 +2575,53 @@ } }, "node_modules/braintrust": { - "version": "3.10.0", - "integrity": "sha512-On1OyvjBoatLI6BRWIQJ2Qpt4ed5MHXonH4vPzrQ3E5d7wiNGQ5DDw0dvEQhPVDIx9+u8f9h14JBCgv2fDMQyQ==", + "version": "3.17.0", + "integrity": "sha512-nyV+j/FJJJsWnkiSn9tAoNSTsMtDfbH4v8EQpBTYGj1120eXFPcPPs66kkkKcYuN0tEo/Ai7VO8Ujcy5j3SrUQ==", + "hasInstallScript": true, "license": "MIT", "dependencies": { - "@ai-sdk/provider": "^1.1.3", "@apm-js-collab/code-transformer": "^0.12.0", "@next/env": "^14.2.3", "@vercel/functions": "^1.0.2", - "ajv": "^8.17.1", + "ajv": "^8.20.0", "argparse": "^2.0.1", - "boxen": "^8.0.1", - "chalk": "^4.1.2", "cli-progress": "^3.12.0", "cli-table3": "^0.6.5", "cors": "^2.8.5", "dc-browser": "^1.0.4", "dotenv": "^16.4.5", - "esbuild": "^0.27.0", + "esbuild": "0.28.0", "eventsource-parser": "^1.1.2", - "express": "^4.21.2", - "graceful-fs": "^4.2.11", + "express": "^5.2.1", "http-errors": "^2.0.0", - "minimatch": "^9.0.3", + "minimatch": "^10.2.5", "module-details-from-path": "^1.0.4", "mustache": "^4.2.0", "pluralize": "^8.0.0", - "simple-git": "^3.21.0", + "simple-git": "^3.36.0", "source-map": "^0.7.4", "termi-link": "^1.0.1", "unplugin": "^2.3.5", - "uuid": "^9.0.1", + "uuid": "^11.1.1", "zod-to-json-schema": "^3.25.0" }, "bin": { - "braintrust": "dist/cli.js" + "braintrust": "dist/cli.js", + "bt": "bin/bt" + }, + "optionalDependencies": { + "@braintrust/bt-darwin-arm64": "0.11.1", + "@braintrust/bt-darwin-x64": "0.11.1", + "@braintrust/bt-linux-arm64": "0.11.1", + "@braintrust/bt-linux-x64": "0.11.1", + "@braintrust/bt-linux-x64-musl": "0.11.1", + "@braintrust/bt-win32-arm64": "0.11.1", + "@braintrust/bt-win32-x64": "0.11.1" }, "peerDependencies": { "zod": "^3.25.34 || ^4.0" } }, - "node_modules/braintrust/node_modules/accepts": { - "version": "1.3.8", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", - "license": "MIT", - "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/balanced-match": { - "version": "1.0.2", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/braintrust/node_modules/body-parser": { - "version": "1.20.5", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/braintrust/node_modules/brace-expansion": { - "version": "2.1.0", - "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/braintrust/node_modules/content-disposition": { - "version": "0.5.4", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/cookie-signature": { - "version": "1.0.7", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", - "license": "MIT" - }, - "node_modules/braintrust/node_modules/debug": { - "version": "2.6.9", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/braintrust/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/braintrust/node_modules/dotenv": { "version": "16.6.1", "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", @@ -2744,212 +2641,6 @@ "node": ">=14.18" } }, - "node_modules/braintrust/node_modules/express": { - "version": "4.22.2", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/braintrust/node_modules/finalhandler": { - "version": "1.3.2", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/braintrust/node_modules/fresh": { - "version": "0.5.2", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/iconv-lite": { - "version": "0.4.24", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/braintrust/node_modules/media-typer": { - "version": "0.3.0", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/merge-descriptors": { - "version": "1.0.3", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/braintrust/node_modules/mime-db": { - "version": "1.52.0", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/mime-types": { - "version": "2.1.35", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/minimatch": { - "version": "9.0.9", - "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.2" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/braintrust/node_modules/negotiator": { - "version": "0.6.3", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/braintrust/node_modules/path-to-regexp": { - "version": "0.1.13", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" - }, - "node_modules/braintrust/node_modules/raw-body": { - "version": "2.5.3", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/braintrust/node_modules/send": { - "version": "0.19.2", - "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.1", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "~2.4.1", - "range-parser": "~1.2.1", - "statuses": "~2.0.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/braintrust/node_modules/serve-static": { - "version": "1.16.3", - "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", - "license": "MIT", - "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "~0.19.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/braintrust/node_modules/type-is": { - "version": "1.6.18", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "license": "MIT", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/bytes": { "version": "3.1.2", "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", @@ -2985,17 +2676,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/camelcase": { - "version": "8.0.0", - "integrity": "sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==", - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/chai": { "version": "6.2.2", "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", @@ -3005,46 +2685,6 @@ "node": ">=18" } }, - "node_modules/chalk": { - "version": "4.1.2", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/ansi-styles": { - "version": "4.3.0", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cli-boxes": { - "version": "3.0.0", - "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/cli-cursor": { "version": "5.0.0", "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", @@ -3191,22 +2831,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/color-convert": { - "version": "2.0.1", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, "node_modules/commander": { "version": "15.0.0", "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", @@ -3323,15 +2947,6 @@ "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", @@ -3373,6 +2988,7 @@ "node_modules/emoji-regex": { "version": "10.6.0", "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "dev": true, "license": "MIT" }, "node_modules/encodeurl": { @@ -3429,8 +3045,8 @@ } }, "node_modules/esbuild": { - "version": "0.27.7", - "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "version": "0.28.0", + "integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==", "hasInstallScript": true, "license": "MIT", "bin": { @@ -3440,32 +3056,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.7", - "@esbuild/android-arm": "0.27.7", - "@esbuild/android-arm64": "0.27.7", - "@esbuild/android-x64": "0.27.7", - "@esbuild/darwin-arm64": "0.27.7", - "@esbuild/darwin-x64": "0.27.7", - "@esbuild/freebsd-arm64": "0.27.7", - "@esbuild/freebsd-x64": "0.27.7", - "@esbuild/linux-arm": "0.27.7", - "@esbuild/linux-arm64": "0.27.7", - "@esbuild/linux-ia32": "0.27.7", - "@esbuild/linux-loong64": "0.27.7", - "@esbuild/linux-mips64el": "0.27.7", - "@esbuild/linux-ppc64": "0.27.7", - "@esbuild/linux-riscv64": "0.27.7", - "@esbuild/linux-s390x": "0.27.7", - "@esbuild/linux-x64": "0.27.7", - "@esbuild/netbsd-arm64": "0.27.7", - "@esbuild/netbsd-x64": "0.27.7", - "@esbuild/openbsd-arm64": "0.27.7", - "@esbuild/openbsd-x64": "0.27.7", - "@esbuild/openharmony-arm64": "0.27.7", - "@esbuild/sunos-x64": "0.27.7", - "@esbuild/win32-arm64": "0.27.7", - "@esbuild/win32-ia32": "0.27.7", - "@esbuild/win32-x64": "0.27.7" + "@esbuild/aix-ppc64": "0.28.0", + "@esbuild/android-arm": "0.28.0", + "@esbuild/android-arm64": "0.28.0", + "@esbuild/android-x64": "0.28.0", + "@esbuild/darwin-arm64": "0.28.0", + "@esbuild/darwin-x64": "0.28.0", + "@esbuild/freebsd-arm64": "0.28.0", + "@esbuild/freebsd-x64": "0.28.0", + "@esbuild/linux-arm": "0.28.0", + "@esbuild/linux-arm64": "0.28.0", + "@esbuild/linux-ia32": "0.28.0", + "@esbuild/linux-loong64": "0.28.0", + "@esbuild/linux-mips64el": "0.28.0", + "@esbuild/linux-ppc64": "0.28.0", + "@esbuild/linux-riscv64": "0.28.0", + "@esbuild/linux-s390x": "0.28.0", + "@esbuild/linux-x64": "0.28.0", + "@esbuild/netbsd-arm64": "0.28.0", + "@esbuild/netbsd-x64": "0.28.0", + "@esbuild/openbsd-arm64": "0.28.0", + "@esbuild/openbsd-x64": "0.28.0", + "@esbuild/openharmony-arm64": "0.28.0", + "@esbuild/sunos-x64": "0.28.0", + "@esbuild/win32-arm64": "0.28.0", + "@esbuild/win32-ia32": "0.28.0", + "@esbuild/win32-x64": "0.28.0" } }, "node_modules/escape-html": { @@ -3921,6 +3537,7 @@ "node_modules/get-east-asian-width": { "version": "1.6.0", "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -3987,14 +3604,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "license": "ISC" - }, "node_modules/has-flag": { "version": "4.0.0", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4727,25 +4340,6 @@ "node": ">=18.0.0" } }, - "node_modules/methods": { - "version": "1.1.2", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/mime-db": { "version": "1.54.0", "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", @@ -4786,7 +4380,6 @@ "node_modules/minimatch": { "version": "10.2.5", "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.5" @@ -5281,25 +4874,6 @@ "node": ">= 18" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/safer-buffer": { "version": "2.1.2", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", @@ -5574,6 +5148,7 @@ "node_modules/strip-ansi": { "version": "7.2.0", "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^6.2.2" @@ -5588,6 +5163,7 @@ "node_modules/supports-color": { "version": "7.2.0", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -5715,17 +5291,6 @@ "node": ">= 0.8.0" } }, - "node_modules/type-fest": { - "version": "4.41.0", - "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/type-is": { "version": "2.0.1", "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", @@ -5822,24 +5387,16 @@ "punycode": "^2.1.0" } }, - "node_modules/utils-merge": { - "version": "1.0.1", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/uuid": { - "version": "14.0.0", - "integrity": "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==", + "version": "11.1.1", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], "license": "MIT", "bin": { - "uuid": "dist-node/bin/uuid" + "uuid": "dist/esm/bin/uuid" } }, "node_modules/vary": { @@ -5970,36 +5527,6 @@ "node": ">=8" } }, - "node_modules/widest-line": { - "version": "5.0.0", - "integrity": "sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==", - "license": "MIT", - "dependencies": { - "string-width": "^7.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/widest-line/node_modules/string-width": { - "version": "7.2.0", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/word-wrap": { "version": "1.2.5", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", @@ -6012,6 +5539,7 @@ "node_modules/wrap-ansi": { "version": "9.0.2", "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^6.2.1", @@ -6028,6 +5556,7 @@ "node_modules/wrap-ansi/node_modules/string-width": { "version": "7.2.0", "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^10.3.0", @@ -6105,7 +5634,7 @@ "@openai/codex": "^0.139.0", "@openai/codex-sdk": "^0.139.0", "ai": "^6.0.201", - "braintrust": "^3.9.0", + "braintrust": "^3.17.0", "commander": "^15.0.0", "dotenv": "^17.4.2", "p-limit": "^7.3.0" diff --git a/packages/eval/package.json b/packages/eval/package.json index 71953e87..1548f1c2 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -24,7 +24,7 @@ "@openai/codex": "^0.139.0", "@openai/codex-sdk": "^0.139.0", "ai": "^6.0.201", - "braintrust": "^3.9.0", + "braintrust": "^3.17.0", "commander": "^15.0.0", "dotenv": "^17.4.2", "p-limit": "^7.3.0" From 9a9b1e48048071ef1dbc87a9949136b8042bdfed Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Mon, 15 Jun 2026 15:46:45 +0200 Subject: [PATCH 04/23] fix(core): don't apply Bedrock model map to judge and recommendations (#33) The modelIds map holds Bedrock IDs (global.anthropic.*) needed only by the Claude Code agent runner, which routes through the proxy's /anthropic pass-through endpoint. The LLM judge and the recommendation generator both POST to the LiteLLM /chat/completions endpoint, which serves models under their plain alias. Applying the map there rewrote e.g. claude-opus-4-8 to global.anthropic.claude-opus-4-8, which LiteLLM rejects with a 400. Send the model alias as-is from both paths and drop the now-unused modelMap plumbing from the judge. Add regression tests asserting the alias is sent unchanged even when a Bedrock map is configured. --- packages/eval-core/src/graders/engine.ts | 3 -- .../src/graders/executors/llm-judge.ts | 3 +- .../eval-core/src/graders/executors/types.ts | 2 - packages/eval-core/src/graders/llm-judge.ts | 7 ++- .../eval-core/tests/graders/engine.test.ts | 22 ++++++++ .../eval/src/recommendations/generator.ts | 9 ++-- packages/eval/tests/recommendations.test.ts | 51 +++++++++++++------ 7 files changed, 66 insertions(+), 31 deletions(-) diff --git a/packages/eval-core/src/graders/engine.ts b/packages/eval-core/src/graders/engine.ts index 5277336f..9b9ac994 100644 --- a/packages/eval-core/src/graders/engine.ts +++ b/packages/eval-core/src/graders/engine.ts @@ -10,7 +10,6 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { getFrameworkConfig } from '../config/framework-config.js'; -import { getModelIdMap } from '../config/settings.js'; import { collectFiles as collectFilePaths } from '../workspace/index.js'; import type { GraderDef, GraderResult, EventToolCall } from '@a0/eval-graders'; import { GraderLevel } from '@a0/eval-graders'; @@ -122,7 +121,6 @@ export async function runGraders( const judgeMaxCodeChars = config.judge.maxCodeChars ?? 32_768; const judgeMaxTokens = config.judge.maxTokens ?? 1024; const judgeBaseUrl = config.proxy.baseUrl; - const judgeModelMap = getModelIdMap(); const active = allowedLevels ? graderDefs.filter((g) => g.level === undefined || allowedLevels.has(g.level)) : graderDefs; @@ -143,7 +141,6 @@ export async function runGraders( baseUrl: judgeBaseUrl, maxTokens: judgeMaxTokens, maxCodeChars: judgeMaxCodeChars, - modelMap: judgeModelMap, enforceMaxChars, }, toolCalls, diff --git a/packages/eval-core/src/graders/executors/llm-judge.ts b/packages/eval-core/src/graders/executors/llm-judge.ts index 41e9c946..e9870e99 100644 --- a/packages/eval-core/src/graders/executors/llm-judge.ts +++ b/packages/eval-core/src/graders/executors/llm-judge.ts @@ -48,7 +48,7 @@ export const llmJudgeExecutor: GraderExecutor = { }; } - const { model, baseUrl, maxTokens, maxCodeChars, modelMap, enforceMaxChars } = ctx.judge; + const { model, baseUrl, maxTokens, maxCodeChars, enforceMaxChars } = ctx.judge; const judgeEntries = Object.entries(ctx.files).filter(([k]) => !isJudgeExcluded(k)); const judgeText = judgeEntries.map(([k, v]) => `// FILE: ${k}\n${v}`).join('\n\n'); @@ -68,7 +68,6 @@ export const llmJudgeExecutor: GraderExecutor = { model, baseUrl, maxTokens, - modelMap, enforceMaxChars, maxCodeChars, }); diff --git a/packages/eval-core/src/graders/executors/types.ts b/packages/eval-core/src/graders/executors/types.ts index f08f7994..17a2252c 100644 --- a/packages/eval-core/src/graders/executors/types.ts +++ b/packages/eval-core/src/graders/executors/types.ts @@ -32,8 +32,6 @@ export interface GraderContext { baseUrl: string; /** Maximum tokens for judge response. */ maxTokens: number; - /** LiteLLM model alias map resolved from FrameworkConfig. */ - modelMap: Record; /** Maximum code characters for judge input. */ maxCodeChars: number; /** Whether to enforce the max chars limit (throws vs warns). */ diff --git a/packages/eval-core/src/graders/llm-judge.ts b/packages/eval-core/src/graders/llm-judge.ts index 14bbbc34..cf3b806e 100644 --- a/packages/eval-core/src/graders/llm-judge.ts +++ b/packages/eval-core/src/graders/llm-judge.ts @@ -21,8 +21,6 @@ export interface LlmJudgeOptions { maxTokens?: number; enforceMaxChars?: boolean; maxCodeChars?: number; - /** LiteLLM model alias map. When provided, model is resolved via this map before the API call. */ - modelMap?: Record; } export interface LlmJudgeResult { @@ -61,10 +59,11 @@ export async function llmJudge(opts: LlmJudgeOptions): Promise { logger.warn(`[judge] Code corpus exceeds limit (${code.length} > ${judgeMaxCodeChars} chars) — proceeding anyway`); } const user = USER_TEMPLATE.replace('{question}', question).replace('{code}', code); - const apiModel = (opts.modelMap ?? {})[model] ?? model; + // The judge hits the /chat/completions endpoint, which serves models under + // their plain alias, so the model is sent as-is (no Bedrock ID mapping). const payload = JSON.stringify({ - model: apiModel, + model, messages: [ { role: 'system', content: system }, { role: 'user', content: user }, diff --git a/packages/eval-core/tests/graders/engine.test.ts b/packages/eval-core/tests/graders/engine.test.ts index 18f8b9cd..11ebdcee 100644 --- a/packages/eval-core/tests/graders/engine.test.ts +++ b/packages/eval-core/tests/graders/engine.test.ts @@ -465,6 +465,28 @@ describe('llmJudge', () => { expect(capturedBody?.max_tokens).toBe(512); }); + it('sends the model as-is (no Bedrock ID mapping on the chat endpoint)', async () => { + // Regression: the judge hits the /chat/completions endpoint, which serves + // models under their plain alias. Mapping the alias to a Bedrock ID here + // produced model="global.anthropic.*" → 400. + let capturedBody: Record | undefined; + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(async (_url: string, opts: RequestInit) => { + capturedBody = JSON.parse(opts.body as string) as Record; + return { ok: true, json: async () => ({ choices: [{ message: { content: 'yes' } }] }) }; + }), + ); + await llmJudge({ + question: 'question', + code: 'code', + apiKey: 'key', + model: 'claude-opus-4-8', + baseUrl: 'http://test', + }); + expect(capturedBody?.model).toBe('claude-opus-4-8'); + }); + it('throws JudgeError when baseUrl is empty', async () => { await expect( llmJudge({ question: 'question', code: 'code', apiKey: 'key', model: 'model', baseUrl: '' }), diff --git a/packages/eval/src/recommendations/generator.ts b/packages/eval/src/recommendations/generator.ts index ef32d300..af7a7efc 100644 --- a/packages/eval/src/recommendations/generator.ts +++ b/packages/eval/src/recommendations/generator.ts @@ -5,7 +5,7 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { collectFiles, logger, getModelIdMap } from '@a0/eval-core'; +import { collectFiles, logger } from '@a0/eval-core'; import type { RunRecord, ScoredResult, Recommendations, Recommendation } from '@a0/eval-core'; /** Maximum characters of workspace code to include in the prompt. */ @@ -165,11 +165,12 @@ Analyze this run and provide your recommendations as JSON.`; // ── LLM call ────────────────────────────────────────────────────────────────── async function callLlm(system: string, user: string, apiKey: string, baseUrl: string, model: string): Promise { - const modelMap = getModelIdMap(); - const apiModel = modelMap[model] ?? model; + // The modelIds map holds Bedrock IDs for the agent runner's /anthropic + // endpoint. This call hits the /chat/completions endpoint, which serves + // models under their plain alias — so the alias is sent as-is. const url = `${baseUrl}/chat/completions`; const body = { - model: apiModel, + model, messages: [ { role: 'system', content: system }, { role: 'user', content: user }, diff --git a/packages/eval/tests/recommendations.test.ts b/packages/eval/tests/recommendations.test.ts index 6becdd27..62b42d98 100644 --- a/packages/eval/tests/recommendations.test.ts +++ b/packages/eval/tests/recommendations.test.ts @@ -329,24 +329,43 @@ describe('generateRecommendations', () => { expect(result).toBeUndefined(); }); - it('resolves model through LiteLLM model map when present', async () => { - const { generateRecommendations } = await import('../src/recommendations/generator.js'); - const dir = tmpDir(); - - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ - choices: [{ message: { content: JSON.stringify({ recommendations: [], summary: '' }) } }], - }), + it('sends the model alias as-is, ignoring the Bedrock modelIds map', async () => { + // The modelIds map holds Bedrock IDs for the agent runner's /anthropic + // endpoint. Recommendations hit the /chat/completions endpoint, which serves + // models under their plain alias — so even when a Bedrock map is configured, + // the alias must be sent unchanged (regression: applying the map here + // produced model="global.anthropic.claude-opus-4-8" → 400). + const { setFrameworkConfig } = await import('@a0/eval-core'); + const { TEST_CONFIG } = await import('./test-config.js'); + setFrameworkConfig({ + ...TEST_CONFIG, + models: { + ...TEST_CONFIG.models, + modelIds: { 'claude-sonnet-4-6': 'global.anthropic.claude-sonnet-4-6' }, + }, }); - globalThis.fetch = fetchMock; - const input = makeInput(dir); - input.judgeModel = 'claude-sonnet-4-6'; - await generateRecommendations(input); - - const body = JSON.parse(fetchMock.mock.calls[0][1].body); - expect(body.model).toBe('claude-sonnet-4-6'); // empty modelIds map → alias passes through + try { + const { generateRecommendations } = await import('../src/recommendations/generator.js'); + const dir = tmpDir(); + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + choices: [{ message: { content: JSON.stringify({ recommendations: [], summary: '' }) } }], + }), + }); + globalThis.fetch = fetchMock; + + const input = makeInput(dir); + input.judgeModel = 'claude-sonnet-4-6'; + await generateRecommendations(input); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.model).toBe('claude-sonnet-4-6'); + } finally { + setFrameworkConfig(TEST_CONFIG); + } }); it('truncates workspace files at MAX_WORKSPACE_CHARS', async () => { From f49b04c9f193b24fd61963487b5eaf07f041e68a Mon Sep 17 00:00:00 2001 From: sanchitmehtagit Date: Mon, 15 Jun 2026 19:32:55 +0530 Subject: [PATCH 05/23] fix(swift-eval): accept fluent webAuth().start() chain in L5 grader (#34) --- apps/auth0-evals/src/evals/quickstarts/swift/graders.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/auth0-evals/src/evals/quickstarts/swift/graders.ts b/apps/auth0-evals/src/evals/quickstarts/swift/graders.ts index 56fcf0a8..6b4e8023 100644 --- a/apps/auth0-evals/src/evals/quickstarts/swift/graders.ts +++ b/apps/auth0-evals/src/evals/quickstarts/swift/graders.ts @@ -36,8 +36,11 @@ export function defineGraders() { ), // ── L5: Version-specific API correctness ────────────────────────────────── + // Allow chained builder calls (e.g. `.scope(...)`, `.useEphemeralSession()`) + // and newlines between webAuth() and .start() — the idiomatic fluent form + // `Auth0.webAuth().scope("...").start()` is correct async/await usage. matches( - String.raw`webAuth\(\)\.start\(\)`, + String.raw`webAuth\(\)(?:\s*\.\w+\([^)]*\))*\s*\.start\(\)`, 'Uses async/await webAuth().start() syntax (not completion handlers)', GraderLevel.L5, ), From 3e5e82eea20d2a59197f1994467ce7184cf6324b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:31:35 +0200 Subject: [PATCH 06/23] chore(deps-dev): bump typescript-eslint from 8.61.0 to 8.61.1 (#40) Bumps [typescript-eslint](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/typescript-eslint) from 8.61.0 to 8.61.1. - [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases) - [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/main/packages/typescript-eslint/CHANGELOG.md) - [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v8.61.1/packages/typescript-eslint) --- updated-dependencies: - dependency-name: typescript-eslint dependency-version: 8.61.1 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 128 +++++++++++++++++++--------------------------- package.json | 2 +- 2 files changed, 54 insertions(+), 76 deletions(-) diff --git a/package-lock.json b/package-lock.json index c884bcb6..f403f193 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "prettier": "^3.8.4", "turbo": "^2.9.18", "typescript": "^6.0.3", - "typescript-eslint": "^8.61.0" + "typescript-eslint": "^8.61.1" } }, "apps/auth0-evals": { @@ -2110,16 +2110,16 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.61.0", - "integrity": "sha512-bFNvl9ZczlVb+wR2Akszf3gHfKVj/8WanXaGJ3UstTA7brNKg0cNdk6X1Psu5V7MZ2oQtzZKOEzIUehaoxbDGw==", + "version": "8.61.1", + "integrity": "sha512-ZPlVl3PB3et/59Ne0fv/sci6ZXz4T4Hp4nTJ56i/Y0gR89ARb+KphojTq6j+56E5PIezmOIOOWyY+aWQFd+IkQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.61.0", - "@typescript-eslint/type-utils": "8.61.0", - "@typescript-eslint/utils": "8.61.0", - "@typescript-eslint/visitor-keys": "8.61.0", + "@typescript-eslint/scope-manager": "8.61.1", + "@typescript-eslint/type-utils": "8.61.1", + "@typescript-eslint/utils": "8.61.1", + "@typescript-eslint/visitor-keys": "8.61.1", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" @@ -2132,7 +2132,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.61.0", + "@typescript-eslint/parser": "^8.61.1", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } @@ -2147,15 +2147,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.61.0", - "integrity": "sha512-5B7PfA2e1NQGCnDHd/0lW7W3gvp3d59Ryw54FYO8Uswxo9f6ikw3AZV+Xj/TvpImmpsiYyUqAfhC6kJID1jF6w==", + "version": "8.61.1", + "integrity": "sha512-PJ5vePq5/ognBbrIcoC5+SHO5dfpeLPzP9FpLkzWrguoYQEeeSjlJpVwOpo1JRSTEi7dRcwNy4h4dzV70PqHcg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.61.0", - "@typescript-eslint/types": "8.61.0", - "@typescript-eslint/typescript-estree": "8.61.0", - "@typescript-eslint/visitor-keys": "8.61.0", + "@typescript-eslint/scope-manager": "8.61.1", + "@typescript-eslint/types": "8.61.1", + "@typescript-eslint/typescript-estree": "8.61.1", + "@typescript-eslint/visitor-keys": "8.61.1", "debug": "^4.4.3" }, "engines": { @@ -2171,13 +2171,13 @@ } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.61.0", - "integrity": "sha512-DV42F7MLJO6Rax7SK1yg43tcnEfGUrurSpSxKuVX+a3RCTzBlH3fuxprrOJXKCJGAaw82xXocikJ0uQaqwXgGA==", + "version": "8.61.1", + "integrity": "sha512-PrC4JYGmR241lYnfhmKGTXkFqv8+ymbTFgSAY0fVXpY82/QkMw5TZPl+vGzuDDU2QYJk9fIDOBTntF+yDv9LEA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.61.0", - "@typescript-eslint/types": "^8.61.0", + "@typescript-eslint/tsconfig-utils": "^8.61.1", + "@typescript-eslint/types": "^8.61.1", "debug": "^4.4.3" }, "engines": { @@ -2192,13 +2192,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.61.0", - "integrity": "sha512-IWdXFHFSb6mlC3HPc7QsLDm5zYEbUla6trDEHf32D3/dnuUyXd87plScSNXSbm0/RxMvObpI17sv/EDTGrGZkA==", + "version": "8.61.1", + "integrity": "sha512-L2bdIeoQS8FlKAvONAr20w6OcLXeB+qiDKbAooS9A0Ben+iSIkBef0FxqwKWYqt5sa0i4KJtxVyVmhMylKzF5w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.61.0", - "@typescript-eslint/visitor-keys": "8.61.0" + "@typescript-eslint/types": "8.61.1", + "@typescript-eslint/visitor-keys": "8.61.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2209,8 +2209,8 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.61.0", - "integrity": "sha512-O5Amvdv9ztMpxpf+vmFULGG78IE6Qwdr3bCGvqwG4nwc9H2qXkOYJJnRbRHyMkQTjv1d03olqwwwzHLMqpFePQ==", + "version": "8.61.1", + "integrity": "sha512-UN/H4di+OO7EWx2ovME+8t31YO+KVnK0RRKEHR3kOt21/Ay8BOq3M1OMvWs5vNiqcFCYGYoxK3MXPZzmMUE+yg==", "dev": true, "license": "MIT", "engines": { @@ -2225,14 +2225,14 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.61.0", - "integrity": "sha512-TuBiQYIkd97yBfInHCTKVYMbX4kvEmpOEuixIuzCU9p8BGT1SfyyO0d0IfDMbPIHcjn/hWnusUX5e8v5Xg+X8A==", + "version": "8.61.1", + "integrity": "sha512-GYRicKmVK0C4fsKgaACaknOUAq9Oa2kwsjnpFhFcS/5p4Ht5IP9OVLbgIgcK4SRk92nVHFluurg1lumD9dBcLw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.61.0", - "@typescript-eslint/typescript-estree": "8.61.0", - "@typescript-eslint/utils": "8.61.0", + "@typescript-eslint/types": "8.61.1", + "@typescript-eslint/typescript-estree": "8.61.1", + "@typescript-eslint/utils": "8.61.1", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, @@ -2249,8 +2249,8 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.61.0", - "integrity": "sha512-9QTQpZ5Iin4CdIodfbDQFSeiSJKidgYJYug1P9CC2xWgUTvlmixViqDZNciMjwLBZyJnG4tGmPl97rVAFb1AJg==", + "version": "8.61.1", + "integrity": "sha512-G+CRlPqLv7Bz1IZVs03x5K59F1veqL0EJUROAdGhKsEq8qOiRiZbI+HUojPq5l0fEGOKModD9br6lObhB8zkoA==", "dev": true, "license": "MIT", "engines": { @@ -2262,15 +2262,15 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.61.0", - "integrity": "sha512-42zatd5qSvvcV1JdDBCLxYRznvP4eIHpPoZXdkPFnAmanA4FuZ5dibSnCBggY8hQnqajPpoGjXFdZ7fIJKQnlA==", + "version": "8.61.1", + "integrity": "sha512-u+oQD3BqYWPc8YV9Zab4vaJElJuwOLPRc10Jm1o/qS+6Qwen14HCWwx0Seo4LnSn2wxea2Ik8DxPt2/FHmuhrg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.61.0", - "@typescript-eslint/tsconfig-utils": "8.61.0", - "@typescript-eslint/types": "8.61.0", - "@typescript-eslint/visitor-keys": "8.61.0", + "@typescript-eslint/project-service": "8.61.1", + "@typescript-eslint/tsconfig-utils": "8.61.1", + "@typescript-eslint/types": "8.61.1", + "@typescript-eslint/visitor-keys": "8.61.1", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", @@ -2289,15 +2289,15 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.61.0", - "integrity": "sha512-3bzFt7ImFMW/jVYwJamDoe/dMOdFLSC6pom6rRjdh4SZJEYupyMzem8e7vKZLclLfpHjlwSAXOUxtKxGXUiLqA==", + "version": "8.61.1", + "integrity": "sha512-1+P/3Dj6jvtybE1q0HQ6yBt/gq+oKJyLdEv4HdnqasaEXRSYCAsD59mXEVQnM/ULNdQxbX77tdG4jPRjIS6knA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.61.0", - "@typescript-eslint/types": "8.61.0", - "@typescript-eslint/typescript-estree": "8.61.0" + "@typescript-eslint/scope-manager": "8.61.1", + "@typescript-eslint/types": "8.61.1", + "@typescript-eslint/typescript-estree": "8.61.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2312,12 +2312,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.61.0", - "integrity": "sha512-QVLZu3ZPQEE+HICQyAMZ2yLQhxf0meY/wx6Hx14YcTNj13JB3qHlX3lJ02L3fLGHgERRH71kvYDwiXIguT3AjQ==", + "version": "8.61.1", + "integrity": "sha512-6fJ9MHWtK14C1DSkiMlHUSOmrVebL7150xZJBlJiL62jjhIA4JmOq6flwBgDxIdBKKdoiZRel+dfPD5MLfny3w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.61.0", + "@typescript-eslint/types": "8.61.1", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -2361,7 +2361,6 @@ "version": "2.0.0", "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "license": "MIT", - "peer": true, "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" @@ -2543,7 +2542,6 @@ "version": "2.2.2", "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", "license": "MIT", - "peer": true, "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", @@ -2843,7 +2841,6 @@ "version": "1.1.0", "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -2878,7 +2875,6 @@ "version": "1.2.2", "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", "license": "MIT", - "peer": true, "engines": { "node": ">=6.6.0" } @@ -3321,7 +3317,6 @@ "version": "5.2.1", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3443,7 +3438,6 @@ "version": "2.1.1", "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "license": "MIT", - "peer": true, "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", @@ -3507,7 +3501,6 @@ "version": "2.0.0", "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.8" } @@ -3688,7 +3681,6 @@ "version": "0.7.2", "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "license": "MIT", - "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -3779,8 +3771,7 @@ "node_modules/is-promise": { "version": "4.0.0", "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/isexe": { "version": "2.0.0", @@ -4315,7 +4306,6 @@ "version": "1.1.0", "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.8" } @@ -4324,7 +4314,6 @@ "version": "2.0.0", "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -4344,7 +4333,6 @@ "version": "1.54.0", "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.6" } @@ -4353,7 +4341,6 @@ "version": "3.0.2", "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "license": "MIT", - "peer": true, "dependencies": { "mime-db": "^1.54.0" }, @@ -4437,7 +4424,6 @@ "version": "1.0.0", "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.6" } @@ -4543,7 +4529,6 @@ "version": "1.4.0", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "license": "ISC", - "peer": true, "dependencies": { "wrappy": "1" } @@ -4639,7 +4624,6 @@ "version": "8.4.2", "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "license": "MIT", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" @@ -4784,7 +4768,6 @@ "version": "3.0.2", "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", - "peer": true, "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", @@ -4862,7 +4845,6 @@ "version": "2.2.0", "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", "license": "MIT", - "peer": true, "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", @@ -4900,7 +4882,6 @@ "version": "1.2.1", "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", "license": "MIT", - "peer": true, "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", @@ -4926,7 +4907,6 @@ "version": "2.2.1", "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "license": "MIT", - "peer": true, "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", @@ -5295,7 +5275,6 @@ "version": "2.0.1", "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "license": "MIT", - "peer": true, "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", @@ -5319,15 +5298,15 @@ } }, "node_modules/typescript-eslint": { - "version": "8.61.0", - "integrity": "sha512-8y31Rd0eGTrDKqhy6vT0HtzhN+YLjQizwX3aA3hPXP/ynSfnrBXcQY5IzsP9/DM7+klX4IUncZZjkchP0z+rUw==", + "version": "8.61.1", + "integrity": "sha512-V7PayAfJokV3pEHgN7/v03D1SpujhRfQtYLbLIiBfDDncdg4PAiRBfoS4cnCANK4jmAPncczi59QO3afiXUlNw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.61.0", - "@typescript-eslint/parser": "8.61.0", - "@typescript-eslint/typescript-estree": "8.61.0", - "@typescript-eslint/utils": "8.61.0" + "@typescript-eslint/eslint-plugin": "8.61.1", + "@typescript-eslint/parser": "8.61.1", + "@typescript-eslint/typescript-estree": "8.61.1", + "@typescript-eslint/utils": "8.61.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -5573,8 +5552,7 @@ "node_modules/wrappy": { "version": "1.0.2", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/yaml": { "version": "2.9.0", diff --git a/package.json b/package.json index 571b6994..537aa5a0 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "prettier": "^3.8.4", "turbo": "^2.9.18", "typescript": "^6.0.3", - "typescript-eslint": "^8.61.0" + "typescript-eslint": "^8.61.1" }, "overrides": { "uuid": "^14.0.0" From 5b30051c7bb73570c9be7b683b5657a775d9b49c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:31:54 +0200 Subject: [PATCH 07/23] chore(deps-dev): bump @vitest/coverage-v8 from 4.1.8 to 4.1.9 (#41) Bumps [@vitest/coverage-v8](https://github.com/vitest-dev/vitest/tree/HEAD/packages/coverage-v8) from 4.1.8 to 4.1.9. - [Release notes](https://github.com/vitest-dev/vitest/releases) - [Changelog](https://github.com/vitest-dev/vitest/blob/main/docs/releases.md) - [Commits](https://github.com/vitest-dev/vitest/commits/HEAD/packages/coverage-v8) --- updated-dependencies: - dependency-name: "@vitest/coverage-v8" dependency-version: 4.1.9 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 1177 ++++++--------------------- packages/eval-core/package.json | 2 +- packages/eval-graders/package.json | 2 +- packages/eval-reporter/package.json | 2 +- packages/eval/package.json | 2 +- 5 files changed, 255 insertions(+), 930 deletions(-) diff --git a/package-lock.json b/package-lock.json index f403f193..11bc87eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2352,6 +2352,142 @@ "node": ">= 20" } }, + "node_modules/@vitest/coverage-v8": { + "version": "4.1.9", + "integrity": "sha512-G9/lgqibheLVBDRuya45EbsEXTYcWoSG+TLg7i2axuzx0Eq62eXn+aWXyaVdV5vKvFSWd6ywcX8hA7la9Pvu8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.9", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.9", + "vitest": "4.1.9" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.9", + "integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.9", + "integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.9", + "integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.9", + "integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.9", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.9", + "integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.9", + "@vitest/utils": "4.1.9", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.9", + "integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.9", + "integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.9", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/a-sync-waterfall": { "version": "1.0.1", "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==", @@ -5463,6 +5599,95 @@ } } }, + "node_modules/vitest": { + "version": "4.1.9", + "integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.9", + "@vitest/mocker": "4.1.9", + "@vitest/pretty-format": "4.1.9", + "@vitest/runner": "4.1.9", + "@vitest/snapshot": "4.1.9", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.9", + "@vitest/browser-preview": "4.1.9", + "@vitest/browser-webdriverio": "4.1.9", + "@vitest/coverage-istanbul": "4.1.9", + "@vitest/coverage-v8": "4.1.9", + "@vitest/ui": "4.1.9", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, "node_modules/vscode-jsonrpc": { "version": "8.2.1", "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", @@ -5623,7 +5848,7 @@ "devDependencies": { "@a0/eval-graders": "*", "@types/node": "^25.9.3", - "@vitest/coverage-v8": "^4.1.6", + "@vitest/coverage-v8": "^4.1.9", "vitest": "^4.1.8" } }, @@ -5635,948 +5860,48 @@ }, "devDependencies": { "@types/node": "^25.9.3", - "@vitest/coverage-v8": "^4.1.6", + "@vitest/coverage-v8": "^4.1.9", "vitest": "^4.1.8" } }, - "packages/eval-core/node_modules/@vitest/coverage-v8": { - "version": "4.1.8", - "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.8", - "ast-v8-to-istanbul": "^1.0.0", - "istanbul-lib-coverage": "^3.2.2", - "istanbul-lib-report": "^3.0.1", - "istanbul-reports": "^3.2.0", - "magicast": "^0.5.2", - "obug": "^2.1.1", - "std-env": "^4.0.0-rc.1", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@vitest/browser": "4.1.8", - "vitest": "4.1.8" - }, - "peerDependenciesMeta": { - "@vitest/browser": { - "optional": true - } + "packages/eval-graders": { + "name": "@a0/eval-graders", + "version": "0.1.0", + "devDependencies": { + "@types/node": "^25.9.3", + "@vitest/coverage-v8": "^4.1.9", + "vitest": "^4.1.8" } }, - "packages/eval-core/node_modules/@vitest/expect": { - "version": "4.1.8", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", - "dev": true, - "license": "MIT", + "packages/eval-reporter": { + "name": "@a0/eval-reporter", + "version": "0.1.0", "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" + "@a0/eval-core": "*", + "@a0/eval-graders": "*", + "marked": "^18.0.5", + "nunjucks": "^3.2.4" }, - "funding": { - "url": "https://opencollective.com/vitest" + "devDependencies": { + "@types/node": "^25.9.3", + "@types/nunjucks": "^3.2.6", + "@vitest/coverage-v8": "^4.1.9", + "vitest": "^4.1.8" } }, - "packages/eval-core/node_modules/@vitest/mocker": { - "version": "4.1.8", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", - "dev": true, + "packages/eval/node_modules/p-limit": { + "version": "7.3.0", "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.8", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + "yocto-queue": "^1.2.1" }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "packages/eval-core/node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-core/node_modules/@vitest/runner": { - "version": "4.1.8", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.8", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-core/node_modules/@vitest/snapshot": { - "version": "4.1.8", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-core/node_modules/@vitest/spy": { - "version": "4.1.8", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-core/node_modules/@vitest/utils": { - "version": "4.1.8", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-core/node_modules/vitest": { - "version": "4.1.8", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, - "packages/eval-graders": { - "name": "@a0/eval-graders", - "version": "0.1.0", - "devDependencies": { - "@types/node": "^25.9.3", - "@vitest/coverage-v8": "^4.1.6", - "vitest": "^4.1.8" - } - }, - "packages/eval-graders/node_modules/@vitest/coverage-v8": { - "version": "4.1.8", - "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.8", - "ast-v8-to-istanbul": "^1.0.0", - "istanbul-lib-coverage": "^3.2.2", - "istanbul-lib-report": "^3.0.1", - "istanbul-reports": "^3.2.0", - "magicast": "^0.5.2", - "obug": "^2.1.1", - "std-env": "^4.0.0-rc.1", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@vitest/browser": "4.1.8", - "vitest": "4.1.8" - }, - "peerDependenciesMeta": { - "@vitest/browser": { - "optional": true - } - } - }, - "packages/eval-graders/node_modules/@vitest/expect": { - "version": "4.1.8", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-graders/node_modules/@vitest/mocker": { - "version": "4.1.8", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.8", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "packages/eval-graders/node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-graders/node_modules/@vitest/runner": { - "version": "4.1.8", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.8", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-graders/node_modules/@vitest/snapshot": { - "version": "4.1.8", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-graders/node_modules/@vitest/spy": { - "version": "4.1.8", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-graders/node_modules/@vitest/utils": { - "version": "4.1.8", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-graders/node_modules/vitest": { - "version": "4.1.8", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, - "packages/eval-reporter": { - "name": "@a0/eval-reporter", - "version": "0.1.0", - "dependencies": { - "@a0/eval-core": "*", - "@a0/eval-graders": "*", - "marked": "^18.0.5", - "nunjucks": "^3.2.4" - }, - "devDependencies": { - "@types/node": "^25.9.3", - "@types/nunjucks": "^3.2.6", - "@vitest/coverage-v8": "^4.1.6", - "vitest": "^4.1.8" - } - }, - "packages/eval-reporter/node_modules/@vitest/coverage-v8": { - "version": "4.1.8", - "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.8", - "ast-v8-to-istanbul": "^1.0.0", - "istanbul-lib-coverage": "^3.2.2", - "istanbul-lib-report": "^3.0.1", - "istanbul-reports": "^3.2.0", - "magicast": "^0.5.2", - "obug": "^2.1.1", - "std-env": "^4.0.0-rc.1", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@vitest/browser": "4.1.8", - "vitest": "4.1.8" - }, - "peerDependenciesMeta": { - "@vitest/browser": { - "optional": true - } - } - }, - "packages/eval-reporter/node_modules/@vitest/expect": { - "version": "4.1.8", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-reporter/node_modules/@vitest/mocker": { - "version": "4.1.8", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.8", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "packages/eval-reporter/node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-reporter/node_modules/@vitest/runner": { - "version": "4.1.8", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.8", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-reporter/node_modules/@vitest/snapshot": { - "version": "4.1.8", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-reporter/node_modules/@vitest/spy": { - "version": "4.1.8", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-reporter/node_modules/@vitest/utils": { - "version": "4.1.8", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval-reporter/node_modules/vitest": { - "version": "4.1.8", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, - "packages/eval/node_modules/@vitest/coverage-v8": { - "version": "4.1.8", - "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.8", - "ast-v8-to-istanbul": "^1.0.0", - "istanbul-lib-coverage": "^3.2.2", - "istanbul-lib-report": "^3.0.1", - "istanbul-reports": "^3.2.0", - "magicast": "^0.5.2", - "obug": "^2.1.1", - "std-env": "^4.0.0-rc.1", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@vitest/browser": "4.1.8", - "vitest": "4.1.8" - }, - "peerDependenciesMeta": { - "@vitest/browser": { - "optional": true - } - } - }, - "packages/eval/node_modules/@vitest/expect": { - "version": "4.1.8", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.1.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "chai": "^6.2.2", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval/node_modules/@vitest/mocker": { - "version": "4.1.8", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.1.8", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "packages/eval/node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval/node_modules/@vitest/runner": { - "version": "4.1.8", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.1.8", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval/node_modules/@vitest/snapshot": { - "version": "4.1.8", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval/node_modules/@vitest/spy": { - "version": "4.1.8", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval/node_modules/@vitest/utils": { - "version": "4.1.8", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.1.8", - "convert-source-map": "^2.0.0", - "tinyrainbow": "^3.1.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "packages/eval/node_modules/p-limit": { - "version": "7.3.0", - "license": "MIT", - "dependencies": { - "yocto-queue": "^1.2.1" - }, - "engines": { - "node": ">=20" + "engines": { + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "packages/eval/node_modules/vitest": { - "version": "4.1.8", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", - "es-module-lexer": "^2.0.0", - "expect-type": "^1.3.0", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^4.0.0-rc.1", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.1.0", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", - "happy-dom": "*", - "jsdom": "*", - "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/coverage-istanbul": { - "optional": true - }, - "@vitest/coverage-v8": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - }, - "vite": { - "optional": false - } - } - }, "packages/eval/node_modules/yocto-queue": { "version": "1.2.2", "license": "MIT", diff --git a/packages/eval-core/package.json b/packages/eval-core/package.json index c3b386e8..36ed6ef1 100644 --- a/packages/eval-core/package.json +++ b/packages/eval-core/package.json @@ -21,7 +21,7 @@ }, "devDependencies": { "@types/node": "^25.9.3", - "@vitest/coverage-v8": "^4.1.6", + "@vitest/coverage-v8": "^4.1.9", "vitest": "^4.1.8" } } diff --git a/packages/eval-graders/package.json b/packages/eval-graders/package.json index 1404bc7a..511f28a4 100644 --- a/packages/eval-graders/package.json +++ b/packages/eval-graders/package.json @@ -18,7 +18,7 @@ }, "devDependencies": { "@types/node": "^25.9.3", - "@vitest/coverage-v8": "^4.1.6", + "@vitest/coverage-v8": "^4.1.9", "vitest": "^4.1.8" } } diff --git a/packages/eval-reporter/package.json b/packages/eval-reporter/package.json index 863da285..f1859d75 100644 --- a/packages/eval-reporter/package.json +++ b/packages/eval-reporter/package.json @@ -25,7 +25,7 @@ "devDependencies": { "@types/node": "^25.9.3", "@types/nunjucks": "^3.2.6", - "@vitest/coverage-v8": "^4.1.6", + "@vitest/coverage-v8": "^4.1.9", "vitest": "^4.1.8" } } diff --git a/packages/eval/package.json b/packages/eval/package.json index 1548f1c2..dc2b7329 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -38,7 +38,7 @@ "devDependencies": { "@a0/eval-graders": "*", "@types/node": "^25.9.3", - "@vitest/coverage-v8": "^4.1.6", + "@vitest/coverage-v8": "^4.1.9", "vitest": "^4.1.8" } } From de9a473d2b3d6edf8c39401427adc5858159dcd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 13:17:38 +0200 Subject: [PATCH 08/23] chore(deps-dev): bump eslint from 10.4.1 to 10.5.0 (#39) Bumps [eslint](https://github.com/eslint/eslint) from 10.4.1 to 10.5.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v10.4.1...v10.5.0) --- updated-dependencies: - dependency-name: eslint dependency-version: 10.5.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 9 ++++++--- package.json | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 11bc87eb..b67b42d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "devDependencies": { "@actions/core": "^3.0.1", "@eslint/js": "^10.0.1", - "eslint": "^10.4.1", + "eslint": "^10.5.0", "husky": "^9.1.7", "lint-staged": "^17.0.7", "prettier": "^3.8.4", @@ -3234,10 +3234,13 @@ } }, "node_modules/eslint": { - "version": "10.4.1", - "integrity": "sha512-AyIKhnOBuOAdueD7RB3xB+YeAWScb9jHsJBgH2Hcde8InP5JYhqrRR6iTMHyTEwgENK54Cp44e4v8BwNhsuHuw==", + "version": "10.5.0", + "integrity": "sha512-1y+7C+vi12bUK1IpZeaV3gsH9fHLBmPvYmPx42pvT/E9yG0IC8g3PUZZgp0+JLJl7ZDK0flc2gc+Aw9dpCvIsQ==", "dev": true, "license": "MIT", + "workspaces": [ + "packages/*" + ], "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", diff --git a/package.json b/package.json index 537aa5a0..4b0970f5 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "devDependencies": { "@actions/core": "^3.0.1", "@eslint/js": "^10.0.1", - "eslint": "^10.4.1", + "eslint": "^10.5.0", "husky": "^9.1.7", "lint-staged": "^17.0.7", "prettier": "^3.8.4", From f8443c60e8f7fb7ba2d8ea8978428a324b5e0471 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 13:18:12 +0200 Subject: [PATCH 09/23] chore(deps): bump @ai-sdk/openai from 3.0.69 to 3.0.71 (#42) Bumps [@ai-sdk/openai](https://github.com/vercel/ai/tree/HEAD/packages/openai) from 3.0.69 to 3.0.71. - [Release notes](https://github.com/vercel/ai/releases) - [Changelog](https://github.com/vercel/ai/blob/@ai-sdk/openai@3.0.71/packages/openai/CHANGELOG.md) - [Commits](https://github.com/vercel/ai/commits/@ai-sdk/openai@3.0.71/packages/openai) --- updated-dependencies: - dependency-name: "@ai-sdk/openai" dependency-version: 3.0.71 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 24 ++++++++++++++++++++---- packages/eval/package.json | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index b67b42d8..71afcc7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -112,12 +112,12 @@ } }, "node_modules/@ai-sdk/openai": { - "version": "3.0.69", - "integrity": "sha512-T/J3ED6ERDsine+crkXHfraisSG20k6BkBdgjvXCvySYnnJ19IaLIOsQPYfvXdOsKrl0LVNghooTV/nKCA7SmQ==", + "version": "3.0.71", + "integrity": "sha512-j6eBAa5oHFZ4U5CxpIV3T4zXNM/BviodNCZCL1qHkA4aqkwK9iQ18TWYz2DZcXpw4BO5pikKzqpXORxb1EnZGA==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27" + "@ai-sdk/provider-utils": "4.0.29" }, "engines": { "node": ">=18" @@ -137,6 +137,22 @@ "node": ">=18" } }, + "node_modules/@ai-sdk/openai/node_modules/@ai-sdk/provider-utils": { + "version": "4.0.29", + "integrity": "sha512-uhukHaCBvqkwBHkT8C2PrnqKTCoLn3pdHXqtcR9I8ErH+flbzgW4o7VHSNIup9LRu+WBvZIZDQLsx6rwl2tiOA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/provider-utils": { "version": "4.0.27", "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", @@ -5832,7 +5848,7 @@ "dependencies": { "@a0/eval-core": "*", "@a0/eval-reporter": "*", - "@ai-sdk/openai": "^3.0.69", + "@ai-sdk/openai": "^3.0.71", "@anthropic-ai/claude-agent-sdk": "^0.3.173", "@anthropic-ai/claude-code": "2.1.173", "@github/copilot-sdk": "^0.3.0", diff --git a/packages/eval/package.json b/packages/eval/package.json index dc2b7329..6b552bfb 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -16,7 +16,7 @@ "dependencies": { "@a0/eval-core": "*", "@a0/eval-reporter": "*", - "@ai-sdk/openai": "^3.0.69", + "@ai-sdk/openai": "^3.0.71", "@anthropic-ai/claude-agent-sdk": "^0.3.173", "@anthropic-ai/claude-code": "2.1.173", "@github/copilot-sdk": "^0.3.0", From 4df6dc1241b3a80dcecc1b11fe688a4ce44c0a36 Mon Sep 17 00:00:00 2001 From: sanchitmehtagit Date: Tue, 16 Jun 2026 17:46:25 +0530 Subject: [PATCH 10/23] fix: strengthen express-api graders: verify issuer/audience via .env event, not source substrings (#45) --- .../src/evals/quickstarts/express-api/graders.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/auth0-evals/src/evals/quickstarts/express-api/graders.ts b/apps/auth0-evals/src/evals/quickstarts/express-api/graders.ts index 804211ca..b1cfa904 100644 --- a/apps/auth0-evals/src/evals/quickstarts/express-api/graders.ts +++ b/apps/auth0-evals/src/evals/quickstarts/express-api/graders.ts @@ -4,8 +4,6 @@ export function defineGraders() { return [ // ── L1: Positive presence ────────────────────────────────────────────────── contains('express-oauth2-jwt-bearer', 'Uses express-oauth2-jwt-bearer SDK', GraderLevel.L1), - contains('issuerBaseURL', 'Configures issuerBaseURL', GraderLevel.L1), - contains('audience', 'Configures audience claim', GraderLevel.L1), contains('requiredScopes', 'Uses requiredScopes() for scope-based route protection', GraderLevel.L1), contains('req.auth', 'Accesses JWT data via req.auth', GraderLevel.L1), @@ -64,16 +62,19 @@ export function defineGraders() { notContains('req.user', 'No req.user (express-oauth2-jwt-bearer uses req.auth, not req.user)', GraderLevel.L5), judge( 'Does the solution use current express-oauth2-jwt-bearer patterns? ' + - 'Specifically: does it configure auth() with issuerBaseURL and audience, ' + + 'Specifically: does it apply the auth() middleware to protect routes, ' + 'use requiredScopes() for scope checks (not manual payload inspection), ' + - 'and access token data via req.auth.payload (not req.user or manually decoded tokens)?', + 'and access token data via req.auth.payload (not req.user or manually decoded tokens)? ' + + 'Judge only from the source code; the issuer and audience may be supplied via ' + + 'environment variables (ISSUER_BASE_URL / AUDIENCE), so do not assume the contents of any .env file.', GraderLevel.L5, ), // ── Holistic judge ──────────────────────────────────────────────────────── judge( 'Does the solution correctly protect an Express.js API using express-oauth2-jwt-bearer? ' + - 'It should configure auth() middleware with issuerBaseURL and audience, ' + + 'It should apply the auth() middleware (issuer and audience may come from ISSUER_BASE_URL / AUDIENCE ' + + 'environment variables — judge only from the source code and do not assume the contents of any .env file), ' + 'protect GET /api/messages with read:messages scope, ' + 'protect POST /api/messages with write:messages scope, ' + 'and return user profile info from req.auth.payload at GET /api/profile.', From 59d7429249466bb379f37e9d2c6d54ea4f746c77 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 14:16:48 +0200 Subject: [PATCH 11/23] chore(deps): bump ai from 6.0.201 to 6.0.206 (#44) Bumps [ai](https://github.com/vercel/ai/tree/HEAD/packages/ai) from 6.0.201 to 6.0.206. - [Release notes](https://github.com/vercel/ai/releases) - [Changelog](https://github.com/vercel/ai/blob/ai@6.0.206/packages/ai/CHANGELOG.md) - [Commits](https://github.com/vercel/ai/commits/ai@6.0.206/packages/ai) --- updated-dependencies: - dependency-name: ai dependency-version: 6.0.206 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 69 ++++++-------------------------------- packages/eval/package.json | 2 +- 2 files changed, 11 insertions(+), 60 deletions(-) diff --git a/package-lock.json b/package-lock.json index 71afcc7b..8235619d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -85,12 +85,12 @@ "license": "MIT" }, "node_modules/@ai-sdk/gateway": { - "version": "3.0.127", - "integrity": "sha512-Obmw5hmE5x+ccRrMp/Djx5r0rpFVX87YqE6OY06g5fwYlRI30dA84ARfTzX45ivCvkW4eCnBpOVXVWQ/pjH85w==", + "version": "3.0.132", + "integrity": "sha512-sFZFGk6aVSNKmgDrb14efaAbvM71AB3UUvaTsU+bvhWP9jrkRrwix/jFX8IoOAJk0/X7Xf7p3m0J2O2G6ljZbA==", "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27", + "@ai-sdk/provider-utils": "4.0.29", "@vercel/oidc": "3.2.0" }, "engines": { @@ -100,17 +100,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@ai-sdk/gateway/node_modules/@ai-sdk/provider": { - "version": "3.0.10", - "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/@ai-sdk/openai": { "version": "3.0.71", "integrity": "sha512-j6eBAa5oHFZ4U5CxpIV3T4zXNM/BviodNCZCL1qHkA4aqkwK9iQ18TWYz2DZcXpw4BO5pikKzqpXORxb1EnZGA==", @@ -126,7 +115,7 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@ai-sdk/openai/node_modules/@ai-sdk/provider": { + "node_modules/@ai-sdk/provider": { "version": "3.0.10", "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", "license": "Apache-2.0", @@ -137,7 +126,7 @@ "node": ">=18" } }, - "node_modules/@ai-sdk/openai/node_modules/@ai-sdk/provider-utils": { + "node_modules/@ai-sdk/provider-utils": { "version": "4.0.29", "integrity": "sha512-uhukHaCBvqkwBHkT8C2PrnqKTCoLn3pdHXqtcR9I8ErH+flbzgW4o7VHSNIup9LRu+WBvZIZDQLsx6rwl2tiOA==", "license": "Apache-2.0", @@ -153,33 +142,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@ai-sdk/provider-utils": { - "version": "4.0.27", - "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.10", - "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.8" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/provider-utils/node_modules/@ai-sdk/provider": { - "version": "3.0.10", - "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/@anthropic-ai/claude-agent-sdk": { "version": "0.3.173", "integrity": "sha512-BsdL223y7vCUJA9uBW9osSrhufvwIT+J94IBkh83v+wjyjoBIwLXREdacFabair70bGNdtkw6cWCaYNThSQg7A==", @@ -2542,13 +2504,13 @@ } }, "node_modules/ai": { - "version": "6.0.201", - "integrity": "sha512-iDmXSy7csk+SYaCkTUMwsOE/73pksR1spctV+u+DW97v+R/Cj01gchgCUNVrn9QyNHqOJQuTg+JBVsBsp6lOag==", + "version": "6.0.206", + "integrity": "sha512-hVdB5PozMMxmkPBfbCxkFOn0eVeCMi/imkfPvk65cxL4O8oIrvjUxDUX8dGkdkWG1No+R7e7D9ti2DHO61Z4YQ==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/gateway": "3.0.127", + "@ai-sdk/gateway": "3.0.132", "@ai-sdk/provider": "3.0.10", - "@ai-sdk/provider-utils": "4.0.27", + "@ai-sdk/provider-utils": "4.0.29", "@opentelemetry/api": "^1.9.0" }, "engines": { @@ -2558,17 +2520,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/ai/node_modules/@ai-sdk/provider": { - "version": "3.0.10", - "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/ajv": { "version": "8.20.0", "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", @@ -5855,7 +5806,7 @@ "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.139.0", "@openai/codex-sdk": "^0.139.0", - "ai": "^6.0.201", + "ai": "^6.0.206", "braintrust": "^3.17.0", "commander": "^15.0.0", "dotenv": "^17.4.2", diff --git a/packages/eval/package.json b/packages/eval/package.json index 6b552bfb..49161529 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -23,7 +23,7 @@ "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.139.0", "@openai/codex-sdk": "^0.139.0", - "ai": "^6.0.201", + "ai": "^6.0.206", "braintrust": "^3.17.0", "commander": "^15.0.0", "dotenv": "^17.4.2", From dcb1c3139d87110f401645d6c9f860e387f740fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 14:54:12 +0200 Subject: [PATCH 12/23] chore(deps): bump the openai-codex group across 1 directory with 2 updates (#36) --- updated-dependencies: - dependency-name: "@openai/codex" dependency-version: 0.140.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: openai-codex - dependency-name: "@openai/codex-sdk" dependency-version: 0.140.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: openai-codex ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 50 +++++++++++++++++++------------------- packages/eval/package.json | 4 +-- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8235619d..f9065687 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1543,8 +1543,8 @@ "license": "MIT" }, "node_modules/@openai/codex": { - "version": "0.139.0", - "integrity": "sha512-wr2fRE+fzW0CjEbfFsLh1ftarVEcw0CMLWS7QyA0nyOz5qacQPVq3cq2+/U7oEbwm1TOqoi0Fm1nxniB5FkpmA==", + "version": "0.140.0", + "integrity": "sha512-FMnN12kJzVPljMTYRydLCNgd0cXXmVasNSfq2PtS42RMEIxoQ3dHtMvmno35hu2tfwrKNAAPCm4s+2PaFTEBGg==", "license": "Apache-2.0", "bin": { "codex": "bin/codex.js" @@ -1553,18 +1553,18 @@ "node": ">=16" }, "optionalDependencies": { - "@openai/codex-darwin-arm64": "npm:@openai/codex@0.139.0-darwin-arm64", - "@openai/codex-darwin-x64": "npm:@openai/codex@0.139.0-darwin-x64", - "@openai/codex-linux-arm64": "npm:@openai/codex@0.139.0-linux-arm64", - "@openai/codex-linux-x64": "npm:@openai/codex@0.139.0-linux-x64", - "@openai/codex-win32-arm64": "npm:@openai/codex@0.139.0-win32-arm64", - "@openai/codex-win32-x64": "npm:@openai/codex@0.139.0-win32-x64" + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.140.0-darwin-arm64", + "@openai/codex-darwin-x64": "npm:@openai/codex@0.140.0-darwin-x64", + "@openai/codex-linux-arm64": "npm:@openai/codex@0.140.0-linux-arm64", + "@openai/codex-linux-x64": "npm:@openai/codex@0.140.0-linux-x64", + "@openai/codex-win32-arm64": "npm:@openai/codex@0.140.0-win32-arm64", + "@openai/codex-win32-x64": "npm:@openai/codex@0.140.0-win32-x64" } }, "node_modules/@openai/codex-darwin-arm64": { "name": "@openai/codex", - "version": "0.139.0-darwin-arm64", - "integrity": "sha512-o+0ZKWwgDFMMLO7rwinzO0PQsgK+Vme1pMN2GeAxsX29ZgGZcyPICfpJbeGSUO1mb2a36Skjx6nfdRnxMY0r7w==", + "version": "0.140.0-darwin-arm64", + "integrity": "sha512-KDyQHsxdc8FHZKziSBXs82ABgben/8lLPdhi2Nu+wj6qs2RAp4k/IvE8foafVnp3OeGqhtEFbhlZp0H4Dg/Slg==", "cpu": [ "arm64" ], @@ -1579,8 +1579,8 @@ }, "node_modules/@openai/codex-darwin-x64": { "name": "@openai/codex", - "version": "0.139.0-darwin-x64", - "integrity": "sha512-9gkBWzu6DB2rqU4DbpxD3DE5bofGpsK46Lp0h0I+bKWc2IIcxvSi8K2utKmBLoJCbKrn4JQu7dFNGRqEfENung==", + "version": "0.140.0-darwin-x64", + "integrity": "sha512-xA77AcKbP8BKxKqaJz8bqXtU1dUtanEKpWCMJ68LuYU054EC31BD7NftFe5/vpLUQR95fhRr7V9a91SLtCuLAg==", "cpu": [ "x64" ], @@ -1595,8 +1595,8 @@ }, "node_modules/@openai/codex-linux-arm64": { "name": "@openai/codex", - "version": "0.139.0-linux-arm64", - "integrity": "sha512-tBQE5lZciRHeWZGuURgjP9S717MvTIpQMc593+DNxY2LQxozkngOkzFSQd1+/UmQKGrCqdFLu5irIwPXpSZyEw==", + "version": "0.140.0-linux-arm64", + "integrity": "sha512-rGOgWEONilm+pQoQgcGpPRzvnou1CawyBOe8gvtuS32PQ00Pn+9nZF4O7iKBVlNh6Jeun8kpdJSjFdULm2wr4A==", "cpu": [ "arm64" ], @@ -1611,8 +1611,8 @@ }, "node_modules/@openai/codex-linux-x64": { "name": "@openai/codex", - "version": "0.139.0-linux-x64", - "integrity": "sha512-14UgzDS+X4crkvdt6S02A/ZZOrS8ZyWiuTRpguCtnhNamb7unSuDxy86BWgpAl3sqiTaN2CP8VLyp2ohQ8Nbzw==", + "version": "0.140.0-linux-x64", + "integrity": "sha512-7+N/cHB74nsDkOoL+VQVFVFRlfGj6GFSIAQHgs9DQIsvG+UdzWgUeeDE3l926taJqmzcP9NH8bysptKlZ2Ff6g==", "cpu": [ "x64" ], @@ -1626,11 +1626,11 @@ } }, "node_modules/@openai/codex-sdk": { - "version": "0.139.0", - "integrity": "sha512-r4lDckaVx4mVZy1v/7ykEhkeWjVfM/4oGJfhG0AP4+zN3Sa+jcf5hdY4EHfJasofBcp0tIF/7JCKHpv534R+tw==", + "version": "0.140.0", + "integrity": "sha512-sHErDmCss08GMFo3FYVDFtVYiDpf1f3VRdhB4PBILBauZVYuFVYr3INvf6xc6t+oD2MbjuU+gtrw0it9aGesdA==", "license": "Apache-2.0", "dependencies": { - "@openai/codex": "0.139.0" + "@openai/codex": "0.140.0" }, "engines": { "node": ">=18" @@ -1638,8 +1638,8 @@ }, "node_modules/@openai/codex-win32-arm64": { "name": "@openai/codex", - "version": "0.139.0-win32-arm64", - "integrity": "sha512-nlwRjsYotH1Rtqu/Q0VwQbIeO2UX1mkHK84Ov9qn/hl29QqqoBtno0tRyqIPbkXFIVQuWiAYXlV3ugLwH5fTrQ==", + "version": "0.140.0-win32-arm64", + "integrity": "sha512-vs5Ed5OF+4671SZoO0MN5WoHl/K9aOSNzLgzbyyDyM7Jwm/PZYvF6OmIPRWf5AGatYqEOWt8Ovp5+df5PFPM7A==", "cpu": [ "arm64" ], @@ -1654,8 +1654,8 @@ }, "node_modules/@openai/codex-win32-x64": { "name": "@openai/codex", - "version": "0.139.0-win32-x64", - "integrity": "sha512-lQrVLNz+90wdvWVNFDvCkHQRiAK9ZllmkTka3c8eqSDqdJk35Gpgppfv9Xtw5M2ZBtTq0sBdWBiCMyzGDBSpmQ==", + "version": "0.140.0-win32-x64", + "integrity": "sha512-dP+nzd8UQ3Gdby+F5x0Sxd0hu6V9s6/cZYFsGtmmA6eCpU+IIu5tCOnUfgSu5HDw4BvXg046yd8Ihy5bOhwO4A==", "cpu": [ "x64" ], @@ -5804,8 +5804,8 @@ "@anthropic-ai/claude-code": "2.1.173", "@github/copilot-sdk": "^0.3.0", "@google/gemini-cli": "^0.46.0", - "@openai/codex": "^0.139.0", - "@openai/codex-sdk": "^0.139.0", + "@openai/codex": "^0.140.0", + "@openai/codex-sdk": "^0.140.0", "ai": "^6.0.206", "braintrust": "^3.17.0", "commander": "^15.0.0", diff --git a/packages/eval/package.json b/packages/eval/package.json index 49161529..8d9a5a32 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -21,8 +21,8 @@ "@anthropic-ai/claude-code": "2.1.173", "@github/copilot-sdk": "^0.3.0", "@google/gemini-cli": "^0.46.0", - "@openai/codex": "^0.139.0", - "@openai/codex-sdk": "^0.139.0", + "@openai/codex": "^0.140.0", + "@openai/codex-sdk": "^0.140.0", "ai": "^6.0.206", "braintrust": "^3.17.0", "commander": "^15.0.0", From 37ea5493f0ca0ab7416be05c5604fd5b084165d0 Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Tue, 16 Jun 2026 15:12:49 +0200 Subject: [PATCH 13/23] fix(gemini-cli): normalize mcp_ tool names to the mcp__ convention (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gemini CLI >=0.46 emits MCP tool names as `mcp__` with a single-underscore prefix (see generateValidName in the bundle). Every other runner and the trace-based MCP graders use the double-underscore `mcp__` prefix (Claude Code emits it natively; codex/copilot translators normalize to it). The Gemini translator's mapMcpName was identity, so its tool calls were stored with a single underscore and never matched calledTool / calledToolOneOf — a successful MCP invocation scored 0%. Normalize the prefix in mapMcpName (idempotent for names already on the mcp__ convention). --- .../eval/src/runners/gemini-cli/translator.ts | 8 ++++++- .../tests/runners/gemini-cli-agent.test.ts | 22 +++++++++++++++++++ packages/eval/tests/tool-translator.test.ts | 7 ++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/eval/src/runners/gemini-cli/translator.ts b/packages/eval/src/runners/gemini-cli/translator.ts index cd49bdac..c731e858 100644 --- a/packages/eval/src/runners/gemini-cli/translator.ts +++ b/packages/eval/src/runners/gemini-cli/translator.ts @@ -32,7 +32,13 @@ export class GeminiCliTranslator extends BaseToolTranslator { } protected override mapMcpName(name: string): string { - return name; + // Gemini CLI (>=0.46) emits MCP tool names as `mcp__` with a + // single-underscore prefix; the framework convention used by the trace-based + // MCP graders (and every other runner) is the double-underscore `mcp__` + // prefix. Normalize the prefix so `calledTool`/`calledToolOneOf` match. + // Idempotent: names already on the `mcp__` convention are left untouched. + if (name.startsWith('mcp__')) return name; + return `mcp__${name.slice('mcp_'.length)}`; } override isDocLookup(name: string): boolean { diff --git a/packages/eval/tests/runners/gemini-cli-agent.test.ts b/packages/eval/tests/runners/gemini-cli-agent.test.ts index a8999635..2cc38182 100644 --- a/packages/eval/tests/runners/gemini-cli-agent.test.ts +++ b/packages/eval/tests/runners/gemini-cli-agent.test.ts @@ -218,6 +218,28 @@ describe('tool events', () => { expect(record.toolCalls[0].isDocLookup).toBe(true); }); + it('normalizes Gemini single-underscore mcp_ names to the mcp__ convention', async () => { + // Gemini CLI >=0.46 emits `mcp__` (single underscore); the + // trace-based MCP graders require the `mcp__` double-underscore prefix. + mockSpawn.mockReturnValue( + makeChild([ + { + type: 'tool_use', + tool_id: 't1', + tool_name: 'mcp_auth0-hosted-mcp_auth0_list_applications', + parameters: {}, + }, + { type: 'tool_result', tool_id: 't1', status: 'success', output: '{"applications":[]}' }, + { type: 'message', role: 'assistant', content: 'Done.', delta: true }, + resultEvent(), + ]), + ); + + const record = await runGeminiCliAgent(evalDef, workspace); + expect(record.toolCalls[0].name).toBe('mcp__auth0-hosted-mcp_auth0_list_applications'); + expect(record.toolCalls[0].name.startsWith('mcp__')).toBe(true); + }); + it('web_fetch tool is classified as a doc lookup', async () => { mockSpawn.mockReturnValue( makeChild([ diff --git a/packages/eval/tests/tool-translator.test.ts b/packages/eval/tests/tool-translator.test.ts index 98b4d068..23d44563 100644 --- a/packages/eval/tests/tool-translator.test.ts +++ b/packages/eval/tests/tool-translator.test.ts @@ -166,6 +166,13 @@ describe('GeminiCliTranslator — mapping', () => { expect(translator.mapName('mcp__anything__tool')).toBe('mcp__anything__tool'); }); + it('normalizes single-underscore mcp_ names (Gemini >=0.46) to the mcp__ prefix', () => { + expect(translator.mapName('mcp_auth0-hosted-mcp_auth0_list_applications')).toBe( + 'mcp__auth0-hosted-mcp_auth0_list_applications', + ); + expect(translator.mapName('mcp_server_tool')).toBe('mcp__server_tool'); + }); + it('passes through unknown tool names', () => { expect(translator.mapName('some_unknown_tool')).toBe('some_unknown_tool'); }); From 4ce53a8bcf9607eb545a1cebb6058efa8579d5c9 Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Tue, 16 Jun 2026 15:51:09 +0200 Subject: [PATCH 14/23] chore: group Anthropic claude-code and claude-agent-sdk dependabot updates (#48) Keep @anthropic-ai/claude-code and @anthropic-ai/claude-agent-sdk in lockstep so they are bumped in a single PR, mirroring the openai-codex group. Easier to test the two together than separately. --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5677487c..dbb48abf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -25,3 +25,9 @@ updates: patterns: - "@openai/codex" - "@openai/codex-sdk" + # Keep the Claude Code CLI and Agent SDK in lockstep — they share a + # release cadence and are easier to test together than separately. + anthropic: + patterns: + - "@anthropic-ai/claude-code" + - "@anthropic-ai/claude-agent-sdk" From c2426b9e8a2b4d6419513739251c570cc73f7dad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 19:31:54 +0530 Subject: [PATCH 15/23] chore(deps): bump the anthropic group with 2 updates (#49) Bumps the anthropic group with 2 updates: [@anthropic-ai/claude-agent-sdk](https://github.com/anthropics/claude-agent-sdk-typescript) and [@anthropic-ai/claude-code](https://github.com/anthropics/claude-code). Updates `@anthropic-ai/claude-agent-sdk` from 0.3.173 to 0.3.178 - [Release notes](https://github.com/anthropics/claude-agent-sdk-typescript/releases) - [Changelog](https://github.com/anthropics/claude-agent-sdk-typescript/blob/main/CHANGELOG.md) - [Commits](https://github.com/anthropics/claude-agent-sdk-typescript/compare/v0.3.173...v0.3.178) Updates `@anthropic-ai/claude-code` from 2.1.173 to 2.1.178 - [Release notes](https://github.com/anthropics/claude-code/releases) - [Changelog](https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md) - [Commits](https://github.com/anthropics/claude-code/compare/v2.1.173...v2.1.178) --- updated-dependencies: - dependency-name: "@anthropic-ai/claude-agent-sdk" dependency-version: 0.3.178 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: anthropic - dependency-name: "@anthropic-ai/claude-code" dependency-version: 2.1.178 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: anthropic ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 108 ++++++++++++++++++------------------- packages/eval/package.json | 4 +- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index f9065687..e5bfbb01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -143,21 +143,21 @@ } }, "node_modules/@anthropic-ai/claude-agent-sdk": { - "version": "0.3.173", - "integrity": "sha512-BsdL223y7vCUJA9uBW9osSrhufvwIT+J94IBkh83v+wjyjoBIwLXREdacFabair70bGNdtkw6cWCaYNThSQg7A==", + "version": "0.3.178", + "integrity": "sha512-PNsz20jWuahDWq7OU+pjrgYzr2TnpC1oj5yCZDxGWJ8OvucXIdD2AYlf/vUo7oE2JJwGbMTBFqXErNrfPi6Ffw==", "license": "SEE LICENSE IN README.md", "engines": { "node": ">=18.0.0" }, "optionalDependencies": { - "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.173", - "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.173", - "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.173", - "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.173", - "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.173", - "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.173", - "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.173", - "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.173" + "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.178", + "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.178", + "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.178", + "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.178", + "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.178", + "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.178", + "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.178", + "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.178" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", @@ -166,8 +166,8 @@ } }, "node_modules/@anthropic-ai/claude-agent-sdk-darwin-arm64": { - "version": "0.3.173", - "integrity": "sha512-McW1toJ4Qdo/i7bHnsiFMm2AtyCiK/5V90WgL7M9ZO9llrJr3riGRdBlRGHvWnS7rKuv5ttj4/SuBkLoL9o75A==", + "version": "0.3.178", + "integrity": "sha512-RmIJRoZfjwcrd7cHR3fJOL1d4L8SN7oB0REcQPbIuPM1vav2Ft3g5hBX7I86u52A4LLFKBc3SMom3+E6lR1YtQ==", "cpu": [ "arm64" ], @@ -178,8 +178,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-darwin-x64": { - "version": "0.3.173", - "integrity": "sha512-m2Oh1pQ69IUg6DSH6n1nAAAtRq+G7J2B/CKTbTfDAEmg5t0lzakZV9l28GZrlP21xl+PIg71dkiJ5u94KYgpRw==", + "version": "0.3.178", + "integrity": "sha512-wfNl7JoaUk9IzKWQPr+hyEOX8qnkM+e4GBZnKISh60xVhW9wOOp5RdfnYj5MoJzaLYivMSCbo5rb4ThWguhOMw==", "cpu": [ "x64" ], @@ -190,8 +190,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-arm64": { - "version": "0.3.173", - "integrity": "sha512-Vb64WJOD2D9tKn/i0ErmLbO6I1187xTwL/kxEANjg4L1FGQnvdYBnZ6J6jU6+x8UgcuIi82imHROQp80gbPW2w==", + "version": "0.3.178", + "integrity": "sha512-ktBU6EdJoZivf67AxRXe2uSwj0g5tCe20So9QvWVKuEEtRA4sXW5EpgxSWT6LXkgfwUoeNsJK7VAOuVjZumKmg==", "cpu": [ "arm64" ], @@ -202,8 +202,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-arm64-musl": { - "version": "0.3.173", - "integrity": "sha512-uu8MnPwFBc9ayFg5c94aHaqJVLS51oHNVxHwud4nK27GliP5WoME3F7pmm1N/Jyy2ry2E2CLNnsAm5l3zemfYA==", + "version": "0.3.178", + "integrity": "sha512-Z3U0fNatVK3vkki3e5sjlLJMjros+cT6uq//tdohB2kjNK1CugUuPQFQxd6GU1Lq1DokmdSEPL4OGdKiHckNdQ==", "cpu": [ "arm64" ], @@ -214,8 +214,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-x64": { - "version": "0.3.173", - "integrity": "sha512-0l9L5O+Uiw8gq9mu2NqGSYcSmX8RJMAh9GkGacTJqJbkfHCiFxqZmWBWODOt6HP7PTFCV+yMzQK/xJQjnag/jw==", + "version": "0.3.178", + "integrity": "sha512-CPTivDz27LMn5m9iQnJSoWkztLo41e8QTbuYKlAC+CTr98gaYy3Mao0M2tirEK/nGno0xRJw/EpZ1G61yEM3yw==", "cpu": [ "x64" ], @@ -226,8 +226,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-x64-musl": { - "version": "0.3.173", - "integrity": "sha512-ofKxyp/N8+LLSrClt+dJo0laCHDzmBgmN8+Q+zabJ54Jqc1KXee68UsziE7kLCqGbOSY7rsIK9d22T1uQyZkUw==", + "version": "0.3.178", + "integrity": "sha512-K84Ybyr0Olsslg2I1Tzd7KIcSkZgFqqDHn7q1Dfqi7bXVxjMjJ4SaV+LnEAeHSM2es7H8A7ejoTEl89xMZde4Q==", "cpu": [ "x64" ], @@ -238,8 +238,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-win32-arm64": { - "version": "0.3.173", - "integrity": "sha512-aulIrBYjFDm+S5kl4CxC8A3KUiTDKLHoKV46Mat64E+skGEyBkC7WzZAGbpGKB2y8KZo1WbrwJTGefgyHMpDhw==", + "version": "0.3.178", + "integrity": "sha512-uKH3vgv7cQV3d9BPU4lrUKrFgoU/flmy/1QI62j9JzgE6uWVQKWC+BI4V7iceJ36tYVneRhjjuux+l+Q8egTHA==", "cpu": [ "arm64" ], @@ -250,8 +250,8 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-win32-x64": { - "version": "0.3.173", - "integrity": "sha512-HjGkfDlNLI3Kh9NIUfevJ3SZY8Xv3td4zx5Cz3YE0VPrrjIkRvdEv9K6WTjT94tlS0TUXQS/kFT+NCmoGXuvZQ==", + "version": "0.3.178", + "integrity": "sha512-wSC5qG6UA1laWGylOU7axuC4NQxZJtibGQ79sYeOQCc05G7CfkUKSzszLOzsPP3eK63cKi/bX5PA6dd4nA5Wow==", "cpu": [ "x64" ], @@ -262,8 +262,8 @@ ] }, "node_modules/@anthropic-ai/claude-code": { - "version": "2.1.173", - "integrity": "sha512-bivKe/lhaK908Is16AG5xXWxwOtDhvJ46/u7U2j4xxQOhM3jC/4TikD4jJFRNZaeY8jjaReklhFg/aoXRd5vIg==", + "version": "2.1.178", + "integrity": "sha512-XhNQu2QkthrwznlzQA1ZrnRmApA2fgjQ2jxlcPP5B581Tg/E5+hUcbWTkW4OkPDgXg9KljtK7XZ20KszNxt8xg==", "hasInstallScript": true, "license": "SEE LICENSE IN README.md", "bin": { @@ -273,19 +273,19 @@ "node": ">=18.0.0" }, "optionalDependencies": { - "@anthropic-ai/claude-code-darwin-arm64": "2.1.173", - "@anthropic-ai/claude-code-darwin-x64": "2.1.173", - "@anthropic-ai/claude-code-linux-arm64": "2.1.173", - "@anthropic-ai/claude-code-linux-arm64-musl": "2.1.173", - "@anthropic-ai/claude-code-linux-x64": "2.1.173", - "@anthropic-ai/claude-code-linux-x64-musl": "2.1.173", - "@anthropic-ai/claude-code-win32-arm64": "2.1.173", - "@anthropic-ai/claude-code-win32-x64": "2.1.173" + "@anthropic-ai/claude-code-darwin-arm64": "2.1.178", + "@anthropic-ai/claude-code-darwin-x64": "2.1.178", + "@anthropic-ai/claude-code-linux-arm64": "2.1.178", + "@anthropic-ai/claude-code-linux-arm64-musl": "2.1.178", + "@anthropic-ai/claude-code-linux-x64": "2.1.178", + "@anthropic-ai/claude-code-linux-x64-musl": "2.1.178", + "@anthropic-ai/claude-code-win32-arm64": "2.1.178", + "@anthropic-ai/claude-code-win32-x64": "2.1.178" } }, "node_modules/@anthropic-ai/claude-code-darwin-arm64": { - "version": "2.1.173", - "integrity": "sha512-OSigC9K7UvkJFZIK9GEVT54GaTYUa59BhS26m/m+TdAaEMErKUqyv1v3Y6Su3jQcGU2J5voiLRXPd2gne5PdHw==", + "version": "2.1.178", + "integrity": "sha512-y2oFhS2EPfUBl7RMaM1fcVZ9ZMCsSfKxst5j+kmy0WUtS0FE0538KN/ECLDlZkpslAg2WdBNz01M0woMLk6uqw==", "cpu": [ "arm64" ], @@ -296,8 +296,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-darwin-x64": { - "version": "2.1.173", - "integrity": "sha512-lLpWIKqWv6k9Uj2bpd8+E4WNcTGLdvvbXOvZxfBTF46NS5Wx3VgJhCPNg5v0hI+ok62nYMwUaqXDd0ctEaI6pA==", + "version": "2.1.178", + "integrity": "sha512-QjVYEoyEgBU4xRvMHKKSlafl1rG/4dENlpZEDG/9sinGftFdzgm0kuzmq7l/heOM0+ZXJ1LQJdI6xIRaa1lGag==", "cpu": [ "x64" ], @@ -308,8 +308,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-linux-arm64": { - "version": "2.1.173", - "integrity": "sha512-yxZ0QP9HXw6hT8Cyy7pCgQJI4eGj04fo1DNN1Iuvjl3vb6rQm0IfX65ckQwkuX2Sz1o2MuPxBdPsaNLYo7O0LQ==", + "version": "2.1.178", + "integrity": "sha512-tDdcLyUNahnj71lj5c/BEsAHt6Ad1KjRznfzE2IuHflUrHgQtCpObjV8QK2YtoDVgDaPT7frGekse/SK6eKweQ==", "cpu": [ "arm64" ], @@ -320,8 +320,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-linux-arm64-musl": { - "version": "2.1.173", - "integrity": "sha512-TqBZHiojHw7rGNrYcfuAgCpYhiHLGPk4xJCttNYuK5OHc5ROOJmuY+B/HpHKaW7UIOJNjIYL3qPrbkQ1stIplA==", + "version": "2.1.178", + "integrity": "sha512-ICUv3w1tYPU5KZOmdwn1vihMyqmkITbu3VFUZInYU7LlBpOFq8WwzO/G61mRQih2iKciN6WE2tJ06WeJhGXJFA==", "cpu": [ "arm64" ], @@ -332,8 +332,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-linux-x64": { - "version": "2.1.173", - "integrity": "sha512-DRAW6QUAuKlkXlwXsZH1Hiau5qtrFdd9JucZlxbO6PtvKisH0XTv0l/8EGYkSN/1IiVuu5titKGQC99yazxkAw==", + "version": "2.1.178", + "integrity": "sha512-W3XUZDHi3XtsftWK+phsnPyKWx5y7ULfyCiNQFLH5LNih73v0/wRg5t/Kqtj8+rl4pI0LCgcJW2USeK/7CCmvQ==", "cpu": [ "x64" ], @@ -344,8 +344,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-linux-x64-musl": { - "version": "2.1.173", - "integrity": "sha512-dXJLKlaOs+i/+W9Pl58MfqHF2wnTLYC60UcjhOa5Rqv56CbqyLSoGZ5CDbIipJc405vAXsBYjB7S6Kwyye75Kg==", + "version": "2.1.178", + "integrity": "sha512-qkNHcT9XVM5NegMMm5gbJ3SFb4uvwVHPUU3RAVWXFGaGIN74n1G/Pvfoo9Qn+wp9owzlh3NAGnav3Css9AUtFQ==", "cpu": [ "x64" ], @@ -356,8 +356,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-win32-arm64": { - "version": "2.1.173", - "integrity": "sha512-VkiMFsvPbZsA1gW/fQmSjXhLMGz4OaIpPYXLZivmXlIGSmzAMLDCoEAWz67o3q5T4IzG6bCO94Pbl0burMf+Fw==", + "version": "2.1.178", + "integrity": "sha512-oEOSzGhzbPEHK863q+87ojdLmvo6jKVJAcCM5/CX4Bdcf/qtZqY7EuZW2fJ5RQTqy6GRUUbFjGuf5No8GVJdOQ==", "cpu": [ "arm64" ], @@ -368,8 +368,8 @@ ] }, "node_modules/@anthropic-ai/claude-code-win32-x64": { - "version": "2.1.173", - "integrity": "sha512-KLPbApMl8wpN76Vn3l4ZPXyuL9incHELY80sivIPqmZwJv9uKTYv5gwoDvGWuRj5R8tkftzcijbPh5K0fPmeJg==", + "version": "2.1.178", + "integrity": "sha512-WXOoz69alLEaXS4uqeipjT1sArOhuhCbDy08+gTAjPCohGBoyisxEbDxpGUM0jnr2IGg6du0tvd2AM0ZBMqqAg==", "cpu": [ "x64" ], @@ -5800,8 +5800,8 @@ "@a0/eval-core": "*", "@a0/eval-reporter": "*", "@ai-sdk/openai": "^3.0.71", - "@anthropic-ai/claude-agent-sdk": "^0.3.173", - "@anthropic-ai/claude-code": "2.1.173", + "@anthropic-ai/claude-agent-sdk": "^0.3.178", + "@anthropic-ai/claude-code": "^2.1.178", "@github/copilot-sdk": "^0.3.0", "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.140.0", diff --git a/packages/eval/package.json b/packages/eval/package.json index 8d9a5a32..480b8995 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -17,8 +17,8 @@ "@a0/eval-core": "*", "@a0/eval-reporter": "*", "@ai-sdk/openai": "^3.0.71", - "@anthropic-ai/claude-agent-sdk": "^0.3.173", - "@anthropic-ai/claude-code": "2.1.173", + "@anthropic-ai/claude-agent-sdk": "^0.3.178", + "@anthropic-ai/claude-code": "2.1.178", "@github/copilot-sdk": "^0.3.0", "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.140.0", From fb640640f14508ac77318b5c1a867a4da151e28e Mon Sep 17 00:00:00 2001 From: sanchitmehtagit Date: Tue, 16 Jun 2026 19:44:15 +0530 Subject: [PATCH 16/23] docs: Add Architecture.md file (#35) --- README.md | 1 + docs/ARCHITECTURE.md | 281 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 docs/ARCHITECTURE.md diff --git a/README.md b/README.md index c5b57167..f84e708e 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Agent runs are scored across 8 dimensions (process + output quality) into a JSON ## Documentation +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) - System architecture, layers, data flow, scoring, and recommendations - [`packages/eval/README.md`](packages/eval/) - CLI usage, configuration, runners, scoring methodology - [`apps/auth0-evals/README.md`](apps/auth0-evals/) - Auth0 eval suite, available evals, how to add new ones - [`docs/ADDING_EVALS.md`](docs/ADDING_EVALS.md) - Full guide to writing evals diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..624d0a9c --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,281 @@ +# Architecture + +`auth0-evals` is a **TypeScript monorepo** (npm workspaces + Turbo) that runs LLM coding agents against Auth0 SDK integration tasks and scores the code they produce. + +It does two things: + +1. **Measures the Agent Experience (AX) of integrating Auth0** — how well AI coding agents complete real Auth0 integration tasks. +2. **Produces actionable insights to improve it** — concrete fixes for the three investments behind Auth0's [Agent Experience](https://auth0.com/agent-experience): **Auth0 skills** ([auth0/agent-skills](https://github.com/auth0/agent-skills)), the **Auth0 docs MCP server**, and the **Auth0 docs**. + +The loop: run a realistic integration task across multiple agents and investment levels, grade the generated code, and turn each score into a fix. The guiding belief — **every score must point to a fix**. + +## Architecture Diagram + +An `a0-eval run` expands into a job matrix (eval × model × mode × tools); each job walks the same six stages — **Kick off → Prepare → Run the agent → Grade → Score → Report** — fanning out under a `pLimit(workers)` gate (one subprocess and Docker sandbox per job) and converging at `mergeResults()`. Every box names the function behind it, so the diagram doubles as a call map. Colours mark the layer: control plane (`@a0/eval` + `@a0/eval-core`), execution plane (the runner + Auth0 tools it reads), data plane (the artifacts it leaves behind). + +```mermaid +%%{init: {'theme':'base', 'themeVariables': { + 'fontFamily':'ui-sans-serif, system-ui, sans-serif', + 'fontSize':'15px', + 'lineColor':'#495057','primaryTextColor':'#1A1A2E' +}}}%% +flowchart TB + subgraph RowA[" "] + direction LR + subgraph CLI["1 · Kick off — @a0/eval"] + direction TB + Bin["Start a run
bin.ts (local or CI)"] + Matrix["Plan the work
buildJobList()
eval × model × mode × tools"] + Pool["Run jobs in parallel
pLimit(workers) · spawnEval()"] + Bin --> Matrix --> Pool + end + + subgraph Setup["2 · Prepare each job — @a0/eval-core"] + direction TB + Discover["Find the eval
discoverEvals() + loadEval()"] + WS["Clean workspace + scaffold
setupWorkspace()"] + Discover --> WS + end + + subgraph Exec["3 · Run the agent — @a0/eval"] + direction TB + Runners["Coding agent (runner)
Claude Code · Codex · Gemini CLI · Copilot"] + Invest["Helped by Auth0 tools
agent skills + Auth0 docs MCP server"] + Runners -. reads .-> Invest + end + end + + subgraph RowB[" "] + direction LR + subgraph Grade["4 · Grade the code — @a0/eval-core"] + direction TB + Engine["Automated checks
runGraders() · L1–L5"] + Judge["AI judge
llmJudge() · claude-opus-4-8"] + Engine --> Judge + end + + subgraph Score["5 · Score the run — @a0/eval"] + direction TB + Scorer["8-dimension score
score() + A–F grade"] + Recs["Suggested fixes
generateRunRecommendations()"] + Scorer --> Recs + end + + subgraph Storage["6 · Report"] + direction TB + Results[("Results
scores-*.json")] + Report["Leaderboard
renderHtml() → report.html"] + BT[("Braintrust
--braintrust (optional)")] + Results --> Report + Results -. optional .-> BT + end + end + + Pool ==> Discover + WS ==> Runners + Runners ==> Engine + Judge ==> Scorer + Recs ==> Results + + classDef control fill:#E7F0FF,stroke:#4C6EF5,color:#1A1A2E; + classDef work fill:#FFE8CC,stroke:#E8590C,color:#1A1A2E; + classDef data fill:#D3F9D8,stroke:#2F9E44,color:#1A1A2E; + + class Bin,Matrix,Pool,Discover,WS,Engine,Judge,Scorer,Recs control; + class Runners,Invest work; + class Results,Report,BT data; + + classDef cluster fill:#FFFDF2,stroke:#E9D8A6,color:#5C4A1A; + class CLI,Setup,Exec,Grade,Score,Storage cluster; + style RowA fill:none,stroke:none; + style RowB fill:none,stroke:none; +``` + +## Component responsibilities + +Bottom-up, with a clean acyclic dependency graph (`@a0/eval-graders` is the leaf, built first): + +### `@a0/eval-graders` +- **Purpose**: Grader primitive factories + level taxonomy. +- **Responsibilities**: Produce `GraderDef` descriptors (`contains`, `notContains`, `notContainsInSource`, `matches`, `judge`, `ranCommand`, `ranCommandOneOf`, `wroteFile`); define `GraderLevel` (L1–L5) and validate that event graders use L4/L5 only. +- **Dependencies**: None (leaf). +- **Type**: Shared library / SDK. + +### `@a0/eval-core` +- **Purpose**: Evaluation engine. +- **Responsibilities**: Eval discovery (`discoverEvals`) and loading (`loadEval`); framework config load/merge (`loadConfig`, `defineConfig`); workspace lifecycle (`setupWorkspace`, `cleanupWorkspace`, `writeAgentGuidance`); grader engine with a pluggable executor **registry** (`registerExecutor`/`getExecutor`/`executeGrader`); LLM judge; the `AgentRunner` and `ToolTranslator` interfaces; trace classification; result serializers. +- **Dependencies**: `@a0/eval-graders`. +- **Type**: Application infrastructure / engine. + +### `@a0/eval` +- **Purpose**: CLI, orchestration, runners, scoring, insight, persistence, reporting glue. +- **Responsibilities**: Flag parsing (`commander`); job-matrix build with model-prefix auto-routing; worker-pool parallelism (`p-limit`) + per-job subprocess spawning; Docker sandbox lifecycle; four concrete agent runners + baseline; 8-dimension scorer + waste analysis; recommendation generator; result persistence/merge; Braintrust reporter. +- **Sandbox entry point**: `cli/sandbox-runner.ts` (invoked by `docker/entrypoint.sh`) scores and generates recommendations **inside** the sandbox, so the host only reads back the resulting JSON. +- **Dependencies**: `@a0/eval-core`, `@a0/eval-reporter`, agent SDKs (`@anthropic-ai/claude-agent-sdk`, `@openai/codex-sdk`, `@github/copilot-sdk`, `@google/gemini-cli`), `ai` + `@ai-sdk/openai`, `braintrust`, `commander`, `p-limit`, `dotenv`. +- **Type**: Application (publishes the `a0-eval` binary). + +### `@a0/eval-reporter` +- **Purpose**: HTML leaderboard rendering. +- **Responsibilities**: `groupResults`/`groupByVariant`/`computeDeltas`; Nunjucks template (`report.html.j2`) with CSS-class and markdown filters; aggregate cost/run stats; ordered variant display. +- **Dependencies**: `@a0/eval-core`, `@a0/eval-graders`, `marked`, `nunjucks`. +- **Type**: Application infrastructure / reporting. + +### `apps/auth0-evals` +- **Purpose**: Auth0's concrete deployment. +- **Responsibilities**: The 14-eval suite (`src/evals/**`), `eval.config.js` (proxy, models, MCP server, skills sources, judge, scoring allowlist), the React scaffolds, and the local skills dir. Publishes thin `evals`/`report` npm scripts that shell out to `a0-eval`. +- **Dependencies**: `@a0/eval`, `@a0/eval-graders`, `@a0/eval-reporter`, `commander`. +- **Type**: Application / deployment. + +## Runners (auto-routed by model prefix) + +| Runner | Models | SDK | +|---|---|---| +| `claude-code` | `claude-*` | `@anthropic-ai/claude-agent-sdk` (`query()`) | +| `codex` | `gpt-*` | `@openai/codex-sdk` (`thread.runStreamed()`) | +| `gemini-cli` | `gemini-*` | `@google/gemini-cli` | +| `copilot` | else (default) | `@github/copilot-sdk` | +| `baseline` | any (no tools) | `ai` + `@ai-sdk/openai` single-shot | + +New runners plug in via `registerRunner(id, impl)` with no dispatcher changes (Registry + Strategy). + +Each runner ships a `ToolTranslator` that normalizes its SDK's tool names to one **canonical vocabulary** — `run_command`, `read_file`, `write_file`, `list_files`, `fetch_url`, `ask_user` — so trace classification, waste analysis, and scoring see the same signals regardless of vendor (`base-translator.ts`, `runners/classify.ts`). + +## Grader levels + +| Level | Enum | Tests | Runs in | +|---|---|---|---| +| L1 | `positive_presence` | required SDK symbols/imports present | all configs | +| L2 | `hallucination` | hallucinated packages absent | all configs | +| L3 | `security` | no hardcoded secrets | all configs | +| L4 | `structural` | code correctly wired | agent configs | +| L5 | `version_correctness` | current API, not deprecated | agent+mcp configs | + +Every eval ends with one holistic `judge()` with **no level** — it always runs. + +> Full authoring detail (per-level intent, code examples): [`docs/ADDING_EVALS.md`](ADDING_EVALS.md). + +## The 5 configurations + +Each configuration adds **exactly one variable**, so the delta between two adjacent columns *is* the measured value of that investment. + +| Configuration | CLI flags | Isolates | Grader levels | +|---|---|---|---| +| `baseline` | `--mode baseline` | Training-data knowledge | L1–L3 | +| `agent` | `--mode agent` | + agentic loop / tools | L1–L4 | +| `agent+skills` | `--mode agent --tools skills` | + SKILL.md in context | L1–L4 | +| `agent+mcp` | `--mode agent --tools mcp` | + Auth0 docs MCP | L1–L5 | +| `agent+mcp+skills` | `--mode agent --tools mcp,skills` | full investment | L1–L5 | + +## End-to-end data flow + +```mermaid +%%{init: {'theme':'base', 'mirrorActors': false, 'themeVariables': { + 'primaryColor':'#E7F0FF','primaryBorderColor':'#4C6EF5','primaryTextColor':'#1A1A2E', + 'actorBkg':'#E7F0FF','actorBorder':'#4C6EF5','actorTextColor':'#1A1A2E', + 'signalColor':'#495057','signalTextColor':'#1A1A2E','noteBkgColor':'#FFF9DB','noteBorderColor':'#F08C00', + 'fontFamily':'ui-sans-serif, system-ui, sans-serif' +}}}%% +sequenceDiagram + box rgba(231,240,255,0.6) CLI and Orchestration + participant U as User / CI + participant CLI as run.ts + end + box rgba(255,232,204,0.55) Execution + participant Exec as Sandbox / Local + participant Agent as Runner + end + box rgba(211,249,216,0.5) Engine + participant Grade as runGraders + participant Score as score() + end + box rgba(243,232,255,0.55) Insight and Output + participant Recs as recommendations + participant Rep as reporter + end + + U->>CLI: a0-eval --eval react_quickstart --mode agent --tools mcp + CLI->>CLI: buildJobList() → matrix + + loop per job [eval × model × mode × tools] + CLI->>CLI: spawnEval() — one subprocess + CLI->>Exec: setupWorkspace() + dispatch (Docker / local) + Exec->>Agent: run task in workspace + Agent-->>Exec: edited workspace + RunRecord trace + Exec->>Grade: runGraders(workspace, levels) + Grade->>Grade: LLM-judge for judge graders + Grade-->>Score: GraderResult[] + Score->>Score: 8 dimensions → overall + grade + opt skills or MCP active + Score->>Recs: ask judge LLM for fixes + Note over Recs: see "Recommendations":
grader / skill / mcp / efficiency + end + Recs-->>CLI: scores-*.json (+ recommendations) + end + + CLI->>CLI: mergeIntoOutput() — dedup by eval|model|mode|tools + CLI->>Rep: a0-eval report → report.html +``` + +## Scoring — 8 dimensions + +The overall score is a **weighted sum** of 8 dimensions, split evenly between *how* the agent worked (Process, 50%) and *what* it produced (Output, 50%). Process dimensions are **zeroed when the agent never executed** (0 tool calls), so a no-op run can't score well on efficiency. + +**Process — how the agent worked (50%)** + +| Dimension | Weight | Gist | +|---|---|---| +| Setup Friction | 12% | penalize interruptions + provider errors | +| Setup Speed | 12% | active tool time vs. 60s ideal | +| Efficiency | 12% | waste = dup reads + errors + overwrites + interruptions | +| Error Recovery | 7% | penalize provider errors | +| Docs Quality | 7% | valid doc URLs, no error, no rewrite-after-fetch | + +**Output — what the agent produced (50%)** + +| Dimension | Weight | Gist | +|---|---|---| +| Correctness | 25% | L1/L4/L5 grader pass rate (excludes L2/L3) | +| Hallucination | 15% | L2 grader pass rate | +| Security | 10% | L3 grader pass rate | + +**Letter grades:** A ≥ 90 · B ≥ 75 · C ≥ 60 · D ≥ 40 · F < 40 + +> This is a summary. Decision rationale: [`docs/SCORING_METHODOLOGY.md`](SCORING_METHODOLOGY.md). + +## Recommendations — turning scores into fixes + +Scores diagnose; **recommendations prescribe** — the "every score must point to a fix" principle, in code. + +When a run had **skills or MCP enabled**, `generateRunRecommendations` hands the judge LLM the full run context (task, workspace output, injected skill content, grader results, scoring dimensions, efficiency breakdown) and gets back structured JSON: a `severity`-ranked list of fixes, each targeting one of four things to improve. + +| Category | What it flags | Example | +|---|---|---| +| `grader` | Missing checks, false pos/neg, over-strict criteria | "L4 grader misses the `audience` config key" | +| `skill` | Skill doc gaps, confusing or outdated instructions | "SKILL.md omits the `cacheLocation` option" | +| `mcp` | Missing MCP tools, unhelpful responses, poor tool UX | "Add a `get_quickstart` tool returning the canonical snippet" | +| `efficiency` | Thrashing that better docs/tools would prevent | "Agent retried the redirect-URI config 3× — document it" | + +Recommendations are scoped to **custom** skills/MCP tools (never the agent's built-in tools), then persisted alongside scores and surfaced in the leaderboard. The step is safe by construction: it never throws (returns `undefined` on failure) and strips `.env*` from the prompt. + +## Sandbox — running untrusted agent code safely + +By default each job runs inside a hardened, ephemeral **Docker sandbox**. The container starts as root only long enough for the entrypoint (`docker/entrypoint.sh`) to apply network rules, then drops to an unprivileged user before any agent code runs. `cli/sandbox-runner.ts` scores and generates recommendations **inside** the box and writes `.eval-results.json`, so the host only reads the JSON back — agent output never executes on the host. + +| Control | How it's enforced | +|---|---| +| **Network fail-closed** | `iptables` default-DROP on INPUT/OUTPUT/FORWARD; only established/related, loopback, and explicit DNS are allowed (`entrypoint.sh`). | +| **Read-only code** | `chmod -R a-w /app/node_modules /app/packages /app/apps` in the image — the agent can't mutate framework code (`docker/Dockerfile`). | +| **Validated mount** | The workspace mount is checked to live under the OS temp dir (`realpathSync(tmpdir())`) to prevent mounting arbitrary host paths (`sandbox/docker.ts`). | +| **Dropped privileges** | Runs as UID 1000 (`node`) with `--cap-drop=ALL` and `setpriv --inh-caps=-all`, so the process holds zero capabilities. | + +`--dangerously-skip-sandbox` runs directly on the host instead (debugging only). + +## Framework vs. consumer + +The **framework** ([auth0/auth0-evals](https://github.com/auth0/auth0-evals)) — the engine, graders, runners, and eval suite — is environment-agnostic. A **consumer** supplies only two things to run it: a settings file (`eval.config.js` — which models, docs MCP server, and skills to use) and an access key (`LLM_API_KEY`, read from the env). Everything environment-specific lives in the consumer, so the framework stays clean and behaves identically wherever it runs. + +| Consumer | What it is | +|---|---| +| **Laptop** | Clone + `npm install`, point `eval.config.js` at the models you want, put `LLM_API_KEY` in `.env`, run `npm run evals -- --eval …`, then `npm run report`. For developing and spot-checking. | +| **CI pipeline** | A thin wrapper that **pins a framework version** (reproducible runs), **builds once** and shares `dist/` across workers, **fans out the full matrix** as sandboxed jobs, and **merges** every `scores-*.json` into one leaderboard. | + From a29b6d6b53005e22decde859438fc3370ea18c47 Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Wed, 17 Jun 2026 09:21:25 +0200 Subject: [PATCH 17/23] fix(copilot): bump @github/copilot-sdk to 1.0.1 and route through LLM proxy (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps @github/copilot-sdk from 0.3.0 to 1.0.1 (supersedes dependabot #25). The major version has breaking API changes: `cwd` -> `workingDirectory` and `cliPath` -> `connection: RuntimeConnection.forStdio({ path })`. While migrating, route the Copilot runner's inference through the configured LLM proxy via an OpenAI-compatible BYOK provider, instead of GitHub's Copilot backend. The runner previously authenticated via the logged-in gh user, whose backend does not serve the GPT models the eval matrix expects (e.g. gpt-5.4). Uses the Responses API because Copilot emits freeform/custom tool calls (apply_patch) that the chat-completions API rejects — same as the codex runner. --- AGENTS.md | 6 + apps/auth0-evals/eval.config.js | 4 + package-lock.json | 151 ++++++++++++++---- packages/eval/package.json | 2 +- packages/eval/src/runners/copilot/agent.ts | 31 +++- .../eval/tests/runners/copilot-agent.test.ts | 50 ++++++ 6 files changed, 212 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d5845885..cb9cad98 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -336,6 +336,12 @@ Set `CLAUDE_CODE_USE_BEDROCK_PROXY=1` to route through the Bedrock proxy instead - `claude-opus-4-5` → `global.anthropic.claude-opus-4-5-20251101-v1:0` - `claude-haiku-4-5` → `global.anthropic.claude-haiku-4-5-20251001-v1:0` +### Copilot SDK runner details + +Uses `@github/copilot-sdk` to drive the bundled Copilot runtime over JSON-RPC. Inference is routed through the configured LLM proxy via an OpenAI-compatible BYOK provider (`provider: { type: 'openai', wireApi: 'responses', baseUrl, apiKey }`) — **not** GitHub's Copilot backend. The Responses API is required because Copilot emits freeform/custom tool calls (e.g. `apply_patch`) that the chat-completions API rejects — the same reason the Codex runner uses `wire_api = "responses"`. This mirrors the Codex runner: the proxy base URL resolves from `agents.copilot.proxy.baseUrl` (falling back to the top-level `proxy.baseUrl`) and is normalized to end in `/v1`; the API key comes from the `LLM_API_KEY` env var. Set `COPILOT_PROXY_BASE_URL` to override the proxy for this runner only. + +The runner is GPT-only: `gpt-*` / `o*` models pass through, anything else (e.g. `claude-*`, `gemini-*`, or the `copilot` sentinel) falls back to the default GPT model (`gpt-5.4`). + --- ## Agent tools diff --git a/apps/auth0-evals/eval.config.js b/apps/auth0-evals/eval.config.js index 5d96cc0a..dc28af70 100644 --- a/apps/auth0-evals/eval.config.js +++ b/apps/auth0-evals/eval.config.js @@ -11,6 +11,7 @@ const PROXY_BASE_URL = process.env.LLM_PROXY_BASE_URL ?? ''; const CLAUDE_PROXY_BASE_URL = process.env.CLAUDE_PROXY_BASE_URL ?? PROXY_BASE_URL; const GEMINI_PROXY_BASE_URL = process.env.GEMINI_PROXY_BASE_URL ?? PROXY_BASE_URL; const CODEX_PROXY_BASE_URL = process.env.CODEX_PROXY_BASE_URL ?? PROXY_BASE_URL; +const COPILOT_PROXY_BASE_URL = process.env.COPILOT_PROXY_BASE_URL ?? PROXY_BASE_URL; // When set, route Claude models through the Bedrock proxy, which needs full // `global.anthropic.*` model IDs instead of the short aliases. @@ -36,6 +37,9 @@ export default { 'codex': { proxy: { baseUrl: CODEX_PROXY_BASE_URL }, }, + 'copilot': { + proxy: { baseUrl: COPILOT_PROXY_BASE_URL }, + }, }, mcp: { diff --git a/package-lock.json b/package-lock.json index e5bfbb01..b6d0640d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1118,24 +1118,30 @@ } }, "node_modules/@github/copilot": { - "version": "1.0.46", - "integrity": "sha512-e3gxCj8DLGesTAZQ5+jCCbCxe3lMyjKfs5eLgER/SID8Rcb7YpgBXoUvOn3eXxLSsJEmJ3GagHaaHDkf3Zm+Ng==", + "version": "1.0.63", + "integrity": "sha512-e8DRYiWJQc4kepVXsXjC8vpDU2FXS/TfR+Z6p/KAojfcwIUZzKMAfCV5D1lD25hV4CryVH1Z9t7mHqChickj0Q==", "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2", + "os-theme": "^0.0.8" + }, "bin": { "copilot": "npm-loader.js" }, "optionalDependencies": { - "@github/copilot-darwin-arm64": "1.0.46", - "@github/copilot-darwin-x64": "1.0.46", - "@github/copilot-linux-arm64": "1.0.46", - "@github/copilot-linux-x64": "1.0.46", - "@github/copilot-win32-arm64": "1.0.46", - "@github/copilot-win32-x64": "1.0.46" + "@github/copilot-darwin-arm64": "1.0.63", + "@github/copilot-darwin-x64": "1.0.63", + "@github/copilot-linux-arm64": "1.0.63", + "@github/copilot-linux-x64": "1.0.63", + "@github/copilot-linuxmusl-arm64": "1.0.63", + "@github/copilot-linuxmusl-x64": "1.0.63", + "@github/copilot-win32-arm64": "1.0.63", + "@github/copilot-win32-x64": "1.0.63" } }, "node_modules/@github/copilot-darwin-arm64": { - "version": "1.0.46", - "integrity": "sha512-zbhXuRguCdDgeIZKH+rjgBM/6CDMUmhLMck8w9XFDxUY2wrP7MSWXuX8yA4/1H3ySOTZMIH1G5DQpWh+npmR2Q==", + "version": "1.0.63", + "integrity": "sha512-z6CMBxNDlKvT6bvOpqhu4M2bhb0daEbVwSe9SN9WfDUJbt7bpoL7OKKas428iyPSWHoL2WXwxSsy/FjIwSLV6w==", "cpu": [ "arm64" ], @@ -1149,8 +1155,8 @@ } }, "node_modules/@github/copilot-darwin-x64": { - "version": "1.0.46", - "integrity": "sha512-kSUcV6cARhM+b/BuNSQtazbORTetRjIWpO3SqWSmH+2UoeZP5A5x+ipr7mhshq+E+pcWPeQKMGbKGY3lrCSMFw==", + "version": "1.0.63", + "integrity": "sha512-YKd7cXZgAGxhudzrtWdWh2NS35p2G5bV22Gz3jhEyBTqmq45o4sD4OwO87+UpkvM+3nZpwsHaLd3a+ILYX6OXg==", "cpu": [ "x64" ], @@ -1164,8 +1170,8 @@ } }, "node_modules/@github/copilot-linux-arm64": { - "version": "1.0.46", - "integrity": "sha512-Tz3F0LuGFbOvvv0VKQJ4E5XYBsTdqTNMAwOhbkwX6TuKMX88uLJNKP5uPf6yuu1z3J3nt/5rfEd9CxVrZbnqLA==", + "version": "1.0.63", + "integrity": "sha512-A3DOeEfmsJH9j1N+QLc7WXmESBskbezmhDyhyAJcHkw0ngRbKctuWQf/evUHFMh/kgwy1Lr/+9jXJm3NZqr0MA==", "cpu": [ "arm64" ], @@ -1179,8 +1185,8 @@ } }, "node_modules/@github/copilot-linux-x64": { - "version": "1.0.46", - "integrity": "sha512-s9JWe/YE78I7QEeXrvDGHB5x2XnnkegUJYVE9QR2DI/qLXviHMarM3akOUhed21uVqzoiLPacXKZcTcaDO8tOg==", + "version": "1.0.63", + "integrity": "sha512-OMKfZJRoDaJOV7vuWX/nFPNdLa9/H+nhajdE83v4YT9mKLXr86aWrkXE3pPoDYsKWvgQFHg4APA6oZPao0Fyow==", "cpu": [ "x64" ], @@ -1193,22 +1199,52 @@ "copilot-linux-x64": "copilot" } }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.63", + "integrity": "sha512-jcIo6B3uHgcOluNfUHp+6atShKKrXYBPLaRyF6aDT699lwI83gW9KTDuEvDs5FDg8qWsWFfOl+al2dkWDYD3CQ==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.63", + "integrity": "sha512-BEdBbEF3fG7VqXzuaAY4JtmbdGSkpJFeb2ZQYaMpq7OP3aS7ssGe1cCX8ehZNegcMM/eb4GC6PXNXsvl3X/PAQ==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, "node_modules/@github/copilot-sdk": { - "version": "0.3.0", - "integrity": "sha512-SUo35k56pzzgYgwmDPHcu7kZxPrzXbH66IWXaEf6pmb94DlA709F82HrrDeja087TL4djJ9OuvRFWWOKCosAsg==", + "version": "1.0.1", + "integrity": "sha512-w6AaS0WqqTE/3iyUrZznvgCLQhsUF7ZmEVCneacuHCfOzlH0r6ww9WUmyA0zgqmXO75V0IYrkIcnFke/qJkkDg==", "license": "MIT", "dependencies": { - "@github/copilot": "^1.0.36-0", + "@github/copilot": "^1.0.61", "vscode-jsonrpc": "^8.2.1", "zod": "^4.3.6" }, "engines": { - "node": ">=20.0.0" + "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@github/copilot-win32-arm64": { - "version": "1.0.46", - "integrity": "sha512-auX8o8vG8A+rdSthvey1D8q3o6lNlNIfHFjoBU0Z9Fxid6Ghz2paaAn0/Uwz9Ev8W8cn/5C5kEPs3niMXSh4Jw==", + "version": "1.0.63", + "integrity": "sha512-7FqUwOmtoeBoOn4zkKQqRL+WGFwektVRSr5Po2FvPAbKxGXGyFXApZTmRLqVcHhMKDRzMb8KLST1LU1TMTY/wg==", "cpu": [ "arm64" ], @@ -1222,8 +1258,8 @@ } }, "node_modules/@github/copilot-win32-x64": { - "version": "1.0.46", - "integrity": "sha512-iXo9TUqtSxqlBfC+SZSQMrctKJpWR19zr+8dk7hczE42gOVB0/A+NySJwCmY3UFAEY98lbLDjIC+NCbYFcpEHA==", + "version": "1.0.63", + "integrity": "sha512-RC/6y9KHdw/YRCrCEksF2RzbeblfBUNE7bkYZxygaQGYThuv1GeZL2YD2jVqxC2LxKzsUmWGvwEMxerfR6pmeQ==", "cpu": [ "x64" ], @@ -1236,6 +1272,32 @@ "copilot-win32-x64": "copilot.exe" } }, + "node_modules/@github/copilot/node_modules/os-theme": { + "version": "0.0.8", + "integrity": "sha512-u1q3bLSv5uMHNIiPItkfDrHXu6ZFs2juwqxWREFM/uVBa+7Kkhy2v49LmJev2JcinGwqiEccElB/XsH9gwasuA==", + "license": "MIT", + "optionalDependencies": { + "@os-theme/darwin-arm64": "0.0.8", + "@os-theme/linux-x64": "0.0.8", + "@os-theme/win32-x64": "0.0.8" + }, + "peerDependencies": { + "typescript": "^5" + } + }, + "node_modules/@github/copilot/node_modules/typescript": { + "version": "5.9.3", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/@github/keytar": { "version": "7.10.6", "integrity": "sha512-mRW6cUsSG+nj4jp5gp8e91zPySaT73r+2JM6VyMZfrEgksjPmjSMr+tPGNOK3HUHV+GUU9B1LAiiYy/wmAnIxA==", @@ -1676,6 +1738,42 @@ "node": ">=8.0.0" } }, + "node_modules/@os-theme/darwin-arm64": { + "version": "0.0.8", + "integrity": "sha512-gMsOs+8Ju396a5yyMWigkbA0dMTxD78U3HzG3mlpiAyn6hfd5dbyI4VGP+sfTB82KGgWLzIhWWTFX5UYY6iX0A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@os-theme/linux-x64": { + "version": "0.0.8", + "integrity": "sha512-zvjmBUiSQPjM1RbhpsfCDYMJxW4eLlGmkFPnpteC/03X2lz6CjiX2hfbN2EWLxXjNnIje3Jqaen8IsqEnWrRBg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@os-theme/win32-x64": { + "version": "0.0.8", + "integrity": "sha512-N3yxKNbVl2IBa/ncDuq55QhwqwUjnYLJxDKMEmYeJbLIV950qZNojPw3scXA6PbfxPZfIiRa8iz1pzNg9XxP8w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@oxc-project/types": { "version": "0.129.0", "integrity": "sha512-3oz8m3FGdr2nDXVqmFUw7jolKliC4MoyXYIG2c7gpjBnzUWQpUGIYcXYKxTdTi+N2jusvt610ckTMkxdwHkYEg==", @@ -3049,7 +3147,6 @@ "node_modules/detect-libc": { "version": "2.1.2", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -5801,8 +5898,8 @@ "@a0/eval-reporter": "*", "@ai-sdk/openai": "^3.0.71", "@anthropic-ai/claude-agent-sdk": "^0.3.178", - "@anthropic-ai/claude-code": "^2.1.178", - "@github/copilot-sdk": "^0.3.0", + "@anthropic-ai/claude-code": "2.1.178", + "@github/copilot-sdk": "^1.0.1", "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.140.0", "@openai/codex-sdk": "^0.140.0", diff --git a/packages/eval/package.json b/packages/eval/package.json index 480b8995..2e97641b 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -19,7 +19,7 @@ "@ai-sdk/openai": "^3.0.71", "@anthropic-ai/claude-agent-sdk": "^0.3.178", "@anthropic-ai/claude-code": "2.1.178", - "@github/copilot-sdk": "^0.3.0", + "@github/copilot-sdk": "^1.0.1", "@google/gemini-cli": "^0.46.0", "@openai/codex": "^0.140.0", "@openai/codex-sdk": "^0.140.0", diff --git a/packages/eval/src/runners/copilot/agent.ts b/packages/eval/src/runners/copilot/agent.ts index 638d9578..ff9152cf 100644 --- a/packages/eval/src/runners/copilot/agent.ts +++ b/packages/eval/src/runners/copilot/agent.ts @@ -15,19 +15,21 @@ */ import { join } from 'node:path'; -import { CopilotClient, approveAll } from '@github/copilot-sdk'; -import type { MCPServerConfig } from '@github/copilot-sdk'; +import { CopilotClient, RuntimeConnection, approveAll } from '@github/copilot-sdk'; +import type { MCPServerConfig, ProviderConfig } from '@github/copilot-sdk'; import type { EvalDefinition, RunRecord, ToolCallRecord, TurnMetric } from '@a0/eval-core'; import { COPILOT_TASK_TIMEOUT_MS, MAX_TURNS, getFrameworkConfig, + getAgentProxyBaseUrl, estimateCost, logger, makeSessionId, filteredEnv, } from '@a0/eval-core'; import { classifyActionType, classifyErrorCategory, detectRetry } from '@a0/eval-core'; +import { LLM_API_KEY_ENV } from '../../cli/constants.js'; import { CopilotCliTranslator } from './translator.js'; const translator = new CopilotCliTranslator(); @@ -112,9 +114,19 @@ export async function runCopilotAgent( copilotEnv.GH_TOKEN = process.env.GH_TOKEN; } + // Route inference through the configured LLM proxy via a BYOK provider, just + // like the codex runner. The proxy exposes an OpenAI-compatible API under /v1. + const proxyBaseUrl = getAgentProxyBaseUrl('copilot'); + const normalizedBaseUrl = proxyBaseUrl.replace(/\/+$/, ''); + const proxyApiUrl = normalizedBaseUrl.endsWith('/v1') ? normalizedBaseUrl : `${normalizedBaseUrl}/v1`; + const apiKey = process.env[LLM_API_KEY_ENV]; + if (!apiKey) { + logger.warn(`[Copilot] ${LLM_API_KEY_ENV} not set — requests will fail.`); + } + const client = new CopilotClient({ - ...(copilotBin ? { cliPath: copilotBin } : {}), - cwd: workspace, + ...(copilotBin ? { connection: RuntimeConnection.forStdio({ path: copilotBin }) } : {}), + workingDirectory: workspace, // useLoggedInUser defaults to true — the Copilot CLI picks up auth from // the gh CLI's stored credentials (set via `gh auth login` in CI). env: { ...filteredEnv(), ...copilotEnv, ...env }, @@ -122,6 +134,7 @@ export async function runCopilotAgent( logger.info(`\n[Copilot] Starting task: ${evalDef.id}`); logger.info(`[Copilot] Workspace: ${workspace}`); + logger.info(`[Copilot] Proxy: ${proxyBaseUrl}`); if (model) { logger.info(`[Copilot] Model: ${model}`); } @@ -132,6 +145,16 @@ export async function runCopilotAgent( model, workingDirectory: workspace, onPermissionRequest: approveAll, + // Route inference through the LLM proxy (OpenAI-compatible BYOK provider). + // Use the Responses API: Copilot emits freeform/custom tool calls (e.g. + // apply_patch) which the chat-completions API rejects. + provider: { + type: 'openai', + wireApi: 'responses', + baseUrl: proxyApiUrl, + apiKey, + modelId: model, + } satisfies ProviderConfig, // Suppress ask_user to prevent eval runs from blocking on interactive input. excludedTools: ['ask_user'], ...(tools.includes('mcp') ? { mcpServers: getMcpServers() } : {}), diff --git a/packages/eval/tests/runners/copilot-agent.test.ts b/packages/eval/tests/runners/copilot-agent.test.ts index a054c76e..654abed6 100644 --- a/packages/eval/tests/runners/copilot-agent.test.ts +++ b/packages/eval/tests/runners/copilot-agent.test.ts @@ -795,6 +795,56 @@ describe('runCopilotAgent — session configuration', () => { }); }); +// ── Proxy / BYOK provider ───────────────────────────────────────────────────── + +describe('runCopilotAgent — proxy provider', () => { + let prevKey: string | undefined; + + beforeEach(() => { + prevKey = process.env.LLM_API_KEY; + }); + + afterEach(() => { + if (prevKey === undefined) delete process.env.LLM_API_KEY; + else process.env.LLM_API_KEY = prevKey; + }); + + it('passes an OpenAI-compatible provider pointing at the proxy', async () => { + process.env.LLM_API_KEY = 'test-key'; + fakeSession.setScenario(async () => {}); + await runCopilotAgent(evalDef, workspace, { model: 'gpt-5.4' }); + expect(mockCreateSession).toHaveBeenCalledWith( + expect.objectContaining({ + provider: expect.objectContaining({ + type: 'openai', + wireApi: 'responses', + baseUrl: expect.stringContaining('/v1'), + apiKey: 'test-key', + modelId: 'gpt-5.4', + }), + }), + ); + }); + + it('does not double-append /v1 when the proxy base URL already ends in /v1', async () => { + // TEST_CONFIG.proxy.baseUrl is https://llm.example.com/v1 + process.env.LLM_API_KEY = 'test-key'; + fakeSession.setScenario(async () => {}); + await runCopilotAgent(evalDef, workspace, { model: 'gpt-5.4' }); + const config = mockCreateSession.mock.calls[0][0]; + expect(config.provider.baseUrl).toBe('https://llm.example.com/v1'); + }); + + it('still calls createSession when LLM_API_KEY is unset (warns, does not throw)', async () => { + delete process.env.LLM_API_KEY; + fakeSession.setScenario(async () => {}); + await runCopilotAgent(evalDef, workspace, { model: 'gpt-5.4' }); + expect(mockCreateSession).toHaveBeenCalled(); + const config = mockCreateSession.mock.calls[0][0]; + expect(config.provider.apiKey).toBeUndefined(); + }); +}); + // ── CopilotCliRunner ────────────────────────────────────────────────────────── describe('CopilotCliRunner — model routing', () => { From 14eef3945b46e5c77468a02a5ff95f8331175883 Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Wed, 17 Jun 2026 11:25:40 +0200 Subject: [PATCH 18/23] feat: inject compile_command guidance into agent context files (#18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: inject compile_command guidance into agent context files Add an optional `compile_command` PROMPT.md frontmatter field. When set, a verify-compiles instruction is appended to the agent's native context file (CLAUDE.md / GEMINI.md / AGENTS.md / copilot-instructions.md) alongside the existing "no docs files" guidance, so the agent verifies the project compiles and the command appears in the tool trace. Wires the field into all 10 quickstart evals. * fix: make compile_command guidance imperative so agents run it The injected compile-verification guidance used permissive wording ("you can use this command"), so capable models produced correct code but skipped the build — failing the mandatory build-verification grader. Rephrase as a "you MUST run" instruction and assert the mandatory wording in tests. --- AGENTS.md | 3 +- .../src/evals/quickstarts/angular/PROMPT.md | 1 + .../src/evals/quickstarts/express/PROMPT.md | 1 + .../src/evals/quickstarts/fastapi/PROMPT.md | 1 + .../evals/quickstarts/fastify-api/PROMPT.md | 1 + .../src/evals/quickstarts/flask/PROMPT.md | 1 + .../src/evals/quickstarts/nextjs/PROMPT.md | 1 + .../src/evals/quickstarts/nuxt/PROMPT.md | 1 + .../src/evals/quickstarts/react/PROMPT.md | 1 + .../src/evals/quickstarts/spa-js/PROMPT.md | 1 + .../src/evals/quickstarts/vue/PROMPT.md | 1 + docs/ADDING_EVALS.md | 3 ++ packages/eval-core/src/index.ts | 1 + packages/eval-core/src/loader.ts | 2 + packages/eval-core/src/types/eval.ts | 1 + packages/eval-core/src/workspace/index.ts | 1 + packages/eval-core/src/workspace/workspace.ts | 20 ++++++++-- packages/eval-core/tests/loader.test.ts | 22 +++++++++- packages/eval-core/tests/workspace.test.ts | 40 +++++++++++++++++++ packages/eval/src/cli/run.ts | 2 +- 20 files changed, 97 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cb9cad98..fe112b01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,7 @@ When you make a change, update every doc whose described behavior is affected. T |---|---| | New eval added (`PROMPT.md` + `graders.ts`) | `AGENTS.md` eval list (if maintaining one); `docs/ADDING_EVALS.md` if the change reveals a gap in the guide | | `setup_command` behaviour changed (e.g. new syntax supported) | `docs/ADDING_EVALS.md` — frontmatter table and example; `AGENTS.md` checklist if relevant | +| `compile_command` added to an eval or its context-injection behaviour changed | `docs/ADDING_EVALS.md` — frontmatter table and example; `AGENTS.md` checklist if relevant | | New skill added or skill resolution logic changed | `docs/TESTING_SKILLS.md`; `AGENTS.md` if skill tooling or config changed | | New CLI flag or runner added | `AGENTS.md` CLI flags table and Agent runners table; `README.md` quick-start if the flag is commonly used | | Scoring dimension added, changed, or removed | `docs/SCORING_METHODOLOGY.md` first (per the workflow); then `AGENTS.md` scoring section once merged | @@ -151,7 +152,7 @@ When you make a change, update every doc whose described behavior is affected. T ## Adding an eval — checklist 1. `src/evals///PROMPT.md` + `graders.ts` -2. Add `id` (required) and optionally `name`/`category` to `PROMPT.md` frontmatter — the framework auto-discovers evals from `evalsDir` +2. Add `id` (required) and optionally `name`/`category`/`compile_command` to `PROMPT.md` frontmatter — the framework auto-discovers evals from `evalsDir`. Set `compile_command` to point the agent at a verify-compiles command (injected into the agent's context file, e.g. `CLAUDE.md`); omit for evals with no CLI compile step (e.g. mobile). 3. All imports use `.js` extensions; `import type` for type-only 4. All graders have `GraderLevel`; one final holistic `judge` with no level 5. `npm run build && npm test` passes diff --git a/apps/auth0-evals/src/evals/quickstarts/angular/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/angular/PROMPT.md index ed416e0b..671b0745 100644 --- a/apps/auth0-evals/src/evals/quickstarts/angular/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/angular/PROMPT.md @@ -3,6 +3,7 @@ id: angular_quickstart name: Angular Quickstart skills: auth0-angular setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/express/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/express/PROMPT.md index 47b3d612..27b66e30 100644 --- a/apps/auth0-evals/src/evals/quickstarts/express/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/express/PROMPT.md @@ -3,6 +3,7 @@ id: express_quickstart name: Express Quickstart skills: auth0-express setup_command: npm install +compile_command: node --check server.js --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/fastapi/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/fastapi/PROMPT.md index a1f475b5..0f3fa1a0 100644 --- a/apps/auth0-evals/src/evals/quickstarts/fastapi/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/fastapi/PROMPT.md @@ -3,6 +3,7 @@ id: fastapi_quickstart name: FastAPI Quickstart skills: auth0-fastapi-api setup_command: python3 -m venv .venv && .venv/bin/pip install -r requirements.txt +compile_command: .venv/bin/python -m py_compile main.py --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/fastify-api/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/fastify-api/PROMPT.md index ce6e8ceb..531afdfb 100644 --- a/apps/auth0-evals/src/evals/quickstarts/fastify-api/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/fastify-api/PROMPT.md @@ -3,6 +3,7 @@ id: fastify_api_quickstart name: Fastify API Quickstart skills: auth0-fastify-api setup_command: npm install +compile_command: node --check server.js --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/flask/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/flask/PROMPT.md index 19bd9bea..51710e4f 100644 --- a/apps/auth0-evals/src/evals/quickstarts/flask/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/flask/PROMPT.md @@ -3,6 +3,7 @@ id: flask_quickstart name: Flask Quickstart skills: auth0-flask setup_command: python3 -m venv .venv && .venv/bin/pip install -r requirements.txt +compile_command: .venv/bin/python -m py_compile app.py --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/nextjs/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/nextjs/PROMPT.md index 14629d1c..049ddc6c 100644 --- a/apps/auth0-evals/src/evals/quickstarts/nextjs/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/nextjs/PROMPT.md @@ -3,6 +3,7 @@ id: nextjs_quickstart name: Next.js App Router Quickstart skills: auth0-nextjs setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/nuxt/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/nuxt/PROMPT.md index 0f7a6b19..2dd5a460 100644 --- a/apps/auth0-evals/src/evals/quickstarts/nuxt/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/nuxt/PROMPT.md @@ -3,6 +3,7 @@ id: nuxt_quickstart name: Nuxt Quickstart skills: auth0-nuxt setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/react/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/react/PROMPT.md index 35f5f915..926e56e5 100644 --- a/apps/auth0-evals/src/evals/quickstarts/react/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/react/PROMPT.md @@ -4,6 +4,7 @@ name: React Quickstart scaffold: src/evals/scaffolds/react/basic skills: auth0-react setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/spa-js/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/spa-js/PROMPT.md index 5712b20f..a26f79c4 100644 --- a/apps/auth0-evals/src/evals/quickstarts/spa-js/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/spa-js/PROMPT.md @@ -3,6 +3,7 @@ id: spa_js_quickstart name: SPA JS Quickstart skills: auth0-spa-js setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/quickstarts/vue/PROMPT.md b/apps/auth0-evals/src/evals/quickstarts/vue/PROMPT.md index 812b87b2..28b0eb0d 100644 --- a/apps/auth0-evals/src/evals/quickstarts/vue/PROMPT.md +++ b/apps/auth0-evals/src/evals/quickstarts/vue/PROMPT.md @@ -3,6 +3,7 @@ id: vue_quickstart name: Vue Quickstart skills: auth0-vue setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/docs/ADDING_EVALS.md b/docs/ADDING_EVALS.md index fa045bd7..f1b89356 100644 --- a/docs/ADDING_EVALS.md +++ b/docs/ADDING_EVALS.md @@ -31,6 +31,7 @@ id: my_new_eval name: My New Eval skills: auth0-react setup_command: npm install +compile_command: npm run build --- ``` @@ -41,6 +42,7 @@ setup_command: npm install | `category` | no | Defaults to the parent directory name (e.g. `quickstarts`) | | `skills` | no | Comma-separated skill names from [auth0/agent-skills](https://github.com/auth0/agent-skills). Injected into agent context when running with `--tools skills` | | `setup_command` | no | Command run before the agent starts (e.g. `npm install`). Split on whitespace and executed directly via `spawnSync` — no shell, no operators (`&&`, `\|`, etc.), no quoting. One command only. | +| `compile_command` | no | Compile/build command (e.g. `npm run build`, `node --check server.js`, `.venv/bin/python -m py_compile main.py`). When set, an instruction pointing the agent at this command is appended to the agent's native context file (`CLAUDE.md` / `GEMINI.md` / `AGENTS.md` / `.github/copilot-instructions.md`) alongside the "no docs files" guidance, so the agent verifies the project compiles and the command shows up in the tool trace. Agent modes only — baseline ignores it. Omit for evals with no CLI compile step (e.g. mobile). | To test a skill before it is pushed to the remote repo, see [TESTING_SKILLS.md](TESTING_SKILLS.md). @@ -61,6 +63,7 @@ id: react_quickstart name: React Quickstart skills: auth0-react setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/packages/eval-core/src/index.ts b/packages/eval-core/src/index.ts index e6147132..dcb39c17 100644 --- a/packages/eval-core/src/index.ts +++ b/packages/eval-core/src/index.ts @@ -82,6 +82,7 @@ export { writeAgentGuidance, AGENT_GUIDANCE, AGENT_CONTEXT_FILENAMES, + compileGuidance, collectFiles, readWorkspaceFile, isPathInside, diff --git a/packages/eval-core/src/loader.ts b/packages/eval-core/src/loader.ts index 4979b65f..51f33416 100644 --- a/packages/eval-core/src/loader.ts +++ b/packages/eval-core/src/loader.ts @@ -63,6 +63,7 @@ export async function loadEval( .filter(Boolean); const setupCommand = meta.setup_command || undefined; + const compileCommand = meta.compile_command || undefined; return { id: evalConfig.id, @@ -74,6 +75,7 @@ export async function loadEval( graders, scaffold, setupCommand, + compileCommand, skills, metadata: { provider_name: meta.provider_name ?? 'Auth0', diff --git a/packages/eval-core/src/types/eval.ts b/packages/eval-core/src/types/eval.ts index ab11647e..1f518ad0 100644 --- a/packages/eval-core/src/types/eval.ts +++ b/packages/eval-core/src/types/eval.ts @@ -19,6 +19,7 @@ export interface EvalDefinition { graders: GraderDef[]; scaffold: Record; setupCommand?: string; + compileCommand?: string; skills: string[]; metadata: Record; } diff --git a/packages/eval-core/src/workspace/index.ts b/packages/eval-core/src/workspace/index.ts index dc4e26af..5a69b748 100644 --- a/packages/eval-core/src/workspace/index.ts +++ b/packages/eval-core/src/workspace/index.ts @@ -5,6 +5,7 @@ export { writeAgentGuidance, AGENT_GUIDANCE, AGENT_CONTEXT_FILENAMES, + compileGuidance, } from './workspace.js'; export type { SetupWorkspaceOptions, RunSetupCommandOptions } from './workspace.js'; export { collectFiles, readWorkspaceFile } from './file-utils.js'; diff --git a/packages/eval-core/src/workspace/workspace.ts b/packages/eval-core/src/workspace/workspace.ts index c3890d6e..edf03895 100644 --- a/packages/eval-core/src/workspace/workspace.ts +++ b/packages/eval-core/src/workspace/workspace.ts @@ -32,6 +32,15 @@ import { resolveInside } from './path-utils.js'; export const AGENT_GUIDANCE = `Do not create any documentation files (README.md, SETUP.md, QUICKSTART.md, IMPLEMENTATION_SUMMARY.md, or any other .md files). Do not create any .txt summary or verification files. Do not create standalone summary or status files of any kind (e.g. AUTH0_SETUP.ts, IMPLEMENTATION_COMPLETE.ts, QUICK_START.ts, FILES_CREATED.txt) — these are not application source code. Only create and modify source code files that are part of the application. `; +/** + * Builds the compile-verification guidance appended to the agent's context file + * when the eval declares a `compileCommand`. Pointing the agent at the command + * means it appears in the tool trace and the agent can fix any failures. + */ +export function compileGuidance(compileCommand: string): string { + return `After making your changes, you MUST run this command to verify your integration compiles, and fix any errors it reports before finishing:\n\n\`${compileCommand}\`\n`; +} + /** * The context/memory file each runner reads, relative to the workspace root. * Writing guidance to the wrong file means the agent silently ignores it: @@ -51,12 +60,15 @@ export const AGENT_CONTEXT_FILENAMES: Record = { /** * Writes {@link AGENT_GUIDANCE} into the context file the given runner reads. * Appends (preserving any scaffold-provided content) when the file already - * exists; creates it otherwise. + * exists; creates it otherwise. When `compileCommand` is provided, the + * compile-verification guidance (see {@link compileGuidance}) is appended too. */ -export function writeAgentGuidance(workspace: string, agentType: AgentType): void { +export function writeAgentGuidance(workspace: string, agentType: AgentType, compileCommand?: string): void { const filename = AGENT_CONTEXT_FILENAMES[agentType]; const dest = join(workspace, filename); + const guidance = compileCommand ? `${AGENT_GUIDANCE}\n${compileGuidance(compileCommand)}` : AGENT_GUIDANCE; + // If the scaffold shipped AGENTS.md but the active runner reads a different // file, rename it so the guidance reaches the right runner. const scaffoldAgentsMd = join(workspace, 'AGENTS.md'); @@ -66,10 +78,10 @@ export function writeAgentGuidance(workspace: string, agentType: AgentType): voi } if (existsSync(dest)) { - appendFileSync(dest, `\n${AGENT_GUIDANCE}`, 'utf-8'); + appendFileSync(dest, `\n${guidance}`, 'utf-8'); } else { mkdirSync(join(dest, '..'), { recursive: true }); - writeFileSync(dest, AGENT_GUIDANCE, 'utf-8'); + writeFileSync(dest, guidance, 'utf-8'); } } diff --git a/packages/eval-core/tests/loader.test.ts b/packages/eval-core/tests/loader.test.ts index fb9fb916..dbee9f40 100644 --- a/packages/eval-core/tests/loader.test.ts +++ b/packages/eval-core/tests/loader.test.ts @@ -151,6 +151,26 @@ describe('loadEval - setupCommand', () => { }); }); +// ── compile_command frontmatter tests ──────────────────────────────────────── + +describe('loadEval - compileCommand', () => { + it('parses compile_command from frontmatter', async () => { + makeEvalDir(tmpBase, '---\nskills: auth0-react\ncompile_command: npm run build\n---\n\n## Task\nDo the task.\n'); + + const result = await loadEval(EVAL_CONFIG, tmpBase); + + expect(result.compileCommand).toBe('npm run build'); + }); + + it('returns undefined when compile_command is absent', async () => { + makeEvalDir(tmpBase, '---\nskills: auth0-react\n---\n\n## Task\nDo the task.\n'); + + const result = await loadEval(EVAL_CONFIG, tmpBase); + + expect(result.compileCommand).toBeUndefined(); + }); +}); + // ── System prompt tests ─────────────────────────────────────────────────────── describe('loadEval - system prompt', () => { @@ -239,10 +259,8 @@ describe('loadEval - scaffold loading', () => { expect(result.scaffold['readable.txt']).toBe('this is fine'); expect(result.scaffold['unreadable.txt']).toBeUndefined(); }); - }); - // ── frontmatter scaffold field tests ───────────────────────────────────────── function makePromptWithScaffold(scaffoldPath: string): string { diff --git a/packages/eval-core/tests/workspace.test.ts b/packages/eval-core/tests/workspace.test.ts index 9120771e..ee8d4cfd 100644 --- a/packages/eval-core/tests/workspace.test.ts +++ b/packages/eval-core/tests/workspace.test.ts @@ -8,6 +8,7 @@ import { writeAgentGuidance, AGENT_GUIDANCE, AGENT_CONTEXT_FILENAMES, + compileGuidance, } from '../src/workspace/workspace.js'; import { resolveInside } from '../src/workspace/path-utils.js'; @@ -139,6 +140,45 @@ describe('writeAgentGuidance - runner-aware context file', () => { cleanupWorkspace(workspace); }); + + it('appends compile guidance containing the command when compileCommand is provided', () => { + const workspace = setupWorkspace({ 'index.js': 'ok' }); + writeAgentGuidance(workspace, 'claude-code', 'npm run build'); + + const content = readFileSync(join(workspace, 'CLAUDE.md'), 'utf-8'); + expect(content).toContain(AGENT_GUIDANCE); + expect(content).toContain('npm run build'); + expect(content).toContain('verify your integration compiles'); + + cleanupWorkspace(workspace); + }); + + it('omits compile guidance when compileCommand is not provided', () => { + const workspace = setupWorkspace({ 'index.js': 'ok' }); + writeAgentGuidance(workspace, 'claude-code'); + + const content = readFileSync(join(workspace, 'CLAUDE.md'), 'utf-8'); + expect(content).toBe(AGENT_GUIDANCE); + expect(content).not.toContain('verify your integration compiles'); + + cleanupWorkspace(workspace); + }); +}); + +describe('compileGuidance', () => { + it('embeds the command and the verification instruction', () => { + const result = compileGuidance('npm run build'); + + expect(result).toContain('npm run build'); + expect(result).toContain('verify your integration compiles'); + }); + + it('phrases the instruction as mandatory, not optional', () => { + const result = compileGuidance('npm run build'); + + expect(result).toContain('MUST'); + expect(result).not.toContain('you can use'); + }); }); describe('resolveInside - symlink escape protection', () => { diff --git a/packages/eval/src/cli/run.ts b/packages/eval/src/cli/run.ts index 19da6f06..a2cd082b 100644 --- a/packages/eval/src/cli/run.ts +++ b/packages/eval/src/cli/run.ts @@ -134,7 +134,7 @@ async function runAgentJob( // Inject "no docs files" guidance into the context file this runner reads // (CLAUDE.md / GEMINI.md / AGENTS.md). Must run before both the docker and // local execution paths so every runner picks it up. - writeAgentGuidance(workspace, agentType); + writeAgentGuidance(workspace, agentType, evalDef.compileCommand); try { if (!sandbox && evalDef.setupCommand) { runSetupCommand(workspace, evalDef.setupCommand); From 7286310f2bfd7f7d487bdda5ca45f0ec22dda5d2 Mon Sep 17 00:00:00 2001 From: sanchitmehtagit Date: Wed, 17 Jun 2026 15:39:58 +0530 Subject: [PATCH 19/23] docs(agents): require ARCHITECTURE.md diagram updates on drift (#52) Add a trigger row mapping package/runner/scoring/config/data-flow changes to docs/ARCHITECTURE.md, and extend the rule-of-thumb so the Mermaid diagrams are kept in sync with the code, not just the prose. --- AGENTS.md | 194 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 100 insertions(+), 94 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fe112b01..122088a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,13 +4,13 @@ `auth0-evals` is an evaluation framework that measures how accurately LLM agents complete Auth0 SDK integration tasks. It runs each task across 5 configurations (2 modes × optional tool flags) and compares results: -| Configuration | CLI flags | What it tests | Grader levels | -|---|---|---|---| -| `baseline` | `--mode baseline` | Single LLM call, no tools — training data knowledge only | L1-L3 | -| `agent` | `--mode agent` | Full agentic loop with file/shell tools | L1-L4 | -| `agent+skills` | `--mode agent --tools skills` | Agent + SKILL.md injected into context | L1-L4 | -| `agent+mcp` | `--mode agent --tools mcp` | Agent + Auth0 MCP server tools | L1-L5 | -| `agent+mcp+skills` | `--mode agent --tools mcp,skills` | Agent + MCP + skills (full investment) | L1-L5 | +| Configuration | CLI flags | What it tests | Grader levels | +| ------------------ | --------------------------------- | -------------------------------------------------------- | ------------- | +| `baseline` | `--mode baseline` | Single LLM call, no tools — training data knowledge only | L1-L3 | +| `agent` | `--mode agent` | Full agentic loop with file/shell tools | L1-L4 | +| `agent+skills` | `--mode agent --tools skills` | Agent + SKILL.md injected into context | L1-L4 | +| `agent+mcp` | `--mode agent --tools mcp` | Agent + Auth0 MCP server tools | L1-L5 | +| `agent+mcp+skills` | `--mode agent --tools mcp,skills` | Agent + MCP + skills (full investment) | L1-L5 | Each eval lives in `src/evals///` and consists of a `PROMPT.md` (task description) and a `graders.ts` (acceptance criteria). The framework auto-discovers evals by scanning `evalsDir` for directories containing both files. The eval's snake_case config ID (e.g. `react_quickstart`) is declared in `PROMPT.md` frontmatter via the `id` field — this ID is used with `--eval`. Agent-mode runs are scored across 8 dimensions into a JSON results file; baseline runs only produce grader pass rates (no 8-dimension scoring). @@ -35,8 +35,8 @@ Full guide for adding evals: [docs/ADDING_EVALS.md](docs/ADDING_EVALS.md) `package.json` sets `"type": "module"`. Every import needs a `.js` extension, even when importing `.ts` source files. Use the `node:` prefix for builtins. Use `import type` for type-only imports. ```typescript -import { contains } from '@a0/eval-graders'; // ✓ -import { readFileSync } from 'node:fs'; // ✓ +import { contains } from '@a0/eval-graders'; // ✓ +import { readFileSync } from 'node:fs'; // ✓ import type { GraderDef } from '@a0/eval-graders'; // ✓ type-only ``` @@ -47,8 +47,8 @@ For dynamic imports of absolute paths, use `pathToFileURL(path).href` — bare a Tools return `[message, isDoc, isInterrupt, isError]`. Throwing crashes the agent loop: ```typescript -return ['path argument is required', false, false, true]; // ✓ -throw new Error('path required'); // ✗ crashes the loop +return ['path argument is required', false, false, true]; // ✓ +throw new Error('path required'); // ✗ crashes the loop ``` Always resolve file paths with `resolveInside(context.workspace, args.path)` — not `join()`. It throws on path traversal attempts. @@ -59,13 +59,13 @@ Always resolve file paths with `resolveInside(context.workspace, args.path)` — Every grader must have a `GraderLevel`. End every eval with one holistic `judge` with **no level** — it always runs regardless of level filtering: -| Level | Enum value | What it tests | Runs in | -|---|---|---|---| -| L1 | `positive_presence` | Required SDK symbols, imports, config keys are present | All configs | -| L2 | `hallucination` | Hallucinated packages / wrong SDK variants are absent | All configs | -| L3 | `security` | No hardcoded credentials or tokens in insecure storage | All configs | -| L4 | `structural` | Code is correctly wired — right components, lifecycle handled | Agent configs only | -| L5 | `version_correctness` | Uses current API, not deprecated patterns | Agent+MCP configs only | +| Level | Enum value | What it tests | Runs in | +| ----- | --------------------- | ------------------------------------------------------------- | ---------------------- | +| L1 | `positive_presence` | Required SDK symbols, imports, config keys are present | All configs | +| L2 | `hallucination` | Hallucinated packages / wrong SDK variants are absent | All configs | +| L3 | `security` | No hardcoded credentials or tokens in insecure storage | All configs | +| L4 | `structural` | Code is correctly wired — right components, lifecycle handled | Agent configs only | +| L5 | `version_correctness` | Uses current API, not deprecated patterns | Agent+MCP configs only | For rationale on why each level runs in specific configurations, see `docs/SCORING_METHODOLOGY.md`. @@ -73,15 +73,15 @@ Use `notContainsInSource` (not `notContains`) when a value like a client ID is a ### Grader primitives -| Primitive | What it does | -|---|---| -| `contains(needle)` | Substring present in any non-excluded workspace file | -| `notContains(needle)` | Substring must NOT appear in any non-excluded workspace file | -| `notContainsInSource(needle)` | Substring must NOT appear in source files (allowed in config) | -| `matches(pattern)` | Regex match in any non-excluded workspace file | -| `judge(question, framework?)` | LLM-as-judge yes/no question — uses `claude-opus-4-7` | -| `ranCommand(command, args, description, level)` | Agent ran a shell command containing `command` (and all `args`) — event-based, level required (L4 or L5) | -| `ranCommandOneOf(commands, description, level)` | Agent ran at least one command from the list — event-based, level required (L4 or L5) | +| Primitive | What it does | +| ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `contains(needle)` | Substring present in any non-excluded workspace file | +| `notContains(needle)` | Substring must NOT appear in any non-excluded workspace file | +| `notContainsInSource(needle)` | Substring must NOT appear in source files (allowed in config) | +| `matches(pattern)` | Regex match in any non-excluded workspace file | +| `judge(question, framework?)` | LLM-as-judge yes/no question — uses `claude-opus-4-7` | +| `ranCommand(command, args, description, level)` | Agent ran a shell command containing `command` (and all `args`) — event-based, level required (L4 or L5) | +| `ranCommandOneOf(commands, description, level)` | Agent ran at least one command from the list — event-based, level required (L4 or L5) | | `wroteFile(path, description, level, expected?)` | Agent wrote a file whose path contains the substring. With optional `expected` (string or string array), the combined content of all writes to that path must also contain every `expected` substring — event-based, level required (L4 or L5) | ## Grading exclusions @@ -89,6 +89,7 @@ Use `notContainsInSource` (not `notContains`) when a value like a client ID is a Graders run against all workspace files (scaffold + agent edits) minus the exclusions below. The following directories and files are excluded from the grading corpus: **Directories:** + - `.claude` — injected skill files and agent context - `.codex` — Codex agent context - `.github` — workflow metadata @@ -103,6 +104,7 @@ Graders run against all workspace files (scaffold + agent edits) minus the exclu - `out-tsc` — TypeScript compiled output **Files:** + - `package-lock.json` — noise - `tsconfig.tsbuildinfo` — TypeScript incremental build cache - `CLAUDE.md`, `GEMINI.md`, `AGENTS.md` — injected agent guidance. Before each agent run, the framework writes the "no documentation files" guidance into the context file the chosen runner reads (`CLAUDE.md` for Claude Code, `GEMINI.md` for Gemini CLI, `AGENTS.md` for Codex, and `.github/copilot-instructions.md` for Copilot — the `.github` dir is already excluded). All three are excluded so the injected text is never graded. @@ -133,19 +135,20 @@ Run `npm test` before committing. A change is not done until tests pass. When you make a change, update every doc whose described behavior is affected. The table below maps change types to the docs that must stay in sync. -| Change type | Docs to update | -|---|---| -| New eval added (`PROMPT.md` + `graders.ts`) | `AGENTS.md` eval list (if maintaining one); `docs/ADDING_EVALS.md` if the change reveals a gap in the guide | -| `setup_command` behaviour changed (e.g. new syntax supported) | `docs/ADDING_EVALS.md` — frontmatter table and example; `AGENTS.md` checklist if relevant | -| `compile_command` added to an eval or its context-injection behaviour changed | `docs/ADDING_EVALS.md` — frontmatter table and example; `AGENTS.md` checklist if relevant | -| New skill added or skill resolution logic changed | `docs/TESTING_SKILLS.md`; `AGENTS.md` if skill tooling or config changed | -| New CLI flag or runner added | `AGENTS.md` CLI flags table and Agent runners table; `README.md` quick-start if the flag is commonly used | -| Scoring dimension added, changed, or removed | `docs/SCORING_METHODOLOGY.md` first (per the workflow); then `AGENTS.md` scoring section once merged | -| New grader level or grader primitive added | `AGENTS.md` grader levels table and grader primitives table; `docs/ADDING_EVALS.md` | -| Framework package added or restructured | `README.md` Packages list; `packages/eval/README.md` if it exists | -| Docker/sandbox behaviour changed | `AGENTS.md` if it affects how evals run; no dedicated doc today — add a note here | - -**Rule of thumb**: if a developer reading the doc would get the wrong mental model or follow a broken example after your change, update the doc. If the doc is still accurate, leave it alone. +| Change type | Docs to update | +| ------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| New eval added (`PROMPT.md` + `graders.ts`) | `AGENTS.md` eval list (if maintaining one); `docs/ADDING_EVALS.md` if the change reveals a gap in the guide | +| `setup_command` behaviour changed (e.g. new syntax supported) | `docs/ADDING_EVALS.md` — frontmatter table and example; `AGENTS.md` checklist if relevant | +| `compile_command` added to an eval or its context-injection behaviour changed | `docs/ADDING_EVALS.md` — frontmatter table and example; `AGENTS.md` checklist if relevant | +| New skill added or skill resolution logic changed | `docs/TESTING_SKILLS.md`; `AGENTS.md` if skill tooling or config changed | +| New CLI flag or runner added | `AGENTS.md` CLI flags table and Agent runners table; `README.md` quick-start if the flag is commonly used | +| Scoring dimension added, changed, or removed | `docs/SCORING_METHODOLOGY.md` first (per the workflow); then `AGENTS.md` scoring section once merged | +| New grader level or grader primitive added | `AGENTS.md` grader levels table and grader primitives table; `docs/ADDING_EVALS.md` | +| Framework package added or restructured | `README.md` Packages list; `packages/eval/README.md` if it exists | +| Docker/sandbox behaviour changed | `AGENTS.md` if it affects how evals run; no dedicated doc today — add a note here | +| Package structure, runners, scoring, configurations, or end-to-end data flow changed | `docs/ARCHITECTURE.md` — update the prose **and** the Mermaid diagrams (component diagram + data-flow diagram) so they don't drift from reality | + +**Rule of thumb**: if a developer reading the doc would get the wrong mental model or follow a broken example after your change, update the doc. If the doc is still accurate, leave it alone. This applies to diagrams too — a Mermaid diagram in `docs/ARCHITECTURE.md` that no longer matches the code is as broken as stale prose; keep both in sync. --- @@ -165,17 +168,17 @@ When you make a change, update every doc whose described behavior is affected. T ### Overview -8 dimensions, each scored 0–100, combined by weighted sum into an overall score. Process dimensions (50%) measure *how* the agent worked. Output dimensions (50%) measure *what* it produced. Process dimensions are **zeroed when the agent didn't execute** (0 tool calls) — this prevents broken runs from scoring high on "efficiency" by doing nothing. +8 dimensions, each scored 0–100, combined by weighted sum into an overall score. Process dimensions (50%) measure _how_ the agent worked. Output dimensions (50%) measure _what_ it produced. Process dimensions are **zeroed when the agent didn't execute** (0 tool calls) — this prevents broken runs from scoring high on "efficiency" by doing nothing. ### Grade thresholds | Grade | Min score | -|-------|-----------| -| A | 90 | -| B | 75 | -| C | 60 | -| D | 40 | -| F | < 40 | +| ----- | --------- | +| A | 90 | +| B | 75 | +| C | 60 | +| D | 40 | +| F | < 40 | ### Process dimensions (50%) @@ -218,6 +221,7 @@ efficiency (%) = max(0, 100 × (1 - waste_count / total_calls)) ``` Waste categories (a single call can match multiple, but is counted at most once): + 1. **Duplicate reads** — same path read twice with no intervening `write_file` or `run_command`. A `run_command` resets duplicate-read tracking for all paths (it may mutate any file). 2. **Errored calls** — any call where `causedError = true` OR `isRetry = true`. 3. **Overwritten writes** — a `write_file` to path X followed by another `write_file` to path X with no intervening `read_file` (the first write was discarded). @@ -250,11 +254,11 @@ else: Each lookup scores up to 100 points across three signals: -| Signal | Points | How detected | -|---|---|---| -| URL is a valid Auth0 domain | +34 | URL `startsWith` one of the allowed prefixes (`https://auth0.github.io`, `https://auth0.com/docs`, `https://auth0.com/blog`, `https://community.auth0.com`, `https://npmjs.com/package/@auth0`, `https://github.com/auth0/`, `https://github.com/auth0-samples`, `https://jwt.io`) | -| Fetch did not error or 404 | +33 | `causedError == false` on the tool call | -| No file overwrite after this fetch | +33 | No `write_file` to an already-written path between this fetch and the next (or end-of-trace for the final fetch) — agent got it right first time | +| Signal | Points | How detected | +| ---------------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| URL is a valid Auth0 domain | +34 | URL `startsWith` one of the allowed prefixes (`https://auth0.github.io`, `https://auth0.com/docs`, `https://auth0.com/blog`, `https://community.auth0.com`, `https://npmjs.com/package/@auth0`, `https://github.com/auth0/`, `https://github.com/auth0-samples`, `https://jwt.io`) | +| Fetch did not error or 404 | +33 | `causedError == false` on the tool call | +| No file overwrite after this fetch | +33 | No `write_file` to an already-written path between this fetch and the next (or end-of-trace for the final fetch) — agent got it right first time | - All signals are pure trace sequence analysis — no LLM judge, no added cost. - Notes show per-lookup breakdown and total score. @@ -304,16 +308,17 @@ else: score = 100 × passed / relevant.length ## Agent runners -| Runner | ID | Used for | How it's selected | -|---|---|---|---| -| Claude Code | `claude-code` | Claude models via Agent SDK | Auto-selected for `claude-*` models when no `--agent-type` flag | -| Copilot SDK | `copilot` | GPT models via `@github/copilot-sdk` | Not auto-selected; available via `--agent-type copilot` | -| Gemini CLI | `gemini-cli` | Gemini models via Gemini CLI | Auto-selected for `gemini-*` models when no `--agent-type` flag | -| Codex CLI | `codex` | GPT models via OpenAI Codex CLI | Auto-selected for `gpt-*` models when no `--agent-type` flag | +| Runner | ID | Used for | How it's selected | +| ----------- | ------------- | ------------------------------------ | --------------------------------------------------------------- | +| Claude Code | `claude-code` | Claude models via Agent SDK | Auto-selected for `claude-*` models when no `--agent-type` flag | +| Copilot SDK | `copilot` | GPT models via `@github/copilot-sdk` | Not auto-selected; available via `--agent-type copilot` | +| Gemini CLI | `gemini-cli` | Gemini models via Gemini CLI | Auto-selected for `gemini-*` models when no `--agent-type` flag | +| Codex CLI | `codex` | GPT models via OpenAI Codex CLI | Auto-selected for `gpt-*` models when no `--agent-type` flag | ### Auto-routing logic When `--agent-type` is **not** specified, the runner is selected by model prefix: + - `claude-*` → `claude-code` - `gemini-*` → `gemini-cli` - `gpt-*` → `codex` @@ -330,6 +335,7 @@ Short aliases are resolved to the ID the active proxy expects via the single `mo By default (LiteLLM proxy) the `modelIds` map is empty — the proxy serves models under the alias itself, so aliases pass through unchanged. Set `CLAUDE_CODE_USE_BEDROCK_PROXY=1` to route through the Bedrock proxy instead (`/anthropic` endpoint). The config then populates `modelIds` to map supported short aliases to full Bedrock model IDs: + - `claude-sonnet-4-6` → `global.anthropic.claude-sonnet-4-6` - `claude-opus-4-6` → `global.anthropic.claude-opus-4-6-v1` - `claude-opus-4-7` → `global.anthropic.claude-opus-4-7` @@ -375,22 +381,22 @@ All LLM-as-judge graders use `claude-sonnet-4-5` via the configured LLM proxy (` ### Settings -| Setting | Value | -|---|---| -| Base URL | Configured in `eval.config.js` (`proxy.baseUrl`) | -| Judge model | `claude-sonnet-4-5` | -| Judge max tokens | 1024 | -| Judge max code chars | 32,768 | -| Max agent turns | 75 | -| Runner task timeout | 30 min (per eval, graceful abort) | -| Docker host timeout | 35 min (per container, hard kill — sandbox only) | +| Setting | Value | +| -------------------- | ------------------------------------------------ | +| Base URL | Configured in `eval.config.js` (`proxy.baseUrl`) | +| Judge model | `claude-sonnet-4-5` | +| Judge max tokens | 1024 | +| Judge max code chars | 32,768 | +| Max agent turns | 75 | +| Runner task timeout | 30 min (per eval, graceful abort) | +| Docker host timeout | 35 min (per container, hard kill — sandbox only) | --- ## Slash commands -| Command | Purpose | -|---|---| +| Command | Purpose | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `/evals-smoke-test` | End-to-end smoke test: builds the project, runs the full `react_quickstart` matrix across all models and configurations, generates an HTML report, and reports a PASS/FAIL verdict. Use after making framework changes to verify nothing is broken. | ## Key commands @@ -421,33 +427,33 @@ npm run report ### CLI flags -| Flag | Values | Default | Notes | -|---|---|---|---| -| `--eval ` | Any registered eval ID | all evals | Repeatable | -| `--model ` | Any model string | `gpt-5.4` | Repeatable; `all` expands to known working models | -| `--mode ` | `baseline`, `agent`, `all` | `baseline` | `all` expands to both | -| `--tools ` | `skills`, `mcp`, or comma-separated | none | Only applies to agent mode | -| `--agent-type ` | `claude-code`, `copilot`, `gemini-cli`, `codex` | auto-routed by model | Overrides auto-routing | -| `--workers ` | number | 4 | Parallel job limit | -| `--output ` | file path | auto-named | JSON results output | -| `--keep-workspace` | flag | off | Don't delete temp workspace after run | -| `--dangerously-skip-sandbox` | flag | off | Disable Docker sandbox — run agent jobs directly on host | -| `--braintrust` | flag | off | Log results to Braintrust experiment | +| Flag | Values | Default | Notes | +| ---------------------------- | ----------------------------------------------- | -------------------- | -------------------------------------------------------- | +| `--eval ` | Any registered eval ID | all evals | Repeatable | +| `--model ` | Any model string | `gpt-5.4` | Repeatable; `all` expands to known working models | +| `--mode ` | `baseline`, `agent`, `all` | `baseline` | `all` expands to both | +| `--tools ` | `skills`, `mcp`, or comma-separated | none | Only applies to agent mode | +| `--agent-type ` | `claude-code`, `copilot`, `gemini-cli`, `codex` | auto-routed by model | Overrides auto-routing | +| `--workers ` | number | 4 | Parallel job limit | +| `--output ` | file path | auto-named | JSON results output | +| `--keep-workspace` | flag | off | Don't delete temp workspace after run | +| `--dangerously-skip-sandbox` | flag | off | Disable Docker sandbox — run agent jobs directly on host | +| `--braintrust` | flag | off | Log results to Braintrust experiment | --- ## Glossary -| Term | Definition | -|---|---| -| **Workspace** | Temporary directory created per eval run. Contains the scaffold plus everything the agent writes. Deleted after the run unless `--keep-workspace`. | -| **Scaffold** | Starter project seeded into the workspace before the agent runs — e.g., a bare `create-react-app` or an empty Express server. The agent builds on top of it. | -| **Grader** | A single pass/fail check run against workspace output. Defined in each eval's `graders.ts`. Has a level (L1–L5) or no level (holistic judge). | -| **Grader primitive** | Factory function that creates a grader: `contains`, `notContains`, `notContainsInSource`, `matches`, `judge`. | -| **Needle** | The substring or pattern a grader searches for — as in "needle in a haystack." The first argument to `contains`, `notContains`, and `notContainsInSource`. | -| **Configuration** | A specific combination of mode + tools — e.g., `agent+mcp+skills`. Determines which grader levels are active. | -| **Mode** | `baseline` (single LLM call, no tools) or `agent` (full agentic loop with file/shell tools). | -| **Runner** | The agent runtime that executes the task: Claude Code, Copilot SDK, or Gemini CLI. | -| **Interruption** | An `ask_user` tool call — the agent asking for human input (credentials, domains). Penalized in Setup Friction scoring. | -| **Provider error** | LLM API failure: rate limit, timeout, malformed response. Penalized in both Setup Friction and Error Recovery. | -| **Holistic judge** | The final `judge()` grader in every eval with no level assigned. Always runs regardless of configuration. Asks the LLM judge a high-level yes/no question about overall correctness. | +| Term | Definition | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Workspace** | Temporary directory created per eval run. Contains the scaffold plus everything the agent writes. Deleted after the run unless `--keep-workspace`. | +| **Scaffold** | Starter project seeded into the workspace before the agent runs — e.g., a bare `create-react-app` or an empty Express server. The agent builds on top of it. | +| **Grader** | A single pass/fail check run against workspace output. Defined in each eval's `graders.ts`. Has a level (L1–L5) or no level (holistic judge). | +| **Grader primitive** | Factory function that creates a grader: `contains`, `notContains`, `notContainsInSource`, `matches`, `judge`. | +| **Needle** | The substring or pattern a grader searches for — as in "needle in a haystack." The first argument to `contains`, `notContains`, and `notContainsInSource`. | +| **Configuration** | A specific combination of mode + tools — e.g., `agent+mcp+skills`. Determines which grader levels are active. | +| **Mode** | `baseline` (single LLM call, no tools) or `agent` (full agentic loop with file/shell tools). | +| **Runner** | The agent runtime that executes the task: Claude Code, Copilot SDK, or Gemini CLI. | +| **Interruption** | An `ask_user` tool call — the agent asking for human input (credentials, domains). Penalized in Setup Friction scoring. | +| **Provider error** | LLM API failure: rate limit, timeout, malformed response. Penalized in both Setup Friction and Error Recovery. | +| **Holistic judge** | The final `judge()` grader in every eval with no level assigned. Always runs regardless of configuration. Asks the LLM judge a high-level yes/no question about overall correctness. | From 9dfb7c52703fcff5bac90831fb652de55b71592e Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Wed, 17 Jun 2026 15:57:24 +0200 Subject: [PATCH 20/23] feat: post-run compiles() grader (#51) * docs: spec for post-run compile grader Design for running compile_command against the workspace after the agent finishes and driving a compiles() grader from the captured result, so agents are graded on whether their output compiles rather than on whether they ran the build themselves. * docs: implementation plan for post-run compile grader * feat(graders): add CompileResult type * feat(graders): add compiles() grader primitive * feat(core): thread compileResult through GraderContext * feat(core): add compile grader executor * feat(core): add compileResult parameter to runGraders * feat(core): add non-throwing runCompileCommand workspace helper * feat(eval): run compile_command post-agent on the host path * feat(eval): run compile_command post-agent in the sandbox path * test: enable compiles() grader for frontend quickstarts * docs: document compiles() grader and post-run compile behaviour * chore(graders): format long type export added for CompileResult * chore(core): format long type export line for RunCompileCommandOptions * chore: remove superpowers spec and plan docs --- AGENTS.md | 3 +- .../src/evals/quickstarts/angular/graders.ts | 8 +-- .../src/evals/quickstarts/nextjs/graders.ts | 13 +++- .../src/evals/quickstarts/nuxt/graders.ts | 13 +++- .../src/evals/quickstarts/react/graders.ts | 4 +- .../src/evals/quickstarts/spa-js/graders.ts | 4 +- .../src/evals/quickstarts/vue/graders.ts | 4 +- docs/ADDING_EVALS.md | 5 +- packages/eval-core/src/config/defaults.ts | 1 + packages/eval-core/src/config/framework.ts | 2 + packages/eval-core/src/graders/engine.ts | 6 +- .../src/graders/executors/compile.ts | 49 +++++++++++++++ .../eval-core/src/graders/executors/types.ts | 4 +- packages/eval-core/src/index.ts | 9 ++- packages/eval-core/src/workspace/index.ts | 3 +- packages/eval-core/src/workspace/workspace.ts | 63 +++++++++++++++++++ .../eval-core/tests/graders/engine.test.ts | 36 +++++++++++ .../eval-core/tests/graders/executors.test.ts | 43 ++++++++++++- .../tests/workspace-commands.test.ts | 60 +++++++++++++++++- packages/eval-graders/src/index.ts | 10 ++- packages/eval-graders/src/primitives.ts | 15 +++++ packages/eval-graders/src/types.ts | 14 +++++ .../eval-graders/tests/primitives.test.ts | 28 ++++++++- packages/eval/src/cli/run.ts | 7 ++- packages/eval/src/cli/sandbox-runner.ts | 5 ++ 25 files changed, 381 insertions(+), 28 deletions(-) create mode 100644 packages/eval-core/src/graders/executors/compile.ts diff --git a/AGENTS.md b/AGENTS.md index 122088a2..dfa2c482 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,7 @@ Use `notContainsInSource` (not `notContains`) when a value like a client ID is a | `ranCommand(command, args, description, level)` | Agent ran a shell command containing `command` (and all `args`) — event-based, level required (L4 or L5) | | `ranCommandOneOf(commands, description, level)` | Agent ran at least one command from the list — event-based, level required (L4 or L5) | | `wroteFile(path, description, level, expected?)` | Agent wrote a file whose path contains the substring. With optional `expected` (string or string array), the combined content of all writes to that path must also contain every `expected` substring — event-based, level required (L4 or L5) | +| `compiles(description, level)` | Framework runs the eval's `compile_command` against the workspace after the agent finishes and passes/fails on its exit code — level required (L4 or L5). Decoupled from whether the agent ran the build itself, so output that compiles passes even if the agent never ran the build. Requires `compile_command` in PROMPT.md frontmatter or the grader fails. | ## Grading exclusions @@ -449,7 +450,7 @@ npm run report | **Workspace** | Temporary directory created per eval run. Contains the scaffold plus everything the agent writes. Deleted after the run unless `--keep-workspace`. | | **Scaffold** | Starter project seeded into the workspace before the agent runs — e.g., a bare `create-react-app` or an empty Express server. The agent builds on top of it. | | **Grader** | A single pass/fail check run against workspace output. Defined in each eval's `graders.ts`. Has a level (L1–L5) or no level (holistic judge). | -| **Grader primitive** | Factory function that creates a grader: `contains`, `notContains`, `notContainsInSource`, `matches`, `judge`. | +| **Grader primitive** | Factory function that creates a grader: `contains`, `notContains`, `notContainsInSource`, `matches`, `judge`, `compiles`. | | **Needle** | The substring or pattern a grader searches for — as in "needle in a haystack." The first argument to `contains`, `notContains`, and `notContainsInSource`. | | **Configuration** | A specific combination of mode + tools — e.g., `agent+mcp+skills`. Determines which grader levels are active. | | **Mode** | `baseline` (single LLM call, no tools) or `agent` (full agentic loop with file/shell tools). | diff --git a/apps/auth0-evals/src/evals/quickstarts/angular/graders.ts b/apps/auth0-evals/src/evals/quickstarts/angular/graders.ts index 8f057e25..c76a81ef 100644 --- a/apps/auth0-evals/src/evals/quickstarts/angular/graders.ts +++ b/apps/auth0-evals/src/evals/quickstarts/angular/graders.ts @@ -1,4 +1,4 @@ -import { contains, notContains, matches, judge, GraderLevel } from '@a0/eval-graders'; +import { contains, notContains, matches, judge, compiles, GraderLevel } from '@a0/eval-graders'; export function defineGraders() { return [ @@ -22,11 +22,7 @@ export function defineGraders() { // ── L4: Structural / behavioral correctness ─────────────────────────────── // Event-based install/build verification temporarily disabled — see PR scoping discussion. // ranCommand('npm install', '@auth0/auth0-angular', 'Ran npm install for @auth0/auth0-angular', GraderLevel.L4), - // ranCommandOneOf( - // ['npm run build', 'ng build'], - // 'Ran build to verify compilation (npm run build, ng build, or npx ng build)', - // GraderLevel.L4, - // ), + compiles('Project compiles (build succeeds)', GraderLevel.L4), matches(String.raw`provideAuth0\s*\(`, 'Auth0 configured via provideAuth0()', GraderLevel.L4), matches( String.raw`canActivate\s*:\s*\[?\s*(AuthGuard|authGuardFn)`, diff --git a/apps/auth0-evals/src/evals/quickstarts/nextjs/graders.ts b/apps/auth0-evals/src/evals/quickstarts/nextjs/graders.ts index 842a116a..b0997521 100644 --- a/apps/auth0-evals/src/evals/quickstarts/nextjs/graders.ts +++ b/apps/auth0-evals/src/evals/quickstarts/nextjs/graders.ts @@ -1,4 +1,13 @@ -import { contains, notContains, notContainsInSource, matches, judge, wroteFile, GraderLevel } from '@a0/eval-graders'; +import { + contains, + notContains, + notContainsInSource, + matches, + judge, + wroteFile, + compiles, + GraderLevel, +} from '@a0/eval-graders'; export function defineGraders() { return [ @@ -37,7 +46,7 @@ export function defineGraders() { // ── L4: Structural / behavioral correctness ─────────────────────────────── // Event-based install/build verification temporarily disabled — see PR scoping discussion. // ranCommand('npm install', '@auth0/nextjs-auth0', 'Ran npm install for @auth0/nextjs-auth0', GraderLevel.L4), - // ranCommandOneOf(['npm run build', 'next build'], 'Ran build to verify compilation', GraderLevel.L4), + compiles('Project compiles (build succeeds)', GraderLevel.L4), wroteFile('.env', 'Wrote Auth0 credentials to .env file', GraderLevel.L4, [ 'dev-barkbook.us.auth0.com', 'barkbook_client_abc123xyz', diff --git a/apps/auth0-evals/src/evals/quickstarts/nuxt/graders.ts b/apps/auth0-evals/src/evals/quickstarts/nuxt/graders.ts index 95cff92d..80d8afdb 100644 --- a/apps/auth0-evals/src/evals/quickstarts/nuxt/graders.ts +++ b/apps/auth0-evals/src/evals/quickstarts/nuxt/graders.ts @@ -1,4 +1,13 @@ -import { contains, notContains, notContainsInSource, matches, judge, wroteFile, GraderLevel } from '@a0/eval-graders'; +import { + contains, + notContains, + notContainsInSource, + matches, + judge, + wroteFile, + compiles, + GraderLevel, +} from '@a0/eval-graders'; export function defineGraders() { return [ @@ -46,7 +55,7 @@ export function defineGraders() { // ── L4: Structural / behavioral correctness ─────────────────────────────── // Event-based install/build verification temporarily disabled — see PR scoping discussion. // ranCommand('npm install', '@auth0/auth0-nuxt', 'Ran npm install for @auth0/auth0-nuxt', GraderLevel.L4), - // ranCommandOneOf(['npm run build', 'nuxt build'], 'Ran build to verify compilation', GraderLevel.L4), + compiles('Project compiles (build succeeds)', GraderLevel.L4), wroteFile('.env', 'Wrote Auth0 credentials to .env file', GraderLevel.L4, [ 'dev-playground.us.auth0.com', 'playground_client_abc123xyz', diff --git a/apps/auth0-evals/src/evals/quickstarts/react/graders.ts b/apps/auth0-evals/src/evals/quickstarts/react/graders.ts index f9981a16..e61424ec 100644 --- a/apps/auth0-evals/src/evals/quickstarts/react/graders.ts +++ b/apps/auth0-evals/src/evals/quickstarts/react/graders.ts @@ -1,4 +1,4 @@ -import { contains, notContains, matches, judge, GraderLevel } from '@a0/eval-graders'; +import { contains, notContains, matches, judge, compiles, GraderLevel } from '@a0/eval-graders'; export function defineGraders() { return [ @@ -23,7 +23,7 @@ export function defineGraders() { // ── L4: Structural / behavioral correctness ─────────────────────────────── // Event-based install/build verification temporarily disabled — see PR scoping discussion. // ranCommand('npm install', '@auth0/auth0-react', 'Ran npm install for @auth0/auth0-react', GraderLevel.L4), - // ranCommand('npm run', 'build', 'Ran build to verify compilation', GraderLevel.L4), + compiles('Project compiles (build succeeds)', GraderLevel.L4), matches(String.raw` = { maxListedFiles: 200, tempDirPrefix: 'auth0_eval_', setupCommandTimeoutMs: 300_000, + compileCommandTimeoutMs: 300_000, }, models: { diff --git a/packages/eval-core/src/config/framework.ts b/packages/eval-core/src/config/framework.ts index 09666de0..c058293c 100644 --- a/packages/eval-core/src/config/framework.ts +++ b/packages/eval-core/src/config/framework.ts @@ -99,6 +99,8 @@ export interface WorkspaceConfig { tempDirPrefix?: string; /** Timeout (ms) for setup commands like `npm install`. */ setupCommandTimeoutMs?: number; + /** Timeout (ms) for the post-agent compile_command. Builds are slower than installs. */ + compileCommandTimeoutMs?: number; } export interface BraintrustConfig { diff --git a/packages/eval-core/src/graders/engine.ts b/packages/eval-core/src/graders/engine.ts index 9b9ac994..72365135 100644 --- a/packages/eval-core/src/graders/engine.ts +++ b/packages/eval-core/src/graders/engine.ts @@ -11,7 +11,7 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { getFrameworkConfig } from '../config/framework-config.js'; import { collectFiles as collectFilePaths } from '../workspace/index.js'; -import type { GraderDef, GraderResult, EventToolCall } from '@a0/eval-graders'; +import type { GraderDef, GraderResult, EventToolCall, CompileResult } from '@a0/eval-graders'; import { GraderLevel } from '@a0/eval-graders'; import { registerExecutor, executeGrader } from './executors/index.js'; import { containsExecutor } from './executors/contains.js'; @@ -20,6 +20,7 @@ import { notContainsInSourceExecutor } from './executors/not-contains-in-source. import { matchesExecutor } from './executors/matches.js'; import { llmJudgeExecutor } from './executors/llm-judge.js'; import { eventExecutor } from './executors/event.js'; +import { compileExecutor } from './executors/compile.js'; // Re-export llmJudge from its dedicated module for backward compatibility. export { llmJudge } from './llm-judge.js'; @@ -32,6 +33,7 @@ registerExecutor(notContainsInSourceExecutor); registerExecutor(matchesExecutor); registerExecutor(llmJudgeExecutor); registerExecutor(eventExecutor); +registerExecutor(compileExecutor); // ── Workspace helpers ───────────────────────────────────────────────────────── @@ -115,6 +117,7 @@ export async function runGraders( allowedLevels?: Set, enforceMaxChars: boolean = true, toolCalls?: EventToolCall[], + compileResult?: CompileResult, ): Promise { const config = getFrameworkConfig(); const resolvedJudgeModel = judgeModel ?? config.judge.model ?? ''; @@ -144,6 +147,7 @@ export async function runGraders( enforceMaxChars, }, toolCalls, + compileResult, }; const results: GraderResult[] = []; diff --git a/packages/eval-core/src/graders/executors/compile.ts b/packages/eval-core/src/graders/executors/compile.ts new file mode 100644 index 00000000..57d12cce --- /dev/null +++ b/packages/eval-core/src/graders/executors/compile.ts @@ -0,0 +1,49 @@ +/** + * Grader executor: compile + * + * Evaluates the result of running the eval's compile_command against the + * workspace after the agent finishes. The framework runs the command and + * populates ctx.compileResult before grading. + */ + +import type { GraderDef, GraderResult } from '@a0/eval-graders'; +import type { GraderContext, GraderExecutor } from './types.js'; + +const MAX_OUTPUT_TAIL = 500; + +export const compileExecutor: GraderExecutor = { + kind: 'compile', + + async execute(def: GraderDef, ctx: GraderContext): Promise { + const result = ctx.compileResult; + if (result === undefined) { + return { + name: def.name, + kind: def.kind, + passed: false, + detail: 'compile was not run (no compile_command declared for this eval)', + level: def.level, + }; + } + + if (result.ok) { + return { + name: def.name, + kind: def.kind, + passed: true, + detail: `compile_command '${result.command}' succeeded`, + level: def.level, + }; + } + + const tail = result.output.slice(-MAX_OUTPUT_TAIL); + const cause = result.signal ? `signal ${result.signal}` : `exit code ${result.exitCode}`; + return { + name: def.name, + kind: def.kind, + passed: false, + detail: `compile_command '${result.command}' failed (${cause}): ${tail}`, + level: def.level, + }; + }, +}; diff --git a/packages/eval-core/src/graders/executors/types.ts b/packages/eval-core/src/graders/executors/types.ts index 17a2252c..6d906b01 100644 --- a/packages/eval-core/src/graders/executors/types.ts +++ b/packages/eval-core/src/graders/executors/types.ts @@ -5,7 +5,7 @@ * kinds to executors and dispatches grader evaluation through them. */ -import type { GraderDef, GraderResult, EventToolCall } from '@a0/eval-graders'; +import type { GraderDef, GraderResult, EventToolCall, CompileResult } from '@a0/eval-graders'; /** * Context passed to every executor. Each executor uses what it needs: @@ -39,6 +39,8 @@ export interface GraderContext { }; /** Tool call trace from the agent run — used by event-based graders. */ toolCalls?: EventToolCall[]; + /** Result of running the eval's compile_command post-agent — used by the compile grader. */ + compileResult?: CompileResult; } /** diff --git a/packages/eval-core/src/index.ts b/packages/eval-core/src/index.ts index dcb39c17..9fa9eda9 100644 --- a/packages/eval-core/src/index.ts +++ b/packages/eval-core/src/index.ts @@ -78,6 +78,7 @@ export type { LoadConfigOptions } from './config/loader.js'; export { setupWorkspace, runSetupCommand, + runCompileCommand, cleanupWorkspace, writeAgentGuidance, AGENT_GUIDANCE, @@ -89,7 +90,12 @@ export { resolveInside, validatePathFormat, } from './workspace/index.js'; -export type { SetupWorkspaceOptions, RunSetupCommandOptions, CollectFilesOptions } from './workspace/index.js'; +export type { + SetupWorkspaceOptions, + RunSetupCommandOptions, + RunCompileCommandOptions, + CollectFilesOptions, +} from './workspace/index.js'; // Types — eval definition export type { EvalDefinition, GraderDef } from './types/eval.js'; @@ -152,6 +158,7 @@ export { } from './graders/index.js'; export type { GraderContext, GraderExecutor } from './graders/index.js'; +export type { CompileResult } from '@a0/eval-graders'; // Mode export { ALL_MODES } from './types/mode.js'; diff --git a/packages/eval-core/src/workspace/index.ts b/packages/eval-core/src/workspace/index.ts index 5a69b748..93ead1e2 100644 --- a/packages/eval-core/src/workspace/index.ts +++ b/packages/eval-core/src/workspace/index.ts @@ -1,13 +1,14 @@ export { setupWorkspace, runSetupCommand, + runCompileCommand, cleanupWorkspace, writeAgentGuidance, AGENT_GUIDANCE, AGENT_CONTEXT_FILENAMES, compileGuidance, } from './workspace.js'; -export type { SetupWorkspaceOptions, RunSetupCommandOptions } from './workspace.js'; +export type { SetupWorkspaceOptions, RunSetupCommandOptions, RunCompileCommandOptions } from './workspace.js'; export { collectFiles, readWorkspaceFile } from './file-utils.js'; export type { CollectFilesOptions } from './file-utils.js'; export { isPathInside, resolveInside, validatePathFormat } from './path-utils.js'; diff --git a/packages/eval-core/src/workspace/workspace.ts b/packages/eval-core/src/workspace/workspace.ts index edf03895..f015baf4 100644 --- a/packages/eval-core/src/workspace/workspace.ts +++ b/packages/eval-core/src/workspace/workspace.ts @@ -22,6 +22,7 @@ import { tmpdir } from 'node:os'; import { logger } from '../utils/logger.js'; import { DEFAULT_FRAMEWORK_CONFIG } from '../config/defaults.js'; import type { AgentType } from '../types/agents.js'; +import type { CompileResult } from '@a0/eval-graders'; import { resolveInside } from './path-utils.js'; /** @@ -95,6 +96,11 @@ export interface RunSetupCommandOptions { timeoutMs?: number; } +export interface RunCompileCommandOptions { + /** Timeout in ms for the compile command. Defaults to {@link DEFAULT_FRAMEWORK_CONFIG}.workspace.compileCommandTimeoutMs. */ + timeoutMs?: number; +} + /** * Creates a fresh temp directory and seeds it with the scaffold files from the * eval definition. Returns the absolute path to the workspace. @@ -159,6 +165,63 @@ export function runSetupCommand(workspace: string, command: string, options?: Ru } } +/** + * Runs the eval's compile_command in the workspace AFTER the agent finishes and + * captures the outcome. Unlike runSetupCommand, this never throws — a failed + * compile is a valid graded result, not an infrastructure error. + * + * Splits on `&&` and runs each sub-command in sequence (same argv tokenisation + * as runSetupCommand). Short-circuits on the first failing sub-command. `ok` is + * true only if every sub-command exits 0. Output (stdout+stderr) is captured. + */ +export function runCompileCommand( + workspace: string, + command: string, + options?: RunCompileCommandOptions, +): CompileResult { + const timeout = options?.timeoutMs ?? DEFAULT_FRAMEWORK_CONFIG.workspace.compileCommandTimeoutMs!; + const base: CompileResult = { ok: false, exitCode: null, signal: null, output: '', command }; + + if (!command.trim()) { + return { ...base, output: 'compile command is empty' }; + } + + const subCommands = command.split('&&').map((s) => s.trim()); + if (subCommands.some((s) => !s)) { + return { ...base, output: `compile command has an empty segment: ${command}` }; + } + + let combinedOutput = ''; + for (const subCommand of subCommands) { + logger.info(` [Compile] Running: ${subCommand}`); + const args = subCommand.split(/\s+/); + const cmd = args.shift()!; + const result = spawnSync(cmd, args, { cwd: workspace, encoding: 'utf-8', timeout }); + + combinedOutput += (result.stdout ?? '') + (result.stderr ?? ''); + + if (result.error) { + // ENOENT (command not found) or timeout surfaces here. + const signal = result.signal ?? null; + return { + ok: false, + exitCode: null, + signal, + output: combinedOutput + `\n${result.error.message}`, + command, + }; + } + if (result.signal) { + return { ok: false, exitCode: null, signal: result.signal, output: combinedOutput, command }; + } + if (result.status !== 0) { + return { ok: false, exitCode: result.status, signal: null, output: combinedOutput, command }; + } + } + + return { ok: true, exitCode: 0, signal: null, output: combinedOutput, command }; +} + /** * Removes the workspace directory and all its contents. Silently ignores * errors so a cleanup failure never masks the actual eval result. diff --git a/packages/eval-core/tests/graders/engine.test.ts b/packages/eval-core/tests/graders/engine.test.ts index 11ebdcee..7311a61e 100644 --- a/packages/eval-core/tests/graders/engine.test.ts +++ b/packages/eval-core/tests/graders/engine.test.ts @@ -14,9 +14,11 @@ import { ranCommand, ranCommandOneOf, wroteFile, + compiles, GraderLevel, type GraderResult, type EventToolCall, + type CompileResult, } from '@a0/eval-graders'; import { setFrameworkConfig } from '../../src/config/framework-config.js'; import type { FrameworkConfig } from '../../src/config/framework.js'; @@ -1072,3 +1074,37 @@ describe('runGraders - event graders', () => { expect(results[0]!.passed).toBe(true); }); }); + +describe('runGraders - compile', () => { + it('passes the compile grader through to the compile executor via compileResult', async () => { + const dir = tmpDir(); + writeFileSync(join(dir, 'index.ts'), 'export const x = 1;'); + const compileResult: CompileResult = { + ok: true, + exitCode: 0, + signal: null, + output: '', + command: 'npm run build', + }; + const results = await runGraders( + [compiles('builds', GraderLevel.L4)], + dir, + 'test-key', + undefined, + undefined, + true, + undefined, + compileResult, + ); + expect(results).toHaveLength(1); + expect(results[0]!.passed).toBe(true); + }); + + it('fails the compile grader when no compileResult is threaded', async () => { + const dir = tmpDir(); + writeFileSync(join(dir, 'index.ts'), 'export const x = 1;'); + const results = await runGraders([compiles('builds', GraderLevel.L4)], dir, 'test-key'); + expect(results[0]!.passed).toBe(false); + expect(results[0]!.detail).toContain('compile was not run'); + }); +}); diff --git a/packages/eval-core/tests/graders/executors.test.ts b/packages/eval-core/tests/graders/executors.test.ts index 20c64b12..ae751e11 100644 --- a/packages/eval-core/tests/graders/executors.test.ts +++ b/packages/eval-core/tests/graders/executors.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect } from 'vitest'; import { GraderLevel } from '@a0/eval-graders'; -import type { GraderDef } from '@a0/eval-graders'; +import type { GraderDef, CompileResult } from '@a0/eval-graders'; import { containsExecutor } from '../../src/graders/executors/contains.js'; import { notContainsExecutor } from '../../src/graders/executors/not-contains.js'; import { notContainsInSourceExecutor } from '../../src/graders/executors/not-contains-in-source.js'; import { matchesExecutor } from '../../src/graders/executors/matches.js'; import { isJudgeExcluded } from '../../src/graders/executors/llm-judge.js'; +import { compileExecutor } from '../../src/graders/executors/compile.js'; import type { GraderContext } from '../../src/graders/executors/types.js'; // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -335,3 +336,43 @@ describe('isJudgeExcluded', () => { expect(isJudgeExcluded('CHANGELOG.MD')).toBe(true); }); }); + +// ── compile ─────────────────────────────────────────────────────────────────── + +describe('compile executor', () => { + it('passes when compileResult.ok is true', async () => { + const def = makeDef({ kind: 'compile', level: GraderLevel.L4 }); + const compileResult: CompileResult = { + ok: true, + exitCode: 0, + signal: null, + output: 'done', + command: 'npm run build', + }; + const res = await compileExecutor.execute(def, { ...makeCtx({}), compileResult }); + expect(res.passed).toBe(true); + expect(res.kind).toBe('compile'); + }); + + it('fails when compileResult.ok is false and includes exit code + output tail in detail', async () => { + const def = makeDef({ kind: 'compile', level: GraderLevel.L4 }); + const compileResult: CompileResult = { + ok: false, + exitCode: 2, + signal: null, + output: 'TS2304: Cannot find name foo', + command: 'npm run build', + }; + const res = await compileExecutor.execute(def, { ...makeCtx({}), compileResult }); + expect(res.passed).toBe(false); + expect(res.detail).toContain('2'); + expect(res.detail).toContain('TS2304'); + }); + + it('fails when no compileResult is present (eval misconfigured)', async () => { + const def = makeDef({ kind: 'compile', level: GraderLevel.L4 }); + const res = await compileExecutor.execute(def, makeCtx({})); + expect(res.passed).toBe(false); + expect(res.detail).toContain('compile was not run'); + }); +}); diff --git a/packages/eval-core/tests/workspace-commands.test.ts b/packages/eval-core/tests/workspace-commands.test.ts index 5dd1f593..7d0ef670 100644 --- a/packages/eval-core/tests/workspace-commands.test.ts +++ b/packages/eval-core/tests/workspace-commands.test.ts @@ -9,7 +9,7 @@ import { describe, it, expect } from 'vitest'; import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { makeTmpDir } from './tmp.js'; -import { setupWorkspace, runSetupCommand, cleanupWorkspace } from '../src/workspace/workspace.js'; +import { setupWorkspace, runSetupCommand, runCompileCommand, cleanupWorkspace } from '../src/workspace/workspace.js'; const tmpDir = makeTmpDir('workspace_cmd_test_'); @@ -128,3 +128,61 @@ describe('setupWorkspace — edge cases', () => { cleanupWorkspace(workspace); }); }); + +// ── runCompileCommand ───────────────────────────────────────────────────────── + +describe('runCompileCommand', () => { + it('returns ok:true with exit code 0 on success', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'true'); + expect(res.ok).toBe(true); + expect(res.exitCode).toBe(0); + expect(res.command).toBe('true'); + }); + + it('returns ok:false WITHOUT throwing on non-zero exit', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'false'); + expect(res.ok).toBe(false); + expect(res.exitCode).not.toBe(0); + }); + + it('captures combined stdout/stderr output', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'echo hello-build'); + expect(res.output).toContain('hello-build'); + }); + + it('returns ok:false for a missing command without throwing', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'nonexistent_command_xyz'); + expect(res.ok).toBe(false); + }); + + it('short-circuits an &&-chain on first failure', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'false && touch should_not_exist.txt'); + expect(res.ok).toBe(false); + expect(existsSync(join(dir, 'should_not_exist.txt'))).toBe(false); + }); + + it('returns ok:true when every sub-command in an &&-chain succeeds', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'mkdir sub && touch sub/built.txt'); + expect(res.ok).toBe(true); + expect(existsSync(join(dir, 'sub/built.txt'))).toBe(true); + }); + + it('records the signal and ok:false on timeout', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'sleep 10', { timeoutMs: 100 }); + expect(res.ok).toBe(false); + expect(res.signal).not.toBeNull(); + }); + + it('returns ok:false for an empty command', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, ''); + expect(res.ok).toBe(false); + }); +}); diff --git a/packages/eval-graders/src/index.ts b/packages/eval-graders/src/index.ts index 03381627..a6018814 100644 --- a/packages/eval-graders/src/index.ts +++ b/packages/eval-graders/src/index.ts @@ -2,7 +2,14 @@ // Types export { GraderLevel } from './types.js'; -export type { GraderResult, GraderDef, GraderOptions, EventToolCall, EventGraderLevel } from './types.js'; +export type { + GraderResult, + GraderDef, + GraderOptions, + EventToolCall, + EventGraderLevel, + CompileResult, +} from './types.js'; // Grader factory functions export { @@ -14,4 +21,5 @@ export { ranCommand, ranCommandOneOf, wroteFile, + compiles, } from './primitives.js'; diff --git a/packages/eval-graders/src/primitives.ts b/packages/eval-graders/src/primitives.ts index e61c3ed0..94f2841e 100644 --- a/packages/eval-graders/src/primitives.ts +++ b/packages/eval-graders/src/primitives.ts @@ -191,3 +191,18 @@ export function wroteFile( }, }; } + +/** + * Asserts that the eval's compile_command succeeds when run against the workspace + * after the agent finishes. The framework runs the command and captures the result; + * this grader reads it. The command comes from the eval's `compile_command` + * frontmatter, so no command argument is needed here. + */ +export function compiles(description: string | undefined, level: EventGraderLevel): GraderDef { + validateEventLevel(level, 'compiles'); + return { + kind: 'compile', + name: description ?? 'compiles successfully', + level, + }; +} diff --git a/packages/eval-graders/src/types.ts b/packages/eval-graders/src/types.ts index 6a458e5a..f642f680 100644 --- a/packages/eval-graders/src/types.ts +++ b/packages/eval-graders/src/types.ts @@ -18,6 +18,20 @@ export interface EventToolCall { causedError: boolean; } +/** Outcome of running an eval's compile_command against the workspace post-agent. */ +export interface CompileResult { + /** True only if every sub-command exited 0. */ + ok: boolean; + /** Exit code of the failing (or last) sub-command; null if killed by signal. */ + exitCode: number | null; + /** Signal that killed the command (e.g. 'SIGTERM' on timeout); null otherwise. */ + signal: string | null; + /** Combined stdout+stderr of the command run. */ + output: string; + /** The compile_command string that was executed. */ + command: string; +} + export interface GraderResult { name: string; kind: string; diff --git a/packages/eval-graders/tests/primitives.test.ts b/packages/eval-graders/tests/primitives.test.ts index 6af6a142..81bdb9f7 100644 --- a/packages/eval-graders/tests/primitives.test.ts +++ b/packages/eval-graders/tests/primitives.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { contains, notContains, notContainsInSource, matches, judge } from '../src/primitives.js'; +import { contains, notContains, notContainsInSource, matches, judge, compiles } from '../src/primitives.js'; import { GraderLevel } from '../src/types.js'; // ── contains ────────────────────────────────────────────────────────────────── @@ -161,3 +161,29 @@ describe('judge', () => { expect(def.level).toBeUndefined(); }); }); + +// ── compiles ────────────────────────────────────────────────────────────────── + +describe('compiles', () => { + it('returns a compile-kind grader with the given level and name', () => { + const g = compiles('verifies the build', GraderLevel.L4); + expect(g.kind).toBe('compile'); + expect(g.level).toBe(GraderLevel.L4); + expect(g.name).toBe('verifies the build'); + expect(g.predicate).toBeUndefined(); + }); + + it('falls back to a default name when description is omitted', () => { + const g = compiles(undefined, GraderLevel.L4); + expect(g.name).toBe('compiles successfully'); + }); + + it('accepts L5', () => { + expect(compiles('build', GraderLevel.L5).level).toBe(GraderLevel.L5); + }); + + it('rejects non-event levels', () => { + // @ts-expect-error — L1 is not an EventGraderLevel + expect(() => compiles('build', GraderLevel.L1)).toThrow('event-based graders only support'); + }); +}); diff --git a/packages/eval/src/cli/run.ts b/packages/eval/src/cli/run.ts index a2cd082b..3304ca97 100644 --- a/packages/eval/src/cli/run.ts +++ b/packages/eval/src/cli/run.ts @@ -126,7 +126,8 @@ async function runAgentJob( agentType: AgentType, sandbox: boolean, ): Promise { - const { setupWorkspace, runSetupCommand, cleanupWorkspace, writeAgentGuidance } = await import('@a0/eval-core'); + const { setupWorkspace, runSetupCommand, runCompileCommand, cleanupWorkspace, writeAgentGuidance } = + await import('@a0/eval-core'); const { generateRunRecommendations } = await import('../recommendations/index.js'); await initRunners(); @@ -161,6 +162,9 @@ async function runAgentJob( const preparedEval = tools.includes('skills') ? await runner.prepareSkills(evalDef, workspace) : evalDef; const { record, resolvedModel } = await runner.run({ evalDef: preparedEval, workspace, model, tools, apiKey }); + const compileResult = + evalDef.compileCommand !== undefined ? runCompileCommand(workspace, evalDef.compileCommand) : undefined; + let graderResults: Awaited> = []; if (evalDef.graders.length > 0) { const agentLevels = tools.includes('mcp') ? AGENT_MCP_LEVELS : AGENT_LEVELS; @@ -172,6 +176,7 @@ async function runAgentJob( agentLevels, true, record.toolCalls, + compileResult, ); } diff --git a/packages/eval/src/cli/sandbox-runner.ts b/packages/eval/src/cli/sandbox-runner.ts index f7db0db3..4048845e 100644 --- a/packages/eval/src/cli/sandbox-runner.ts +++ b/packages/eval/src/cli/sandbox-runner.ts @@ -21,6 +21,7 @@ import { setFrameworkConfig, runGraders, runSetupCommand, + runCompileCommand, AGENT_LEVELS, AGENT_MCP_LEVELS, registerRunner, @@ -108,6 +109,9 @@ async function main(): Promise { const preparedEval = tools.includes('skills') ? await runner.prepareSkills(evalDef, workspace) : evalDef; const { record, resolvedModel } = await runner.run({ evalDef: preparedEval, workspace, model, tools, apiKey }); + const compileResult = + evalDef.compileCommand !== undefined ? runCompileCommand(workspace, evalDef.compileCommand) : undefined; + let graderResults: Awaited> = []; if (evalDef.graders.length > 0) { const agentLevels = tools.includes('mcp') ? AGENT_MCP_LEVELS : AGENT_LEVELS; @@ -119,6 +123,7 @@ async function main(): Promise { agentLevels, true, record.toolCalls, + compileResult, ); } From 69bbeb2e34ffbe926af3222563276d698606cf21 Mon Sep 17 00:00:00 2001 From: sanchitmehtagit Date: Fri, 19 Jun 2026 16:52:11 +0530 Subject: [PATCH 21/23] feat(mfa/react): add compile-time check to React MFA eval (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the compile grader already present in react_quickstart — adds compile_command: npm run build to PROMPT.md frontmatter and a compiles() grader at L4 so generated MFA code is verified to build after each agent run. --- apps/auth0-evals/src/evals/mfa/react/PROMPT.md | 1 + apps/auth0-evals/src/evals/mfa/react/graders.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/auth0-evals/src/evals/mfa/react/PROMPT.md b/apps/auth0-evals/src/evals/mfa/react/PROMPT.md index 50fc5cd3..948272b5 100644 --- a/apps/auth0-evals/src/evals/mfa/react/PROMPT.md +++ b/apps/auth0-evals/src/evals/mfa/react/PROMPT.md @@ -4,6 +4,7 @@ name: React MFA Step-Up scaffold: src/evals/scaffolds/react/auth0 skills: auth0-mfa setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/mfa/react/graders.ts b/apps/auth0-evals/src/evals/mfa/react/graders.ts index 363c06c0..40003bac 100644 --- a/apps/auth0-evals/src/evals/mfa/react/graders.ts +++ b/apps/auth0-evals/src/evals/mfa/react/graders.ts @@ -1,4 +1,4 @@ -import { contains, notContains, judge, GraderLevel } from '@a0/eval-graders'; +import { contains, notContains, judge, compiles, GraderLevel } from '@a0/eval-graders'; export function defineGraders() { return [ @@ -27,6 +27,7 @@ export function defineGraders() { ), // ── L4: Structural correctness ──────────────────────────────────────── + compiles('Project compiles (build succeeds)', GraderLevel.L4), judge( 'Does the code check the amr claim before executing the transfer action, and only ' + 'proceed when "mfa" is present in the amr array?', From 97eecbe0bf272334802a4a0a4b7b86b92e0b4400 Mon Sep 17 00:00:00 2001 From: sanchitmehtagit Date: Mon, 22 Jun 2026 12:47:40 +0530 Subject: [PATCH 22/23] fix(compile): auto-run npm install before compile_command (#55) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(compile): auto-run npm install before compile_command If the agent adds new deps to package.json but never runs npm install, runCompileCommand would fail with "module not found". Now we prepend npm install automatically whenever package.json exists in the workspace, so the build always has up-to-date dependencies. * fix(compile): use setup_command instead of heuristic npm install detection Replace the package.json-existence heuristic that unconditionally prepended `npm install` with an explicit `setupCommand` option on `RunCompileCommandOptions`. Callers (run.ts, sandbox-runner.ts) now pass `evalDef.setupCommand` so the eval's own declared setup_command is reused — only unshifted when it exists. * fix(compile): add comment explaining why setupCommand is prepended before compile --- packages/eval-core/src/workspace/workspace.ts | 9 +++++++++ .../eval-core/tests/workspace-commands.test.ts | 15 +++++++++++++++ packages/eval/src/cli/run.ts | 4 +++- packages/eval/src/cli/sandbox-runner.ts | 4 +++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/eval-core/src/workspace/workspace.ts b/packages/eval-core/src/workspace/workspace.ts index f015baf4..de143fcd 100644 --- a/packages/eval-core/src/workspace/workspace.ts +++ b/packages/eval-core/src/workspace/workspace.ts @@ -99,6 +99,8 @@ export interface RunSetupCommandOptions { export interface RunCompileCommandOptions { /** Timeout in ms for the compile command. Defaults to {@link DEFAULT_FRAMEWORK_CONFIG}.workspace.compileCommandTimeoutMs. */ timeoutMs?: number; + /** When provided, this command is prepended to the compile command sequence (e.g. the eval's setup_command to ensure deps are installed). */ + setupCommand?: string; } /** @@ -191,6 +193,13 @@ export function runCompileCommand( return { ...base, output: `compile command has an empty segment: ${command}` }; } + // When a setupCommand is provided, we ensure to run it before verifying compilation. + // This is done because there is no guarantee that an agent did not add a dependency + // without running the installation command. + if (options?.setupCommand) { + subCommands.unshift(options.setupCommand); + } + let combinedOutput = ''; for (const subCommand of subCommands) { logger.info(` [Compile] Running: ${subCommand}`); diff --git a/packages/eval-core/tests/workspace-commands.test.ts b/packages/eval-core/tests/workspace-commands.test.ts index 7d0ef670..3716ca90 100644 --- a/packages/eval-core/tests/workspace-commands.test.ts +++ b/packages/eval-core/tests/workspace-commands.test.ts @@ -185,4 +185,19 @@ describe('runCompileCommand', () => { const res = runCompileCommand(dir, ''); expect(res.ok).toBe(false); }); + + it('prepends setupCommand when provided', () => { + const dir = tmpDir(); + const sentinel = join(dir, 'setup_ran.txt'); + const res = runCompileCommand(dir, 'true', { setupCommand: `touch ${sentinel}` }); + expect(res.ok).toBe(true); + expect(existsSync(sentinel)).toBe(true); + }); + + it('does not prepend any command when setupCommand is absent', () => { + const dir = tmpDir(); + const res = runCompileCommand(dir, 'echo hello-compile'); + expect(res.ok).toBe(true); + expect(res.output).toContain('hello-compile'); + }); }); diff --git a/packages/eval/src/cli/run.ts b/packages/eval/src/cli/run.ts index 3304ca97..045ea62b 100644 --- a/packages/eval/src/cli/run.ts +++ b/packages/eval/src/cli/run.ts @@ -163,7 +163,9 @@ async function runAgentJob( const { record, resolvedModel } = await runner.run({ evalDef: preparedEval, workspace, model, tools, apiKey }); const compileResult = - evalDef.compileCommand !== undefined ? runCompileCommand(workspace, evalDef.compileCommand) : undefined; + evalDef.compileCommand !== undefined + ? runCompileCommand(workspace, evalDef.compileCommand, { setupCommand: evalDef.setupCommand }) + : undefined; let graderResults: Awaited> = []; if (evalDef.graders.length > 0) { diff --git a/packages/eval/src/cli/sandbox-runner.ts b/packages/eval/src/cli/sandbox-runner.ts index 4048845e..dfd9668c 100644 --- a/packages/eval/src/cli/sandbox-runner.ts +++ b/packages/eval/src/cli/sandbox-runner.ts @@ -110,7 +110,9 @@ async function main(): Promise { const { record, resolvedModel } = await runner.run({ evalDef: preparedEval, workspace, model, tools, apiKey }); const compileResult = - evalDef.compileCommand !== undefined ? runCompileCommand(workspace, evalDef.compileCommand) : undefined; + evalDef.compileCommand !== undefined + ? runCompileCommand(workspace, evalDef.compileCommand, { setupCommand: evalDef.setupCommand }) + : undefined; let graderResults: Awaited> = []; if (evalDef.graders.length > 0) { From 6a5d4852bc850b7c826a18e180e1c423f52be8ba Mon Sep 17 00:00:00 2001 From: Bharath Natarajan Date: Mon, 22 Jun 2026 17:11:00 +0530 Subject: [PATCH 23/23] chore(evals): add compile_command and compiles() grader to CTE evals Aligns with new convention introduced in main (MFA eval updated same way). compile_command: npm run build added to both PROMPT.md files. compiles() grader added at L4 in both startup and enterprise graders. --- .../src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md | 1 + .../evals/custom-token-exchange/nextjs-enterprise/graders.ts | 2 ++ .../src/evals/custom-token-exchange/nextjs-startup/PROMPT.md | 1 + .../src/evals/custom-token-exchange/nextjs-startup/graders.ts | 2 ++ 4 files changed, 6 insertions(+) diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md index 59fdb279..0011a7f7 100644 --- a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/PROMPT.md @@ -4,6 +4,7 @@ name: Custom Token Exchange — Next.js (Enterprise) scaffold: src/evals/custom-token-exchange/nextjs-enterprise skills: auth0-custom-token-exchange setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts index e3eb5bdb..c6ba4aa4 100644 --- a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-enterprise/graders.ts @@ -5,6 +5,7 @@ import { judge, ranCommandOneOf, wroteFile, + compiles, GraderLevel, } from '@a0/eval-graders'; @@ -59,6 +60,7 @@ export function defineGraders() { ), // ── L4: Structural correctness + tenant config (any valid method) ────── + compiles('Project compiles without errors', GraderLevel.L4), judge( 'Is auth0.customTokenExchange() called in a server-side context — a Route Handler, ' + 'Server Component, or Server Action — and NOT inside a file marked with "use client"?', diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md index 9f29de6e..1cbf3e4a 100644 --- a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/PROMPT.md @@ -4,6 +4,7 @@ name: Custom Token Exchange — Next.js (Startup) scaffold: src/evals/custom-token-exchange/nextjs-startup skills: auth0-custom-token-exchange setup_command: npm install +compile_command: npm run build --- ## Task diff --git a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts index 352341df..aeeed404 100644 --- a/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts +++ b/apps/auth0-evals/src/evals/custom-token-exchange/nextjs-startup/graders.ts @@ -4,6 +4,7 @@ import { notContainsInSource, judge, ranCommand, + compiles, GraderLevel, } from '@a0/eval-graders'; @@ -58,6 +59,7 @@ export function defineGraders() { ), // ── L4: Structural correctness + tenant config ───────────────────────── + compiles('Project compiles without errors', GraderLevel.L4), judge( 'Is auth0.customTokenExchange() called in a server-side context — a Route Handler, ' + 'Server Component, or Server Action — and NOT inside a file marked with "use client"?',