From 19055067673a0492738ae4b6dced0cf341240bb7 Mon Sep 17 00:00:00 2001 From: Lena Date: Thu, 11 Jun 2026 10:45:24 -0400 Subject: [PATCH 01/19] feat: add hosted_mcp_create_application eval Tests whether an agent correctly creates a SPA in the tenant using the auth0_create_application MCP tool with the right parameters. Graders: - L2: no hallucinated tool names (create_client, create_app, register_application) - L4: calledTool check for auth0_create_application - L5: custom arg predicates verifying app_type=spa and name="My Web App" Grade A (100%) on claude-opus-4-7 with --tools mcp. Co-Authored-By: Claude Sonnet 4.6 --- .../hosted-mcp/create-application/PROMPT.md | 17 ++++++++++ .../hosted-mcp/create-application/graders.ts | 34 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-application/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md new file mode 100644 index 00000000..ea1d70d8 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md @@ -0,0 +1,17 @@ +--- +id: hosted_mcp_create_application +name: Hosted MCP - Create Application +--- + +## Task + +I need to register a new Single Page Application in my Auth0 tenant. + +- Name: **My Web App** +- Type: Single Page Application (SPA) +- Allowed Callback URL: `https://mywebapp.example.com/callback` +- Allowed Logout URL: `https://mywebapp.example.com` + +Domain: mcptesttenant.tus.auth0.com + +Create the application and confirm back to me with the new Client ID. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-application/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/create-application/graders.ts new file mode 100644 index 00000000..2bce61ca --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-application/graders.ts @@ -0,0 +1,34 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +// Returns successful MCP calls that match the given tool name substring. +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + // ── L2: Did not hallucinate wrong tool names ────────────────────────────── + notContains('create_client', 'No hallucinated create_client tool name', GraderLevel.L2), + notContains('create_app', 'No hallucinated create_app tool name', GraderLevel.L2), + notContains('register_application', 'No hallucinated register_application tool name', GraderLevel.L2), + + // ── L4: Called the right tool ───────────────────────────────────────────── + calledTool('auth0_create_application', 'Called the auth0_create_application MCP tool', GraderLevel.L4), + + // ── L5: Correct parameters ──────────────────────────────────────────────── + { + kind: 'event', + name: 'Created application with app_type set to spa', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_application').some((tc) => tc.args['app_type'] === 'spa'), + }, + { + kind: 'event', + name: 'Created application with the correct name "My Web App"', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_application').some((tc) => tc.args['name'] === 'My Web App'), + }, + ]; +} From 25ee21d15ce10ffb47e94dafaee3502e9a05a286 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 15 Jun 2026 13:56:30 -0400 Subject: [PATCH 02/19] feat(eval): add hosted_mcp_setup_m2m_application eval Multi-step eval: create API, create M2M app, grant access. All 3 tools called in sequence with L5 arg checks. Grades A (100%) on claude-opus-4-7. --- .../setup-m2m-application/PROMPT.md | 22 ++++++ .../setup-m2m-application/graders.ts | 70 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md new file mode 100644 index 00000000..30800215 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md @@ -0,0 +1,22 @@ +--- +id: hosted_mcp_setup_m2m_application +name: Hosted MCP - Set Up M2M Application +category: hosted-mcp +--- + +## Task + +I need to set up a machine-to-machine integration in my Auth0 tenant. Please do the following in order: + +1. Create a new API with: + - Name: **Inventory Service** + - Identifier (audience): `https://api.inventory.example.com` + - A scope named `read:inventory` with description "Read inventory data" + +2. Create a new Machine to Machine application named **Warehouse Bot** + +3. Authorize the **Warehouse Bot** application to call the **Inventory Service** API with the `read:inventory` scope + +Domain: mcptesttenant.tus.auth0.com + +Confirm back with the Client ID of the Warehouse Bot and the resource server ID of the Inventory Service. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts new file mode 100644 index 00000000..a6f8185a --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts @@ -0,0 +1,70 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + // ── L2: No hallucinated tool names ──────────────────────────────────────── + notContains('create_api', 'No hallucinated create_api tool name', GraderLevel.L2), + notContains('create_m2m', 'No hallucinated create_m2m tool name', GraderLevel.L2), + notContains('authorize_client', 'No hallucinated authorize_client tool name', GraderLevel.L2), + + // ── L4: Called all three required tools ─────────────────────────────────── + calledTool('auth0_create_resource_server', 'Called auth0_create_resource_server to create the API', GraderLevel.L4), + calledTool('auth0_create_application', 'Called auth0_create_application to create the M2M app', GraderLevel.L4), + calledTool( + 'auth0_create_application_grant', + 'Called auth0_create_application_grant to authorize the app', + GraderLevel.L4, + ), + + // ── L5: Correct parameters ──────────────────────────────────────────────── + { + kind: 'event', + name: 'Created resource server with correct identifier', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_resource_server').some( + (tc) => tc.args['identifier'] === 'https://api.inventory.example.com', + ), + }, + { + kind: 'event', + name: 'Created resource server with read:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'read:inventory'); + }), + }, + { + kind: 'event', + name: 'Created M2M application with app_type non_interactive', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_application').some((tc) => tc.args['app_type'] === 'non_interactive'), + }, + { + kind: 'event', + name: 'Created application grant with read:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => { + const scope = tc.args['scope'] as string[] | undefined; + return Array.isArray(scope) && scope.includes('read:inventory'); + }), + }, + { + kind: 'event', + name: 'Application grant targets the correct audience', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_application_grant').some( + (tc) => tc.args['audience'] === 'https://api.inventory.example.com', + ), + }, + ]; +} From 75efa6febee81f081244e1c0c643cb3f27128a64 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 15 Jun 2026 14:07:16 -0400 Subject: [PATCH 03/19] feat(eval): add hosted_mcp_diagnose_with_logs eval Read-and-reason eval: query logs with filter, report failure types and counts. Grades A (100%) on claude-opus-4-7. --- .../hosted-mcp/diagnose-with-logs/PROMPT.md | 18 ++++++++++ .../hosted-mcp/diagnose-with-logs/graders.ts | 36 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md new file mode 100644 index 00000000..9386c314 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md @@ -0,0 +1,18 @@ +--- +id: hosted_mcp_diagnose_with_logs +name: Hosted MCP - Diagnose Login Failures with Logs +category: hosted-mcp +--- + +## Task + +My Auth0 tenant has been experiencing login failures recently. + +Domain: mcptesttenant.tus.auth0.com + +Check the recent logs and tell me: +1. What types of login failures are occurring (include the event type codes) +2. How many of each failure type you found +3. What the most likely cause is + +Use the logs to give me a concrete answer based on what is actually in the tenant. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/graders.ts new file mode 100644 index 00000000..a2a4cc24 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/graders.ts @@ -0,0 +1,36 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + // ── L2: Did not hallucinate data or make up error types ─────────────────── + notContains('I cannot access', 'Did not refuse to check logs', GraderLevel.L2), + notContains("I don't have access", 'Did not claim lack of access', GraderLevel.L2), + + // ── L4: Actually queried the logs ───────────────────────────────────────── + calledTool('auth0_list_logs', 'Called auth0_list_logs to retrieve tenant logs', GraderLevel.L4), + + // ── L5: Used a failure-focused query and reported concrete findings ─────── + { + kind: 'event', + name: 'Queried logs with a failure filter (q parameter)', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_list_logs').some( + (tc) => typeof tc.args['q'] === 'string' && tc.args['q'].length > 0, + ), + }, + { + kind: 'event', + name: 'Retrieved a meaningful number of log entries (take >= 10)', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_list_logs').some( + (tc) => typeof tc.args['take'] !== 'number' || (tc.args['take'] as number) >= 10, + ), + }, + ]; +} From 0b98a18000785490af3c0782b0f3bcade5f3c761 Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 15 Jun 2026 14:29:25 -0400 Subject: [PATCH 04/19] feat(eval): add hosted_mcp_create_and_deploy_action eval Multi-step eval: create Post-Login action with roles claim, then deploy it. L5 checks trigger ID, event.authorization.roles usage, and idToken assignment. Grades A (100%) on claude-opus-4-7. --- .../create-and-deploy-action/PROMPT.md | 17 +++++++ .../create-and-deploy-action/graders.ts | 50 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md new file mode 100644 index 00000000..d16c75f3 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md @@ -0,0 +1,17 @@ +--- +id: hosted_mcp_create_and_deploy_action +name: Hosted MCP - Create and Deploy Action +category: hosted-mcp +--- + +## Task + +I need a Post-Login action in my Auth0 tenant that adds the user's roles to the ID token. + +Domain: mcptesttenant.tus.auth0.com + +Please: +1. Create an action named **Add Roles to Token** that runs on the `post-login` trigger and adds a `roles` claim to the ID token using `event.authorization.roles` +2. Deploy the action so it is active + +Confirm back with the action ID once it is deployed. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/graders.ts new file mode 100644 index 00000000..ad799d22 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/graders.ts @@ -0,0 +1,50 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + // ── L2: No hallucinated trigger IDs ─────────────────────────────────────── + notContains('post_login', 'No hallucinated post_login trigger ID (correct is post-login)', GraderLevel.L2), + notContains('postLogin', 'No hallucinated postLogin trigger ID (correct is post-login)', GraderLevel.L2), + + // ── L4: Called both required tools ──────────────────────────────────────── + calledTool('auth0_create_action', 'Called auth0_create_action to create the action', GraderLevel.L4), + calledTool('auth0_deploy_action', 'Called auth0_deploy_action to deploy the action', GraderLevel.L4), + + // ── L5: Correct parameters ──────────────────────────────────────────────── + { + kind: 'event', + name: 'Created action with correct post-login trigger ID', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_action').some((tc) => { + const triggers = tc.args['supported_triggers'] as Array<{ id: string }> | undefined; + return Array.isArray(triggers) && triggers.some((t) => t.id === 'post-login'); + }), + }, + { + kind: 'event', + name: 'Action code references event.authorization.roles', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_action').some( + (tc) => typeof tc.args['code'] === 'string' && tc.args['code'].includes('event.authorization.roles'), + ), + }, + { + kind: 'event', + name: 'Action code sets a roles claim on the ID token', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_action').some( + (tc) => + typeof tc.args['code'] === 'string' && + tc.args['code'].includes('idToken') && + tc.args['code'].includes('roles'), + ), + }, + ]; +} From 564be5b82926a3df8be76ebd8298d684ad86ae3a Mon Sep 17 00:00:00 2001 From: Lena Date: Mon, 15 Jun 2026 15:09:14 -0400 Subject: [PATCH 05/19] feat(eval): add hosted_mcp_create_progressive_profile_form eval Tests creating a progressive profiling form with named fields (job_title, company). L5 checks form name and presence of both field keys in nodes. Grades A (100%) on claude-opus-4-7 with MCP. --- .../create-progressive-profile-form/PROMPT.md | 19 ++++++++ .../graders.ts | 45 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md new file mode 100644 index 00000000..b8bbf794 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md @@ -0,0 +1,19 @@ +--- +id: hosted_mcp_create_progressive_profile_form +name: Hosted MCP - Create Progressive Profile Form +category: hosted-mcp +--- + +## Task + +I need a progressive profiling form in my Auth0 tenant that collects additional information from users after they log in. + +Domain: mcptesttenant.tus.auth0.com + +Please create a form with: +- Name: **Collect Work Info** +- Two input fields: + - Job Title (field key: `job_title`) + - Company Name (field key: `company`) + +Confirm back with the form ID once it is created. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/graders.ts new file mode 100644 index 00000000..7cb72cb9 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/graders.ts @@ -0,0 +1,45 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +const nodesContainKey = (nodes: unknown, key: string): boolean => { + if (!Array.isArray(nodes)) return false; + const haystack = JSON.stringify(nodes).toLowerCase(); + return haystack.includes(key.toLowerCase()); +}; + +export function defineGraders(): GraderDef[] { + return [ + // ── L2: No hallucinated tool names ──────────────────────────────────────── + notContains('create_progressive_profile', 'No hallucinated create_progressive_profile tool name', GraderLevel.L2), + notContains('create_custom_form', 'No hallucinated create_custom_form tool name', GraderLevel.L2), + + // ── L4: Called the right tool ───────────────────────────────────────────── + calledTool('auth0_create_form', 'Called auth0_create_form to create the form', GraderLevel.L4), + + // ── L5: Correct parameters ──────────────────────────────────────────────── + { + kind: 'event', + name: 'Created form with the correct name "Collect Work Info"', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_form').some((tc) => tc.args['name'] === 'Collect Work Info'), + }, + { + kind: 'event', + name: 'Form nodes include job_title field', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_form').some((tc) => nodesContainKey(tc.args['nodes'], 'job_title')), + }, + { + kind: 'event', + name: 'Form nodes include company field', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_create_form').some((tc) => nodesContainKey(tc.args['nodes'], 'company')), + }, + ]; +} From a4c1314e893999661566261d0050f7c64c119fba Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 17 Jun 2026 11:24:23 -0400 Subject: [PATCH 06/19] feat(eval): add hosted_mcp_update_application_callbacks eval Read-then-write eval: list apps to find by name, get existing callbacks, update with new URL preserved alongside existing ones. L5 checks new callback present and client_id was looked up dynamically. Grades A (100%) on claude-opus-4-7. --- .../update-application-callbacks/PROMPT.md | 13 +++++++ .../update-application-callbacks/graders.ts | 38 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md new file mode 100644 index 00000000..557650c1 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md @@ -0,0 +1,13 @@ +--- +id: hosted_mcp_update_application_callbacks +name: Hosted MCP - Update Application Callbacks +category: hosted-mcp +--- + +## Task + +My application called **Warehouse Bot** needs a new allowed callback URL added: `https://warehouse.example.com/callback` + +Domain: mcptesttenant.tus.auth0.com + +Find the application by name and update its allowed callback URLs to include the new one. Keep any existing callback URLs — do not replace them. Confirm back with the updated list of callback URLs. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/graders.ts new file mode 100644 index 00000000..1fd32b85 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/graders.ts @@ -0,0 +1,38 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + // ── L2: Did not hallucinate tool names ──────────────────────────────────── + notContains('update_client', 'No hallucinated update_client tool name', GraderLevel.L2), + notContains('patch_application', 'No hallucinated patch_application tool name', GraderLevel.L2), + + // ── L4: Called both tools in the right order (lookup then update) ───────── + calledTool('auth0_list_applications', 'Called auth0_list_applications to find the app by name', GraderLevel.L4), + calledTool('auth0_update_application', 'Called auth0_update_application to add the callback URL', GraderLevel.L4), + + // ── L5: Correct parameters ──────────────────────────────────────────────── + { + kind: 'event', + name: 'Update includes the new callback URL', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_update_application').some((tc) => { + const callbacks = tc.args['callbacks'] as string[] | undefined; + return Array.isArray(callbacks) && callbacks.includes('https://warehouse.example.com/callback'); + }), + }, + { + kind: 'event', + name: 'Update was called with a client_id (looked up dynamically, not hardcoded)', + level: GraderLevel.L5, + predicate: (toolCalls) => + mcpCalls(toolCalls, 'auth0_update_application').some( + (tc) => typeof tc.args['client_id'] === 'string' && tc.args['client_id'].length > 0, + ), + }, + ]; +} From 2f07ee67a3b08979a2d079c95544cd3392eb8516 Mon Sep 17 00:00:00 2001 From: Lena Date: Fri, 19 Jun 2026 14:42:39 -0400 Subject: [PATCH 07/19] feat(eval): remove hardcoded domain from prompts and add hosted_mcp_investigate_api_scopes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed hardcoded tenant domain from all hosted-mcp PROMPT.md files — domain is injected via eval.config.js env vars and is not needed in prompts. Added hosted_mcp_investigate_api_scopes: list resource servers, inspect existing scopes, then update with a new write:inventory scope while preserving read:inventory (read→reason→write chain). --- .../create-and-deploy-action/PROMPT.md | 1 - .../hosted-mcp/create-application/PROMPT.md | 1 - .../create-progressive-profile-form/PROMPT.md | 1 - .../hosted-mcp/diagnose-with-logs/PROMPT.md | 1 - .../investigate-api-scopes/PROMPT.md | 16 ++++++ .../investigate-api-scopes/graders.ts | 52 +++++++++++++++++++ .../setup-m2m-application/PROMPT.md | 1 - .../update-application-callbacks/PROMPT.md | 1 - 8 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md index d16c75f3..cb4c63fb 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md @@ -8,7 +8,6 @@ category: hosted-mcp I need a Post-Login action in my Auth0 tenant that adds the user's roles to the ID token. -Domain: mcptesttenant.tus.auth0.com Please: 1. Create an action named **Add Roles to Token** that runs on the `post-login` trigger and adds a `roles` claim to the ID token using `event.authorization.roles` diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md index ea1d70d8..9561eb13 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-application/PROMPT.md @@ -12,6 +12,5 @@ I need to register a new Single Page Application in my Auth0 tenant. - Allowed Callback URL: `https://mywebapp.example.com/callback` - Allowed Logout URL: `https://mywebapp.example.com` -Domain: mcptesttenant.tus.auth0.com Create the application and confirm back to me with the new Client ID. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md index b8bbf794..1dd1625e 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-progressive-profile-form/PROMPT.md @@ -8,7 +8,6 @@ category: hosted-mcp I need a progressive profiling form in my Auth0 tenant that collects additional information from users after they log in. -Domain: mcptesttenant.tus.auth0.com Please create a form with: - Name: **Collect Work Info** diff --git a/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md index 9386c314..9e241363 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md +++ b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-with-logs/PROMPT.md @@ -8,7 +8,6 @@ category: hosted-mcp My Auth0 tenant has been experiencing login failures recently. -Domain: mcptesttenant.tus.auth0.com Check the recent logs and tell me: 1. What types of login failures are occurring (include the event type codes) diff --git a/apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/PROMPT.md new file mode 100644 index 00000000..e74e2176 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_investigate_api_scopes +name: Hosted MCP - Investigate and Update API Scopes +category: hosted-mcp +--- + +## Task + +My **Inventory Service** API is missing a scope that the team needs. + +Please: +1. Find the **Inventory Service** resource server in my Auth0 tenant +2. Check what scopes it currently has +3. Add a new scope named `write:inventory` with description "Write inventory data" — keeping all existing scopes + +Confirm back with the full updated list of scopes on the Inventory Service. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/graders.ts new file mode 100644 index 00000000..a32efd4b --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/investigate-api-scopes/graders.ts @@ -0,0 +1,52 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('list_apis', 'No hallucinated list_apis tool name', GraderLevel.L2), + notContains('update_api', 'No hallucinated update_api tool name', GraderLevel.L2), + notContains('patch_resource_server', 'No hallucinated patch_resource_server tool name', GraderLevel.L2), + calledTool( + 'auth0_list_resource_servers', + 'Called auth0_list_resource_servers to find the Inventory Service', + GraderLevel.L4, + ), + calledTool( + 'auth0_update_resource_server', + 'Called auth0_update_resource_server to add the new scope', + GraderLevel.L4, + ), + { + kind: 'event', + name: 'Update includes the new write:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'write:inventory'); + }), + }, + { + kind: 'event', + name: 'Update preserves the existing read:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'read:inventory'); + }), + }, + { + kind: 'event', + name: 'Update was called with a resource_server_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some( + (tc) => typeof tc.args['resource_server_id'] === 'string' && tc.args['resource_server_id'].length > 0, + ), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md index 30800215..8e14ccf7 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/PROMPT.md @@ -17,6 +17,5 @@ I need to set up a machine-to-machine integration in my Auth0 tenant. Please do 3. Authorize the **Warehouse Bot** application to call the **Inventory Service** API with the `read:inventory` scope -Domain: mcptesttenant.tus.auth0.com Confirm back with the Client ID of the Warehouse Bot and the resource server ID of the Inventory Service. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md index 557650c1..7358064b 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-application-callbacks/PROMPT.md @@ -8,6 +8,5 @@ category: hosted-mcp My application called **Warehouse Bot** needs a new allowed callback URL added: `https://warehouse.example.com/callback` -Domain: mcptesttenant.tus.auth0.com Find the application by name and update its allowed callback URLs to include the new one. Keep any existing callback URLs — do not replace them. Confirm back with the updated list of callback URLs. From 3348dd31161ed14bddfd6294abd6429beb13b650 Mon Sep 17 00:00:00 2001 From: Lena Date: Tue, 23 Jun 2026 11:32:24 -0400 Subject: [PATCH 08/19] feat(eval): add enable_action_in_flow, audit_tenant_applications, find_and_inspect_application evals --- .../audit-tenant-applications/PROMPT.md | 15 ++++++++ .../audit-tenant-applications/graders.ts | 12 ++++++ .../enable-action-in-flow/PROMPT.md | 16 ++++++++ .../enable-action-in-flow/graders.ts | 32 ++++++++++++++++ .../find-and-inspect-application/PROMPT.md | 15 ++++++++ .../find-and-inspect-application/graders.ts | 38 +++++++++++++++++++ 6 files changed, 128 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/PROMPT.md new file mode 100644 index 00000000..797e8efd --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_audit_tenant_applications +name: Hosted MCP - Audit Tenant Applications +category: hosted-mcp +--- + +## Task + +I need a quick security audit of the applications in my Auth0 tenant. + +Please list all applications and identify: +1. Which ones are Machine to Machine (M2M) apps +2. Which ones have no allowed callback URLs configured + +Give me a summary with the app names and client IDs for each category. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/graders.ts new file mode 100644 index 00000000..89c3fce4 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/graders.ts @@ -0,0 +1,12 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef } from '@a0/eval-graders'; + +export function defineGraders(): GraderDef[] { + return [ + notContains('get_clients', 'No hallucinated get_clients tool name', GraderLevel.L2), + notContains('list_clients', 'No hallucinated list_clients tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to retrieve all apps', GraderLevel.L4), + contains('non_interactive', 'Response identifies M2M (non_interactive) app type', GraderLevel.L5), + contains('Warehouse Bot', 'Response mentions Warehouse Bot (known M2M app)', GraderLevel.L5), + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/PROMPT.md new file mode 100644 index 00000000..414b342f --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_enable_action_in_flow +name: Hosted MCP - Enable Action in Login Flow +category: hosted-mcp +--- + +## Task + +I need to check the status of my Auth0 actions and make sure the right one is deployed. + +Please: +1. List all actions in my Auth0 tenant +2. Find the one named **Add Roles to Token** +3. If it is not already deployed, deploy it + +Confirm back with the action ID and its current status. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/graders.ts new file mode 100644 index 00000000..63153b16 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/enable-action-in-flow/graders.ts @@ -0,0 +1,32 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('list_actions', 'No hallucinated list_actions tool name', GraderLevel.L2), + notContains('deploy_action', 'No hallucinated deploy_action tool name', GraderLevel.L2), + calledTool('auth0_list_actions', 'Called auth0_list_actions to discover existing actions', GraderLevel.L4), + { + kind: 'event', + name: 'Listed actions before deciding whether to deploy', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => { + const listIdx = toolCalls.findIndex((tc) => tc.name.toLowerCase().includes('auth0_list_actions')); + const deployIdx = toolCalls.findIndex((tc) => tc.name.toLowerCase().includes('auth0_deploy_action')); + return listIdx !== -1 && (deployIdx === -1 || listIdx < deployIdx); + }, + }, + { + kind: 'event', + name: 'Deployed with a valid action_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_deploy_action').some( + (tc) => typeof tc.args['action_id'] === 'string' && tc.args['action_id'].length > 0, + ), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/PROMPT.md new file mode 100644 index 00000000..a3644fa6 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_find_and_inspect_application +name: Hosted MCP - Find and Inspect Application +category: hosted-mcp +--- + +## Task + +I need the full details of a specific application in my Auth0 tenant. + +Please: +1. Find the application named **Warehouse Bot** +2. Get its full configuration details + +Tell me its client ID, app type, and what callback URLs it has configured. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/graders.ts new file mode 100644 index 00000000..931d5662 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/find-and-inspect-application/graders.ts @@ -0,0 +1,38 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('get_client', 'No hallucinated get_client tool name', GraderLevel.L2), + notContains('find_client', 'No hallucinated find_client tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to find Warehouse Bot by name', GraderLevel.L4), + { + kind: 'event', + name: 'Called auth0_get_application to fetch full details', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some( + (tc) => + !tc.causedError && + (tc.name.toLowerCase().includes('auth0_get_application') || + tc.name.toLowerCase().includes('auth0_find_application_by_name')), + ), + }, + { + kind: 'event', + name: 'Fetched application using a client_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_get_application').some( + (tc) => typeof tc.args['client_id'] === 'string' && tc.args['client_id'].length > 0, + ) || + mcpCalls(toolCalls, 'auth0_find_application_by_name').some( + (tc) => typeof tc.args['name'] === 'string' && tc.args['name'].length > 0, + ), + }, + contains('non_interactive', 'Response correctly identifies app type as non_interactive (M2M)', GraderLevel.L5), + ]; +} From 8b2bdfd83b091b1e94cbe725c64f1f9a195863d6 Mon Sep 17 00:00:00 2001 From: Lena Date: Tue, 23 Jun 2026 16:56:19 -0400 Subject: [PATCH 09/19] feat(eval): add debug_failed_login, update_action_code, create_api_and_inspect evals --- .../create-api-and-inspect/PROMPT.md | 18 +++++++ .../create-api-and-inspect/graders.ts | 53 ++++++++++++++++++ .../hosted-mcp/debug-failed-login/PROMPT.md | 16 ++++++ .../hosted-mcp/debug-failed-login/graders.ts | 33 ++++++++++++ .../hosted-mcp/update-action-code/PROMPT.md | 17 ++++++ .../hosted-mcp/update-action-code/graders.ts | 54 +++++++++++++++++++ 6 files changed, 191 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-action-code/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-action-code/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/PROMPT.md new file mode 100644 index 00000000..00eb2a6c --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/PROMPT.md @@ -0,0 +1,18 @@ +--- +id: hosted_mcp_create_api_and_inspect +name: Hosted MCP - Create API and Inspect Result +category: hosted-mcp +--- + +## Task + +I need to set up a new API in my Auth0 tenant and verify it was created correctly. + +Please: +1. Create a new API (resource server) with: + - Name: **Reporting Service** + - Identifier (audience): `https://api.reporting.example.com` + - A scope named `read:reports` with description "Read reports" +2. Retrieve the resource server you just created to confirm the identifier and scopes are correct + +Confirm back with the resource server ID and the list of scopes. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/graders.ts new file mode 100644 index 00000000..cf9a7abd --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/create-api-and-inspect/graders.ts @@ -0,0 +1,53 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_api', 'No hallucinated create_api tool name', GraderLevel.L2), + notContains('get_api', 'No hallucinated get_api tool name', GraderLevel.L2), + calledTool( + 'auth0_create_resource_server', + 'Called auth0_create_resource_server to create the Reporting Service API', + GraderLevel.L4, + ), + { + kind: 'event', + name: 'Called auth0_get_resource_server or auth0_list_resource_servers to verify creation', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => { + const createIdx = toolCalls.findIndex((tc) => + tc.name.toLowerCase().includes('auth0_create_resource_server'), + ); + const verifyIdx = toolCalls.findIndex( + (tc) => + !tc.causedError && + (tc.name.toLowerCase().includes('auth0_get_resource_server') || + tc.name.toLowerCase().includes('auth0_list_resource_servers')), + ); + return createIdx !== -1 && verifyIdx > createIdx; + }, + }, + { + kind: 'event', + name: 'Created resource server with correct identifier', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_resource_server').some( + (tc) => tc.args['identifier'] === 'https://api.reporting.example.com', + ), + }, + { + kind: 'event', + name: 'Created resource server with read:reports scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'read:reports'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/PROMPT.md new file mode 100644 index 00000000..28f4fcc9 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_debug_failed_login +name: Hosted MCP - Debug Failed Login for User +category: hosted-mcp +--- + +## Task + +A specific user is reporting they cannot log in. I need to investigate what is happening. + +Please search the Auth0 logs for failed login events and tell me: +1. How many failed login attempts occurred recently +2. What error codes or types were involved +3. What the most recent failure timestamp was + +Focus on events with failure/error types (not successful logins). diff --git a/apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/graders.ts new file mode 100644 index 00000000..4b01fb21 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/debug-failed-login/graders.ts @@ -0,0 +1,33 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('get_logs', 'No hallucinated get_logs tool name', GraderLevel.L2), + notContains('search_logs', 'No hallucinated search_logs tool name', GraderLevel.L2), + calledTool('auth0_list_logs', 'Called auth0_list_logs to search for failed login events', GraderLevel.L4), + { + kind: 'event', + name: 'Filtered logs with a query targeting failure events', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_list_logs').some((tc) => { + const q = tc.args['q'] as string | undefined; + // Common failure type codes: f (failed login), fp (wrong password), fu (blocked), etc. + return typeof q === 'string' && (q.includes('type:f') || q.includes('type:fp') || q.includes('type:fu') || q.toLowerCase().includes('fail')); + }), + }, + { + kind: 'event', + name: 'Used take parameter to limit log results', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_list_logs').some( + (tc) => typeof tc.args['take'] === 'number' && tc.args['take'] > 0, + ), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-action-code/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/update-action-code/PROMPT.md new file mode 100644 index 00000000..e7939585 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-action-code/PROMPT.md @@ -0,0 +1,17 @@ +--- +id: hosted_mcp_update_action_code +name: Hosted MCP - Update Action Code +category: hosted-mcp +--- + +## Task + +The **Add Roles to Token** action in my Auth0 tenant needs its code updated. + +Please: +1. Find the **Add Roles to Token** action +2. Update its code to also add a `permissions` claim to the ID token using `event.authorization.permissions` + +The updated code should still add `roles` as before, and now also add `permissions`. Keep the same trigger and other settings. + +Confirm back with the action ID once updated. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-action-code/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/update-action-code/graders.ts new file mode 100644 index 00000000..86b27039 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-action-code/graders.ts @@ -0,0 +1,54 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('patch_action', 'No hallucinated patch_action tool name', GraderLevel.L2), + notContains('update_action_code', 'No hallucinated update_action_code tool name', GraderLevel.L2), + calledTool('auth0_list_actions', 'Called auth0_list_actions to find the action by name', GraderLevel.L4), + { + kind: 'event', + name: 'Called auth0_update_action or auth0_get_action to inspect/update the action', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some( + (tc) => + !tc.causedError && + (tc.name.toLowerCase().includes('auth0_update_action') || + tc.name.toLowerCase().includes('auth0_get_action')), + ), + }, + { + kind: 'event', + name: 'Updated action code includes event.authorization.permissions', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_action').some((tc) => { + const code = tc.args['code'] as string | undefined; + return typeof code === 'string' && code.includes('event.authorization.permissions'); + }), + }, + { + kind: 'event', + name: 'Updated action code retains event.authorization.roles', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_action').some((tc) => { + const code = tc.args['code'] as string | undefined; + return typeof code === 'string' && code.includes('event.authorization.roles'); + }), + }, + { + kind: 'event', + name: 'Update was called with a valid action_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_action').some( + (tc) => typeof tc.args['action_id'] === 'string' && tc.args['action_id'].length > 0, + ), + }, + ]; +} From a446765b6a07d93297d4cb7d4445b2a76bb7e4f8 Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 09:15:33 -0400 Subject: [PATCH 10/19] feat(eval): add setup_spa_with_api, get_log_details, update_form_fields evals --- .../hosted-mcp/get-log-details/PROMPT.md | 15 +++++ .../hosted-mcp/get-log-details/graders.ts | 38 +++++++++++++ .../hosted-mcp/setup-spa-with-api/PROMPT.md | 22 ++++++++ .../hosted-mcp/setup-spa-with-api/graders.ts | 55 +++++++++++++++++++ .../hosted-mcp/update-form-fields/PROMPT.md | 15 +++++ .../hosted-mcp/update-form-fields/graders.ts | 53 ++++++++++++++++++ 6 files changed, 198 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/get-log-details/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/get-log-details/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/get-log-details/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/get-log-details/PROMPT.md new file mode 100644 index 00000000..4b010c58 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/get-log-details/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_get_log_details +name: Hosted MCP - Get Log Event Details +category: hosted-mcp +--- + +## Task + +I need to investigate a specific recent log event in my Auth0 tenant. + +Please: +1. Search the logs and find the most recent successful login event (type `s`) +2. Get the full details of that specific log event + +Tell me the user who logged in, the client application used, and the timestamp. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/get-log-details/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/get-log-details/graders.ts new file mode 100644 index 00000000..12ea3af7 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/get-log-details/graders.ts @@ -0,0 +1,38 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('search_logs', 'No hallucinated search_logs tool name', GraderLevel.L2), + notContains('get_event', 'No hallucinated get_event tool name', GraderLevel.L2), + calledTool('auth0_list_logs', 'Called auth0_list_logs to find recent successful login events', GraderLevel.L4), + { + kind: 'event', + name: 'Filtered logs for successful login type (type:s)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_list_logs').some((tc) => { + const q = tc.args['q'] as string | undefined; + return typeof q === 'string' && q.includes('type:s'); + }), + }, + { + kind: 'event', + name: 'Fetched individual log event details after listing', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => { + const listIdx = toolCalls.findIndex((tc) => tc.name.toLowerCase().includes('auth0_list_logs')); + const getIdx = toolCalls.findIndex( + (tc, i) => + !tc.causedError && + tc.name.toLowerCase().includes('auth0_get_log') && + i > listIdx, + ); + return listIdx !== -1 && getIdx !== -1; + }, + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/PROMPT.md new file mode 100644 index 00000000..0f30980f --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/PROMPT.md @@ -0,0 +1,22 @@ +--- +id: hosted_mcp_setup_spa_with_api +name: Hosted MCP - Set Up SPA with API Access +category: hosted-mcp +--- + +## Task + +I need to set up a complete web application integration in my Auth0 tenant. Please do the following in order: + +1. Create a new API with: + - Name: **Analytics Service** + - Identifier (audience): `https://api.analytics.example.com` + - A scope named `read:analytics` with description "Read analytics data" + +2. Create a Single Page Application named **Analytics Dashboard** + +3. Authorize the **Analytics Dashboard** to call the **Analytics Service** API with the `read:analytics` scope + +4. Update the **Analytics Dashboard** application to add `https://analytics.example.com/callback` as an allowed callback URL + +Confirm back with the Client ID of the Analytics Dashboard and the audience of the Analytics Service. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/graders.ts new file mode 100644 index 00000000..6a1b4e86 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-spa-with-api/graders.ts @@ -0,0 +1,55 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_api', 'No hallucinated create_api tool name', GraderLevel.L2), + notContains('create_spa', 'No hallucinated create_spa tool name', GraderLevel.L2), + notContains('authorize_client', 'No hallucinated authorize_client tool name', GraderLevel.L2), + calledTool('auth0_create_resource_server', 'Called auth0_create_resource_server to create Analytics Service', GraderLevel.L4), + calledTool('auth0_create_application', 'Called auth0_create_application to create Analytics Dashboard SPA', GraderLevel.L4), + calledTool('auth0_create_application_grant', 'Called auth0_create_application_grant to authorize the SPA', GraderLevel.L4), + calledTool('auth0_update_application', 'Called auth0_update_application to add callback URL', GraderLevel.L4), + { + kind: 'event', + name: 'Created resource server with correct audience', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_resource_server').some( + (tc) => tc.args['identifier'] === 'https://api.analytics.example.com', + ), + }, + { + kind: 'event', + name: 'Created SPA with app_type spa', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some( + (tc) => tc.args['app_type'] === 'spa', + ), + }, + { + kind: 'event', + name: 'Application grant includes read:analytics scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => { + const scope = tc.args['scope'] as string[] | undefined; + return Array.isArray(scope) && scope.includes('read:analytics'); + }), + }, + { + kind: 'event', + name: 'Callback URL update includes the new URL', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_application').some((tc) => { + const callbacks = tc.args['callbacks'] as string[] | undefined; + return Array.isArray(callbacks) && callbacks.includes('https://analytics.example.com/callback'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/PROMPT.md new file mode 100644 index 00000000..3064f073 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_update_form_fields +name: Hosted MCP - Update Form Fields +category: hosted-mcp +--- + +## Task + +The **Collect Work Info** form in my Auth0 tenant needs an additional field. + +Please: +1. Find the **Collect Work Info** form +2. Add a new input field for **Phone Number** (field key: `phone_number`) to the form, keeping the existing `job_title` and `company` fields + +Confirm back with the form ID and the updated list of field keys. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts new file mode 100644 index 00000000..faf1ad3d --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts @@ -0,0 +1,53 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('patch_form', 'No hallucinated patch_form tool name', GraderLevel.L2), + notContains('update_form_field', 'No hallucinated update_form_field tool name', GraderLevel.L2), + { + kind: 'event', + name: 'Called auth0_list_forms or auth0_get_form to find the Collect Work Info form', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some( + (tc) => + !tc.causedError && + (tc.name.toLowerCase().includes('auth0_list_forms') || + tc.name.toLowerCase().includes('auth0_get_form')), + ), + }, + { + kind: 'event', + name: 'Called auth0_update_form to add the new field', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_update_form')), + }, + { + kind: 'event', + name: 'Updated form includes phone_number field', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_form').some((tc) => { + const nodes = tc.args['nodes'] as Array<{ field_key?: string }> | undefined; + return Array.isArray(nodes) && nodes.some((n) => n.field_key === 'phone_number'); + }), + }, + { + kind: 'event', + name: 'Updated form preserves existing job_title and company fields', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_form').some((tc) => { + const nodes = tc.args['nodes'] as Array<{ field_key?: string }> | undefined; + if (!Array.isArray(nodes)) return false; + const keys = nodes.map((n) => n.field_key); + return keys.includes('job_title') && keys.includes('company'); + }), + }, + ]; +} From a1d38d4ef9d6814d8c31174fd77f1cddb17c6cba Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 10:25:36 -0400 Subject: [PATCH 11/19] feat(eval): add troubleshoot_missing_scope, list_and_inspect_forms, bulk_app_audit evals --- .../evals/hosted-mcp/bulk-app-audit/PROMPT.md | 16 ++++++++ .../hosted-mcp/bulk-app-audit/graders.ts | 13 +++++++ .../list-and-inspect-forms/PROMPT.md | 15 ++++++++ .../list-and-inspect-forms/graders.ts | 37 +++++++++++++++++++ .../troubleshoot-missing-scope/PROMPT.md | 16 ++++++++ .../troubleshoot-missing-scope/graders.ts | 14 +++++++ 6 files changed, 111 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/PROMPT.md new file mode 100644 index 00000000..480c5309 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_bulk_app_audit +name: Hosted MCP - Bulk Application Audit +category: hosted-mcp +--- + +## Task + +I need a security review of all applications in my Auth0 tenant before an upcoming audit. + +Please list all applications and for each one tell me: +1. The application name and client ID +2. The application type (spa, native, non_interactive, regular_web, etc.) +3. Whether it is a first-party or third-party app + +Give me a structured summary grouped by application type. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/graders.ts new file mode 100644 index 00000000..e74cafb6 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/graders.ts @@ -0,0 +1,13 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef } from '@a0/eval-graders'; + +export function defineGraders(): GraderDef[] { + return [ + notContains('get_clients', 'No hallucinated get_clients tool name', GraderLevel.L2), + notContains('list_clients', 'No hallucinated list_clients tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to retrieve all apps', GraderLevel.L4), + contains('non_interactive', 'Response includes non_interactive app type in summary', GraderLevel.L5), + contains('spa', 'Response includes spa app type in summary', GraderLevel.L5), + contains('first', 'Response addresses first-party vs third-party distinction', GraderLevel.L5), + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/PROMPT.md new file mode 100644 index 00000000..121b2f52 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_list_and_inspect_forms +name: Hosted MCP - List and Inspect Forms +category: hosted-mcp +--- + +## Task + +I need to review the forms configured in my Auth0 tenant. + +Please: +1. List all forms in the tenant +2. Get the full details of the **Collect Work Info** form including its fields + +Tell me how many forms exist and what input fields the Collect Work Info form has (include the field keys). diff --git a/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts new file mode 100644 index 00000000..3961087f --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts @@ -0,0 +1,37 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('get_forms', 'No hallucinated get_forms tool name', GraderLevel.L2), + notContains('list_form', 'No hallucinated list_form tool name', GraderLevel.L2), + { + kind: 'event', + name: 'Called auth0_list_forms to enumerate all forms', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_list_forms')), + }, + { + kind: 'event', + name: 'Called auth0_get_form to fetch Collect Work Info details', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_get_form')), + }, + { + kind: 'event', + name: 'Fetched form using a form_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_get_form').some( + (tc) => typeof tc.args['form_id'] === 'string' && tc.args['form_id'].length > 0, + ), + }, + contains('job_title', 'Response includes job_title field key', GraderLevel.L5), + contains('company', 'Response includes company field key', GraderLevel.L5), + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/PROMPT.md new file mode 100644 index 00000000..9d1b2e1b --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_troubleshoot_missing_scope +name: Hosted MCP - Troubleshoot Missing Scope +category: hosted-mcp +--- + +## Task + +My **Warehouse Bot** application is supposed to call the **Inventory Service** API with the `read:inventory` scope, but it is not receiving that scope in its tokens. + +Please investigate: +1. Find the **Inventory Service** resource server and confirm what scopes it has defined +2. Find the **Warehouse Bot** application and check what grants it has +3. Identify whether the grant exists and includes the `read:inventory` scope + +Tell me what you find and whether the configuration looks correct. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/graders.ts new file mode 100644 index 00000000..3b355394 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/troubleshoot-missing-scope/graders.ts @@ -0,0 +1,14 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef } from '@a0/eval-graders'; + +export function defineGraders(): GraderDef[] { + return [ + notContains('check_permissions', 'No hallucinated check_permissions tool name', GraderLevel.L2), + notContains('get_grant', 'No hallucinated get_grant tool name', GraderLevel.L2), + calledTool('auth0_list_resource_servers', 'Called auth0_list_resource_servers to inspect Inventory Service scopes', GraderLevel.L4), + calledTool('auth0_list_applications', 'Called auth0_list_applications to find Warehouse Bot', GraderLevel.L4), + contains('read:inventory', 'Response references the read:inventory scope in its analysis', GraderLevel.L5), + contains('Warehouse Bot', 'Response references Warehouse Bot by name', GraderLevel.L5), + contains('Inventory Service', 'Response references Inventory Service by name', GraderLevel.L5), + ]; +} From 610d01c2fe3d4a7e4947a9fff12cbb8cd3c711c3 Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 11:00:11 -0400 Subject: [PATCH 12/19] feat(eval): add full_action_lifecycle, add_multiple_scopes, oidc_conformance_review evals --- .../hosted-mcp/add-multiple-scopes/PROMPT.md | 16 ++++++ .../hosted-mcp/add-multiple-scopes/graders.ts | 54 ++++++++++++++++++ .../full-action-lifecycle/PROMPT.md | 21 +++++++ .../full-action-lifecycle/graders.ts | 56 +++++++++++++++++++ .../oidc-conformance-review/PROMPT.md | 15 +++++ .../oidc-conformance-review/graders.ts | 13 +++++ 6 files changed, 175 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/PROMPT.md new file mode 100644 index 00000000..8b10ddd5 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_add_multiple_scopes +name: Hosted MCP - Add Multiple Scopes to API +category: hosted-mcp +--- + +## Task + +The **Inventory Service** API needs several new scopes added to support a new feature rollout. + +Please add all of the following scopes to the **Inventory Service** resource server, keeping any existing scopes: +- `write:inventory` — "Write inventory data" +- `delete:inventory` — "Delete inventory records" +- `admin:inventory` — "Full inventory administration" + +Confirm back with the complete updated list of scopes on the Inventory Service. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/graders.ts new file mode 100644 index 00000000..39087d00 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/graders.ts @@ -0,0 +1,54 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('add_scope', 'No hallucinated add_scope tool name', GraderLevel.L2), + notContains('patch_resource_server', 'No hallucinated patch_resource_server tool name', GraderLevel.L2), + calledTool('auth0_list_resource_servers', 'Called auth0_list_resource_servers to find Inventory Service', GraderLevel.L4), + calledTool('auth0_update_resource_server', 'Called auth0_update_resource_server to add new scopes', GraderLevel.L4), + { + kind: 'event', + name: 'Update includes write:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'write:inventory'); + }), + }, + { + kind: 'event', + name: 'Update includes delete:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'delete:inventory'); + }), + }, + { + kind: 'event', + name: 'Update includes admin:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'admin:inventory'); + }), + }, + { + kind: 'event', + name: 'Update preserves existing read:inventory scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => { + const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined; + return Array.isArray(scopes) && scopes.some((s) => s.value === 'read:inventory'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/PROMPT.md new file mode 100644 index 00000000..81e8db15 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/PROMPT.md @@ -0,0 +1,21 @@ +--- +id: hosted_mcp_full_action_lifecycle +name: Hosted MCP - Full Action Lifecycle +category: hosted-mcp +--- + +## Task + +I need a brand new Post-Login action set up and ready to go in my Auth0 tenant. + +Please: +1. Create an action named **Enrich Token** that runs on the `post-login` trigger with this code: + ```javascript + exports.onExecutePostLogin = async (event, api) => { + api.idToken.setCustomClaim('tenant', event.tenant.id); + }; + ``` +2. Update the action code to also add the user's email as a claim: `api.idToken.setCustomClaim('email', event.user.email)` +3. Deploy the action so it is active + +Confirm back with the action ID and its deployed status. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/graders.ts new file mode 100644 index 00000000..408bf9b0 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/full-action-lifecycle/graders.ts @@ -0,0 +1,56 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_action_version', 'No hallucinated create_action_version tool name', GraderLevel.L2), + notContains('publish_action', 'No hallucinated publish_action tool name', GraderLevel.L2), + calledTool('auth0_create_action', 'Called auth0_create_action to create the Enrich Token action', GraderLevel.L4), + calledTool('auth0_update_action', 'Called auth0_update_action to update the action code', GraderLevel.L4), + calledTool('auth0_deploy_action', 'Called auth0_deploy_action to deploy the action', GraderLevel.L4), + { + kind: 'event', + name: 'Actions were called in correct order: create → update → deploy', + level: GraderLevel.L4, + predicate: (toolCalls: EventToolCall[]) => { + const createIdx = toolCalls.findIndex((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_create_action')); + const updateIdx = toolCalls.findIndex((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_update_action')); + const deployIdx = toolCalls.findIndex((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_deploy_action')); + return createIdx !== -1 && updateIdx > createIdx && deployIdx > updateIdx; + }, + }, + { + kind: 'event', + name: 'Created action with post-login trigger', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_action').some((tc) => { + const triggers = tc.args['supported_triggers'] as Array<{ id: string }> | undefined; + return Array.isArray(triggers) && triggers.some((t) => t.id === 'post-login'); + }), + }, + { + kind: 'event', + name: 'Updated code includes event.user.email claim', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_action').some((tc) => { + const code = tc.args['code'] as string | undefined; + return typeof code === 'string' && code.includes('event.user.email'); + }), + }, + { + kind: 'event', + name: 'Updated code retains event.tenant.id claim', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_action').some((tc) => { + const code = tc.args['code'] as string | undefined; + return typeof code === 'string' && code.includes('event.tenant.id'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/PROMPT.md new file mode 100644 index 00000000..e1e0e047 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_oidc_conformance_review +name: Hosted MCP - OIDC Conformance Review +category: hosted-mcp +--- + +## Task + +I need to audit my Auth0 tenant for OIDC conformance settings before migrating to a stricter authentication setup. + +Please list all applications and identify: +1. Which applications have OIDC conformance **enabled** +2. Which applications have OIDC conformance **disabled** + +Give me two lists with the app name, client ID, and app type for each. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/graders.ts new file mode 100644 index 00000000..9e72c6fd --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/oidc-conformance-review/graders.ts @@ -0,0 +1,13 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef } from '@a0/eval-graders'; + +export function defineGraders(): GraderDef[] { + return [ + notContains('list_clients', 'No hallucinated list_clients tool name', GraderLevel.L2), + notContains('get_oidc_settings', 'No hallucinated get_oidc_settings tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to retrieve all apps', GraderLevel.L4), + contains('oidc_conformant', 'Response references oidc_conformant field', GraderLevel.L5), + contains('enabled', 'Response includes list of apps with OIDC conformance enabled', GraderLevel.L5), + contains('disabled', 'Response includes list of apps with OIDC conformance disabled', GraderLevel.L5), + ]; +} From d9b19752a1d61a99f5f2f83ca558d1114ee1bef9 Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 11:08:46 -0400 Subject: [PATCH 13/19] feat(eval): add complete_developer_onboarding, update_api_token_settings, setup_native_app evals --- .../complete-developer-onboarding/PROMPT.md | 29 ++++++++++ .../complete-developer-onboarding/graders.ts | 55 +++++++++++++++++++ .../hosted-mcp/setup-native-app/PROMPT.md | 16 ++++++ .../hosted-mcp/setup-native-app/graders.ts | 41 ++++++++++++++ .../update-api-token-settings/PROMPT.md | 15 +++++ .../update-api-token-settings/graders.ts | 41 ++++++++++++++ 6 files changed, 197 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/PROMPT.md new file mode 100644 index 00000000..2848259b --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/PROMPT.md @@ -0,0 +1,29 @@ +--- +id: hosted_mcp_complete_developer_onboarding +name: Hosted MCP - Complete Developer Onboarding +category: hosted-mcp +--- + +## Task + +I need to onboard a new service into my Auth0 tenant. Please set everything up in order: + +1. Create an API (resource server) with: + - Name: **Notifications Service** + - Identifier: `https://api.notifications.example.com` + - Scope: `send:notifications` with description "Send notifications" + +2. Create a Machine to Machine application named **Notifications Worker** + +3. Authorize **Notifications Worker** to call **Notifications Service** with the `send:notifications` scope + +4. Create a Post-Login action named **Log Notification Events** with this code: + ```javascript + exports.onExecutePostLogin = async (event, api) => { + api.idToken.setCustomClaim('notification_enabled', true); + }; + ``` + +5. Deploy the **Log Notification Events** action + +Confirm back with the Client ID of the Notifications Worker and the action ID. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts new file mode 100644 index 00000000..68a51309 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts @@ -0,0 +1,55 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_api', 'No hallucinated create_api tool name', GraderLevel.L2), + notContains('create_m2m', 'No hallucinated create_m2m tool name', GraderLevel.L2), + calledTool('auth0_create_resource_server', 'Called auth0_create_resource_server for Notifications Service', GraderLevel.L4), + calledTool('auth0_create_application', 'Called auth0_create_application for Notifications Worker', GraderLevel.L4), + calledTool('auth0_create_application_grant', 'Called auth0_create_application_grant to authorize the worker', GraderLevel.L4), + calledTool('auth0_create_action', 'Called auth0_create_action for Log Notification Events', GraderLevel.L4), + calledTool('auth0_deploy_action', 'Called auth0_deploy_action to deploy the action', GraderLevel.L4), + { + kind: 'event', + name: 'Created resource server with correct identifier', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_resource_server').some( + (tc) => tc.args['identifier'] === 'https://api.notifications.example.com', + ), + }, + { + kind: 'event', + name: 'Created M2M application with non_interactive app type', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some( + (tc) => tc.args['app_type'] === 'non_interactive', + ), + }, + { + kind: 'event', + name: 'Application grant includes send:notifications scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => { + const scope = tc.args['scope'] as string[] | undefined; + return Array.isArray(scope) && scope.includes('send:notifications'); + }), + }, + { + kind: 'event', + name: 'Action created with post-login trigger', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_action').some((tc) => { + const triggers = tc.args['supported_triggers'] as Array<{ id: string }> | undefined; + return Array.isArray(triggers) && triggers.some((t) => t.id === 'post-login'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/PROMPT.md new file mode 100644 index 00000000..2d0e094b --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_setup_native_app +name: Hosted MCP - Set Up Native Mobile App +category: hosted-mcp +--- + +## Task + +I need to register a new mobile application in my Auth0 tenant. + +Please create a native application with: +- Name: **Mobile Warehouse App** +- App type: native +- Allowed callback URL: `com.example.warehouse://callback` + +Confirm back with the Client ID of the new application. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/graders.ts new file mode 100644 index 00000000..9953b48c --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-native-app/graders.ts @@ -0,0 +1,41 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_mobile_app', 'No hallucinated create_mobile_app tool name', GraderLevel.L2), + notContains('register_app', 'No hallucinated register_app tool name', GraderLevel.L2), + calledTool('auth0_create_application', 'Called auth0_create_application to create the native app', GraderLevel.L4), + { + kind: 'event', + name: 'Created application with native app_type', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some( + (tc) => tc.args['app_type'] === 'native', + ), + }, + { + kind: 'event', + name: 'Created application with correct callback URL', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some((tc) => { + const callbacks = tc.args['callbacks'] as string[] | undefined; + return Array.isArray(callbacks) && callbacks.includes('com.example.warehouse://callback'); + }), + }, + { + kind: 'event', + name: 'Created application named Mobile Warehouse App', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some( + (tc) => tc.args['name'] === 'Mobile Warehouse App', + ), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/PROMPT.md new file mode 100644 index 00000000..7a3e0bbf --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_update_api_token_settings +name: Hosted MCP - Update API Token Settings +category: hosted-mcp +--- + +## Task + +The **Inventory Service** API needs its token expiration settings updated for security compliance. + +Please update the **Inventory Service** resource server to: +- Set the token lifetime to **1800 seconds** (30 minutes) +- Enable offline access (allow refresh tokens) + +Confirm back with the resource server ID and the updated token lifetime. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/graders.ts new file mode 100644 index 00000000..ca434b34 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-api-token-settings/graders.ts @@ -0,0 +1,41 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('update_api_settings', 'No hallucinated update_api_settings tool name', GraderLevel.L2), + notContains('set_token_expiry', 'No hallucinated set_token_expiry tool name', GraderLevel.L2), + calledTool('auth0_list_resource_servers', 'Called auth0_list_resource_servers to find Inventory Service', GraderLevel.L4), + calledTool('auth0_update_resource_server', 'Called auth0_update_resource_server to update token settings', GraderLevel.L4), + { + kind: 'event', + name: 'Update sets token_lifetime to 1800', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some( + (tc) => tc.args['token_lifetime'] === 1800, + ), + }, + { + kind: 'event', + name: 'Update enables allow_offline_access', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some( + (tc) => tc.args['allow_offline_access'] === true, + ), + }, + { + kind: 'event', + name: 'Update was called with a resource_server_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_resource_server').some( + (tc) => typeof tc.args['resource_server_id'] === 'string' && tc.args['resource_server_id'].length > 0, + ), + }, + ]; +} From d08fa6c6701fb0fb0605335d0aa1f05772703d46 Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 11:15:18 -0400 Subject: [PATCH 14/19] feat(eval): add setup_regular_web_app, audit_m2m_grants, rename_application evals --- .../hosted-mcp/audit-m2m-grants/PROMPT.md | 15 +++++++ .../hosted-mcp/audit-m2m-grants/graders.ts | 13 ++++++ .../hosted-mcp/rename-application/PROMPT.md | 15 +++++++ .../hosted-mcp/rename-application/graders.ts | 33 +++++++++++++++ .../setup-regular-web-app/PROMPT.md | 17 ++++++++ .../setup-regular-web-app/graders.ts | 42 +++++++++++++++++++ 6 files changed, 135 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/rename-application/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/PROMPT.md new file mode 100644 index 00000000..2674f481 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_audit_m2m_grants +name: Hosted MCP - Audit M2M Application Grants +category: hosted-mcp +--- + +## Task + +I need to audit which Machine to Machine applications in my Auth0 tenant have API access configured. + +Please: +1. List all applications and identify the M2M (non_interactive) ones +2. For each M2M app, check what APIs it has been granted access to + +Give me a summary of each M2M application and the APIs it can access. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts new file mode 100644 index 00000000..d0c6f60f --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts @@ -0,0 +1,13 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef } from '@a0/eval-graders'; + +export function defineGraders(): GraderDef[] { + return [ + notContains('list_grants', 'No hallucinated list_grants tool name', GraderLevel.L2), + notContains('get_client_grants', 'No hallucinated get_client_grants tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to identify M2M apps', GraderLevel.L4), + contains('non_interactive', 'Response identifies M2M (non_interactive) apps', GraderLevel.L5), + contains('Warehouse Bot', 'Response includes Warehouse Bot (known M2M app) in summary', GraderLevel.L5), + contains('Inventory Service', 'Response references Inventory Service API in grant summary', GraderLevel.L5), + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/rename-application/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/rename-application/PROMPT.md new file mode 100644 index 00000000..af3cadc4 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/rename-application/PROMPT.md @@ -0,0 +1,15 @@ +--- +id: hosted_mcp_rename_application +name: Hosted MCP - Rename Application +category: hosted-mcp +--- + +## Task + +The **Analytics Dashboard** application needs to be renamed as part of a rebranding effort. + +Please: +1. Find the **Analytics Dashboard** application +2. Update its name to **Analytics Hub** + +Confirm back with the client ID and the new application name. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts new file mode 100644 index 00000000..1b754cb6 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts @@ -0,0 +1,33 @@ +import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('rename_app', 'No hallucinated rename_app tool name', GraderLevel.L2), + notContains('patch_client', 'No hallucinated patch_client tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to find Analytics Dashboard', GraderLevel.L4), + calledTool('auth0_update_application', 'Called auth0_update_application to rename the app', GraderLevel.L4), + { + kind: 'event', + name: 'Update sets name to Analytics Hub', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_application').some( + (tc) => tc.args['name'] === 'Analytics Hub', + ), + }, + { + kind: 'event', + name: 'Update was called with a client_id (looked up dynamically)', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_application').some( + (tc) => typeof tc.args['client_id'] === 'string' && tc.args['client_id'].length > 0, + ), + }, + contains('Analytics Hub', 'Response confirms the new name Analytics Hub', GraderLevel.L5), + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/PROMPT.md new file mode 100644 index 00000000..12704eea --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/PROMPT.md @@ -0,0 +1,17 @@ +--- +id: hosted_mcp_setup_regular_web_app +name: Hosted MCP - Set Up Regular Web Application +category: hosted-mcp +--- + +## Task + +I need to register a traditional server-side web application in my Auth0 tenant. + +Please create a regular web application with: +- Name: **Customer Portal** +- App type: regular_web +- Allowed callback URL: `https://portal.example.com/callback` +- Allowed logout URL: `https://portal.example.com` + +Confirm back with the Client ID of the new application. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/graders.ts new file mode 100644 index 00000000..fb0718ac --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-regular-web-app/graders.ts @@ -0,0 +1,42 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_web_app', 'No hallucinated create_web_app tool name', GraderLevel.L2), + notContains('register_client', 'No hallucinated register_client tool name', GraderLevel.L2), + calledTool('auth0_create_application', 'Called auth0_create_application to create the Customer Portal', GraderLevel.L4), + { + kind: 'event', + name: 'Created application with regular_web app_type', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some( + (tc) => tc.args['app_type'] === 'regular_web', + ), + }, + { + kind: 'event', + name: 'Created application with correct callback URL', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some((tc) => { + const callbacks = tc.args['callbacks'] as string[] | undefined; + return Array.isArray(callbacks) && callbacks.includes('https://portal.example.com/callback'); + }), + }, + { + kind: 'event', + name: 'Created application with correct logout URL', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application').some((tc) => { + const logoutUrls = tc.args['allowed_logout_urls'] as string[] | undefined; + return Array.isArray(logoutUrls) && logoutUrls.includes('https://portal.example.com'); + }), + }, + ]; +} From f633788a8fee70e5fc73aa621041a9abebb6e645 Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 11:20:01 -0400 Subject: [PATCH 15/19] feat(eval): add diagnose_and_fix_config, search_logs_by_user, action_and_form_setup evals --- .../action-and-form-setup/PROMPT.md | 25 +++++++++++ .../action-and-form-setup/graders.ts | 45 +++++++++++++++++++ .../diagnose-and-fix-config/PROMPT.md | 17 +++++++ .../diagnose-and-fix-config/graders.ts | 31 +++++++++++++ .../hosted-mcp/search-logs-by-user/PROMPT.md | 16 +++++++ .../hosted-mcp/search-logs-by-user/graders.ts | 32 +++++++++++++ 6 files changed, 166 insertions(+) create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/graders.ts create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/PROMPT.md create mode 100644 apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/graders.ts diff --git a/apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/PROMPT.md new file mode 100644 index 00000000..078c18c9 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/PROMPT.md @@ -0,0 +1,25 @@ +--- +id: hosted_mcp_action_and_form_setup +name: Hosted MCP - Action and Form Setup +category: hosted-mcp +--- + +## Task + +I need to set up progressive profiling for my Auth0 tenant. Please do both of the following: + +1. Create a Post-Login action named **Trigger Profile Collection** with this code: + ```javascript + exports.onExecutePostLogin = async (event, api) => { + if (!event.user.user_metadata?.profile_complete) { + api.redirect.sendUserTo('https://profile.example.com/collect'); + } + }; + ``` + Deploy the action. + +2. Create a form named **Profile Collection Form** with two fields: + - Department (field key: `department`) + - Location (field key: `location`) + +Confirm back with the action ID and form ID. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/graders.ts new file mode 100644 index 00000000..20daeeeb --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/graders.ts @@ -0,0 +1,45 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('create_redirect_action', 'No hallucinated create_redirect_action tool name', GraderLevel.L2), + notContains('create_profile_form', 'No hallucinated create_profile_form tool name', GraderLevel.L2), + calledTool('auth0_create_action', 'Called auth0_create_action to create Trigger Profile Collection', GraderLevel.L4), + calledTool('auth0_deploy_action', 'Called auth0_deploy_action to deploy the action', GraderLevel.L4), + calledTool('auth0_create_form', 'Called auth0_create_form to create Profile Collection Form', GraderLevel.L4), + { + kind: 'event', + name: 'Action created with post-login trigger', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_action').some((tc) => { + const triggers = tc.args['supported_triggers'] as Array<{ id: string }> | undefined; + return Array.isArray(triggers) && triggers.some((t) => t.id === 'post-login'); + }), + }, + { + kind: 'event', + name: 'Form includes department field', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_form').some((tc) => { + const nodes = tc.args['nodes'] as Array<{ field_key?: string }> | undefined; + return Array.isArray(nodes) && nodes.some((n) => n.field_key === 'department'); + }), + }, + { + kind: 'event', + name: 'Form includes location field', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_form').some((tc) => { + const nodes = tc.args['nodes'] as Array<{ field_key?: string }> | undefined; + return Array.isArray(nodes) && nodes.some((n) => n.field_key === 'location'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/PROMPT.md new file mode 100644 index 00000000..1d370e9b --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/PROMPT.md @@ -0,0 +1,17 @@ +--- +id: hosted_mcp_diagnose_and_fix_config +name: Hosted MCP - Diagnose and Fix Configuration +category: hosted-mcp +--- + +## Task + +A developer reports that the **Notifications Worker** application cannot get tokens for the **Notifications Service** API. They think there might be a missing grant. + +Please: +1. Find the **Notifications Worker** application +2. Find the **Notifications Service** resource server +3. Check whether a grant exists from the Notifications Worker to the Notifications Service +4. If the grant is missing, create it with the `send:notifications` scope + +Tell me what you found and what action you took. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/graders.ts new file mode 100644 index 00000000..3cd9ee74 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/diagnose-and-fix-config/graders.ts @@ -0,0 +1,31 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('check_grant', 'No hallucinated check_grant tool name', GraderLevel.L2), + notContains('fix_config', 'No hallucinated fix_config tool name', GraderLevel.L2), + calledTool('auth0_list_applications', 'Called auth0_list_applications to find Notifications Worker', GraderLevel.L4), + calledTool('auth0_list_resource_servers', 'Called auth0_list_resource_servers to find Notifications Service', GraderLevel.L4), + { + kind: 'event', + name: 'Created application grant to fix missing access', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + toolCalls.some((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_create_application_grant')), + }, + { + kind: 'event', + name: 'Grant includes send:notifications scope', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => { + const scope = tc.args['scope'] as string[] | undefined; + return Array.isArray(scope) && scope.includes('send:notifications'); + }), + }, + ]; +} diff --git a/apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/PROMPT.md b/apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/PROMPT.md new file mode 100644 index 00000000..45a63cf0 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/PROMPT.md @@ -0,0 +1,16 @@ +--- +id: hosted_mcp_search_logs_by_user +name: Hosted MCP - Search Logs by User +category: hosted-mcp +--- + +## Task + +A specific user is having trouble logging in and I need to see their recent activity. + +Please search the Auth0 logs for events related to the user with email `testuser@example.com` and tell me: +1. What events occurred for this user +2. Whether there were any failures +3. The timestamps of the most recent events + +Use the logs to give me a concrete answer based on what is actually in the tenant. diff --git a/apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/graders.ts new file mode 100644 index 00000000..e282e032 --- /dev/null +++ b/apps/auth0-evals/src/evals/hosted-mcp/search-logs-by-user/graders.ts @@ -0,0 +1,32 @@ +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; + +const mcpCalls = (toolCalls: EventToolCall[], name: string) => + toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase())); + +export function defineGraders(): GraderDef[] { + return [ + notContains('search_user_logs', 'No hallucinated search_user_logs tool name', GraderLevel.L2), + notContains('get_user_events', 'No hallucinated get_user_events tool name', GraderLevel.L2), + calledTool('auth0_list_logs', 'Called auth0_list_logs to search for user events', GraderLevel.L4), + { + kind: 'event', + name: 'Filtered logs by user email in query', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_list_logs').some((tc) => { + const q = tc.args['q'] as string | undefined; + return typeof q === 'string' && q.includes('testuser@example.com'); + }), + }, + { + kind: 'event', + name: 'Used take parameter to bound results', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_list_logs').some( + (tc) => typeof tc.args['take'] === 'number' && tc.args['take'] > 0, + ), + }, + ]; +} From 14105a85da896117895cf4e059a6d52b820c980a Mon Sep 17 00:00:00 2001 From: Lena Date: Wed, 24 Jun 2026 16:44:15 -0400 Subject: [PATCH 16/19] fix(sandbox): forward MCP credentials into Docker container MCP_TENANT_DOMAIN, MCP_CLIENT_ID, and MCP_CLIENT_SECRET were not passed to the container, so eval.config.js never registered auth0-hosted-mcp. Without the server, the agent fell back to auth0-cli bash commands. --- packages/eval/src/sandbox/docker.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/eval/src/sandbox/docker.ts b/packages/eval/src/sandbox/docker.ts index ad87a304..eddd4b25 100644 --- a/packages/eval/src/sandbox/docker.ts +++ b/packages/eval/src/sandbox/docker.ts @@ -150,6 +150,13 @@ export async function runJobInDocker(options: DockerRunOptions): Promise Date: Fri, 26 Jun 2026 11:50:54 -0400 Subject: [PATCH 17/19] fix(graders): fix brittle L5 checks in 4 hosted-mcp evals - audit-m2m-grants: replace contains() file checks with event predicates that inspect tool call results (no file is written in this eval) - rename-application: same fix for the Analytics Hub confirmation check - setup-m2m-application: accept scope as string or array (Management API returns space-separated string, agent may pass either format) - complete-developer-onboarding: same scope format fix for send:notifications --- .../hosted-mcp/audit-m2m-grants/graders.ts | 34 ++++++++++++++++--- .../complete-developer-onboarding/graders.ts | 6 ++-- .../hosted-mcp/rename-application/graders.ts | 12 +++++-- .../setup-m2m-application/graders.ts | 6 ++-- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts index d0c6f60f..9066683d 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts +++ b/apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts @@ -1,13 +1,37 @@ -import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; -import type { GraderDef } from '@a0/eval-graders'; +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import type { GraderDef, EventToolCall } from '@a0/eval-graders'; export function defineGraders(): GraderDef[] { return [ notContains('list_grants', 'No hallucinated list_grants tool name', GraderLevel.L2), notContains('get_client_grants', 'No hallucinated get_client_grants tool name', GraderLevel.L2), calledTool('auth0_list_applications', 'Called auth0_list_applications to identify M2M apps', GraderLevel.L4), - contains('non_interactive', 'Response identifies M2M (non_interactive) apps', GraderLevel.L5), - contains('Warehouse Bot', 'Response includes Warehouse Bot (known M2M app) in summary', GraderLevel.L5), - contains('Inventory Service', 'Response references Inventory Service API in grant summary', GraderLevel.L5), + { + kind: 'event', + name: 'Response identifies M2M (non_interactive) apps', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + toolCalls + .filter((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_list_applications')) + .some((tc) => typeof tc.result === 'string' && tc.result.includes('non_interactive')), + }, + { + kind: 'event', + name: 'Response includes Warehouse Bot (known M2M app) in summary', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + toolCalls + .filter((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_list_applications')) + .some((tc) => typeof tc.result === 'string' && tc.result.includes('Warehouse Bot')), + }, + { + kind: 'event', + name: 'Response references Inventory Service API in grant summary', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + toolCalls + .filter((tc) => !tc.causedError) + .some((tc) => typeof tc.result === 'string' && tc.result.includes('Inventory Service')), + }, ]; } diff --git a/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts index 68a51309..839c0e5b 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts +++ b/apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts @@ -37,8 +37,10 @@ export function defineGraders(): GraderDef[] { level: GraderLevel.L5, predicate: (toolCalls: EventToolCall[]) => mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => { - const scope = tc.args['scope'] as string[] | undefined; - return Array.isArray(scope) && scope.includes('send:notifications'); + const scope = tc.args['scope']; + if (Array.isArray(scope)) return scope.includes('send:notifications'); + if (typeof scope === 'string') return scope.split(/[\s,]+/).includes('send:notifications'); + return false; }), }, { diff --git a/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts index 1b754cb6..a7883d79 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts +++ b/apps/auth0-evals/src/evals/hosted-mcp/rename-application/graders.ts @@ -1,4 +1,4 @@ -import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; import type { GraderDef, EventToolCall } from '@a0/eval-graders'; const mcpCalls = (toolCalls: EventToolCall[], name: string) => @@ -28,6 +28,14 @@ export function defineGraders(): GraderDef[] { (tc) => typeof tc.args['client_id'] === 'string' && tc.args['client_id'].length > 0, ), }, - contains('Analytics Hub', 'Response confirms the new name Analytics Hub', GraderLevel.L5), + { + kind: 'event', + name: 'Response confirms the new name Analytics Hub', + level: GraderLevel.L5, + predicate: (toolCalls: EventToolCall[]) => + mcpCalls(toolCalls, 'auth0_update_application').some( + (tc) => typeof tc.result === 'string' && tc.result.includes('Analytics Hub'), + ), + }, ]; } diff --git a/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts index a6f8185a..15a56e8f 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts +++ b/apps/auth0-evals/src/evals/hosted-mcp/setup-m2m-application/graders.ts @@ -53,8 +53,10 @@ export function defineGraders(): GraderDef[] { level: GraderLevel.L5, predicate: (toolCalls) => mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => { - const scope = tc.args['scope'] as string[] | undefined; - return Array.isArray(scope) && scope.includes('read:inventory'); + const scope = tc.args['scope']; + if (Array.isArray(scope)) return scope.includes('read:inventory'); + if (typeof scope === 'string') return scope.split(/[\s,]+/).includes('read:inventory'); + return false; }), }, { From 581aee13c88b50a40d6308d3a9471202e7b55ff1 Mon Sep 17 00:00:00 2001 From: Lena Date: Fri, 26 Jun 2026 16:06:36 -0400 Subject: [PATCH 18/19] feat(hosted-mcp): add authenticated MCP server support and calledTool graders - Add MCPOAuthConfig type and optional auth field to MCPHttpServerConfig - Add mintMcpToken client-credentials helper in eval-core - Wire mintMcpToken into claude-code runner (per-job Bearer token forwarding) - Add calledTool, calledToolOneOf, getSuccessfulMcpCalls grader primitives - Register auth0-hosted-mcp server in eval.config.js (gated on env vars) - Forward MCP_TENANT_DOMAIN/CLIENT_ID/CLIENT_SECRET into Docker sandbox --- apps/auth0-evals/eval.config.js | 18 +++++++ packages/eval-core/src/config/framework.ts | 17 +++++++ packages/eval-core/src/config/mcp-auth.ts | 41 +++++++++++++++ packages/eval-core/src/index.ts | 2 + packages/eval-graders/src/index.ts | 3 ++ packages/eval-graders/src/primitives.ts | 51 +++++++++++++++++++ .../eval/src/runners/claude-code/agent.ts | 17 +++++-- 7 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 packages/eval-core/src/config/mcp-auth.ts diff --git a/apps/auth0-evals/eval.config.js b/apps/auth0-evals/eval.config.js index dc28af70..d4523aa9 100644 --- a/apps/auth0-evals/eval.config.js +++ b/apps/auth0-evals/eval.config.js @@ -48,6 +48,24 @@ export default { type: 'http', url: 'https://auth0.com/docs/mcp', }, + + ...(process.env.MCP_TENANT_DOMAIN && + process.env.MCP_CLIENT_ID && + process.env.MCP_CLIENT_SECRET + ? { + 'auth0-hosted-mcp': { + type: 'http', + url: `https://${process.env.MCP_TENANT_DOMAIN}/v1/mcp`, + auth: { + tokenUrl: `https://${process.env.MCP_TENANT_DOMAIN}/oauth/token`, + clientId: process.env.MCP_CLIENT_ID, + clientSecret: process.env.MCP_CLIENT_SECRET, + // Management API audience — /v1/mcp audience returns access_denied for client credentials + audience: `https://${process.env.MCP_TENANT_DOMAIN}/api/v2/`, + }, + }, + } + : {}), }, }, diff --git a/packages/eval-core/src/config/framework.ts b/packages/eval-core/src/config/framework.ts index c058293c..f2262c81 100644 --- a/packages/eval-core/src/config/framework.ts +++ b/packages/eval-core/src/config/framework.ts @@ -37,11 +37,28 @@ export interface MCPStdioServerConfig { env?: Record; } +export interface MCPOAuthConfig { + /** OAuth token endpoint, e.g. https://TENANT/oauth/token */ + tokenUrl: string; + /** OAuth client ID for the client-credentials grant. */ + clientId: string; + /** OAuth client secret for the client-credentials grant. */ + clientSecret: string; + /** API audience the token is requested for, e.g. https://TENANT/api/v2/ */ + audience: string; +} + export interface MCPHttpServerConfig { /** URL-based MCP server. */ type: 'http'; /** HTTP URL for the remote MCP server. */ url: string; + /** + * Optional OAuth config. When present, the framework mints a fresh Bearer + * token per agent job and injects it as an Authorization header. If any + * field is empty (e.g. a missing env var), the server is omitted with a warning. + */ + auth?: MCPOAuthConfig; } /** Discriminated union — either a stdio (command-based) or http (URL-based) MCP server. */ diff --git a/packages/eval-core/src/config/mcp-auth.ts b/packages/eval-core/src/config/mcp-auth.ts new file mode 100644 index 00000000..2c6e2ed2 --- /dev/null +++ b/packages/eval-core/src/config/mcp-auth.ts @@ -0,0 +1,41 @@ +/** + * OAuth token minting for authenticated HTTP MCP servers. + * + * Performs a client-credentials exchange to obtain a short-lived Bearer token. + * Called once per agent job so a long matrix run never reuses an expired token. + */ + +import type { MCPOAuthConfig } from './framework.js'; +import { logger } from '../utils/logger.js'; + +export async function mintMcpToken(auth: MCPOAuthConfig): Promise { + if (!auth.tokenUrl || !auth.clientId || !auth.clientSecret || !auth.audience) { + logger.warn('[mcp-auth] Incomplete OAuth config — skipping token mint'); + return undefined; + } + try { + const res = await fetch(auth.tokenUrl, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + grant_type: 'client_credentials', + client_id: auth.clientId, + client_secret: auth.clientSecret, + audience: auth.audience, + }), + }); + if (!res.ok) { + logger.warn(`[mcp-auth] Token request failed: ${res.status}`); + return undefined; + } + const { access_token } = (await res.json()) as { access_token?: string }; + if (!access_token) { + logger.warn('[mcp-auth] Token response missing access_token'); + return undefined; + } + return access_token; + } catch (err) { + logger.warn(`[mcp-auth] Token request error: ${err instanceof Error ? err.message : String(err)}`); + return undefined; + } +} diff --git a/packages/eval-core/src/index.ts b/packages/eval-core/src/index.ts index 9fa9eda9..51c71a91 100644 --- a/packages/eval-core/src/index.ts +++ b/packages/eval-core/src/index.ts @@ -62,6 +62,7 @@ export type { MCPServerConfig, MCPStdioServerConfig, MCPHttpServerConfig, + MCPOAuthConfig, SkillsConfig, RemoteSkillRepo, JudgeConfig, @@ -72,6 +73,7 @@ export type { } from './config/framework.js'; export { DEFAULT_FRAMEWORK_CONFIG } from './config/defaults.js'; export { defineConfig, loadConfig, deepMerge } from './config/loader.js'; +export { mintMcpToken } from './config/mcp-auth.js'; export type { LoadConfigOptions } from './config/loader.js'; // Workspace diff --git a/packages/eval-graders/src/index.ts b/packages/eval-graders/src/index.ts index a6018814..170ca4ab 100644 --- a/packages/eval-graders/src/index.ts +++ b/packages/eval-graders/src/index.ts @@ -22,4 +22,7 @@ export { ranCommandOneOf, wroteFile, compiles, + calledTool, + calledToolOneOf, + getSuccessfulMcpCalls, } from './primitives.js'; diff --git a/packages/eval-graders/src/primitives.ts b/packages/eval-graders/src/primitives.ts index 94f2841e..ecdf6648 100644 --- a/packages/eval-graders/src/primitives.ts +++ b/packages/eval-graders/src/primitives.ts @@ -206,3 +206,54 @@ export function compiles(description: string | undefined, level: EventGraderLeve level, }; } + +// Tool names from any runner that represent MCP tool invocations are prefixed `mcp__`. +const MCP_TOOL_PREFIX = 'mcp__'; + +/** + * Returns all successful MCP tool calls from a trace (helper used by calledTool/calledToolOneOf). + */ +export function getSuccessfulMcpCalls(toolCalls: EventToolCall[]): EventToolCall[] { + return toolCalls.filter((tc) => tc.name.startsWith(MCP_TOOL_PREFIX) && !tc.causedError); +} + +/** + * Asserts that the agent invoked an MCP tool whose (lowercased) name contains + * the given substring. MCP calls are recorded as `mcp____`. + * Errored calls are excluded — a failed MCP call is not a successful invocation. + */ +export function calledTool( + toolName: string, + description: string | undefined, + level: EventGraderLevel, +): GraderDef { + validateEventLevel(level, 'calledTool'); + const lc = toolName.toLowerCase(); + return { + kind: 'event', + name: description ?? `called MCP tool '${toolName}'`, + level, + predicate: (toolCalls: EventToolCall[]) => + getSuccessfulMcpCalls(toolCalls).some((tc) => tc.name.toLowerCase().includes(lc)), + }; +} + +/** + * Asserts that the agent invoked at least one of the given MCP tools. + * Each name is matched as a (lowercased) substring against `mcp__` tool calls. + */ +export function calledToolOneOf( + toolNames: string[], + description: string | undefined, + level: EventGraderLevel, +): GraderDef { + validateEventLevel(level, 'calledToolOneOf'); + const lcs = toolNames.map((t) => t.toLowerCase()); + return { + kind: 'event', + name: description ?? `called one of MCP tools [${toolNames.join(', ')}]`, + level, + predicate: (toolCalls: EventToolCall[]) => + getSuccessfulMcpCalls(toolCalls).some((tc) => lcs.some((lc) => tc.name.toLowerCase().includes(lc))), + }; +} diff --git a/packages/eval/src/runners/claude-code/agent.ts b/packages/eval/src/runners/claude-code/agent.ts index 0cd40adb..ab2830fa 100644 --- a/packages/eval/src/runners/claude-code/agent.ts +++ b/packages/eval/src/runners/claude-code/agent.ts @@ -30,6 +30,7 @@ import { logger, makeSessionId, filteredEnv, + mintMcpToken, } from '@a0/eval-core'; import { classifyActionType, classifyErrorCategory, detectRetry } from '@a0/eval-core'; import { LLM_API_KEY_ENV } from '../../cli/constants.js'; @@ -137,12 +138,22 @@ export async function runClaudeCodeAgent( } // Build MCP server config when --tools mcp is requested. - let mcpServers: Record | undefined; + // Token is minted here (job start) so a long matrix run never reuses an expired token. + let mcpServers: Record }> | undefined; if (tools.includes('mcp')) { const configServers = getFrameworkConfig().mcp.servers; - const httpServers: Record = {}; + const httpServers: Record }> = {}; for (const [name, server] of Object.entries(configServers)) { - if (server.type === 'http') { + if (server.type !== 'http') continue; + if (server.auth) { + const token = await mintMcpToken(server.auth); + if (!token) { + logger.warn(`[ClaudeCode] MCP server '${name}' skipped — token mint failed or creds missing`); + continue; + } + logger.info(`[ClaudeCode] MCP auth token minted for '${name}'`); + httpServers[name] = { type: 'http' as const, url: server.url, headers: { Authorization: `Bearer ${token}` } }; + } else { httpServers[name] = { type: 'http' as const, url: server.url }; } } From 062feee3517c385c06be6622ddae5f83ce8bd37b Mon Sep 17 00:00:00 2001 From: Lena Date: Fri, 26 Jun 2026 16:17:04 -0400 Subject: [PATCH 19/19] fix(lint): remove unused calledTool import in list-and-inspect-forms and update-form-fields --- .../src/evals/hosted-mcp/list-and-inspect-forms/graders.ts | 2 +- .../src/evals/hosted-mcp/update-form-fields/graders.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts index 3961087f..9d555ac0 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts +++ b/apps/auth0-evals/src/evals/hosted-mcp/list-and-inspect-forms/graders.ts @@ -1,4 +1,4 @@ -import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders'; +import { notContains, contains, GraderLevel } from '@a0/eval-graders'; import type { GraderDef, EventToolCall } from '@a0/eval-graders'; const mcpCalls = (toolCalls: EventToolCall[], name: string) => diff --git a/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts b/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts index faf1ad3d..69d8f32b 100644 --- a/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts +++ b/apps/auth0-evals/src/evals/hosted-mcp/update-form-fields/graders.ts @@ -1,4 +1,4 @@ -import { calledTool, notContains, GraderLevel } from '@a0/eval-graders'; +import { notContains, GraderLevel } from '@a0/eval-graders'; import type { GraderDef, EventToolCall } from '@a0/eval-graders'; const mcpCalls = (toolCalls: EventToolCall[], name: string) =>