feat(hosted-mcp): 32 evals for Auth0 hosted MCP server

apps/auth0-evals/eval.config.js

@@ -48,6 +48,24 @@ export default {
         type: 'http',
         url: 'https://auth0.com/docs/mcp',
       },
+
+      ...(process.env.MCP_TENANT_DOMAIN &&
+      process.env.MCP_CLIENT_ID &&
+      process.env.MCP_CLIENT_SECRET
+        ? {
+            'auth0-hosted-mcp': {
+              type: 'http',
+              url: `https://${process.env.MCP_TENANT_DOMAIN}/v1/mcp`,
+              auth: {
+                tokenUrl: `https://${process.env.MCP_TENANT_DOMAIN}/oauth/token`,
+                clientId: process.env.MCP_CLIENT_ID,
+                clientSecret: process.env.MCP_CLIENT_SECRET,
+                // Management API audience — /v1/mcp audience returns access_denied for client credentials
+                audience: `https://${process.env.MCP_TENANT_DOMAIN}/api/v2/`,
+              },
+            },
+          }
+        : {}),
     },
   },
 

apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/PROMPT.md

@@ -0,0 +1,25 @@
+---
+id: hosted_mcp_action_and_form_setup
+name: Hosted MCP - Action and Form Setup
+category: hosted-mcp
+---
+
+## Task
+
+I need to set up progressive profiling for my Auth0 tenant. Please do both of the following:
+
+1. Create a Post-Login action named **Trigger Profile Collection** with this code:
+   ```javascript
+   exports.onExecutePostLogin = async (event, api) => {
+     if (!event.user.user_metadata?.profile_complete) {
+       api.redirect.sendUserTo('https://profile.example.com/collect');
+     }
+   };
+   ```
+   Deploy the action.
+
+2. Create a form named **Profile Collection Form** with two fields:
+   - Department (field key: `department`)
+   - Location (field key: `location`)
+
+Confirm back with the action ID and form ID.

apps/auth0-evals/src/evals/hosted-mcp/action-and-form-setup/graders.ts

@@ -0,0 +1,45 @@
+import { calledTool, notContains, GraderLevel } from '@a0/eval-graders';
+import type { GraderDef, EventToolCall } from '@a0/eval-graders';
+
+const mcpCalls = (toolCalls: EventToolCall[], name: string) =>
+  toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase()));
+
+export function defineGraders(): GraderDef[] {
+  return [
+    notContains('create_redirect_action', 'No hallucinated create_redirect_action tool name', GraderLevel.L2),
+    notContains('create_profile_form', 'No hallucinated create_profile_form tool name', GraderLevel.L2),
+    calledTool('auth0_create_action', 'Called auth0_create_action to create Trigger Profile Collection', GraderLevel.L4),
+    calledTool('auth0_deploy_action', 'Called auth0_deploy_action to deploy the action', GraderLevel.L4),
+    calledTool('auth0_create_form', 'Called auth0_create_form to create Profile Collection Form', GraderLevel.L4),
+    {
+      kind: 'event',
+      name: 'Action created with post-login trigger',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_action').some((tc) => {
+          const triggers = tc.args['supported_triggers'] as Array<{ id: string }> | undefined;
+          return Array.isArray(triggers) && triggers.some((t) => t.id === 'post-login');
+        }),
+    },
+    {
+      kind: 'event',
+      name: 'Form includes department field',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_form').some((tc) => {
+          const nodes = tc.args['nodes'] as Array<{ field_key?: string }> | undefined;
+          return Array.isArray(nodes) && nodes.some((n) => n.field_key === 'department');
+        }),
+    },
+    {
+      kind: 'event',
+      name: 'Form includes location field',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_form').some((tc) => {
+          const nodes = tc.args['nodes'] as Array<{ field_key?: string }> | undefined;
+          return Array.isArray(nodes) && nodes.some((n) => n.field_key === 'location');
+        }),
+    },
+  ];
+}

apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/PROMPT.md

@@ -0,0 +1,16 @@
+---
+id: hosted_mcp_add_multiple_scopes
+name: Hosted MCP - Add Multiple Scopes to API
+category: hosted-mcp
+---
+
+## Task
+
+The **Inventory Service** API needs several new scopes added to support a new feature rollout.
+
+Please add all of the following scopes to the **Inventory Service** resource server, keeping any existing scopes:
+- `write:inventory` — "Write inventory data"
+- `delete:inventory` — "Delete inventory records"
+- `admin:inventory` — "Full inventory administration"
+
+Confirm back with the complete updated list of scopes on the Inventory Service.

apps/auth0-evals/src/evals/hosted-mcp/add-multiple-scopes/graders.ts

@@ -0,0 +1,54 @@
+import { calledTool, notContains, GraderLevel } from '@a0/eval-graders';
+import type { GraderDef, EventToolCall } from '@a0/eval-graders';
+
+const mcpCalls = (toolCalls: EventToolCall[], name: string) =>
+  toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase()));
+
+export function defineGraders(): GraderDef[] {
+  return [
+    notContains('add_scope', 'No hallucinated add_scope tool name', GraderLevel.L2),
+    notContains('patch_resource_server', 'No hallucinated patch_resource_server tool name', GraderLevel.L2),
+    calledTool('auth0_list_resource_servers', 'Called auth0_list_resource_servers to find Inventory Service', GraderLevel.L4),
+    calledTool('auth0_update_resource_server', 'Called auth0_update_resource_server to add new scopes', GraderLevel.L4),
+    {
+      kind: 'event',
+      name: 'Update includes write:inventory scope',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => {
+          const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined;
+          return Array.isArray(scopes) && scopes.some((s) => s.value === 'write:inventory');
+        }),
+    },
+    {
+      kind: 'event',
+      name: 'Update includes delete:inventory scope',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => {
+          const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined;
+          return Array.isArray(scopes) && scopes.some((s) => s.value === 'delete:inventory');
+        }),
+    },
+    {
+      kind: 'event',
+      name: 'Update includes admin:inventory scope',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => {
+          const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined;
+          return Array.isArray(scopes) && scopes.some((s) => s.value === 'admin:inventory');
+        }),
+    },
+    {
+      kind: 'event',
+      name: 'Update preserves existing read:inventory scope',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_update_resource_server').some((tc) => {
+          const scopes = tc.args['scopes'] as Array<{ value: string }> | undefined;
+          return Array.isArray(scopes) && scopes.some((s) => s.value === 'read:inventory');
+        }),
+    },
+  ];
+}

apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/PROMPT.md

@@ -0,0 +1,15 @@
+---
+id: hosted_mcp_audit_m2m_grants
+name: Hosted MCP - Audit M2M Application Grants
+category: hosted-mcp
+---
+
+## Task
+
+I need to audit which Machine to Machine applications in my Auth0 tenant have API access configured.
+
+Please:
+1. List all applications and identify the M2M (non_interactive) ones
+2. For each M2M app, check what APIs it has been granted access to
+
+Give me a summary of each M2M application and the APIs it can access.

apps/auth0-evals/src/evals/hosted-mcp/audit-m2m-grants/graders.ts

@@ -0,0 +1,37 @@
+import { calledTool, notContains, GraderLevel } from '@a0/eval-graders';
+import type { GraderDef, EventToolCall } from '@a0/eval-graders';
+
+export function defineGraders(): GraderDef[] {
+  return [
+    notContains('list_grants', 'No hallucinated list_grants tool name', GraderLevel.L2),
+    notContains('get_client_grants', 'No hallucinated get_client_grants tool name', GraderLevel.L2),
+    calledTool('auth0_list_applications', 'Called auth0_list_applications to identify M2M apps', GraderLevel.L4),
+    {
+      kind: 'event',
+      name: 'Response identifies M2M (non_interactive) apps',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        toolCalls
+          .filter((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_list_applications'))
+          .some((tc) => typeof tc.result === 'string' && tc.result.includes('non_interactive')),
+    },
+    {
+      kind: 'event',
+      name: 'Response includes Warehouse Bot (known M2M app) in summary',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        toolCalls
+          .filter((tc) => !tc.causedError && tc.name.toLowerCase().includes('auth0_list_applications'))
+          .some((tc) => typeof tc.result === 'string' && tc.result.includes('Warehouse Bot')),
+    },
+    {
+      kind: 'event',
+      name: 'Response references Inventory Service API in grant summary',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        toolCalls
+          .filter((tc) => !tc.causedError)
+          .some((tc) => typeof tc.result === 'string' && tc.result.includes('Inventory Service')),
+    },
+  ];
+}

apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/PROMPT.md

@@ -0,0 +1,15 @@
+---
+id: hosted_mcp_audit_tenant_applications
+name: Hosted MCP - Audit Tenant Applications
+category: hosted-mcp
+---
+
+## Task
+
+I need a quick security audit of the applications in my Auth0 tenant.
+
+Please list all applications and identify:
+1. Which ones are Machine to Machine (M2M) apps
+2. Which ones have no allowed callback URLs configured
+
+Give me a summary with the app names and client IDs for each category.

apps/auth0-evals/src/evals/hosted-mcp/audit-tenant-applications/graders.ts

@@ -0,0 +1,12 @@
+import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders';
+import type { GraderDef } from '@a0/eval-graders';
+
+export function defineGraders(): GraderDef[] {
+  return [
+    notContains('get_clients', 'No hallucinated get_clients tool name', GraderLevel.L2),
+    notContains('list_clients', 'No hallucinated list_clients tool name', GraderLevel.L2),
+    calledTool('auth0_list_applications', 'Called auth0_list_applications to retrieve all apps', GraderLevel.L4),
+    contains('non_interactive', 'Response identifies M2M (non_interactive) app type', GraderLevel.L5),
+    contains('Warehouse Bot', 'Response mentions Warehouse Bot (known M2M app)', GraderLevel.L5),
+  ];
+}

apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/PROMPT.md

@@ -0,0 +1,16 @@
+---
+id: hosted_mcp_bulk_app_audit
+name: Hosted MCP - Bulk Application Audit
+category: hosted-mcp
+---
+
+## Task
+
+I need a security review of all applications in my Auth0 tenant before an upcoming audit.
+
+Please list all applications and for each one tell me:
+1. The application name and client ID
+2. The application type (spa, native, non_interactive, regular_web, etc.)
+3. Whether it is a first-party or third-party app
+
+Give me a structured summary grouped by application type.

apps/auth0-evals/src/evals/hosted-mcp/bulk-app-audit/graders.ts

@@ -0,0 +1,13 @@
+import { calledTool, notContains, contains, GraderLevel } from '@a0/eval-graders';
+import type { GraderDef } from '@a0/eval-graders';
+
+export function defineGraders(): GraderDef[] {
+  return [
+    notContains('get_clients', 'No hallucinated get_clients tool name', GraderLevel.L2),
+    notContains('list_clients', 'No hallucinated list_clients tool name', GraderLevel.L2),
+    calledTool('auth0_list_applications', 'Called auth0_list_applications to retrieve all apps', GraderLevel.L4),
+    contains('non_interactive', 'Response includes non_interactive app type in summary', GraderLevel.L5),
+    contains('spa', 'Response includes spa app type in summary', GraderLevel.L5),
+    contains('first', 'Response addresses first-party vs third-party distinction', GraderLevel.L5),
+  ];
+}

apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/PROMPT.md

@@ -0,0 +1,29 @@
+---
+id: hosted_mcp_complete_developer_onboarding
+name: Hosted MCP - Complete Developer Onboarding
+category: hosted-mcp
+---
+
+## Task
+
+I need to onboard a new service into my Auth0 tenant. Please set everything up in order:
+
+1. Create an API (resource server) with:
+   - Name: **Notifications Service**
+   - Identifier: `https://api.notifications.example.com`
+   - Scope: `send:notifications` with description "Send notifications"
+
+2. Create a Machine to Machine application named **Notifications Worker**
+
+3. Authorize **Notifications Worker** to call **Notifications Service** with the `send:notifications` scope
+
+4. Create a Post-Login action named **Log Notification Events** with this code:
+   ```javascript
+   exports.onExecutePostLogin = async (event, api) => {
+     api.idToken.setCustomClaim('notification_enabled', true);
+   };
+   ```
+
+5. Deploy the **Log Notification Events** action
+
+Confirm back with the Client ID of the Notifications Worker and the action ID.

apps/auth0-evals/src/evals/hosted-mcp/complete-developer-onboarding/graders.ts

@@ -0,0 +1,57 @@
+import { calledTool, notContains, GraderLevel } from '@a0/eval-graders';
+import type { GraderDef, EventToolCall } from '@a0/eval-graders';
+
+const mcpCalls = (toolCalls: EventToolCall[], name: string) =>
+  toolCalls.filter((tc) => !tc.causedError && tc.name.toLowerCase().includes(name.toLowerCase()));
+
+export function defineGraders(): GraderDef[] {
+  return [
+    notContains('create_api', 'No hallucinated create_api tool name', GraderLevel.L2),
+    notContains('create_m2m', 'No hallucinated create_m2m tool name', GraderLevel.L2),
+    calledTool('auth0_create_resource_server', 'Called auth0_create_resource_server for Notifications Service', GraderLevel.L4),
+    calledTool('auth0_create_application', 'Called auth0_create_application for Notifications Worker', GraderLevel.L4),
+    calledTool('auth0_create_application_grant', 'Called auth0_create_application_grant to authorize the worker', GraderLevel.L4),
+    calledTool('auth0_create_action', 'Called auth0_create_action for Log Notification Events', GraderLevel.L4),
+    calledTool('auth0_deploy_action', 'Called auth0_deploy_action to deploy the action', GraderLevel.L4),
+    {
+      kind: 'event',
+      name: 'Created resource server with correct identifier',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_resource_server').some(
+          (tc) => tc.args['identifier'] === 'https://api.notifications.example.com',
+        ),
+    },
+    {
+      kind: 'event',
+      name: 'Created M2M application with non_interactive app type',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_application').some(
+          (tc) => tc.args['app_type'] === 'non_interactive',
+        ),
+    },
+    {
+      kind: 'event',
+      name: 'Application grant includes send:notifications scope',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_application_grant').some((tc) => {
+          const scope = tc.args['scope'];
+          if (Array.isArray(scope)) return scope.includes('send:notifications');
+          if (typeof scope === 'string') return scope.split(/[\s,]+/).includes('send:notifications');
+          return false;
+        }),
+    },
+    {
+      kind: 'event',
+      name: 'Action created with post-login trigger',
+      level: GraderLevel.L5,
+      predicate: (toolCalls: EventToolCall[]) =>
+        mcpCalls(toolCalls, 'auth0_create_action').some((tc) => {
+          const triggers = tc.args['supported_triggers'] as Array<{ id: string }> | undefined;
+          return Array.isArray(triggers) && triggers.some((t) => t.id === 'post-login');
+        }),
+    },
+  ];
+}

apps/auth0-evals/src/evals/hosted-mcp/create-and-deploy-action/PROMPT.md

@@ -0,0 +1,16 @@
+---
+id: hosted_mcp_create_and_deploy_action
+name: Hosted MCP - Create and Deploy Action
+category: hosted-mcp
+---
+
+## Task
+
+I need a Post-Login action in my Auth0 tenant that adds the user's roles to the ID token.
+
+
+Please:
+1. Create an action named **Add Roles to Token** that runs on the `post-login` trigger and adds a `roles` claim to the ID token using `event.authorization.roles`
+2. Deploy the action so it is active
+
+Confirm back with the action ID once it is deployed.