diff --git a/src/scenarios/server/tools.test.ts b/src/scenarios/server/tools.test.ts new file mode 100644 index 0000000..d54daa4 --- /dev/null +++ b/src/scenarios/server/tools.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect } from 'vitest'; +import { buildToolsNameFormatCheck, validateToolNameFormat } from './tools.js'; + +describe('validateToolNameFormat', () => { + it('accepts a typical snake_case name', () => { + expect(validateToolNameFormat('test_simple_text')).toBeNull(); + }); + + it('accepts all allowed character classes', () => { + expect(validateToolNameFormat('Aa0_.-/')).toBeNull(); + }); + + it('accepts a single-character name (lower length boundary)', () => { + expect(validateToolNameFormat('a')).toBeNull(); + }); + + it('accepts a 64-character name (upper length boundary)', () => { + expect(validateToolNameFormat('a'.repeat(64))).toBeNull(); + }); + + it('rejects an empty name', () => { + expect(validateToolNameFormat('')).toMatch(/length 0 is outside/); + }); + + it('rejects a 65-character name', () => { + expect(validateToolNameFormat('a'.repeat(65))).toMatch( + /length 65 is outside/ + ); + }); + + it.each([ + ['space', 'bad name'], + ['colon', 'bad:name'], + ['at sign', 'bad@name'], + ['unicode', 'bad\u00e9name'], + ['backslash', 'bad\\name'], + ['plus', 'bad+name'] + ])('rejects a name with a disallowed character (%s)', (_label, name) => { + expect(validateToolNameFormat(name)).toMatch(/contains characters outside/); + }); +}); + +describe('buildToolsNameFormatCheck', () => { + it('returns INFO when tools is undefined', () => { + const check = buildToolsNameFormatCheck(undefined); + expect(check.status).toBe('INFO'); + expect(check.id).toBe('tools-name-format'); + expect(check.details).toEqual({ toolCount: 0 }); + }); + + it('returns INFO when tools is an empty array', () => { + const check = buildToolsNameFormatCheck([]); + expect(check.status).toBe('INFO'); + expect(check.errorMessage).toBe('No tools advertised; nothing to validate'); + }); + + it('returns SUCCESS when all tool names are valid', () => { + const check = buildToolsNameFormatCheck([ + { name: 'test_simple_text' }, + { name: 'namespace/tool-v1.2' } + ]); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + expect(check.details).toMatchObject({ + toolCount: 2, + results: { + test_simple_text: 'valid', + 'namespace/tool-v1.2': 'valid' + } + }); + }); + + it('returns FAILURE with per-tool details when some names are invalid', () => { + const check = buildToolsNameFormatCheck([ + { name: 'good_tool' }, + { name: 'bad name with spaces' }, + { name: 'a'.repeat(65) } + ]); + + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain( + '2 tool name(s) violate SEP-986 format' + ); + + const results = (check.details as { results: Record }) + .results; + expect(results['good_tool']).toBe('valid'); + expect(results['bad name with spaces']).toMatch(/^invalid: /); + expect(results['a'.repeat(65)]).toMatch(/^invalid: length 65/); + }); + + it('flags a tool whose name is not a string', () => { + const check = buildToolsNameFormatCheck([{ name: 123 as unknown }]); + expect(check.status).toBe('FAILURE'); + const results = (check.details as { results: Record }) + .results; + expect(results['']).toBe( + 'invalid: name is not a string' + ); + }); + + it('includes both MCP-Tools-List and SEP-986 spec references', () => { + const check = buildToolsNameFormatCheck([{ name: 'ok' }]); + const ids = check.specReferences?.map((r) => r.id); + expect(ids).toEqual(['MCP-Tools-List', 'SEP-986']); + }); +}); diff --git a/src/scenarios/server/tools.ts b/src/scenarios/server/tools.ts index 7ecbfdd..de49d78 100644 --- a/src/scenarios/server/tools.ts +++ b/src/scenarios/server/tools.ts @@ -11,6 +11,83 @@ import { Progress } from '@modelcontextprotocol/sdk/types.js'; +const TOOL_NAME_PATTERN = /^[A-Za-z0-9_./-]+$/; +const TOOL_NAME_MAX_LENGTH = 64; + +const TOOLS_NAME_FORMAT_SPEC_REFS = [ + { + id: 'MCP-Tools-List', + url: 'https://modelcontextprotocol.io/specification/2025-11-25/server/tools#listing-tools' + }, + { + id: 'SEP-986', + url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/SEP/SEP-986.md' + } +]; + +export function validateToolNameFormat(name: string): string | null { + if (name.length < 1 || name.length > TOOL_NAME_MAX_LENGTH) { + return `length ${name.length} is outside 1-${TOOL_NAME_MAX_LENGTH}`; + } + if (!TOOL_NAME_PATTERN.test(name)) { + return 'contains characters outside [A-Za-z0-9_./-]'; + } + return null; +} + +export function buildToolsNameFormatCheck( + tools: ReadonlyArray<{ name?: unknown }> | undefined +): ConformanceCheck { + const timestamp = new Date().toISOString(); + const baseCheck = { + id: 'tools-name-format', + name: 'ToolsNameFormat', + description: 'Tool names are 1-64 characters and match ^[A-Za-z0-9_./-]+$', + specReferences: TOOLS_NAME_FORMAT_SPEC_REFS, + timestamp + }; + + if (!Array.isArray(tools) || tools.length === 0) { + return { + ...baseCheck, + status: 'INFO', + errorMessage: 'No tools advertised; nothing to validate', + details: { toolCount: 0 } + }; + } + + const toolResults: Record = {}; + const violations: string[] = []; + + tools.forEach((tool, index) => { + const name = typeof tool.name === 'string' ? tool.name : ''; + const key = name || ``; + const reason = + typeof tool.name === 'string' + ? validateToolNameFormat(tool.name) + : 'name is not a string'; + if (reason) { + toolResults[key] = `invalid: ${reason}`; + violations.push(`${key}: ${reason}`); + } else { + toolResults[key] = 'valid'; + } + }); + + return { + ...baseCheck, + status: violations.length === 0 ? 'SUCCESS' : 'FAILURE', + errorMessage: + violations.length > 0 + ? `${violations.length} tool name(s) violate SEP-986 format: ${violations.join('; ')}` + : undefined, + details: { + toolCount: tools.length, + results: toolResults + } + }; +} + export class ToolsListScenario implements ClientScenario { name = 'tools-list'; specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25']; @@ -23,7 +100,7 @@ export class ToolsListScenario implements ClientScenario { **Requirements**: - Return array of all available tools - Each tool MUST have: - - \`name\` (string) + - \`name\` (string, 1-64 chars, matching \`^[A-Za-z0-9_./-]+$\`) - \`description\` (string) - \`inputSchema\` (valid JSON Schema object)`; @@ -72,6 +149,10 @@ export class ToolsListScenario implements ClientScenario { } }); + // Validate tool name format per SEP-986: + // names MUST be 1-64 chars matching ^[A-Za-z0-9_./-]+$ + checks.push(buildToolsNameFormatCheck(result.tools)); + await connection.close(); } catch (error) { checks.push({