From 7432cfb46b291d85abf594db1394f976c2cda43c Mon Sep 17 00:00:00 2001 From: modesty Date: Thu, 23 Oct 2025 14:18:13 -0700 Subject: [PATCH] feat: add Sampling capability for error anlysis and healing; fix resource/get errors; doc: update with key features --- README.md | 464 ++++++------------ package.json | 2 +- src/config.ts | 27 + src/res/resourceManager.ts | 127 ++++- src/server/fluentMCPServer.ts | 86 +++- src/utils/samplingManager.ts | 225 +++++++++ src/utils/types.ts | 14 + test/res/resourceManager.test.ts | 93 ++++ test/server/errorAnalysis.integration.test.ts | 393 +++++++++++++++ test/utils/samplingManager.test.ts | 317 ++++++++++++ 10 files changed, 1411 insertions(+), 337 deletions(-) create mode 100644 src/utils/samplingManager.ts create mode 100644 test/server/errorAnalysis.integration.test.ts create mode 100644 test/utils/samplingManager.test.ts diff --git a/README.md b/README.md index 47f072b..b076b4c 100644 --- a/README.md +++ b/README.md @@ -1,151 +1,101 @@ -# MCP Server for Fluent (ServiceNow SDK) +# Fluent MCP Server -MCP Server for [Fluent (ServiceNow SDK)](https://www.servicenow.com/docs/bundle/yokohama-application-development/page/build/servicenow-sdk/concept/servicenow-fluent.html), a TypeScript-based declarative domain-specific language for creating and managing metadata, modules, records and tests in ServiceNow platform. It supports all commands available in the ServiceNow SDK CLI and provides access to Fluent Plugin's API specifications, code snippets, and instructions for various metadata types. It can be configured for any MCP client with stdio, such as VSCode Agent mode, Claude Code, Cursor, or Windsurf, for either development or learning purposes. +An [MCP server](https://modelcontextprotocol.io) that brings [ServiceNow Fluent SDK](https://www.servicenow.com/docs/bundle/yokohama-application-development/page/build/servicenow-sdk/concept/servicenow-fluent.html) capabilities to AI-assisted development environments. Enables natural language interaction with ServiceNow SDK commands, API specifications, code snippets, and development resources. -## Overview +## Key Features -Fluent (ServiceNow SDK) MCP bridges development tools with AI-assisted development environments by implementing the [Model Context Protocol](https://github.com/modelcontextprotocol). It enables developers and AI Agents to interact with Fluent commands and access resources like API specifications, code snippets, and instructions through natural language. +- **šŸ¤– AI-Powered Error Analysis** - Intelligent diagnosis with root cause, solutions, and prevention tips (MCP Sampling) +- **Complete SDK Coverage** - All ServiceNow SDK commands: `auth`, `init`, `build`, `install`, `dependencies`, `transform`, `download`, `clean`, `pack` +- **Rich Resources** - API specifications, code snippets, instructions for 35+ metadata types +- **Multi-Environment Auth** - Supports `basic` and `oauth` authentication with profile management +- **Session-Aware** - Maintains working directory context across commands -Key capabilities include: +This MCP server implements the complete [Model Context Protocol](https://modelcontextprotocol.io) specification with the following capabilities: -- **Complete ServiceNow SDK CLI coverage**: All native SDK commands are now available including `auth`, `init`, `build`, `install`, `dependencies`, `transform`, `download`, `clean`, and `pack` -- **Enhanced SDK information access**: Get SDK version, help, and debug information with improved `sdk_info` command -- ServiceNow instance `basic` or `oauth` authentication (optional, only needed for CLI commands, not for resources) -- Resource capability of API specifications for metadata types like `acl`, `business-rule`, `client-script`, `table`, `ui-action` and more -- Code snippets and examples for different metadata types -- Instructions for creating and modifying metadata types +### Core -Example prompt: +- **Resources** - Provides 100+ resources across 35+ ServiceNow metadata types (API specs, instructions, snippets, prompts) +- **Tools** - Exposes 10 ServiceNow SDK commands as MCP tools with full parameter validation +- **Prompts** - Offers development workflow templates for common ServiceNow tasks +- **Roots** - Supports MCP roots protocol for workspace-aware operations -```text -Create a new Fluent app under ~/Downloads/fluent-app to track employee time off requests -``` +### Extended Capabilities -## Tools +- **Sampling** (MCP 2024-11-05) - Leverages client LLM for intelligent error analysis when SDK commands fail + - Automatically analyzes command errors >50 characters + - Provides structured diagnostics: root cause, solutions, prevention tips + - Configurable via `FLUENT_MCP_ENABLE_ERROR_ANALYSIS` environment variable -### ServiceNow SDK Commands +- **Elicitation** (MCP 2024-11-05) - Interactive parameter collection for complex workflows + - **`init_fluent_app`** - Prompts for missing project parameters (workingDirectory, template, appName, etc.) + - Supports both creation and conversion workflows with smart validation + - Handles user acceptance/rejection of elicited data -Note: Use `init_fluent_app` command to switch to a working directory for existing Fluent projects or to create a new one. +- **Session Management** - Tracks working directory per session for multi-project workflows +- **Root Fallback** - Automatically falls back to MCP root context when no session directory is set +- **Error Handling** - Comprehensive error messages with actionable guidance +- **Type Safety** - Full TypeScript implementation with strict typing -| Tool Name | Description | Parameters | -| ------------------------------ | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `sdk_info` | Get Fluent (ServiceNow SDK) information using native SDK flags | `flag`: SDK flag to execute (-v/--version, -h/--help, -d/--debug), `command`: (Optional) Specific command to get help for (only used with -h/--help flag) | -| `manage_fluent_auth` | Manage Fluent (ServiceNow SDK) authentication to instance | `add`: (Optional) Instance URL to add, `type`: (Optional) Authentication method, `alias`: (Optional) Alias for the instance, `list`: (Optional) List auth profiles, `delete`: (Optional) Delete auth profile, `use`: (Optional) Switch default auth | -| `init_fluent_app` | Initialize a new ServiceNow custom application or convert a legacy application | `from`: (Optional) sys_id or path, `appName`: App name, `packageName`: Package name, `scopeName`: Scope name, `workingDirectory`: Directory for the project, `template`: Project template (base, javascript.react, typescript.basic, typescript.react, javascript.basic), `intent`: (Optional) Creation or conversion intent | -| `build_fluent_app` | Build the Fluent (ServiceNow SDK) application | `debug`: (Optional) Print debug output | -| `deploy_fluent_app` | Deploy the Fluent (ServiceNow SDK) application to a ServiceNow instance | `auth`: (Optional) Authentication alias to use, `debug`: (Optional) Print debug output | -| `fluent_transform` | Download and convert XML records from instance or local path into Fluent source code | `from`: (Optional) Path to metadata, `directory`: (Optional) Package path, `preview`: (Optional) Preview only, `auth`: (Optional) Authentication alias, `debug`: (Optional) Print debug output | -| `download_fluent_dependencies` | Download configured dependencies in now.config.json and TypeScript type definitions | `auth`: (Optional) Authentication alias to use, `debug`: (Optional) Print debug output | -| `download_fluent_app` **NEW** | Download application metadata from instance | `directory`: Path to expand application, `source`: (Optional) Path to directory containing package.json configuration, `incremental`: (Optional) Download in incremental mode, `debug`: (Optional) Print debug output | -| `clean_fluent_app` **NEW** | Clean output directory | `source`: (Optional) Path to directory containing package.json configuration, `debug`: (Optional) Print debug output | -| `pack_fluent_app` **NEW** | Zip built app into installable artifact | `source`: (Optional) Path to directory containing package.json configuration, `debug`: (Optional) Print debug output | +## Quick Start -### Recent Updates +```bash +# Test with MCP Inspector +npx @modelcontextprotocol/inspector npx @modesty/fluent-mcp -**Complete ServiceNow SDK Command Coverage** - fluent-mcp now provides 100% coverage of all native ServiceNow SDK commands: +# Or use in your MCP client (see Configuration below) +``` -- **āœ… Enhanced `sdk_info` command**: Get SDK version, help, and debug information with improved working directory resolution and error handling -- **āœ… New `download_fluent_app` command**: Download application metadata from instance with support for incremental downloads and custom source directories -- **āœ… New `clean_fluent_app` command**: Clean output directories with proper package.json configuration detection -- **āœ… New `pack_fluent_app` command**: Create installable application artifacts from built applications -- **āœ… All commands aligned**: Parameter descriptions and functionality match the native SDK specification exactly +**Example prompt:** -### Interactive Commands +```text +Create a new Fluent app in ~/projects/time-off-tracker to manage employee PTO requests +``` -All `manage_fluent_auth`, `init_fluent_app` and `download_fluent_dependencies` commands are interactive CLI commands that require user input. The easier way to use them is to have Fluent MCP generate the shell command then run them in a terminal. Preferably, whenever you start a session with Fluent MCP, specify the working directory please. +## Available Tools -## Resources +### SDK Commands -All resources follow standardized URI patterns according to the MCP specification: +| Tool | Description | Key Parameters | +|------|-------------|----------------| +| `sdk_info` | Get SDK version, help, or debug info | `flag` (-v/-h/-d), `command` (optional) | +| `manage_fluent_auth` | Manage instance authentication profiles | `add`, `list`, `delete`, `use`, `type` (basic/oauth) | +| `init_fluent_app` | Initialize or convert ServiceNow app | `workingDirectory` (required), `template`, `from` (optional) | +| `build_fluent_app` | Build the application | `debug` (optional) | +| `deploy_fluent_app` | Deploy to ServiceNow instance | `auth` (optional), `debug` (optional) | +| `fluent_transform` | Convert XML to Fluent TypeScript | `from`, `auth` (optional) | +| `download_fluent_dependencies` | Download dependencies and type definitions | `auth` (optional) | +| `download_fluent_app` | Download metadata from instance | `directory`, `incremental` (optional) | +| `clean_fluent_app` | Clean output directory | `source` (optional) | +| `pack_fluent_app` | Create installable artifact | `source` (optional) | -1. **API Specifications**: `sn-spec://{metadataType}` - - Example: `sn-spec://business-rule` - - Contains API documentation with parameter descriptions +> **Note:** `manage_fluent_auth`, `init_fluent_app`, and `download_fluent_dependencies` are interactive commands. Use `init_fluent_app` to establish working directory context for subsequent commands. -2. **Instructions**: `sn-instruct://{metadataType}` - - Example: `sn-instruct://script-include` - - Offers guidance and best practices +## Resources -3. **Code Snippets**: `sn-snippet://{metadataType}/{snippetId}` - - Example: `sn-snippet://acl/0001` - - Provides practical examples +Standardized URI patterns following MCP specification: -4. **Prompts**: `sn-prompt://{promptId}` - - Example: `sn-prompt://coding_in_fluent` - - Contains development guides and best practices +| Resource Type | URI Pattern | Example | Purpose | +|---------------|-------------|---------|----------| +| **API Specs** | `sn-spec://{type}` | `sn-spec://business-rule` | API documentation and parameters | +| **Instructions** | `sn-instruct://{type}` | `sn-instruct://script-include` | Best practices and guidance | +| **Code Snippets** | `sn-snippet://{type}/{id}` | `sn-snippet://acl/0001` | Practical code examples | +| **Prompts** | `sn-prompt://{id}` | `sn-prompt://coding_in_fluent` | Development guides | ### Supported Metadata Types -- `acl`: Access Control Lists -- `application-menu`: Application Menus -- `business-rule`: Business Rules -- `client-script`: Client Scripts -- `cross-scope-privilege`: Cross Scope Privileges -- `form`: Forms -- `list`: Lists -- `property`: System Properties -- `role`: Roles -- `scheduled-script`: Scheduled Scripts -- `script-include`: Script Includes -- `scripted-rest`: Scripted REST APIs -- `table`: Tables -- `ui-action`: UI Actions -- `user-preference`: User Preferences -- `atf`: Automated Test Framework, including various ATF components - -Resources can be accessed by direct URI or through the `resources/list` method. - -## Requirements - -- Node.js 22.15.1 or later -- npm 11.4.1 or later - -### Client Integration +**Core Types:** `acl`, `application-menu`, `business-rule`, `client-script`, `cross-scope-privilege`, `form`, `list`, `property`, `role`, `scheduled-script`, `script-action`, `script-include`, `scripted-rest`, `service-portal`, `table`, `ui-action`, `ui-page`, `user-preference` -#### Claude Desktop / Claude on macOS - -```json -{ - "mcpServers": { - "fluent-mcp": { - "command": "npx", - "args": ["-y", "@modesty/fluent-mcp"], - "env": { - "SN_INSTANCE_URL": "http://localhost:8080", - "SN_AUTH_TYPE": "oauth" - } - } - } -} -``` +**Table Types:** `column`, `column-generic` -#### VSCode GitHub Copilot Agent Mode +**ATF (Automated Test Framework):** `atf-appnav`, `atf-catalog-action`, `atf-catalog-validation`, `atf-catalog-variable`, `atf-email`, `atf-form`, `atf-form-action`, `atf-form-declarative-action`, `atf-form-field`, `atf-reporting`, `atf-rest-api`, `atf-rest-assert-payload`, `atf-server`, `atf-server-catalog-item`, `atf-server-record` -1. `Shift + CMD + p` to open command palette, search for `MCP: Add Server...` -2. Select `NPM Package. (Model Assisted)` as server type. -3. Fill in package name as `@modesty/fluent-mcp` and follow prompts. +## Configuration -```json -{ - "mcp": { - "servers": { - "fluent-mcp": { - "type": "stdio", - "command": "npx", - "args": ["-y", "@modesty/fluent-mcp"], - "env": { - "SN_INSTANCE_URL": "http://localhost:8080", - "SN_AUTH_TYPE": "oauth" - } - } - } - } -} -``` +**Requirements:** Node.js 22.15.1+, npm 11.4.1+ -#### Cursor +### MCP Client Setup -Add MCP server configuration in Cursor settings: +Add to your MCP client configuration file: ```json { @@ -154,7 +104,7 @@ Add MCP server configuration in Cursor settings: "command": "npx", "args": ["-y", "@modesty/fluent-mcp"], "env": { - "SN_INSTANCE_URL": "http://localhost:8080", + "SN_INSTANCE_URL": "https://your-instance.service-now.com", "SN_AUTH_TYPE": "oauth" } } @@ -162,248 +112,112 @@ Add MCP server configuration in Cursor settings: } ``` -#### Windsurf +**Client-Specific Locations:** -1. `CMD + ,` to open settings, navigate to Cascade => MCP Servers => Manage MCPs => View raw config -1. Add configuration. +- **Claude Desktop / macOS:** `~/Library/Application Support/Claude/claude_desktop_config.json` +- **VSCode Copilot:** `.vscode/mcp.json` (use Command Palette: `MCP: Add Server...`) +- **Cursor:** Settings ā†’ Features ā†’ MCP Settings +- **Windsurf:** Settings ā†’ Cascade ā†’ MCP Servers ā†’ View raw config +- **Gemini CLI:** `~/.gemini/settings.json` -```json -{ - "mcpServers": { - "fluent-mcp": { - "command": "npx", - "args": ["-y", "@modesty/fluent-mcp"], - "env": { - "SN_INSTANCE_URL": "http://localhost:8080", - "SN_AUTH_TYPE": "oauth" - } - } - } -} -``` +> **VSCode note:** For VSCode, the JSON structure uses `"mcp": { "servers": { ... } }` instead of `"mcpServers"`. -1. Refresh when back to Manage MCPs page. +**Environment Variables:** -#### Gemini CLI +- `SN_INSTANCE_URL` - ServiceNow instance URL (optional, can use auth profiles instead) +- `SN_AUTH_TYPE` - Authentication method: `basic` or `oauth` (optional) +- `FLUENT_MCP_ENABLE_ERROR_ANALYSIS` - Enable AI error analysis (default: `true`) +- `FLUENT_MCP_MIN_ERROR_LENGTH` - Minimum error length for analysis (default: `50`) -Configure in `~/.gemini/settings.json` or `./.gemini/settings.json`: +## Usage Examples -```json -{ - "mcpServers": { - "fluent-mcp": { - "command": "npx", - "args": [ - "-y", - "@modesty/fluent-mcp" - ], - "env": { - "SN_INSTANCE_URL": "http://localhost:8080", - "SN_AUTH_TYPE": "oauth" - } - } - } -} -``` - -## Getting Started - -1. **Authentication** - a. Create a new auth alias: `Use manage_fluent_auth to create a new auth profile for with alias myFluentMcpAuth` - b. List existing aliases: `Use manage_fluent_auth to show all auth profiles` - c. Switch default alias: `Use manage_fluent_auth to set myFluentMcpAuth as the default auth` - -1. **Project Setup** - a. Create a new project: `Use init_fluent_app to create a new Fluent project for in directory ` - b. Convert an existing app: - ā€¢ By sys_id: `Use init_fluent_app to convert the existing scoped app with sys_id to a Fluent project in directory ` - ā€¢ By local path: `Use init_fluent_app to convert the existing scoped app from to a Fluent project in directory ` - c. Continue an existing project: `Use init_fluent_app to initialize the Fluent project in ` or `Set working directory to for the Fluent project` - -### Example Prompts - -When testing with Claude Desktop, check the MCP server logs (typically in `~/Library/Logs/Claude/mcp-server-fluent-mcp.log`) to see the actual resource requests being processed. - -#### Domain-Driven Business Rule - -Prompt: -"I'm building a ServiceNow application for IT asset management. Using Fluent, help me design and implement a domain-driven business rule for the following scenario: -When an IT asset (like a laptop) is assigned to an employee, I need to: - -Validate the asset is available and not already assigned -Update the asset status to 'In Use' -Create an audit trail entry -Notify the asset manager if it's a high-value asset (>$2000) - -Please show me the business rule structure using Fluent's TypeScript API, following SOLID principles. Think of this like designing a service layer in a traditional enterprise application - how would you structure the business logic to be maintainable and testable?" - -#### Legacy Application Modernization with Transform - -Prompt: -"I have a legacy ServiceNow application with scattered XML customizations that violate SOLID principles. Using Fluent MCP's transform capabilities, help me: - -Transform the existing XML-based business rules into modern TypeScript code -Refactor the code to follow single responsibility principle -Show me how to extract reusable script includes for common operations - -Use the transform command to convert my existing 'Incident Management' customizations, then demonstrate how to restructure them using proper separation of concerns - like how you'd refactor a monolithic service into microservices." - -#### Test-Driven Development with ATF Integration - -Prompt: -"Using Fluent MCP tools, help me set up a test-driven development workflow for ServiceNow. I want to: - -Create a new application for 'Employee Onboarding' -Set up the project structure with proper dependency management -Show me how to write ATF tests for business rules before implementing them -Demonstrate the build and install process - -Think of this like setting up a new Spring Boot project with Maven - show me the equivalent best practices for ServiceNow development using Fluent SDK." - -#### API-First Development with Scripted REST - -Prompt: +### Typical Workflow -"Help me design and implement a RESTful API for a ServiceNow application using Fluent MCP tools. The API should handle 'Project Management' operations: +1. **Setup Authentication** -GET /projects - list all projects -POST /projects - create new project -PUT /projects/{id} - update project -DELETE /projects/{id} - delete project + ```text + Create a new auth profile for https://dev12345.service-now.com with alias dev-instance + ``` -Show me how to: +2. **Initialize Project** -Use the scripted-rest metadata type to define the API endpoints -Implement proper error handling and validation -Follow REST best practices and HTTP status codes -Structure the code using dependency injection patterns + ```text + Create a new Fluent app in ~/projects/asset-tracker for IT asset management + ``` -This should be like building a REST controller in Spring Boot - clean, testable, and following OpenAPI standards." +3. **Develop with Resources** -#### Full-Stack Application Development Pipeline + ```text + Show me the business-rule API specification and provide an example snippet + ``` -Prompt: -"Using all Fluent MCP capabilities, walk me through creating a complete ServiceNow application from scratch. I want to build a 'Vendor Management' system with: +4. **Build and Deploy** -Custom tables for vendors, contracts, and purchase orders -Business rules for validation and workflow -Client scripts for UI interactions -REST APIs for external integration -Proper authentication and authorization + ```text + Build the app with debug output, then deploy to dev-instance + ``` -Show me the complete development workflow: +## Testing with MCP Inspector -Initialize the project with proper structure -Set up dependencies and build configuration -Implement the domain models and business logic -Create the UI components and forms -Build and deploy to a development instance +The [MCP Inspector](https://modelcontextprotocol.io/docs/tools/inspector) provides a web interface for testing MCP servers. -Think of this as building a full-stack application with proper CI/CD pipeline - what would be the ServiceNow equivalent using Fluent SDK?" +### Launch Inspector -#### Script Include Dependency Pattern - -Prompt: -"Create a UserService script include that depends on a BaseService script include. Show the TypeScript API calls for both, implementing constructor injection pattern. Use get-api-spec script-include to verify the structure." - -#### Multi-Environment Auth Setup - -Prompt: -"Configure auth profiles for dev/test/prod environments, then create a build script that deploys to each sequentially. Handle auth failures with rollback. Show the exact manage_fluent_auth and deploy_fluent_app commands." - -#### Form Field Validation Chain - -Prompt: -"Create 3 client scripts: onChange validation for 'priority' field, onLoad to set field states, onSubmit for final validation. Make them work together using shared utility functions. Use the client-script API spec." - -#### Legacy Code Transformation - -Prompt: -"Transform existing business rules from instance using fluent_transform, identify SOLID violations, refactor into separate script includes, then rebuild. Show the complete transform ā†’ refactor ā†’ build workflow." - -#### Business Rule Error Handling - -Prompt: -"Create before/after/async business rules for incident creation with proper error handling, logging, and transaction rollback. Show how to coordinate execution order and handle failures gracefully." - -"I'm working on a ServiceNow app. Can you show me how to create a UI Action?" - -## Development & Testing - -### MCP Inspector - -The [MCP Inspector](https://modelcontextprotocol.io/docs/tools/inspector) is an interactive web-based tool for testing and debugging MCP servers. It provides real-time visibility into your server's tools, resources, prompts, and notifications without requiring any permanent installation. - -#### Quick Start - -**Test the published NPM package:** ```bash +# Test published package npx @modelcontextprotocol/inspector npx @modesty/fluent-mcp -``` - -Or use the convenience script: -```bash -npm run inspect:published -``` -**Test the locally built server:** -```bash -npm run build -npm run inspect -``` - -**Test during development (without building):** -```bash -npm run inspect:dev +# Or for local development +npm run build && npm run inspect ``` -#### Using the Inspector +### Test Scenarios -Once launched, the Inspector provides a web interface with: +#### Scenario 1: Explore Business Rule Resources -1. **Server Connection Pane** - Configure environment variables (SN_INSTANCE_URL, SN_AUTH_TYPE) and verify server connectivity -2. **Resources Tab** - Browse and test all resources (sn-spec://, sn-instruct://, sn-snippet://, sn-prompt://) -3. **Prompts Tab** - Test prompt templates with custom arguments -4. **Tools Tab** - Explore all available tools (sdk_info, build_fluent_app, etc.) and execute them with test inputs -5. **Notifications Pane** - Monitor server logs and real-time events +**Objective:** Access API specs and code snippets for business rules -#### Development Workflow +**Steps:** -The recommended workflow for developing with fluent-mcp: +1. Launch Inspector and wait for server connection +2. Navigate to **Resources** tab +3. Find and click `sn-spec://business-rule` in the resource list +4. Review the API specification showing all available methods and parameters +5. Go back and search for `sn-snippet://business-rule/0001` +6. Click the snippet to view a complete TypeScript example +7. Verify content includes proper imports and follows Fluent patterns -1. **Make code changes** to your server implementation -2. **Rebuild** the project: `npm run build` -3. **Launch Inspector** to test: `npm run inspect` -4. **Test your changes** interactively through the web interface -5. **Iterate** - repeat the cycle as needed +**Expected Results:** -#### Environment Configuration +- API spec displays structured documentation with method signatures +- Snippet shows runnable TypeScript code with ServiceNow metadata patterns +- Content is properly formatted and readable -You can configure ServiceNow instance settings when launching the Inspector. The Inspector will pass environment variables to your server: +#### Scenario 2: Test SDK Info Command -```bash -# Example: Test with a specific ServiceNow instance -SN_INSTANCE_URL=https://dev12345.service-now.com SN_AUTH_TYPE=oauth npm run inspect -``` +**Objective:** Verify SDK version and help information retrieval -#### Testing Scenarios +**Steps:** -**Test Tools:** -- Verify all ServiceNow SDK commands are available -- Execute commands with sample parameters -- Validate error handling with invalid inputs +1. Navigate to **Tools** tab +2. Select `sdk_info` from the tool list +3. **Test Version:** + - Set `flag` parameter to `-v` + - Click **Execute** + - Verify response shows version number (e.g., "4.0.1") +4. **Test Help:** + - Set `flag` parameter to `-h` + - Set `command` parameter to `build` + - Click **Execute** + - Verify response shows build command documentation with options +5. Monitor **Notifications** pane for command execution logs -**Test Resources:** -- Browse API specifications: `sn-spec://business-rule`, `sn-spec://client-script` -- View code snippets: `sn-snippet://acl/0001` -- Check instructions: `sn-instruct://table` -- Access prompts: `sn-prompt://coding_in_fluent` +**Expected Results:** -**Test Integration:** -- Verify working directory resolution -- Test authentication flows -- Monitor command execution and output -- Validate resource loading and caching +- Version command returns SDK version string +- Help command returns detailed command documentation +- No errors in notifications pane +- Commands execute within 2-3 seconds ## License diff --git a/package.json b/package.json index 57ddefe..219afde 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@modesty/fluent-mcp", - "version": "0.0.17", + "version": "0.0.18", "description": "MCP server for Fluent (ServiceNow SDK)", "keywords": [ "Servicenow SDK", diff --git a/src/config.ts b/src/config.ts index 34590fc..1df8c25 100644 --- a/src/config.ts +++ b/src/config.ts @@ -78,6 +78,14 @@ export interface McpServerConfig { /** Default timeout for SDK CLI commands in milliseconds */ commandTimeoutMs: number; }; + + /** Sampling configuration for AI-powered features */ + sampling: { + /** Enable AI-powered error analysis using MCP Sampling capability */ + enableErrorAnalysis: boolean; + /** Minimum error length to trigger analysis (avoid analyzing trivial errors) */ + minErrorLength: number; + }; } // Environment variable names @@ -90,6 +98,8 @@ const ENV_VAR = { RESOURCE_PATH_INSTRUCT: `${ENV_PREFIX}RESOURCE_PATH_INSTRUCT`, SERVICENOW_SDK_CLI_PATH: `${ENV_PREFIX}SERVICENOW_SDK_CLI_PATH`, COMMAND_TIMEOUT_MS: `${ENV_PREFIX}COMMAND_TIMEOUT_MS`, + ENABLE_ERROR_ANALYSIS: `${ENV_PREFIX}ENABLE_ERROR_ANALYSIS`, + MIN_ERROR_LENGTH: `${ENV_PREFIX}MIN_ERROR_LENGTH`, }; /** @@ -110,6 +120,10 @@ export const defaultConfig: McpServerConfig = { cliPath: 'snc', // Default command name if installed globally commandTimeoutMs: 30000, // 30 seconds default timeout }, + sampling: { + enableErrorAnalysis: true, // Enable by default + minErrorLength: 50, // Only analyze errors with 50+ characters + }, }; /** @@ -149,6 +163,19 @@ export function getConfig(): McpServerConfig { 10 ), }, + sampling: { + enableErrorAnalysis: getEnvVar( + ENV_VAR.ENABLE_ERROR_ANALYSIS, + defaultConfig.sampling.enableErrorAnalysis.toString() + ) === 'true', + minErrorLength: parseInt( + getEnvVar( + ENV_VAR.MIN_ERROR_LENGTH, + defaultConfig.sampling.minErrorLength.toString() + ), + 10 + ), + }, }; // Add optional logFilePath if specified in environment diff --git a/src/res/resourceManager.ts b/src/res/resourceManager.ts index b69f901..d014d41 100644 --- a/src/res/resourceManager.ts +++ b/src/res/resourceManager.ts @@ -56,6 +56,7 @@ export class ResourceManager { `sn-spec-${type}`, template, { + name: `sn-spec-${type}`, title: `${type} API Specification for Fluent (ServiceNow SDK)`, description: `API specification for Fluent (ServiceNow SDK) ${type}`, mimeType: 'text/markdown' @@ -133,6 +134,7 @@ export class ResourceManager { `sn-snippet-${type}`, template, { + name: `sn-snippet-${type}`, title: `${type} Code Snippets for Fluent (ServiceNow SDK)`, description: `Example code snippets for Fluent (ServiceNow SDK) ${type}`, mimeType: 'text/markdown' @@ -196,6 +198,7 @@ export class ResourceManager { `sn-instruct-${type}`, template, { + name: `sn-instruct-${type}`, title: `${type} Instructions for Fluent (ServiceNow SDK)`, description: `Development instructions for Fluent (ServiceNow SDK) ${type}`, mimeType: 'text/markdown' @@ -249,7 +252,7 @@ export class ResourceManager { * Get all available resources for listing * @returns List of resource objects */ - async listResources(): Promise<{ uri: string, title: string, mimeType: string }[]> { + async listResources(): Promise<{ uri: string, name: string, title: string, mimeType: string }[]> { try { // Make sure metadata types are loaded if (!this.metadataTypes || this.metadataTypes.length === 0) { @@ -262,6 +265,7 @@ export class ResourceManager { for (const type of this.metadataTypes) { resources.push({ uri: `sn-spec://${type}`, + name: `sn-spec-${type}`, title: `${type} API Specification for Fluent (ServiceNow SDK) `, mimeType: 'text/markdown' }); @@ -271,6 +275,7 @@ export class ResourceManager { for (const type of this.metadataTypes) { resources.push({ uri: `sn-instruct://${type}`, + name: `sn-instruct-${type}`, title: `${type} Instructions for Fluent (ServiceNow SDK)`, mimeType: 'text/markdown' }); @@ -283,6 +288,7 @@ export class ResourceManager { if (snippetIds.length > 0) { resources.push({ uri: `sn-snippet://${type}/${snippetIds[0]}`, + name: `sn-snippet-${type}`, title: `${type} Code Snippet for Fluent (ServiceNow SDK)`, mimeType: 'text/markdown' }); @@ -318,4 +324,123 @@ export class ResourceManager { getMetadataTypes(): string[] { return this.metadataTypes; } + + /** + * Read a resource by URI + * This method is called by the MCP server's resources/read handler + * @param uriString The resource URI to read + * @returns Resource contents + */ + async readResource(uriString: string): Promise<{ contents: Array<{ uri: string; text: string; mimeType?: string }> }> { + try { + const uri = new URL(uriString); + const scheme = uri.protocol.replace(':', ''); + const metadataType = uri.host; + + // Determine resource type from URI scheme + if (scheme === 'sn-spec') { + return await this.readSpecResource(uri, metadataType); + } else if (scheme === 'sn-snippet') { + return await this.readSnippetResource(uri, metadataType); + } else if (scheme === 'sn-instruct') { + return await this.readInstructResource(uri, metadataType); + } + + throw new Error(`Unknown resource URI scheme: ${scheme}`); + } catch (error) { + logger.error('Error reading resource', + error instanceof Error ? error : new Error(String(error)) + ); + throw error; + } + } + + /** + * Read a spec resource + */ + private async readSpecResource(uri: URL, metadataType: string) { + const result = await this.resourceLoader.getResource( + ResourceType.SPEC, + metadataType + ); + + if (!result.found) { + return { + contents: [{ + uri: uri.href, + text: `API specification not found for ${metadataType}`, + mimeType: 'text/plain' + }] + }; + } + + return { + contents: [{ + uri: uri.href, + text: result.content, + mimeType: 'text/markdown' + }] + }; + } + + /** + * Read a snippet resource + */ + private async readSnippetResource(uri: URL, metadataType: string) { + // Extract snippet ID from URI path + const pathParts = uri.pathname.split('/').filter(p => p); + const snippetId = pathParts.length > 0 ? pathParts[0] : undefined; + + const result = await this.resourceLoader.getResource( + ResourceType.SNIPPET, + metadataType, + snippetId + ); + + if (!result.found) { + return { + contents: [{ + uri: uri.href, + text: `Snippet not found for ${metadataType}`, + mimeType: 'text/plain' + }] + }; + } + + return { + contents: [{ + uri: uri.href, + text: result.content, + mimeType: 'text/markdown' + }] + }; + } + + /** + * Read an instruct resource + */ + private async readInstructResource(uri: URL, metadataType: string) { + const result = await this.resourceLoader.getResource( + ResourceType.INSTRUCT, + metadataType + ); + + if (!result.found) { + return { + contents: [{ + uri: uri.href, + text: `Instructions not found for ${metadataType}`, + mimeType: 'text/plain' + }] + }; + } + + return { + contents: [{ + uri: uri.href, + text: result.content, + mimeType: 'text/markdown' + }] + }; + } } diff --git a/src/server/fluentMCPServer.ts b/src/server/fluentMCPServer.ts index c2d325f..c2f466c 100644 --- a/src/server/fluentMCPServer.ts +++ b/src/server/fluentMCPServer.ts @@ -6,6 +6,7 @@ import { ListToolsRequestSchema, CallToolResult, ListResourcesRequestSchema, + ReadResourceRequestSchema, ListRootsRequestSchema, RootsListChangedNotificationSchema, InitializedNotificationSchema, @@ -22,6 +23,7 @@ import { ToolsManager } from '../tools/toolsManager.js'; import { ResourceManager } from '../res/resourceManager.js'; import { PromptManager } from '../prompts/promptManager.js'; import { autoValidateAuthIfConfigured } from './fluentInstanceAuth.js'; +import { SamplingManager } from '../utils/samplingManager.js'; /** * Implementation of the Model Context Protocol server for Fluent (ServiceNow SDK) @@ -34,23 +36,26 @@ export class FluentMcpServer { private toolsManager: ToolsManager; private resourceManager: ResourceManager; private promptManager: PromptManager; + private samplingManager: SamplingManager; + private config: ReturnType; private status: ServerStatus = ServerStatus.STOPPED; private roots: { uri: string; name?: string }[] = []; private autoAuthTriggered = false; + private resourcesRegistered = false; /** * Create a new MCP server instance */ constructor() { // Initialize server with configuration - const config = getConfig(); + this.config = getConfig(); // Create MCP server instance with server info from package.json this.mcpServer = new McpServer( { - name: config.name, - version: config.version, - description: config.description, + name: this.config.name, + version: this.config.version, + description: this.config.description, }, { capabilities: { @@ -58,6 +63,7 @@ export class FluentMcpServer { resources: {}, // Enable resources capability logging: {}, // Enable logging capability elicitation: {}, // Enable elicitation capability for structured data collection + sampling: {}, // Enable sampling capability for AI-powered features prompts: { listChanged: true, // Enable prompt list change notifications }, @@ -68,17 +74,19 @@ export class FluentMcpServer { } ); - // Initialize managers for tools, resources, and prompts + // Initialize managers for tools, resources, prompts, and sampling this.toolsManager = new ToolsManager(this.mcpServer); this.resourceManager = new ResourceManager(this.mcpServer); this.promptManager = new PromptManager(this.mcpServer); + this.samplingManager = new SamplingManager(this.mcpServer); // Initialize resources and prompts Promise.all([ this.resourceManager.initialize(), this.promptManager.initialize() ]).then(() => { - // Now that all tools, resources, and prompts are registered, we can set up the handlers + // Set up the handlers after initialization + // Resources will be registered during start() to ensure proper timing this.setupHandlers(); }); } @@ -92,7 +100,14 @@ export class FluentMcpServer { if (result.success) { return `āœ… Output:\n${result.output}`; } else { - return `āŒ Error:\n${result.error || 'Unknown error'}\n(exit code: ${result.exitCode})\n${result.output}`; + let errorOutput = `āŒ Error:\n${result.error || 'Unknown error'}\n(exit code: ${result.exitCode})\n${result.output}`; + + // Append AI error analysis if available + if (result.errorAnalysis) { + errorOutput += '\n' + this.samplingManager.formatAnalysis(result.errorAnalysis); + } + + return errorOutput; } } @@ -235,6 +250,27 @@ export class FluentMcpServer { } }); + // Set up the resources/read handler + // The SDK's registerResource() doesn't automatically set up the read handler + // We need to explicitly handle resource read requests + server.setRequestHandler(ReadResourceRequestSchema, async (request) => { + const { uri } = request.params; + + try { + logger.debug('Reading resource', { uri }); + + // Call ResourceManager to handle the read request + const result = await this.resourceManager.readResource(uri); + return result; + } catch (error) { + logger.error('Error reading resource', + error instanceof Error ? error : new Error(String(error)), + { uri } + ); + throw error; + } + }); + // Set up prompts handlers this.promptManager.setupHandlers(); @@ -286,6 +322,33 @@ export class FluentMcpServer { try { const result = await command.execute(args || {}); + // If command failed and error analysis is enabled, analyze the error + if (!result.success && this.config.sampling.enableErrorAnalysis && result.error) { + const errorMessage = result.error.message; + + if (this.samplingManager.shouldAnalyzeError(errorMessage, this.config.sampling.minErrorLength)) { + logger.info('Triggering error analysis for failed command', { command: name }); + + try { + const analysis = await this.samplingManager.analyzeError({ + command: name, + args: Object.entries(args || {}).map(([key, value]) => `${key}=${value}`), + errorOutput: errorMessage, + exitCode: result.exitCode, + }); + + if (analysis) { + result.errorAnalysis = analysis; + } + } catch (analysisError) { + // Log but don't fail the tool call if analysis fails + logger.warn('Error analysis failed', { + error: analysisError instanceof Error ? analysisError.message : String(analysisError), + }); + } + } + } + return { content: [ { @@ -443,9 +506,12 @@ export class FluentMcpServer { // Configure logging manager with MCP server loggingManager.configure(this.mcpServer); - // Now that we're connected and have set up handlers, register resources - // The prompt handlers are already registered by setupHandlers - this.resourceManager.registerAll(); + // Register resources if not already done + // This sets up the resource read handlers in the MCP SDK + if (!this.resourcesRegistered) { + this.resourceManager.registerAll(); + this.resourcesRegistered = true; + } // Set the server status to running before initializing roots // This ensures that client notifications will be sent correctly diff --git a/src/utils/samplingManager.ts b/src/utils/samplingManager.ts new file mode 100644 index 0000000..48d6de5 --- /dev/null +++ b/src/utils/samplingManager.ts @@ -0,0 +1,225 @@ +/** + * SamplingManager - Handles MCP Sampling operations for AI-powered features + * + * This utility provides error analysis using the MCP Sampling capability, + * which allows the server to request LLM assistance from the client. + */ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { ErrorAnalysis } from './types.js'; +import logger from './logger.js'; + +/** + * Parameters for error analysis + */ +export interface ErrorAnalysisParams { + /** Command that failed */ + command: string; + /** Command arguments */ + args: string[]; + /** Error output from the command */ + errorOutput: string; + /** Exit code of the failed command */ + exitCode: number; +} + +/** + * SamplingManager handles AI-powered analysis using MCP Sampling + */ +export class SamplingManager { + constructor(private mcpServer: McpServer) {} + + /** + * Check if error should be analyzed based on its characteristics + * @param errorOutput The error message to check + * @param minLength Minimum error length to trigger analysis + * @returns True if error should be analyzed + */ + shouldAnalyzeError(errorOutput: string, minLength: number = 50): boolean { + if (!errorOutput || errorOutput.trim().length < minLength) { + return false; + } + + // Skip analysis for common trivial errors + const trivialPatterns = [ + /^command not found/i, + /^permission denied/i, + /^no such file or directory$/i, + ]; + + return !trivialPatterns.some(pattern => pattern.test(errorOutput.trim())); + } + + /** + * Analyze command error and provide AI-powered suggestions + * Uses MCP Sampling to request LLM analysis from the client + * + * @param params Error analysis parameters + * @returns Promise with error analysis result + */ + async analyzeError(params: ErrorAnalysisParams): Promise { + try { + logger.info('Requesting error analysis via MCP Sampling', { + command: params.command, + exitCode: params.exitCode, + }); + + // Construct the analysis prompt + const prompt = this.buildErrorAnalysisPrompt(params); + + // Request LLM analysis via MCP Sampling + const result = await this.mcpServer.server.createMessage({ + messages: [ + { + role: 'user', + content: { + type: 'text', + text: prompt, + }, + }, + ], + maxTokens: 800, + modelPreferences: { + intelligencePriority: 0.8, // Prioritize intelligence for analysis + speedPriority: 0.2, + }, + }); + + // Parse the LLM response + const analysis = this.parseAnalysisResponse(result.content); + + logger.info('Error analysis completed successfully'); + return analysis; + } catch (error) { + // If Sampling fails, log but don't break the flow + logger.warn('Error analysis failed, continuing without it', { + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + } + + /** + * Build the error analysis prompt for the LLM + * @param params Error analysis parameters + * @returns Formatted prompt string + */ + private buildErrorAnalysisPrompt(params: ErrorAnalysisParams): string { + return `You are an expert in ServiceNow SDK and Fluent development. Analyze this command error and provide structured guidance. + +Command: ${params.command} ${params.args.join(' ')} +Exit Code: ${params.exitCode} + +Error Output: +${params.errorOutput} + +Provide your analysis in the following format: + +ROOT CAUSE: +[One clear paragraph explaining what went wrong] + +SOLUTIONS: +1. [First specific solution step] +2. [Second specific solution step] +3. [Additional steps as needed] + +PREVENTION: +1. [First prevention tip] +2. [Second prevention tip] +3. [Additional prevention tips] + +Keep each section concise and actionable. Focus on ServiceNow SDK and Fluent-specific issues.`; + } + + /** + * Parse the LLM response into structured ErrorAnalysis + * @param content Response content from LLM + * @returns Parsed error analysis + */ + private parseAnalysisResponse(content: any): ErrorAnalysis { + // Extract text from content + const text = typeof content === 'string' ? content : content.text || ''; + + // Parse sections using regex + const rootCauseMatch = text.match(/ROOT CAUSE:\s*([\s\S]*?)(?=SOLUTIONS:|$)/i); + const solutionsMatch = text.match(/SOLUTIONS:\s*([\s\S]*?)(?=PREVENTION:|$)/i); + const preventionMatch = text.match(/PREVENTION:\s*([\s\S]*?)$/i); + + // Extract root cause + const rootCause = rootCauseMatch + ? rootCauseMatch[1].trim() + : 'Unable to determine root cause'; + + // Extract solutions (numbered list) + const solutionsText = solutionsMatch ? solutionsMatch[1].trim() : ''; + const suggestions = this.extractListItems(solutionsText); + + // Extract prevention tips (numbered list) + const preventionText = preventionMatch ? preventionMatch[1].trim() : ''; + const preventionTips = this.extractListItems(preventionText); + + return { + rootCause, + suggestions: suggestions.length > 0 ? suggestions : ['No specific solutions provided'], + preventionTips: preventionTips.length > 0 ? preventionTips : ['Review ServiceNow SDK documentation'], + }; + } + + /** + * Extract numbered list items from text + * @param text Text containing numbered list + * @returns Array of list items + */ + private extractListItems(text: string): string[] { + const items: string[] = []; + + // Match numbered items (1., 2., etc.) or bullet points + const lines = text.split('\n'); + + for (const line of lines) { + const trimmed = line.trim(); + // Match patterns like "1. ", "2. ", "- ", "* " + const match = trimmed.match(/^(?:\d+\.|[-*])\s+(.+)$/); + if (match && match[1]) { + items.push(match[1].trim()); + } + } + + // If no numbered items found, try to split by sentence + if (items.length === 0 && text.trim().length > 0) { + const sentences = text + .split(/[.!?]+/) + .map(s => s.trim()) + .filter(s => s.length > 10); + + if (sentences.length > 0) { + return sentences.slice(0, 5); // Limit to 5 items + } + } + + return items; + } + + /** + * Format error analysis for display + * @param analysis Error analysis result + * @returns Formatted string for display + */ + formatAnalysis(analysis: ErrorAnalysis): string { + let output = '\nšŸ¤– AI Error Analysis:\n\n'; + + output += 'šŸ“‹ Root Cause:\n'; + output += `${analysis.rootCause}\n\n`; + + output += 'šŸ’” Suggested Solutions:\n'; + analysis.suggestions.forEach((suggestion, index) => { + output += `${index + 1}. ${suggestion}\n`; + }); + + output += '\nšŸ›”ļø Prevention Tips:\n'; + analysis.preventionTips.forEach((tip, index) => { + output += `${index + 1}. ${tip}\n`; + }); + + return output; + } +} diff --git a/src/utils/types.ts b/src/utils/types.ts index 04ad947..3038bf5 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -11,6 +11,18 @@ export interface CommandArgument { defaultValue?: unknown; } +/** + * AI-powered error analysis result + */ +export interface ErrorAnalysis { + /** Root cause of the error */ + rootCause: string; + /** Suggested solutions */ + suggestions: string[]; + /** Prevention tips for future */ + preventionTips: string[]; +} + /** * Command execution result */ @@ -21,6 +33,8 @@ export interface CommandResult { output: string; /** Error that occurred during execution, if any */ error?: Error; + /** AI-powered error analysis (optional, only when enabled and applicable) */ + errorAnalysis?: ErrorAnalysis; } /** diff --git a/test/res/resourceManager.test.ts b/test/res/resourceManager.test.ts index eb5e4e6..00449a7 100644 --- a/test/res/resourceManager.test.ts +++ b/test/res/resourceManager.test.ts @@ -107,8 +107,101 @@ describe("ResourceManager", () => { // Check that each resource has the required properties resources.forEach(resource => { expect(resource).toHaveProperty('uri'); + expect(resource).toHaveProperty('name'); // Required by MCP protocol expect(resource).toHaveProperty('title'); expect(resource).toHaveProperty('mimeType', 'text/markdown'); }); }); + + test("should register spec resources with required 'name' field", async () => { + await resourceManager.initialize(); + resourceManager.registerAll(); + + // Get all registerResource calls + const calls = mockMcpServer.registerResource.mock.calls; + + // Filter spec resource registrations + const specCalls = calls.filter((call: any[]) => call[0].startsWith('sn-spec-')); + + expect(specCalls.length).toBeGreaterThan(0); + + // Validate each spec resource has required 'name' field + specCalls.forEach((call: any[]) => { + const metadata = call[2]; // 3rd parameter is metadata + expect(metadata).toHaveProperty('name'); + expect(metadata.name).toMatch(/^sn-spec-/); + expect(metadata).toHaveProperty('title'); + expect(metadata).toHaveProperty('description'); + expect(metadata).toHaveProperty('mimeType', 'text/markdown'); + }); + }); + + test("should register snippet resources with required 'name' field", async () => { + await resourceManager.initialize(); + resourceManager.registerAll(); + + // Get all registerResource calls + const calls = mockMcpServer.registerResource.mock.calls; + + // Filter snippet resource registrations + const snippetCalls = calls.filter((call: any[]) => call[0].startsWith('sn-snippet-')); + + expect(snippetCalls.length).toBeGreaterThan(0); + + // Validate each snippet resource has required 'name' field + snippetCalls.forEach((call: any[]) => { + const metadata = call[2]; // 3rd parameter is metadata + expect(metadata).toHaveProperty('name'); + expect(metadata.name).toMatch(/^sn-snippet-/); + expect(metadata).toHaveProperty('title'); + expect(metadata).toHaveProperty('description'); + expect(metadata).toHaveProperty('mimeType', 'text/markdown'); + }); + }); + + test("should register instruct resources with required 'name' field", async () => { + await resourceManager.initialize(); + resourceManager.registerAll(); + + // Get all registerResource calls + const calls = mockMcpServer.registerResource.mock.calls; + + // Filter instruct resource registrations + const instructCalls = calls.filter((call: any[]) => call[0].startsWith('sn-instruct-')); + + expect(instructCalls.length).toBeGreaterThan(0); + + // Validate each instruct resource has required 'name' field + instructCalls.forEach((call: any[]) => { + const metadata = call[2]; // 3rd parameter is metadata + expect(metadata).toHaveProperty('name'); + expect(metadata.name).toMatch(/^sn-instruct-/); + expect(metadata).toHaveProperty('title'); + expect(metadata).toHaveProperty('description'); + expect(metadata).toHaveProperty('mimeType', 'text/markdown'); + }); + }); + + test("should ensure all registered resources have MCP-compliant metadata", async () => { + await resourceManager.initialize(); + resourceManager.registerAll(); + + // Get all registerResource calls + const calls = mockMcpServer.registerResource.mock.calls; + + expect(calls.length).toBeGreaterThan(0); + + // Validate every registered resource has MCP-required fields + calls.forEach((call: any[], index: number) => { + const resourceId = call[0]; + const metadata = call[2]; + + // Required fields for MCP protocol + expect(metadata).toHaveProperty('name'); + expect(metadata.name).toBeDefined(); + + // The name should match the resource ID + expect(metadata.name).toBe(resourceId); + }); + }); }); diff --git a/test/server/errorAnalysis.integration.test.ts b/test/server/errorAnalysis.integration.test.ts new file mode 100644 index 0000000..f311c3a --- /dev/null +++ b/test/server/errorAnalysis.integration.test.ts @@ -0,0 +1,393 @@ +/** + * Integration tests for Error Analysis feature + * Tests component integration without mocking the full MCP handler system + */ +import { SamplingManager } from '../../src/utils/samplingManager.js'; +import { ErrorAnalysis, CommandResult } from '../../src/utils/types.js'; + +// Mock config with sampling enabled +jest.mock('../../src/config.js', () => ({ + __esModule: true, + getConfig: jest.fn().mockReturnValue({ + name: 'test-mcp-server', + version: '1.0.0', + description: 'Test MCP Server', + logLevel: 'info', + resourcePaths: { + spec: '/mock/path/to/spec', + snippet: '/mock/path/to/snippet', + instruct: '/mock/path/to/instruct', + }, + servicenowSdk: { + cliPath: 'snc', + commandTimeoutMs: 30000, + }, + sampling: { + enableErrorAnalysis: true, + minErrorLength: 50, + }, + }), + getProjectRootPath: jest.fn().mockReturnValue('/mock/project/root'), +})); + +// Mock logger +jest.mock('../../src/utils/logger.js', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +describe('Error Analysis Integration', () => { + describe('Configuration Integration', () => { + it('should load sampling configuration with default values', () => { + const { getConfig } = require('../../src/config.js'); + const config = getConfig(); + + expect(config.sampling).toBeDefined(); + expect(config.sampling.enableErrorAnalysis).toBe(true); + expect(config.sampling.minErrorLength).toBe(50); + }); + + it('should support disabling error analysis via config', () => { + const { getConfig } = require('../../src/config.js'); + + // Temporarily override mock + getConfig.mockReturnValueOnce({ + name: 'test-mcp-server', + version: '1.0.0', + description: 'Test MCP Server', + logLevel: 'info', + resourcePaths: { + spec: '/mock/path/to/spec', + snippet: '/mock/path/to/snippet', + instruct: '/mock/path/to/instruct', + }, + servicenowSdk: { + cliPath: 'snc', + commandTimeoutMs: 30000, + }, + sampling: { + enableErrorAnalysis: false, + minErrorLength: 50, + }, + }); + + const config = getConfig(); + expect(config.sampling.enableErrorAnalysis).toBe(false); + }); + }); + + describe('SamplingManager Integration with CommandResult', () => { + let samplingManager: SamplingManager; + let mockMcpServer: any; + + beforeEach(() => { + mockMcpServer = { + server: { + createMessage: jest.fn(), + }, + }; + samplingManager = new SamplingManager(mockMcpServer); + }); + + it('should enhance CommandResult with error analysis', async () => { + // Mock successful error analysis + mockMcpServer.server.createMessage.mockResolvedValue({ + content: { + text: `ROOT CAUSE: +The build failed due to TypeScript compilation errors. + +SOLUTIONS: +1. Check tsconfig.json for correct settings +2. Fix type errors in the code +3. Run npm install to update dependencies + +PREVENTION: +1. Use strict type checking +2. Run build before committing`, + }, + }); + + const initialResult: CommandResult = { + success: false, + exitCode: 1, + output: '', + error: new Error('TypeScript compilation failed with 3 errors in business-rule.ts'), + }; + + // Analyze the error + const analysis = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: ['--debug'], + errorOutput: initialResult.error!.message, + exitCode: initialResult.exitCode, + }); + + // Enhance result with analysis + const enhancedResult: CommandResult = { + ...initialResult, + errorAnalysis: analysis!, + }; + + expect(enhancedResult.errorAnalysis).toBeDefined(); + expect(enhancedResult.errorAnalysis?.rootCause).toContain('TypeScript compilation'); + expect(enhancedResult.errorAnalysis?.suggestions).toHaveLength(3); + expect(enhancedResult.errorAnalysis?.preventionTips).toHaveLength(2); + }); + + it('should handle CommandResult without error analysis when disabled', () => { + const result: CommandResult = { + success: false, + exitCode: 1, + output: '', + error: new Error('Some error'), + }; + + // Result without analysis should still be valid + expect(result.errorAnalysis).toBeUndefined(); + expect(result.success).toBe(false); + expect(result.error).toBeDefined(); + }); + }); + + describe('Error Classification Logic', () => { + let samplingManager: SamplingManager; + let mockMcpServer: any; + + beforeEach(() => { + mockMcpServer = { + server: { + createMessage: jest.fn(), + }, + }; + samplingManager = new SamplingManager(mockMcpServer); + }); + + it('should classify ServiceNow SDK build errors as analyzable', () => { + const { getConfig } = require('../../src/config.js'); + const error = 'ServiceNow SDK build failed: Invalid metadata configuration in business-rule.ts'; + const config = getConfig(); + + const shouldAnalyze = samplingManager.shouldAnalyzeError( + error, + config.sampling.minErrorLength + ); + + expect(shouldAnalyze).toBe(true); + }); + + it('should not classify short errors as analyzable', () => { + const { getConfig } = require('../../src/config.js'); + const error = 'Build failed'; + const config = getConfig(); + + const shouldAnalyze = samplingManager.shouldAnalyzeError( + error, + config.sampling.minErrorLength + ); + + expect(shouldAnalyze).toBe(false); + }); + + it('should not classify trivial errors as analyzable', () => { + const { getConfig } = require('../../src/config.js'); + const errors = [ + 'command not found: some-cmd', + 'Permission denied', + 'no such file or directory', + ]; + const config = getConfig(); + + errors.forEach(error => { + const shouldAnalyze = samplingManager.shouldAnalyzeError( + error, + config.sampling.minErrorLength + ); + expect(shouldAnalyze).toBe(false); + }); + }); + }); + + describe('Error Analysis Formatting', () => { + let samplingManager: SamplingManager; + let mockMcpServer: any; + + beforeEach(() => { + mockMcpServer = { + server: { + createMessage: jest.fn(), + }, + }; + samplingManager = new SamplingManager(mockMcpServer); + }); + + it('should format error analysis with proper structure and emojis', () => { + const analysis: ErrorAnalysis = { + rootCause: 'Build failed due to invalid configuration', + suggestions: [ + 'Check tsconfig.json', + 'Verify package.json dependencies', + 'Run clean before build', + ], + preventionTips: [ + 'Use strict mode', + 'Validate before committing', + ], + }; + + const formatted = samplingManager.formatAnalysis(analysis); + + expect(formatted).toContain('šŸ¤– AI Error Analysis:'); + expect(formatted).toContain('šŸ“‹ Root Cause:'); + expect(formatted).toContain('šŸ’” Suggested Solutions:'); + expect(formatted).toContain('šŸ›”ļø Prevention Tips:'); + expect(formatted).toContain('1. Check tsconfig.json'); + expect(formatted).toContain('2. Verify package.json dependencies'); + expect(formatted).toContain('3. Run clean before build'); + expect(formatted).toContain('1. Use strict mode'); + expect(formatted).toContain('2. Validate before committing'); + }); + + it('should format error analysis for integration with CommandResult output', () => { + const analysis: ErrorAnalysis = { + rootCause: 'Authentication failed', + suggestions: ['Check credentials', 'Verify instance URL'], + preventionTips: ['Store credentials securely'], + }; + + const baseErrorOutput = 'āŒ Error:\nAuthentication failed\n(exit code: 1)'; + const formatted = samplingManager.formatAnalysis(analysis); + const fullOutput = baseErrorOutput + '\n' + formatted; + + expect(fullOutput).toContain('āŒ Error:'); + expect(fullOutput).toContain('šŸ¤– AI Error Analysis:'); + expect(fullOutput).toContain('Authentication failed'); // Appears in both sections + }); + }); + + describe('End-to-End Error Analysis Flow', () => { + let samplingManager: SamplingManager; + let mockMcpServer: any; + + beforeEach(() => { + mockMcpServer = { + server: { + createMessage: jest.fn(), + }, + }; + samplingManager = new SamplingManager(mockMcpServer); + }); + + it('should complete full error analysis workflow', async () => { + const { getConfig } = require('../../src/config.js'); + const config = getConfig(); + const errorMessage = 'ServiceNow SDK: Metadata validation failed in business-rule.ts - invalid "when" field value'; + + // Step 1: Check if error should be analyzed + const shouldAnalyze = samplingManager.shouldAnalyzeError( + errorMessage, + config.sampling.minErrorLength + ); + expect(shouldAnalyze).toBe(true); + + // Step 2: Mock LLM response + mockMcpServer.server.createMessage.mockResolvedValue({ + content: { + text: `ROOT CAUSE: +The business rule has an invalid value in the "when" field that doesn't match ServiceNow's schema. + +SOLUTIONS: +1. Review the business-rule.ts file for the "when" field +2. Check ServiceNow SDK documentation for valid values +3. Use TypeScript types for validation + +PREVENTION: +1. Enable strict TypeScript mode +2. Use SDK type definitions +3. Test locally before deployment`, + }, + }); + + // Step 3: Analyze error + const analysis = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: ['--debug'], + errorOutput: errorMessage, + exitCode: 1, + }); + + expect(analysis).not.toBeNull(); + expect(analysis?.rootCause).toContain('invalid value'); + expect(analysis?.suggestions).toHaveLength(3); + expect(analysis?.preventionTips).toHaveLength(3); + + // Step 4: Format for output + const formatted = samplingManager.formatAnalysis(analysis!); + expect(formatted).toContain('šŸ¤– AI Error Analysis:'); + + // Step 5: Verify createMessage was called with correct parameters + expect(mockMcpServer.server.createMessage).toHaveBeenCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: 'user', + content: expect.objectContaining({ + type: 'text', + text: expect.stringContaining('build_fluent_app'), + }), + }), + ]), + maxTokens: 800, + modelPreferences: { + intelligencePriority: 0.8, + speedPriority: 0.2, + }, + }) + ); + }); + + it('should gracefully handle sampling failure in workflow', async () => { + const { getConfig } = require('../../src/config.js'); + const config = getConfig(); + const errorMessage = 'ServiceNow SDK: Build failed with complex error in business-rule.ts due to metadata validation issues'; + + // Step 1: Error should be analyzed + const shouldAnalyze = samplingManager.shouldAnalyzeError( + errorMessage, + config.sampling.minErrorLength + ); + expect(shouldAnalyze).toBe(true); + + // Step 2: Mock sampling failure + mockMcpServer.server.createMessage.mockRejectedValue( + new Error('Sampling not supported by client') + ); + + // Step 3: Analyze error - should return null + const analysis = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: [], + errorOutput: errorMessage, + exitCode: 1, + }); + + expect(analysis).toBeNull(); + + // Step 4: CommandResult should still be valid without analysis + const result: CommandResult = { + success: false, + exitCode: 1, + output: '', + error: new Error(errorMessage), + errorAnalysis: analysis || undefined, + }; + + expect(result.errorAnalysis).toBeUndefined(); + expect(result.error).toBeDefined(); + }); + }); +}); diff --git a/test/utils/samplingManager.test.ts b/test/utils/samplingManager.test.ts new file mode 100644 index 0000000..619998d --- /dev/null +++ b/test/utils/samplingManager.test.ts @@ -0,0 +1,317 @@ +/** + * Tests for SamplingManager - AI-powered error analysis + */ +import { SamplingManager } from '../../src/utils/samplingManager.js'; +import { ErrorAnalysis } from '../../src/utils/types.js'; + +// Mock the logger +jest.mock('../../src/utils/logger.js', () => ({ + __esModule: true, + default: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +describe('SamplingManager', () => { + let samplingManager: SamplingManager; + let mockMcpServer: any; + let mockCreateMessage: jest.Mock; + + beforeEach(() => { + mockCreateMessage = jest.fn(); + + mockMcpServer = { + server: { + createMessage: mockCreateMessage, + }, + }; + + samplingManager = new SamplingManager(mockMcpServer); + }); + + describe('shouldAnalyzeError', () => { + it('should return true for errors longer than minLength', () => { + const errorOutput = 'This is a sufficiently long error message that should be analyzed'; + const result = samplingManager.shouldAnalyzeError(errorOutput, 30); + + expect(result).toBe(true); + }); + + it('should return false for errors shorter than minLength', () => { + const errorOutput = 'Short error'; + const result = samplingManager.shouldAnalyzeError(errorOutput, 50); + + expect(result).toBe(false); + }); + + it('should return false for empty errors', () => { + const result = samplingManager.shouldAnalyzeError('', 10); + + expect(result).toBe(false); + }); + + it('should return false for "command not found" errors', () => { + const errorOutput = 'command not found: some-command'; + const result = samplingManager.shouldAnalyzeError(errorOutput, 10); + + expect(result).toBe(false); + }); + + it('should return false for "permission denied" errors', () => { + const errorOutput = 'Permission denied'; + const result = samplingManager.shouldAnalyzeError(errorOutput, 10); + + expect(result).toBe(false); + }); + + it('should return false for "no such file or directory" errors', () => { + const errorOutput = 'no such file or directory'; + const result = samplingManager.shouldAnalyzeError(errorOutput, 10); + + expect(result).toBe(false); + }); + + it('should return true for complex ServiceNow SDK errors', () => { + const errorOutput = 'ServiceNow SDK Error: Failed to build application due to invalid metadata configuration in business-rule.ts'; + const result = samplingManager.shouldAnalyzeError(errorOutput, 50); + + expect(result).toBe(true); + }); + }); + + describe('analyzeError', () => { + it('should successfully analyze an error and return structured result', async () => { + const mockResponse = { + content: { + type: 'text', + text: `ROOT CAUSE: +The ServiceNow SDK failed to build because of invalid metadata configuration. + +SOLUTIONS: +1. Check the business-rule.ts file for syntax errors +2. Validate the metadata schema against ServiceNow requirements +3. Run the clean command before rebuilding + +PREVENTION: +1. Use TypeScript strict mode to catch errors early +2. Validate metadata before committing +3. Follow ServiceNow SDK best practices`, + }, + }; + + mockCreateMessage.mockResolvedValue(mockResponse); + + const result = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: ['--debug'], + errorOutput: 'Build failed due to invalid metadata', + exitCode: 1, + }); + + expect(result).not.toBeNull(); + expect(result?.rootCause).toContain('ServiceNow SDK failed'); + expect(result?.suggestions).toHaveLength(3); + expect(result?.suggestions[0]).toContain('Check the business-rule.ts'); + expect(result?.preventionTips).toHaveLength(3); + expect(result?.preventionTips[0]).toContain('TypeScript strict mode'); + + expect(mockCreateMessage).toHaveBeenCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: 'user', + content: expect.objectContaining({ + type: 'text', + text: expect.stringContaining('build_fluent_app'), + }), + }), + ]), + maxTokens: 800, + modelPreferences: { + intelligencePriority: 0.8, + speedPriority: 0.2, + }, + }) + ); + }); + + it('should handle LLM response without proper formatting', async () => { + const mockResponse = { + content: { + type: 'text', + text: 'The error occurred because the configuration is invalid. Try fixing it.', + }, + }; + + mockCreateMessage.mockResolvedValue(mockResponse); + + const result = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: [], + errorOutput: 'Configuration error', + exitCode: 1, + }); + + expect(result).not.toBeNull(); + expect(result?.rootCause).toBeDefined(); + expect(result?.suggestions).toHaveLength(1); + expect(result?.preventionTips).toHaveLength(1); + }); + + it('should return null if createMessage fails', async () => { + mockCreateMessage.mockRejectedValue(new Error('Sampling not supported')); + + const result = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: [], + errorOutput: 'Some error', + exitCode: 1, + }); + + expect(result).toBeNull(); + }); + + it('should handle bullet points in solutions', async () => { + const mockResponse = { + content: { + text: `ROOT CAUSE: +Network connection failed + +SOLUTIONS: +- Check your internet connection +- Verify the instance URL is correct +- Ensure authentication is configured + +PREVENTION: +* Always test connection before deployment +* Use valid instance URLs`, + }, + }; + + mockCreateMessage.mockResolvedValue(mockResponse); + + const result = await samplingManager.analyzeError({ + command: 'deploy_fluent_app', + args: [], + errorOutput: 'Connection timeout', + exitCode: 1, + }); + + expect(result?.suggestions).toHaveLength(3); + expect(result?.suggestions[0]).toBe('Check your internet connection'); + expect(result?.preventionTips).toHaveLength(2); + expect(result?.preventionTips[0]).toBe('Always test connection before deployment'); + }); + }); + + describe('formatAnalysis', () => { + it('should format error analysis with proper structure', () => { + const analysis: ErrorAnalysis = { + rootCause: 'The build failed due to invalid TypeScript configuration', + suggestions: [ + 'Update tsconfig.json with correct settings', + 'Run npm install to ensure dependencies are installed', + 'Clean the output directory before rebuilding', + ], + preventionTips: [ + 'Use strict type checking', + 'Validate configuration files before committing', + ], + }; + + const formatted = samplingManager.formatAnalysis(analysis); + + expect(formatted).toContain('šŸ¤– AI Error Analysis:'); + expect(formatted).toContain('šŸ“‹ Root Cause:'); + expect(formatted).toContain('šŸ’” Suggested Solutions:'); + expect(formatted).toContain('šŸ›”ļø Prevention Tips:'); + expect(formatted).toContain('1. Update tsconfig.json'); + expect(formatted).toContain('2. Run npm install'); + expect(formatted).toContain('1. Use strict type checking'); + }); + + it('should handle analysis with single suggestion', () => { + const analysis: ErrorAnalysis = { + rootCause: 'Missing dependency', + suggestions: ['Install the missing package'], + preventionTips: ['Check package.json regularly'], + }; + + const formatted = samplingManager.formatAnalysis(analysis); + + expect(formatted).toContain('1. Install the missing package'); + expect(formatted).not.toContain('2.'); + }); + }); + + describe('integration scenarios', () => { + it('should handle complex ServiceNow SDK build error', async () => { + const mockResponse = { + content: { + text: `ROOT CAUSE: +The Fluent SDK encountered a metadata validation error in the business-rule definition. The "when" field contains an invalid value that doesn't match the expected schema. + +SOLUTIONS: +1. Review the business-rule.ts file and check the "when" field value +2. Refer to ServiceNow SDK documentation for valid "when" field values +3. Use the sdk_info command with --help flag to see valid options +4. Run "npm run lint" to catch schema violations + +PREVENTION: +1. Enable TypeScript strict mode in tsconfig.json +2. Use ServiceNow SDK type definitions for autocompletion +3. Run build locally before pushing to version control +4. Set up pre-commit hooks to validate metadata`, + }, + }; + + mockCreateMessage.mockResolvedValue(mockResponse); + + const result = await samplingManager.analyzeError({ + command: 'build_fluent_app', + args: ['--debug'], + errorOutput: 'Metadata validation failed: Invalid value in business-rule "when" field', + exitCode: 1, + }); + + expect(result).not.toBeNull(); + expect(result?.rootCause).toContain('metadata validation error'); + expect(result?.suggestions).toHaveLength(4); + expect(result?.preventionTips).toHaveLength(4); + }); + + it('should handle authentication errors', async () => { + const mockResponse = { + content: { + text: `ROOT CAUSE: +Authentication failed because the provided credentials are invalid or the instance URL is incorrect. + +SOLUTIONS: +1. Run manage_fluent_auth to reconfigure authentication +2. Verify the instance URL is accessible +3. Check if your ServiceNow account has required permissions + +PREVENTION: +1. Store credentials securely +2. Test authentication before deployment`, + }, + }; + + mockCreateMessage.mockResolvedValue(mockResponse); + + const result = await samplingManager.analyzeError({ + command: 'deploy_fluent_app', + args: ['auth=prod'], + errorOutput: 'Authentication failed: 401 Unauthorized', + exitCode: 1, + }); + + expect(result).not.toBeNull(); + expect(result?.rootCause).toContain('Authentication failed'); + expect(result?.suggestions[0]).toContain('manage_fluent_auth'); + }); + }); +});