diff --git a/MCP/McpServer/Data/ProjectRepository.cs b/MCP/McpServer/Data/ProjectRepository.cs index 116f060..a7a1687 100644 --- a/MCP/McpServer/Data/ProjectRepository.cs +++ b/MCP/McpServer/Data/ProjectRepository.cs @@ -268,6 +268,158 @@ private static IEnumerable FilterQuestionnaires( p.RadarCategoryOrder.AsReadOnly()); } + // ── Quality assessment ──────────────────────────────────────────────────── + + /// + /// Evaluates the response quality of a single questionnaire. + /// Returns a consistency score (0–1), a completeness percentage (0–100), + /// and a list of human-readable warnings describing detected issues. + /// Returns when no workspace is loaded or the + /// questionnaire cannot be found. + /// + public EvaluateResponsesResult? EvaluateResponses(string questionnaireId) + { + if (_workspace is null) return null; + + var questionnaire = _workspace.Questionnaires.FirstOrDefault(q => + q.Id.Equals(questionnaireId, StringComparison.OrdinalIgnoreCase) || + q.Name.Equals(questionnaireId, StringComparison.OrdinalIgnoreCase)); + + if (questionnaire is null) return null; + + var warnings = new List(); + + // ── Completeness ────────────────────────────────────────────────────── + + var metaCat = questionnaire.Categories.FirstOrDefault(c => c.IsMetadata == true); + var meta = metaCat?.Metadata; + + // Mandatory metadata fields + var mandatoryFields = new Dictionary + { + ["productName"] = meta?.ProductName, + ["company"] = meta?.Company, + ["department"] = meta?.Department, + ["contactPerson"] = meta?.ContactPerson, + ["executionType"] = meta?.ExecutionType, + ["architecturalRole"] = meta?.ArchitecturalRole, + }; + + int filledMetadata = 0; + foreach (var (field, value) in mandatoryFields) + { + if (string.IsNullOrWhiteSpace(value)) + warnings.Add($"Missing mandatory metadata field: '{field}'."); + else + filledMetadata++; + } + + // Non-metadata entries + var nonMetaCategories = questionnaire.Categories + .Where(c => c.IsMetadata != true) + .ToList(); + + var allEntries = nonMetaCategories + .SelectMany(c => c.Entries ?? []) + .ToList(); + + int entriesWithAnswers = 0; + foreach (var cat in nonMetaCategories) + { + foreach (var entry in cat.Entries ?? []) + { + if (entry.Answers is { Count: > 0 }) + entriesWithAnswers++; + else + warnings.Add( + $"Entry '{entry.Aspect}' ('{entry.Id}') in category '{cat.Title}' has no answers."); + } + } + + int totalCompletable = mandatoryFields.Count + allEntries.Count; + float completeness = totalCompletable > 0 + ? (float)(filledMetadata + entriesWithAnswers) / totalCompletable * 100f + : 100f; + + // ── Consistency ─────────────────────────────────────────────────────── + + // Collect all statuses per technology name across the questionnaire. + var techStatusMap = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var cat in nonMetaCategories) + { + foreach (var entry in cat.Entries ?? []) + { + if (entry.Answers is null) continue; + + // Within-entry: flag duplicate technology entries. + var duplicates = entry.Answers + .GroupBy(a => a.Technology, StringComparer.OrdinalIgnoreCase) + .Where(g => g.Count() > 1); + + foreach (var group in duplicates) + { + var statuses = group + .Select(a => a.Status) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + warnings.Add(statuses.Count > 1 + ? $"Technology '{group.Key}' in entry '{entry.Aspect}' has conflicting statuses: {string.Join(", ", statuses)}." + : $"Technology '{group.Key}' is listed {group.Count()} times in entry '{entry.Aspect}'."); + } + + // Collect statuses for cross-entry consistency check. + foreach (var answer in entry.Answers) + { + if (!techStatusMap.TryGetValue(answer.Technology, out var set)) + { + set = new HashSet(StringComparer.OrdinalIgnoreCase); + techStatusMap[answer.Technology] = set; + } + if (!string.IsNullOrWhiteSpace(answer.Status)) + set.Add(answer.Status); + } + + // Flag Hold / Retire answers without an explanatory comment. + foreach (var answer in entry.Answers) + { + bool isCritical = + answer.Status?.Equals("Hold", StringComparison.OrdinalIgnoreCase) == true || + answer.Status?.Equals("Retire", StringComparison.OrdinalIgnoreCase) == true; + + if (isCritical && string.IsNullOrWhiteSpace(answer.Comments)) + warnings.Add( + $"Technology '{answer.Technology}' has status '{answer.Status}' in entry '{entry.Aspect}' without an explanatory comment."); + } + } + } + + // Cross-entry consistency: same technology, different statuses. + int consistent = 0, inconsistent = 0; + foreach (var (tech, statuses) in techStatusMap) + { + if (statuses.Count > 1) + { + inconsistent++; + warnings.Add( + $"Technology '{tech}' appears with inconsistent statuses across the questionnaire: {string.Join(", ", statuses)}."); + } + else + { + consistent++; + } + } + + int total = consistent + inconsistent; + float consistencyScore = total > 0 ? (float)consistent / total : 1f; + + return new EvaluateResponsesResult( + (float)Math.Round(consistencyScore, 2, MidpointRounding.AwayFromZero), + (float)Math.Round(completeness, 2, MidpointRounding.AwayFromZero), + warnings.AsReadOnly()); + } + // ── Summary helper (used by the UI API) ─────────────────────────────────── public WorkspaceSummary? GetSummary( diff --git a/MCP/McpServer/Models/WorkspaceQueryModels.cs b/MCP/McpServer/Models/WorkspaceQueryModels.cs index 475e43b..46e44af 100644 --- a/MCP/McpServer/Models/WorkspaceQueryModels.cs +++ b/MCP/McpServer/Models/WorkspaceQueryModels.cs @@ -43,3 +43,9 @@ public record TechRadarData( IReadOnlyList Refs, IReadOnlyList CategoryOrder ); + +public record EvaluateResponsesResult( + float ConsistencyScore, + float CompletenessPercentage, + IReadOnlyList Warnings +); diff --git a/MCP/McpServer/Services/McpSessionManager.cs b/MCP/McpServer/Services/McpSessionManager.cs index 63b803d..4f03c76 100644 --- a/MCP/McpServer/Services/McpSessionManager.cs +++ b/MCP/McpServer/Services/McpSessionManager.cs @@ -317,6 +317,24 @@ private static string BuildToolListResponse(JsonNode id) => ["properties"] = new JsonObject(), ["required"] = new JsonArray() } + }, + new JsonObject + { + ["name"] = "evaluate_responses", + ["description"] = "Evaluates the response quality of a single questionnaire. Returns a consistency_score (0.0–1.0) measuring internal consistency of answers, a completeness_% (0–100) representing the fraction of mandatory fields and entries that have been filled in, and a warnings array listing detected issues such as missing metadata, unanswered entries, conflicting technology statuses, or critical statuses (Hold/Retire) without explanatory comments.", + ["inputSchema"] = new JsonObject + { + ["type"] = "object", + ["properties"] = new JsonObject + { + ["questionnaire_id"] = new JsonObject + { + ["type"] = "string", + ["description"] = "The unique identifier (ID or name) of the questionnaire to evaluate." + } + }, + ["required"] = new JsonArray { "questionnaire_id" } + } } } }); @@ -335,6 +353,7 @@ private async Task BuildToolCallResponseAsync(JsonNode id, JsonNode? @pa "list_questionnaires" => BuildListQuestionnairesResponse(id, excludedIds, referenceId), "get_answers_for_category" => BuildAnswersForCategoryResponse(id, args, excludedIds), "get_tech_radar" => BuildTechRadarResponse(id), + "evaluate_responses" => BuildEvaluateResponsesResponse(id, args), _ => BuildError(id, -32602, $"Unknown tool: {toolName}") }; } @@ -464,7 +483,6 @@ private string BuildTechRadarResponse(JsonNode id) var radar = _repo.GetTechRadar(); if (radar is null) return BuildTextToolResponse(id, NotLoadedMessage); - var sb = new StringBuilder(); sb.AppendLine("# Tech Radar"); sb.AppendLine(); @@ -508,6 +526,34 @@ private string BuildTechRadarResponse(JsonNode id) return BuildTextToolResponse(id, sb.ToString()); } + private string BuildEvaluateResponsesResponse(JsonNode id, JsonNode? args) + { + var questionnaireId = args?["questionnaire_id"]?.GetValue(); + if (string.IsNullOrWhiteSpace(questionnaireId)) + return BuildError(id, -32602, "Parameter 'questionnaire_id' is required."); + + var result = _repo.EvaluateResponses(questionnaireId); + if (result is null) + { + return BuildTextToolResponse(id, _repo.IsLoaded + ? $"No questionnaire found with ID or name '{questionnaireId}'." + : NotLoadedMessage); + } + + var warningsArray = new JsonArray(); + foreach (var w in result.Warnings) + warningsArray.Add(JsonValue.Create(w)); + + var json = new JsonObject + { + ["consistency_score"] = result.ConsistencyScore, + ["completeness_%"] = result.CompletenessPercentage, + ["warnings"] = warningsArray + }; + + return BuildTextToolResponse(id, json.ToJsonString()); + } + private const string NotLoadedMessage = "No workspace loaded. Use the management UI (Workspace tab → Load Example or Load File) to load a workspace first.";