feat(models): implement structured JSON output for OpenAI

.changeset/structured-json-output.md

@@ -0,0 +1,15 @@
+---
+"diff-hound": minor
+---
+
+feat(models): implement structured JSON output for OpenAI
+
+Replace regex-based response parsing with JSON Schema structured output:
+- Add StructuredReviewResponse type with per-comment severity, category, confidence
+- Add JSON Schema (src/schemas/review-response.json) for OpenAI response_format
+- Update OpenAI adapter to use json_schema response format
+- Add validation utilities for structured responses
+- Maintain backward compatibility with legacy free-text parsing as fallback
+
+This eliminates the fragile regex parsing that could silently drop comments
+if the LLM formatted responses slightly differently.

src/models/openai.ts

@@ -1,8 +1,21 @@
 import OpenAI from "openai";
 import { CodeReviewModel, FileChange, AIComment, ReviewConfig } from "../types";
+import {
+  StructuredReviewResponse,
+  toAIComment,
+  CommentSeverity,
+} from "../schemas/review-response";
+import {
+  parseStructuredResponse,
+  looksLikeStructuredResponse,
+} from "../schemas/validate";
+
+// Import the JSON schema for OpenAI
+import reviewResponseSchema from "../schemas/review-response.json";
 
 /**
  * OpenAI model adapter for code review
+ * Uses structured JSON output via response_format for reliable parsing
  */
 export class OpenAIModel implements CodeReviewModel {
   private client: OpenAI;
@@ -47,18 +60,45 @@ ${file.patch || "No changes"}
       .join("\n\n");
 
     return `
-Only provide brief, actionable, file-specific feedback related to the actual code diff.
-Do not include general advice, documentation-style summaries, or best practices unless they directly relate to the diff.
-Use a direct tone. No greetings. No summaries. No repeated advice.
-
-Important formatting rules:
-- Do not comment on lines that are unchanged or just context unless it's directly impacted by a change.
-${
-  config.commentStyle === "inline"
-    ? "- Output only inline comments, Use this format: 'filename.py:<line number in the new file> — comment'"
-    : "- Provide a single short summary of your review."
+Review the following code changes and provide specific, actionable feedback.
+
+Output your response as a JSON object matching this structure:
+{
+  "summary": "Brief overall assessment (optional)",
+  "comments": [
+    {
+      "file": "path/to/file.ts",
+      "line": 42,
+      "severity": "critical|warning|suggestion|nitpick",
+      "category": "bug|security|performance|style|architecture|testing",
+      "confidence": 0.95,
+      "title": "One-line summary of the issue (max 80 chars)",
+      "explanation": "Detailed explanation of why this is an issue",
+      "suggestion": "Suggested fix or improvement (use empty string if none)"
+    }
+  ]
 }
 
+Severity levels:
+- critical: Bugs, security vulnerabilities, or data loss risks
+- warning: Potential issues or code smells
+- suggestion: Improvements that would be nice to have
+- nitpick: Minor style preferences
+
+Categories:
+- bug: Logic errors, incorrect behavior
+- security: Security vulnerabilities, unsafe practices
+- performance: Performance bottlenecks, inefficiencies
+- style: Code style, formatting, naming
+- architecture: Design patterns, code organization
+- testing: Test coverage, test quality
+
+Important rules:
+- Only comment on lines that are part of the diff (added or modified)
+- Do not comment on unchanged context lines unless directly impacted
+- Be specific and actionable in your feedback
+- Use direct, professional tone
+- Include a suggestion when you can provide a concrete improvement
 ${rules}
 
 Here are the changes to review:
@@ -70,15 +110,85 @@ ${config.customPrompt || ""}`;
 
   /**
    * Parse the AI response into comments
+   * Supports both structured JSON and legacy free-text format for backward compatibility
    * @param response AI generated response
    * @param config Review configuration
    * @returns List of comments
    */
   private parseResponse(response: string, config: ReviewConfig): AIComment[] {
+    // Try structured JSON parsing first
+    if (looksLikeStructuredResponse(response)) {
+      const result = parseStructuredResponse(response);
+      if (result.success && result.data) {
+        return this.convertStructuredResponse(result.data, config);
+      }
+      // If structured parsing fails, fall through to legacy parsing
+      console.warn(
+        "Structured response parsing failed, falling back to legacy parsing:",
+        result.error
+      );
+    }
+
+    // Legacy free-text parsing (fallback)
+    return this.parseLegacyResponse(response, config);
+  }
+
+  /**
+   * Convert structured response to AIComment array
+   */
+  private convertStructuredResponse(
+    response: StructuredReviewResponse,
+    config: ReviewConfig
+  ): AIComment[] {
+    const comments: AIComment[] = [];
+
+    // Add summary comment if present
+    if (response.summary) {
+      comments.push({
+        type: "summary",
+        content: response.summary,
+        severity: config.severity,
+      });
+    }
+
+    // Convert structured comments to AIComment format
+    for (const structuredComment of response.comments) {
+      // Apply severity filtering
+      const severityOrder: CommentSeverity[] = [
+        "critical",
+        "warning",
+        "suggestion",
+        "nitpick",
+      ];
+      const minSeverity = (config.severity as CommentSeverity) || "suggestion";
+      const commentSeverityIndex = severityOrder.indexOf(structuredComment.severity);
+      const minSeverityIndex = severityOrder.indexOf(minSeverity);
+
+      // Skip comments below the minimum severity threshold
+      if (commentSeverityIndex > minSeverityIndex) {
+        continue;
+      }
+
+      // Apply confidence filtering (default min: 0.6)
+      const minConfidence = 0.6;
+      if (structuredComment.confidence < minConfidence) {
+        continue;
+      }
+
+      const aiComment = toAIComment(structuredComment);
+      comments.push(aiComment);
+    }
+
+    return comments;
+  }
+
+  /**
+   * Legacy free-text response parsing (fallback mode)
+   */
+  private parseLegacyResponse(response: string, config: ReviewConfig): AIComment[] {
     const comments: AIComment[] = [];
 
     if (config.commentStyle === "summary") {
-      // Generate a single summary comment
       comments.push({
         type: "summary",
         content: response.trim(),
@@ -152,19 +262,29 @@ ${config.customPrompt || ""}`;
     const prompt = this.generatePrompt(diff, config);
 
     try {
+      // Use structured output with JSON schema
       const response = await this.client.chat.completions.create({
         model: this.model,
         messages: [
           {
             role: "system",
             content:
-              "You are a senior software engineer doing a peer code review. Your job is to spot all logic, syntax, and semantic issues in a code diff.",
+              "You are a senior software engineer doing a peer code review. Your job is to spot all logic, syntax, and semantic issues in a code diff. Always respond with valid JSON matching the requested schema.",
           },
           { role: "user", content: prompt },
         ],
         temperature: 0.1,
         max_tokens: 4000,
         store: true,
+        response_format: {
+          type: "json_schema",
+          json_schema: {
+            name: "review_response",
+            description: "Structured code review response",
+            schema: reviewResponseSchema,
+            strict: true,
+          },
+        },
       });
 
       const content = response.choices[0]?.message.content;

src/schemas/review-response.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "StructuredReviewResponse",
+  "description": "Structured output format for code review responses",
+  "type": "object",
+  "properties": {
+    "summary": {
+      "type": "string",
+      "description": "Optional overall summary of the review"
+    },
+    "comments": {
+      "type": "array",
+      "description": "List of review comments",
+      "items": {
+        "$ref": "#/definitions/StructuredComment"
+      }
+    }
+  },
+  "required": ["summary", "comments"],
+  "additionalProperties": false,
+  "definitions": {
+    "StructuredComment": {
+      "type": "object",
+      "properties": {
+        "file": {
+          "type": "string",
+          "description": "Path to the file being commented on"
+        },
+        "line": {
+          "type": "integer",
+          "description": "Line number in the file (1-based)",
+          "minimum": 1
+        },
+        "severity": {
+          "type": "string",
+          "enum": ["critical", "warning", "suggestion", "nitpick"],
+          "description": "Severity level of the issue"
+        },
+        "category": {
+          "type": "string",
+          "enum": ["bug", "security", "performance", "style", "architecture", "testing"],
+          "description": "Category of the issue"
+        },
+        "confidence": {
+          "type": "number",
+          "description": "Confidence score from 0.0 to 1.0",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "title": {
+          "type": "string",
+          "description": "One-line summary (max 80 characters)",
+          "maxLength": 80
+        },
+        "explanation": {
+          "type": "string",
+          "description": "Detailed explanation of the issue"
+        },
+        "suggestion": {
+          "type": "string",
+          "description": "Suggested code fix (optional)"
+        }
+      },
+      "required": ["file", "line", "severity", "category", "confidence", "title", "explanation", "suggestion"],
+      "additionalProperties": false
+    }
+  }
+}

src/schemas/review-response.ts

@@ -0,0 +1,82 @@
+/**
+ * Types for structured LLM review responses
+ * Replaces free-text parsing with JSON Schema validation
+ */
+
+export type CommentSeverity = "critical" | "warning" | "suggestion" | "nitpick";
+export type CommentCategory =
+  | "bug"
+  | "security"
+  | "performance"
+  | "style"
+  | "architecture"
+  | "testing";
+
+/**
+ * A single structured comment from the AI review
+ */
+export interface StructuredComment {
+  file: string;
+  line: number;
+  severity: CommentSeverity;
+  category: CommentCategory;
+  confidence: number; // 0.0–1.0
+  title: string; // One-line summary (max 80 chars)
+  explanation: string; // Detailed explanation
+  suggestion: string; // Suggested code fix (empty string if none)
+}
+
+/**
+ * Structured review response from LLM
+ */
+export interface StructuredReviewResponse {
+  summary: string;  // Can be empty string if no summary
+  comments: StructuredComment[];
+}
+
+/**
+ * Convert a StructuredComment to the legacy AIComment format
+ * for backward compatibility with platform adapters
+ */
+export function toAIComment(
+  comment: StructuredComment
+): {
+  type: "inline";
+  path: string;
+  line: number;
+  content: string;
+  severity: "error" | "warning" | "suggestion";
+} {
+  // Map severity to legacy format
+  const severityMap: Record<CommentSeverity, "error" | "warning" | "suggestion"> = {
+    critical: "error",
+    warning: "warning",
+    suggestion: "suggestion",
+    nitpick: "suggestion",
+  };
+
+  // Build content with rich formatting
+  let content = `**[${capitalize(comment.category)}] ${comment.title}**`;
+
+  // Add confidence badge
+  const confidencePercent = Math.round(comment.confidence * 100);
+  content += ` (confidence: ${confidencePercent}%)`;
+
+  content += `\n\n${comment.explanation}`;
+
+  if (comment.suggestion && comment.suggestion.trim().length > 0) {
+    content += `\n\n**Suggestion:**\n\`\`\`\n${comment.suggestion}\n\`\`\``;
+  }
+
+  return {
+    type: "inline",
+    path: comment.file,
+    line: comment.line,
+    content,
+    severity: severityMap[comment.severity],
+  };
+}
+
+function capitalize(str: string): string {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}