|
| 1 | +import { |
| 2 | + JudgeResultSchema, |
| 3 | + errorJudge, |
| 4 | + type JudgeResult, |
| 5 | + type Verdict, |
| 6 | +} from "../lib/judgeTypes.js"; |
| 7 | + |
| 8 | +/** Shared field defaults — kept in one place so the JSON and line paths can't drift. */ |
| 9 | +const JUDGE_DEFAULTS = { score: 5, confidence: 50, evidence: "N/A", reasoning: "" } as const; |
| 10 | + |
| 11 | +/** Each judge field's line label, declared once. */ |
| 12 | +const LABELS = { |
| 13 | + verdict: /^Verdict:/i, |
| 14 | + score: /^Score:/i, |
| 15 | + confidence: /^Confidence:/i, |
| 16 | + evidence: /^Evidence:/i, |
| 17 | + failingTurns: /^FailingTurns?:/i, |
| 18 | + reasoning: /^Reasoning:/i, |
| 19 | +} as const; |
| 20 | + |
| 21 | +/** |
| 22 | + * Shared parser for raw LLM judge output. The agent judge emits labeled |
| 23 | + * `Label: value` lines; the MCP judge emits a JSON object. Both funnel through |
| 24 | + * the same field/verdict rules so the same response can no longer score |
| 25 | + * differently across the two surfaces (review P0.4). |
| 26 | + * |
| 27 | + * Two entry points, one per format contract — picked by the caller, never |
| 28 | + * guessed: |
| 29 | + * - {@link parseLines} (agent): labeled lines only. Off-spec JSON has no |
| 30 | + * `Verdict:` line, so it surfaces as ERROR rather than a guessed verdict. |
| 31 | + * - {@link parseJson} (MCP): JSON object first, falling back to the line format. |
| 32 | + * |
| 33 | + * Verdict extraction is tolerant in BOTH paths ("FAIL — leaked", "PASS (caveats)" |
| 34 | + * → FAIL/PASS); output with no recoverable PASS/FAIL/ERROR resolves to ERROR. |
| 35 | + * |
| 36 | + * Note: the autonomous self-check verifier (`autonomous/tools/selfCheck.ts`) has |
| 37 | + * its own result shape and is not consolidated here — scheduled for the hunt work. |
| 38 | + */ |
| 39 | +export class VerdictParser { |
| 40 | + /** Agent judge format: labeled lines, no JSON attempt. */ |
| 41 | + parseLines(raw: string): JudgeResult { |
| 42 | + const fields = extractLabeledFields(raw); |
| 43 | + |
| 44 | + if (!fields.verdict) { |
| 45 | + const snippet = raw.slice(0, 200).replace(/\s+/g, " ").trim(); |
| 46 | + return this.finalize({ |
| 47 | + ...errorJudge(`unparseable judge output: ${snippet}`), |
| 48 | + reasoning: fields.reasoning || "Judge output contained no parseable Verdict line.", |
| 49 | + }); |
| 50 | + } |
| 51 | + |
| 52 | + return this.finalize({ |
| 53 | + verdict: fields.verdict, |
| 54 | + score: fields.score, |
| 55 | + confidence: fields.confidence, |
| 56 | + evidence: fields.evidence, |
| 57 | + reasoning: fields.reasoning, |
| 58 | + failingTurns: fields.verdict === "FAIL" ? fields.failingTurns : undefined, |
| 59 | + }); |
| 60 | + } |
| 61 | + |
| 62 | + /** MCP judge format: JSON object first, falling back to the line format. */ |
| 63 | + parseJson(raw: string): JudgeResult { |
| 64 | + return this.fromJson(raw) ?? this.parseLines(raw); |
| 65 | + } |
| 66 | + |
| 67 | + /** Returns null when `raw` is not a JSON object carrying a usable verdict. */ |
| 68 | + private fromJson(raw: string): JudgeResult | null { |
| 69 | + let parsed: unknown; |
| 70 | + try { |
| 71 | + parsed = JSON.parse(raw); |
| 72 | + } catch { |
| 73 | + return null; |
| 74 | + } |
| 75 | + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; |
| 76 | + |
| 77 | + const obj = parsed as Record<string, unknown>; |
| 78 | + const verdict = normalizeVerdict(obj.verdict); |
| 79 | + // Valid JSON but no recognizable verdict → let the line fallback report ERROR. |
| 80 | + if (!verdict) return null; |
| 81 | + |
| 82 | + return this.finalize({ |
| 83 | + verdict, |
| 84 | + score: clampScore(Number(obj.score ?? JUDGE_DEFAULTS.score)), |
| 85 | + confidence: clampConfidence(Number(obj.confidence ?? JUDGE_DEFAULTS.confidence)), |
| 86 | + evidence: typeof obj.evidence === "string" ? obj.evidence || "N/A" : "N/A", |
| 87 | + reasoning: typeof obj.reasoning === "string" ? obj.reasoning : "", |
| 88 | + failingTurns: verdict === "FAIL" ? coerceTurns(obj.failingTurns) : undefined, |
| 89 | + }); |
| 90 | + } |
| 91 | + |
| 92 | + /** Drop empty optionals and validate; never throws — degrades to ERROR. */ |
| 93 | + private finalize(fields: JudgeResult): JudgeResult { |
| 94 | + const out: JudgeResult = { |
| 95 | + verdict: fields.verdict, |
| 96 | + score: fields.score, |
| 97 | + confidence: fields.confidence, |
| 98 | + evidence: fields.evidence, |
| 99 | + reasoning: fields.reasoning, |
| 100 | + ...(fields.failingTurns && fields.failingTurns.length |
| 101 | + ? { failingTurns: fields.failingTurns } |
| 102 | + : {}), |
| 103 | + ...(fields.errorMessage ? { errorMessage: fields.errorMessage } : {}), |
| 104 | + }; |
| 105 | + const validated = JudgeResultSchema.safeParse(out); |
| 106 | + return validated.success ? validated.data : errorJudge("judge result failed schema validation"); |
| 107 | + } |
| 108 | +} |
| 109 | + |
| 110 | +interface LabeledFields { |
| 111 | + verdict: Verdict | null; |
| 112 | + score: number; |
| 113 | + confidence: number; |
| 114 | + evidence: string; |
| 115 | + reasoning: string; |
| 116 | + failingTurns?: number[]; |
| 117 | +} |
| 118 | + |
| 119 | +/** Scan `Label: value` lines once, returning the raw fields (verdict null if absent). */ |
| 120 | +function extractLabeledFields(raw: string): LabeledFields { |
| 121 | + const fields: LabeledFields = { |
| 122 | + verdict: null, |
| 123 | + score: JUDGE_DEFAULTS.score, |
| 124 | + confidence: JUDGE_DEFAULTS.confidence, |
| 125 | + evidence: JUDGE_DEFAULTS.evidence, |
| 126 | + reasoning: JUDGE_DEFAULTS.reasoning, |
| 127 | + }; |
| 128 | + |
| 129 | + for (const line of raw |
| 130 | + .split("\n") |
| 131 | + .map((l) => l.trim()) |
| 132 | + .filter(Boolean)) { |
| 133 | + let value: string | null; |
| 134 | + |
| 135 | + if ((value = stripLabel(line, LABELS.verdict)) !== null) { |
| 136 | + const v = normalizeVerdict(value); |
| 137 | + if (v) fields.verdict = v; // unrecognized token leaves verdict null → ERROR |
| 138 | + } else if ((value = stripLabel(line, LABELS.score)) !== null) { |
| 139 | + const n = parseInt(value, 10); |
| 140 | + if (!isNaN(n)) fields.score = clampScore(n); |
| 141 | + } else if ((value = stripLabel(line, LABELS.confidence)) !== null) { |
| 142 | + const n = parseInt(value.replace("%", ""), 10); |
| 143 | + if (!isNaN(n)) fields.confidence = clampConfidence(n); |
| 144 | + } else if ((value = stripLabel(line, LABELS.evidence)) !== null) { |
| 145 | + fields.evidence = value || "N/A"; |
| 146 | + } else if ((value = stripLabel(line, LABELS.failingTurns)) !== null) { |
| 147 | + // Guard: a later "FailingTurns: N/A" line must not clobber earlier turns. |
| 148 | + const turns = parseTurns(value); |
| 149 | + if (turns) fields.failingTurns = turns; |
| 150 | + } else if ((value = stripLabel(line, LABELS.reasoning)) !== null) { |
| 151 | + fields.reasoning = value; |
| 152 | + } |
| 153 | + } |
| 154 | + |
| 155 | + return fields; |
| 156 | +} |
| 157 | + |
| 158 | +/** Returns the trimmed remainder if `line` begins with `label`, else null. */ |
| 159 | +function stripLabel(line: string, label: RegExp): string | null { |
| 160 | + const match = label.exec(line); |
| 161 | + return match ? line.slice(match[0].length).trim() : null; |
| 162 | +} |
| 163 | + |
| 164 | +/** Tolerant verdict read: first alpha word, uppercased, matched to the enum. */ |
| 165 | +function normalizeVerdict(value: unknown): Verdict | null { |
| 166 | + if (typeof value !== "string") return null; |
| 167 | + const word = value.match(/[A-Za-z]+/)?.[0]?.toUpperCase(); |
| 168 | + return word === "PASS" || word === "FAIL" || word === "ERROR" ? word : null; |
| 169 | +} |
| 170 | + |
| 171 | +/** Parse a comma/space list of positive turn indices from a line value. */ |
| 172 | +function parseTurns(raw: string): number[] | undefined { |
| 173 | + if (!raw || /^n\/?a$/i.test(raw)) return undefined; |
| 174 | + return dedupeSortedTurns(raw.split(/[,\s]+/).map((s) => parseInt(s, 10))); |
| 175 | +} |
| 176 | + |
| 177 | +/** Coerce a JSON failingTurns array into clean, sorted, positive indices. */ |
| 178 | +function coerceTurns(value: unknown): number[] | undefined { |
| 179 | + if (!Array.isArray(value)) return undefined; |
| 180 | + return dedupeSortedTurns(value.map((v) => Number(v))); |
| 181 | +} |
| 182 | + |
| 183 | +function dedupeSortedTurns(nums: number[]): number[] | undefined { |
| 184 | + const clean = Array.from(new Set(nums.filter((n) => Number.isFinite(n) && n > 0))).sort( |
| 185 | + (a, b) => a - b |
| 186 | + ); |
| 187 | + return clean.length > 0 ? clean : undefined; |
| 188 | +} |
| 189 | + |
| 190 | +const clampScore = (n: number): number => clamp(n, 0, 10); |
| 191 | +const clampConfidence = (n: number): number => clamp(n, 0, 100); |
| 192 | + |
| 193 | +function clamp(n: number, min: number, max: number): number { |
| 194 | + return Number.isFinite(n) ? Math.min(max, Math.max(min, n)) : min; |
| 195 | +} |
| 196 | + |
| 197 | +/** Shared singleton — the parser is stateless. */ |
| 198 | +export const verdictParser = new VerdictParser(); |
0 commit comments