From 39ebae8b074e13c3ddcf5168dd5fc7b6a044a48d Mon Sep 17 00:00:00 2001
From: Sanchit Mehta <sanchit.mehta@okta.com>
Date: Thu, 23 Jul 2026 15:57:14 +0530
Subject: [PATCH] fix(judge): exclude .txt files from the LLM judge corpus
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Agents routinely emit large standalone .txt docs (implementation
summaries, verification checklists) alongside the actual source. These
are not code, add no signal the judge needs, and inflate the judged
corpus — sometimes past judge.maxCodeChars, which errors the judge
grader out and scores it as a hard fail.

The vue_quickstart run is a concrete example: its judged corpus was
44,931 chars (over the 32,768 limit), so both LLM-judge graders errored
with "Code corpus exceeds limit" and flipped the run to status=failure
despite 18/20 deterministic graders passing. The overflow was driven by
a 13KB IMPLEMENTATION_SUMMARY.txt and a VERIFICATION_CHECKLIST.txt.

.md is already excluded for the same reason; extend the pattern to .txt.
Deterministic contains/notContains graders are unaffected — they read
the full file set, not the judge-filtered one.
---
 packages/evals-core/src/graders/executors/llm-judge.ts | 6 ++++++
 packages/evals-core/tests/graders/executors.test.ts    | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/packages/evals-core/src/graders/executors/llm-judge.ts b/packages/evals-core/src/graders/executors/llm-judge.ts
index 274a8ae2..f3113fcf 100644
--- a/packages/evals-core/src/graders/executors/llm-judge.ts
+++ b/packages/evals-core/src/graders/executors/llm-judge.ts
@@ -16,12 +16,18 @@ import { logger } from '../../utils/logger.js';
  * are dropped so credential values never reach the judge LLM — security graders verify
  * secret absence in source deterministically via `notContainsInSource`, and "wired into
  * .env" is an event-based (`wroteFile`) concern, so the judge never needs `.env` content.
+ *
+ * `.md` / `.txt` files are dropped because agents routinely emit large standalone docs
+ * (integration guides, summaries, checklists) that are not source code. They inflate the
+ * corpus — sometimes past `maxCodeChars`, which fails the judge outright — while adding no
+ * signal the judge needs.
  */
 const JUDGE_EXCLUDED_PATTERNS = [
   /^tsconfig(\.\w+)?\.json$/,
   /^angular\.json$/,
   /^tsconfig\.tsbuildinfo$/,
   /\.md$/i,
+  /\.txt$/i,
   /^\.env(\..*)?$/,
 ];
 
diff --git a/packages/evals-core/tests/graders/executors.test.ts b/packages/evals-core/tests/graders/executors.test.ts
index 37c235c5..582969cb 100644
--- a/packages/evals-core/tests/graders/executors.test.ts
+++ b/packages/evals-core/tests/graders/executors.test.ts
@@ -349,6 +349,12 @@ describe('isJudgeExcluded', () => {
     expect(isJudgeExcluded('docs/SETUP.md')).toBe(true);
     expect(isJudgeExcluded('CHANGELOG.MD')).toBe(true);
   });
+
+  it('excludes text files (agent-emitted docs, summaries, checklists)', () => {
+    expect(isJudgeExcluded('IMPLEMENTATION_SUMMARY.txt')).toBe(true);
+    expect(isJudgeExcluded('docs/VERIFICATION_CHECKLIST.txt')).toBe(true);
+    expect(isJudgeExcluded('NOTES.TXT')).toBe(true);
+  });
 });
 
 // ── compile ───────────────────────────────────────────────────────────────────