diff --git a/docs/behavioral-evals.md b/docs/behavioral-evals.md new file mode 100644 index 00000000000..9311486d565 --- /dev/null +++ b/docs/behavioral-evals.md @@ -0,0 +1,186 @@ +# Behavioral Evaluations & EDK Guide + +This guide introduces the **Eval Development Kit (EDK)** and details how to +write, validate, run, and report on **behavioral evaluations** in the Gemini CLI +codebase. + +--- + +## Overview + +Behavioral evaluations are automated tests designed to assert on the +**behavior** of the Gemini CLI agent (e.g., verifying which tools are called, +checking call ordering, or avoiding destructive commands) rather than checking +the final prose output. + +Evaluating agent behavior is critical because: + +1. Model responses are non-deterministic, making exact prose matching highly + fragile. +2. We must ensure the model utilizes the most efficient tools (e.g., batching + files via `read_many_files` instead of sequential `read_file` calls). +3. We must enforce safety boundaries (e.g., preventing execution of raw shell + commands when safe alternatives exist). + +All behavioral evaluations are stored under the +[evals/](file:///c:/coding/gemini-cli/evals/) directory. + +--- + +## EDK Developer Commands + +The EDK provides CLI tools under `scripts/` to help contributors audit, check, +and monitor evals. + +### 1. `npm run eval:inventory` + +Scans all eval files under `evals/`, statically parses them, and provides a +structured overview of what exists in the repository. + +- **Usage:** + ```bash + npm run eval:inventory + ``` +- **JSON Output:** For CI integration or inventory indexing, generate a + machine-readable JSON report: + ```bash + npm run eval:inventory -- --json + ``` +- **Custom Root:** Run against another directory or repository: + ```bash + npm run eval:inventory -- --root /path/to/other/repo + ``` + +--- + +### 2. `npm run eval:validate` + +A lint-like checker that validates eval source files against standard structural +guidelines and best practices. + +- **Usage:** + ```bash + npm run eval:validate + ``` +- **Custom Scopes:** Validate a specific file: + ```bash + npm run eval:validate -- evals/my-test.eval.ts + ``` + +#### Validation Rules & Severities + +| Rule ID | Severity | Description | +| :------------------- | :---------- | :--------------------------------------------------------------------------------------------------------------------- | +| `file-naming` | **Error** | File must match `*.eval.ts` or `*.eval.tsx` naming conventions. | +| `valid-policy` | **Error** | Policy must be one of `ALWAYS_PASSES`, `USUALLY_PASSES`, or `USUALLY_FAILS`. | +| `suite-metadata` | **Error** | Both `suiteName` and `suiteType` must be present as static string literals. | +| `prompt-presence` | **Error** | Every eval case must have a non-empty `prompt` string. | +| `case-name-static` | **Error** | The case name must be a static string literal, not computed dynamically. | +| `invalid-tool-refs` | **Error** | All tools referenced in assertions must match known built-in or legacy tools. | +| `positive-assertion` | **Error** | Evaluation cases must assert on at least one tool call (e.g., check `waitForToolCall` has been invoked). | +| `workspace-setup` | **Error** | Workspace behaviors (like file-system edits/reads) must set up a `files` object. | +| `new-evals-policy` | **Warning** | New evals must not use `ALWAYS_PASSES` policy initially (they should be promoted after nightly data proves stability). | + +Warnings (`new-evals-policy`) will be logged with `⚠` and will **not** cause +the CLI process to exit with status `1`. Errors (`✗`) will block CI builds and +return exit status `1`. + +--- + +### 3. `npm run eval:report` + +Aggregates local vitest `report.json` artifacts, maps them against inventory +policies, and summarizes the pass rates per model. + +- **Usage:** + ```bash + npm run eval:report + ``` + By default, it scans `evals/logs/` recursively for `report.json` files. +- **Specifying Directory:** + ```bash + npm run eval:report -- /path/to/logs + ``` +- **JSON Output:** + ```bash + npm run eval:report -- --json + ``` + +--- + +## Contributor Workflow + +When writing a new behavioral evaluation, adhere to this workflow to ensure +high-quality, non-flaky test runs. + +### Step-by-Step Guide + +1. **Identify the Target Behavior**: Determine which tool calls need + verification (e.g., `web_fetch` must be called). +2. **Author the Eval File**: Create your file under `evals/.eval.ts` + naming it properly. +3. **Configure Workspace Files**: If the eval reads or edits files, define them + inside the `files` metadata field. +4. **Assert Behavior, Not Prose**: Ensure the `assert` block checks tool + interactions using `rig.waitForToolCall` or similar. Do not check final + prose. +5. **Run Locally**: + ```bash + RUN_EVALS=true npx vitest run evals/my-test.eval.ts + ``` +6. **Deflake**: Run your eval at least 3 times locally to verify it does not + fail due to model variance. +7. **Run Validation**: Run `npm run eval:validate` to ensure no linting errors + are present. + +### Acceptance Criteria Checklist + +- [ ] **Naming**: File ends with `.eval.ts` or `.eval.tsx`. +- [ ] **Policy**: New evals start as `USUALLY_PASSES`. +- [ ] **Metadata**: Static `suiteName` and `suiteType` (e.g. `'behavioral'`) are + specified. +- [ ] **Assertions**: Uses `rig.waitForToolCall` or asserts tool arguments + explicitly. +- [ ] **Clean workspace**: Does not write to files outside `rig.testDir`. + +### Common Anti-Patterns to Avoid + +- **Restricting core tools**: Never override `settings.tools.core` to limit + tools. Evals must run against the default toolset. +- **Checking model prose**: Avoid `expect(result).toContain('something')` since + model wording is non-deterministic. +- **Integration-only testing**: Evals that only write files without checking + realistic model prompts are integration tests and belong under + `integration-tests/`. + +--- + +## CI & Dashboard Integration + +You can easily automate behavioral evaluations or compile dashboard data using +EDK's JSON reporters. + +### CI Validation Block + +Add a step in your PR checks or GitHub workflows to automatically lint new evals +and block pull requests containing validation errors: + +```yaml +- name: Run Eval Validator + run: npm run eval:validate +``` + +### Publishing to a Dashboard + +To record nightly performance metrics across multiple models: + +1. Configure your workflow to run evaluations with the JSON reporter: + ```bash + cross-env GEMINI_MODEL=gemini-2.5-pro npx vitest run --config evals/vitest.config.ts --reporter=json --outputFile="evals/logs/eval-logs-gemini-2.5-pro/report.json" + ``` +2. Aggregate all test runs using the reporting tool: + ```bash + npm run eval:report -- evals/logs --json > aggregated_report.json + ``` +3. Upload `aggregated_report.json` to your dashboard storage backend to + visualize pass rates over time. diff --git a/docs/sidebar.json b/docs/sidebar.json index 9f2e3241d04..bf82bc8220b 100644 --- a/docs/sidebar.json +++ b/docs/sidebar.json @@ -217,6 +217,10 @@ { "label": "Development", "items": [ + { + "label": "Behavioral evaluations", + "slug": "docs/behavioral-evals" + }, { "label": "Contribution guide", "slug": "docs/contributing" }, { "label": "Integration testing", "slug": "docs/integration-tests" }, { diff --git a/package.json b/package.json index def289795f4..11fede6406e 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "docs:keybindings": "tsx ./scripts/generate-keybindings-doc.ts", "eval:inventory": "tsx ./scripts/eval-inventory-cli.ts", "eval:inventory:json": "tsx ./scripts/eval-inventory-cli.ts --json", + "eval:report": "tsx ./scripts/eval-report-cli.ts", "build": "node scripts/build.js", "build-and-start": "npm run build && npm run start --", "build:vscode": "node scripts/build_vscode_companion.js", diff --git a/scripts/eval-report-cli.ts b/scripts/eval-report-cli.ts new file mode 100644 index 00000000000..c438aa366e6 --- /dev/null +++ b/scripts/eval-report-cli.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env tsx + +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @fileoverview CLI entry point to summarize eval report.json files. + * + * Scans a directory for report.json files, groups them by model name, + * and prints pass rate summaries. Integrates with static inventory data + * to display static policies. + * + * Usage: + * npm run eval:report + * npm run eval:report -- [--json] [--root ] + */ + +import path from 'node:path'; +import fs from 'node:fs'; +import { collectInventory } from './utils/eval-inventory.js'; +import { + summarizeReports, + formatReportSummary, + formatReportSummaryJson, +} from './utils/eval-report.js'; + +async function main() { + const args = process.argv.slice(2); + + const jsonFlagIndex = args.indexOf('--json'); + const jsonMode = jsonFlagIndex !== -1; + if (jsonMode) args.splice(jsonFlagIndex, 1); + + const rootFlagIndex = args.indexOf('--root'); + let repoRoot: string | undefined; + if (rootFlagIndex !== -1) { + repoRoot = args[rootFlagIndex + 1]; + if (repoRoot === undefined || repoRoot.startsWith('--')) { + console.error('Error: --root requires a valid directory path.'); + process.exit(1); + } + args.splice(rootFlagIndex, 2); + } + + const resolvedRoot = repoRoot ? path.resolve(repoRoot) : process.cwd(); + + // The first positional argument is the directory of reports + const reportsDirArg = args.find((a) => !a.startsWith('--')); + const reportsDir = reportsDirArg + ? path.resolve(reportsDirArg) + : path.join(resolvedRoot, 'evals', 'logs'); + + if (!fs.existsSync(reportsDir)) { + console.error(`Error: Reports directory does not exist: ${reportsDir}`); + process.exit(1); + } + + // Try to load inventory if available to match policies + let inventory; + try { + inventory = await collectInventory(resolvedRoot); + } catch { + // If inventory fails to load (e.g. running outside repo), proceed without it + } + + const summary = await summarizeReports(reportsDir, inventory); + + if (jsonMode) { + console.log(formatReportSummaryJson(summary, resolvedRoot)); + } else { + console.log(formatReportSummary(summary, resolvedRoot)); + } +} + +main().catch((error) => { + console.error('Fatal error:', error); + process.exit(1); +}); diff --git a/scripts/tests/eval-report.test.ts b/scripts/tests/eval-report.test.ts new file mode 100644 index 00000000000..7f757e4ab48 --- /dev/null +++ b/scripts/tests/eval-report.test.ts @@ -0,0 +1,333 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import fs from 'node:fs'; +import os from 'node:os'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + findReportFiles, + getModelFromPath, + summarizeReports, + formatReportSummary, + formatReportSummaryJson, +} from '../utils/eval-report.js'; +import type { InventoryResult } from '../utils/eval-inventory.js'; + +describe('eval-report utility', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gemini-eval-report-test-')); + vi.unstubAllEnvs(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + describe('findReportFiles', () => { + it('returns an empty array if directory does not exist', () => { + const nonExistent = path.join(tmpDir, 'does-not-exist'); + expect(findReportFiles(nonExistent)).toEqual([]); + }); + + it('recursively finds report.json files', () => { + const sub1 = path.join(tmpDir, 'sub1'); + const sub2 = path.join(tmpDir, 'sub2'); + fs.mkdirSync(sub1); + fs.mkdirSync(sub2); + + fs.writeFileSync(path.join(sub1, 'report.json'), '{}'); + fs.writeFileSync(path.join(sub2, 'report.json'), '{}'); + fs.writeFileSync(path.join(tmpDir, 'other.txt'), '{}'); + + const found = findReportFiles(tmpDir).map((p) => + path.basename(path.dirname(p)), + ); + expect(found.sort()).toEqual(['sub1', 'sub2']); + }); + }); + + describe('getModelFromPath', () => { + it('extracts model name from eval-logs- prefix', () => { + const reportPath = path.join( + tmpDir, + 'eval-logs-gemini-2.5-pro-12345', + 'report.json', + ); + expect(getModelFromPath(reportPath)).toBe('gemini-2.5-pro'); + }); + + it('extracts model name from simple eval-logs- directory name without timestamp', () => { + const reportPath = path.join( + tmpDir, + 'eval-logs-gemini-2.5-flash', + 'report.json', + ); + expect(getModelFromPath(reportPath)).toBe('gemini-2.5-flash'); + }); + + it('falls back to GEMINI_MODEL env var if prefix is not matched', () => { + vi.stubEnv('GEMINI_MODEL', 'env-model'); + const reportPath = path.join(tmpDir, 'some-other-folder', 'report.json'); + expect(getModelFromPath(reportPath)).toBe('env-model'); + }); + + it('falls back to unknown-model if no env var or folder match exists', () => { + const reportPath = path.join(tmpDir, 'some-other-folder', 'report.json'); + expect(getModelFromPath(reportPath)).toBe('unknown-model'); + }); + }); + + describe('summarizeReports', () => { + it('correctly parses vitest JSON and aggregates pass rate metrics by model', async () => { + const modelDir = path.join(tmpDir, 'eval-logs-gemini-2.5-flash-999'); + fs.mkdirSync(modelDir); + + const dummyReport = { + testResults: [ + { + name: '/repo/evals/test-one.eval.ts', + status: 'passed', + assertionResults: [ + { title: 'should retrieve memory', status: 'passed' }, + { title: 'should plan task', status: 'failed' }, + ], + }, + ], + }; + + fs.writeFileSync( + path.join(modelDir, 'report.json'), + JSON.stringify(dummyReport), + ); + + const mockInventory: InventoryResult = { + totalFiles: 1, + totalCases: 2, + repoRoot: '/repo', + files: [], + cases: [ + { + filePath: '/repo/evals/test-one.eval.ts', + relativePath: 'evals/test-one.eval.ts', + helperName: 'evalTest', + baseHelperName: 'evalTest', + policy: 'ALWAYS_PASSES', + name: 'should retrieve memory', + hasFiles: false, + hasPrompt: true, + hasAssert: true, + toolReferences: [], + location: { line: 1, column: 1 }, + }, + { + filePath: '/repo/evals/test-one.eval.ts', + relativePath: 'evals/test-one.eval.ts', + helperName: 'evalTest', + baseHelperName: 'evalTest', + policy: 'USUALLY_PASSES', + name: 'should plan task', + hasFiles: false, + hasPrompt: true, + hasAssert: true, + toolReferences: [], + location: { line: 10, column: 1 }, + }, + ], + diagnostics: [], + }; + + const result = await summarizeReports(tmpDir, mockInventory); + + expect(result.totalFiles).toBe(1); + expect(result.models).toHaveLength(1); + const modelSummary = result.models[0]; + expect(modelSummary.modelName).toBe('gemini-2.5-flash'); + expect(modelSummary.totalCases).toBe(2); + expect(modelSummary.totalRuns).toBe(2); + expect(modelSummary.passedRuns).toBe(1); + expect(modelSummary.overallPassRate).toBe(0.5); + + const cases = modelSummary.cases; + expect(cases[0].name).toBe('should plan task'); + expect(cases[0].policy).toBe('USUALLY_PASSES'); + expect(cases[0].passed).toBe(0); + expect(cases[0].total).toBe(1); + expect(cases[0].passRate).toBe(0.0); + + expect(cases[1].name).toBe('should retrieve memory'); + expect(cases[1].policy).toBe('ALWAYS_PASSES'); + expect(cases[1].passed).toBe(1); + expect(cases[1].total).toBe(1); + expect(cases[1].passRate).toBe(1.0); + }); + + it('does not collide test cases with duplicate names in different files', async () => { + const modelDir = path.join(tmpDir, 'eval-logs-gemini-2.5-flash-999'); + fs.mkdirSync(modelDir); + + const dummyReport = { + testResults: [ + { + name: '/repo/evals/test-one.eval.ts', + status: 'passed', + assertionResults: [{ title: 'duplicate case', status: 'passed' }], + }, + { + name: '/repo/evals/test-two.eval.ts', + status: 'passed', + assertionResults: [{ title: 'duplicate case', status: 'failed' }], + }, + ], + }; + + fs.writeFileSync( + path.join(modelDir, 'report.json'), + JSON.stringify(dummyReport), + ); + + const mockInventory: InventoryResult = { + totalFiles: 2, + totalCases: 2, + repoRoot: '/repo', + files: [], + cases: [ + { + filePath: '/repo/evals/test-one.eval.ts', + relativePath: 'evals/test-one.eval.ts', + helperName: 'evalTest', + baseHelperName: 'evalTest', + policy: 'ALWAYS_PASSES', + name: 'duplicate case', + hasFiles: false, + hasPrompt: true, + hasAssert: true, + toolReferences: [], + location: { line: 1, column: 1 }, + }, + { + filePath: '/repo/evals/test-two.eval.ts', + relativePath: 'evals/test-two.eval.ts', + helperName: 'evalTest', + baseHelperName: 'evalTest', + policy: 'USUALLY_PASSES', + name: 'duplicate case', + hasFiles: false, + hasPrompt: true, + hasAssert: true, + toolReferences: [], + location: { line: 1, column: 1 }, + }, + ], + diagnostics: [], + }; + + const result = await summarizeReports(tmpDir, mockInventory); + + expect(result.models).toHaveLength(1); + const modelSummary = result.models[0]; + expect(modelSummary.totalCases).toBe(2); + expect(modelSummary.cases).toHaveLength(2); + + const c1 = modelSummary.cases.find( + (c) => c.filePath === '/repo/evals/test-one.eval.ts', + )!; + expect(c1.name).toBe('duplicate case'); + expect(c1.policy).toBe('ALWAYS_PASSES'); + expect(c1.passed).toBe(1); + expect(c1.total).toBe(1); + + const c2 = modelSummary.cases.find( + (c) => c.filePath === '/repo/evals/test-two.eval.ts', + )!; + expect(c2.name).toBe('duplicate case'); + expect(c2.policy).toBe('USUALLY_PASSES'); + expect(c2.passed).toBe(0); + expect(c2.total).toBe(1); + }); + }); + + describe('formatReportSummary', () => { + it('handles empty results gracefully', () => { + const empty = { totalFiles: 0, models: [] }; + const out = formatReportSummary(empty); + expect(out).toContain('No report data found.'); + }); + + it('formats a report beautifully with correct details', () => { + const summary = { + totalFiles: 2, + models: [ + { + modelName: 'test-model', + totalCases: 1, + passedCount: 1, + totalRuns: 1, + passedRuns: 1, + overallPassRate: 1.0, + cases: [ + { + name: 'should work', + passed: 1, + total: 1, + passRate: 1.0, + policy: 'ALWAYS_PASSES', + filePath: '/repo/evals/test.eval.ts', + }, + ], + }, + ], + }; + + const formatted = formatReportSummary(summary, '/repo'); + expect(formatted).toContain('Model: test-model'); + expect(formatted).toContain( + '✓ [ALWAYS_PASSES] should work — 100.0% (1/1)', + ); + expect(formatted).toContain('[evals/test.eval.ts]'); + }); + }); + + describe('formatReportSummaryJson', () => { + it('formats deterministic JSON output', () => { + const summary = { + totalFiles: 1, + models: [ + { + modelName: 'test-model', + totalCases: 1, + passedCount: 1, + totalRuns: 1, + passedRuns: 1, + overallPassRate: 1.0, + cases: [ + { + name: 'should work', + passed: 1, + total: 1, + passRate: 1.0, + policy: 'ALWAYS_PASSES', + filePath: '/repo/evals/test.eval.ts', + }, + ], + }, + ], + }; + + const fixedDate = new Date('2026-07-06T00:00:00.000Z'); + const formatted = formatReportSummaryJson(summary, '/repo', fixedDate); + const parsed = JSON.parse(formatted); + + expect(parsed.version).toBe(1); + expect(parsed.totalFiles).toBe(1); + expect(parsed.generated).toBe('2026-07-06T00:00:00.000Z'); + expect(parsed.models[0].modelName).toBe('test-model'); + expect(parsed.models[0].cases[0].filePath).toBe('evals/test.eval.ts'); + }); + }); +}); diff --git a/scripts/utils/eval-report.ts b/scripts/utils/eval-report.ts new file mode 100644 index 00000000000..c964c9dcb67 --- /dev/null +++ b/scripts/utils/eval-report.ts @@ -0,0 +1,268 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import type { InventoryResult } from './eval-inventory.js'; + +export interface ReportCaseSummary { + name: string; + passed: number; + total: number; + passRate: number; + policy: string; + filePath: string; +} + +export interface ModelSummary { + modelName: string; + totalCases: number; + passedCount: number; + totalRuns: number; + passedRuns: number; + overallPassRate: number; + cases: ReportCaseSummary[]; +} + +export interface ReportSummaryResult { + totalFiles: number; + models: ModelSummary[]; +} + +/** + * Recursively scans a directory for report.json files. + */ +export function findReportFiles(dir: string): string[] { + const reports: string[] = []; + if (!fs.existsSync(dir)) return reports; + + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + reports.push(...findReportFiles(fullPath)); + } else if (entry.isFile() && entry.name === 'report.json') { + reports.push(fullPath); + } + } + return reports; +} + +/** + * Parses the model name from the directory path or defaults to env. + */ +export function getModelFromPath(reportPath: string): string { + const normalized = path.normalize(reportPath).replace(/\\/g, '/'); + const parts = normalized.split('/'); + const logDir = parts.find((p) => p.startsWith('eval-logs-')); + if (logDir) { + const match = logDir.match(/^eval-logs-(.+)-(\d+)$/); + if (match) return match[1]; + const matchSimple = logDir.match(/^eval-logs-(.+)$/); + if (matchSimple) return matchSimple[1]; + } + return process.env.GEMINI_MODEL || 'unknown-model'; +} + +/** + * Summarizes the pass rate stats by test case and model from report.json files. + */ +export async function summarizeReports( + reportsDir: string, + inventory?: InventoryResult, +): Promise { + const reportPaths = findReportFiles(reportsDir); + const modelSummariesMap = new Map< + string, + Map< + string, + { name: string; passed: number; total: number; filePath: string } + > + >(); + + for (const reportPath of reportPaths) { + try { + const model = getModelFromPath(reportPath); + if (!modelSummariesMap.has(model)) { + modelSummariesMap.set(model, new Map()); + } + const testCasesMap = modelSummariesMap.get(model)!; + + const data = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + if (data && Array.isArray(data.testResults)) { + for (const fileResult of data.testResults) { + const filePath = fileResult.name; + const normalizedPath = path.resolve(filePath).replace(/\\/g, '/'); + if (Array.isArray(fileResult.assertionResults)) { + for (const assertion of fileResult.assertionResults) { + const testName = assertion.title; + const compoundKey = `${normalizedPath}::${testName}`; + if (!testCasesMap.has(compoundKey)) { + testCasesMap.set(compoundKey, { + name: testName, + passed: 0, + total: 0, + filePath, + }); + } + const stats = testCasesMap.get(compoundKey)!; + stats.total += 1; + if (assertion.status === 'passed') { + stats.passed += 1; + } + } + } + } + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + console.error(`Error reading or parsing report at ${reportPath}: ${msg}`); + } + } + + const policyMap = new Map(); + if (inventory) { + for (const caseRec of inventory.cases) { + const normalizedCasePath = path + .resolve(caseRec.filePath) + .replace(/\\/g, '/'); + const key = `${normalizedCasePath}::${caseRec.name}`; + policyMap.set(key, caseRec.policy); + } + } + + const models: ModelSummary[] = []; + for (const [modelName, testCasesMap] of modelSummariesMap.entries()) { + const cases: ReportCaseSummary[] = []; + let totalRuns = 0; + let passedRuns = 0; + + for (const stats of testCasesMap.values()) { + const normalizedPath = path.resolve(stats.filePath).replace(/\\/g, '/'); + const key = `${normalizedPath}::${stats.name}`; + const policy = policyMap.get(key) || 'unknown'; + const passRate = stats.total > 0 ? stats.passed / stats.total : 0; + cases.push({ + name: stats.name, + passed: stats.passed, + total: stats.total, + passRate, + policy, + filePath: stats.filePath, + }); + totalRuns += stats.total; + passedRuns += stats.passed; + } + + cases.sort((a, b) => a.name.localeCompare(b.name, 'en')); + + models.push({ + modelName, + totalCases: testCasesMap.size, + passedCount: cases.filter((c) => c.passRate === 1.0).length, + totalRuns, + passedRuns, + overallPassRate: totalRuns > 0 ? passedRuns / totalRuns : 0, + cases, + }); + } + + models.sort((a, b) => a.modelName.localeCompare(b.modelName, 'en')); + + return { + totalFiles: reportPaths.length, + models, + }; +} + +/** + * Formats report summary to human-readable string. + */ +export function formatReportSummary( + result: ReportSummaryResult, + repoRoot?: string, +): string { + const lines: string[] = []; + lines.push('Eval Nightly Pass Rate Report'); + lines.push('═════════════════════════════'); + lines.push(''); + lines.push(`Processed ${result.totalFiles} report(s).`); + lines.push(''); + + if (result.models.length === 0) { + lines.push('No report data found.'); + return lines.join('\n'); + } + + for (const model of result.models) { + lines.push(`Model: ${model.modelName}`); + lines.push('───────────────────────'); + lines.push(` Unique cases: ${model.totalCases}`); + lines.push( + ` Total runs: ${model.totalRuns} (Passed: ${model.passedRuns}, Failed: ${ + model.totalRuns - model.passedRuns + })`, + ); + lines.push(` Pass rate: ${(model.overallPassRate * 100).toFixed(1)}%`); + lines.push(''); + lines.push(' Test Cases:'); + for (const c of model.cases) { + const relPath = + repoRoot && path.isAbsolute(c.filePath) + ? path.relative(repoRoot, c.filePath).replace(/\\/g, '/') + : c.filePath; + const indicator = + c.passRate === 1.0 ? '✓' : c.passRate === 0 ? '✗' : '⚠'; + lines.push( + ` ${indicator} [${c.policy}] ${c.name} — ${( + c.passRate * 100 + ).toFixed(1)}% (${c.passed}/${c.total}) [${relPath}]`, + ); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Formats report summary to deterministic JSON. + */ +export function formatReportSummaryJson( + result: ReportSummaryResult, + repoRoot?: string, + now?: Date, +): string { + let generatedDate = now || new Date(); + if (process.env.EVAL_INVENTORY_DETERMINISTIC) { + generatedDate = new Date(0); + } + const output = { + version: 1, + generated: generatedDate.toISOString(), + totalFiles: result.totalFiles, + models: result.models.map((m) => ({ + modelName: m.modelName, + totalCases: m.totalCases, + passedCount: m.passedCount, + totalRuns: m.totalRuns, + passedRuns: m.passedRuns, + overallPassRate: m.overallPassRate, + cases: m.cases.map((c) => ({ + name: c.name, + passed: c.passed, + total: c.total, + passRate: c.passRate, + policy: c.policy, + filePath: + repoRoot && path.isAbsolute(c.filePath) + ? path.relative(repoRoot, c.filePath).replace(/\\/g, '/') + : c.filePath, + })), + })), + }; + return JSON.stringify(output, null, 2); +}