From 0fd63ef7e5d1f2ff87468f2cf67c80abe1920275 Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 08:27:39 +0530
Subject: [PATCH 01/10] feat(eval): add --runs N flag with median scoring

Adds a --runs <n> CLI option (default: 1) that runs each eval job N times
in parallel and aggregates the results using median scoring. Scores are
medianed per-dimension; costs and tokens are summed across all runs. Raw
individual run results are embedded in a runs[] field for debugging, and
run_count records how many runs were aggregated.

When runs > 1, a run index suffix is appended to each temp file name to
avoid collisions, and collectFromTempFiles + aggregateRuns replaces the
previous mergeIntoOutput call in the subprocess path.
---
 packages/evals-core/src/types/results.ts    |   8 ++
 packages/evals/src/cli/config.ts            |   6 +
 packages/evals/src/cli/index.ts             |   2 +-
 packages/evals/src/cli/run.ts               |  25 +++-
 packages/evals/src/cli/subprocess-runner.ts |  27 ++--
 packages/evals/src/cli/validators.ts        |  10 ++
 packages/evals/src/index.ts                 |   1 +
 packages/evals/src/persistence/index.ts     |   2 +-
 packages/evals/src/persistence/results.ts   | 111 ++++++++++++++-
 packages/evals/tests/cli-config.test.ts     |  34 +++++
 packages/evals/tests/persistence.test.ts    | 150 +++++++++++++++++++-
 11 files changed, 356 insertions(+), 20 deletions(-)

diff --git a/packages/evals-core/src/types/results.ts b/packages/evals-core/src/types/results.ts
index ba313812..c90a8239 100644
--- a/packages/evals-core/src/types/results.ts
+++ b/packages/evals-core/src/types/results.ts
@@ -80,6 +80,10 @@ export interface BaselineJobResult {
   error: string;
   /** Per-grader pass/fail detail. */
   graders: GraderSummary[];
+  /** Number of runs aggregated into this result (absent when run_count is 1). */
+  run_count?: number;
+  /** Raw individual run results when run_count > 1. */
+  runs?: BaselineJobResult[];
 }
 
 /**
@@ -141,6 +145,10 @@ export interface AgentJobResult {
   turn_metrics: TurnMetricEntry[];
   /** Structured recommendations for improving graders, skills, MCP, and efficiency. Present only when skills or MCP tools are enabled. */
   recommendations?: Recommendations;
+  /** Number of runs aggregated into this result (absent when run_count is 1). */
+  run_count?: number;
+  /** Raw individual run results when run_count > 1. */
+  runs?: AgentJobResult[];
 }
 
 /**
diff --git a/packages/evals/src/cli/config.ts b/packages/evals/src/cli/config.ts
index f1349e30..a64e48d2 100644
--- a/packages/evals/src/cli/config.ts
+++ b/packages/evals/src/cli/config.ts
@@ -21,6 +21,7 @@ import {
   validateEvalIds,
   validateTools,
   validateWorkers,
+  validateRuns,
   validateAgentType,
 } from './validators.js';
 
@@ -36,6 +37,8 @@ export interface RunConfig {
   evalIds: string[];
   /** Maximum number of concurrent jobs. */
   workers: number;
+  /** Number of times to run each job — the median score is used as the final result. */
+  runs: number;
   /** Caller-supplied `--output` path, or `undefined` to use the default name. */
   outputPath: string | undefined;
   /** When `true`, agent workspaces are not deleted after the run. */
@@ -122,6 +125,7 @@ export function parseRunConfig(argv: string[], options: ParseRunConfigOptions =
       `Agent runner for agent mode: ${KNOWN_AGENT_TYPES.join(' | ')}. Auto-routed by model prefix when omitted (claude-* → claude-code, gemini-* → gemini-cli, gpt-* → codex, else ${DEFAULT_AGENT_TYPE}).`,
     )
     .option('--workers <n>', 'Parallel workers (default: 4)')
+    .option('--runs <n>', 'Number of times to run each job — median score is used (default: 1)')
     .option('--output <path>', 'JSON output path')
     .option('--keep-workspace', '(agent mode) Keep temp workspace after run', false)
     .option('--braintrust', 'Log results to Braintrust experiment', false)
@@ -144,6 +148,7 @@ export function parseRunConfig(argv: string[], options: ParseRunConfigOptions =
     : (opts.eval as string[]);
   const tools = validateTools(opts.tools as string);
   const workers = validateWorkers(opts.workers as string | undefined);
+  const runs = validateRuns(opts.runs as string | undefined);
   const agentType = validateAgentType(opts.agentType as string | undefined);
 
   return {
@@ -152,6 +157,7 @@ export function parseRunConfig(argv: string[], options: ParseRunConfigOptions =
     tools,
     evalIds,
     workers,
+    runs,
     outputPath: opts.output as string | undefined,
     keepWorkspace: opts.keepWorkspace as boolean,
     braintrust: opts.braintrust as boolean,
diff --git a/packages/evals/src/cli/index.ts b/packages/evals/src/cli/index.ts
index b59d9571..0d605b75 100644
--- a/packages/evals/src/cli/index.ts
+++ b/packages/evals/src/cli/index.ts
@@ -10,6 +10,6 @@ export {
 
 export { parseRunConfig, extractConfigPath, type RunConfig } from './config.js';
 
-export { spawnEval, mergeIntoOutput } from './subprocess-runner.js';
+export { spawnEval, mergeIntoOutput, collectFromTempFiles } from './subprocess-runner.js';
 
 export { runCli, runJob, buildJobList, buildSubprocessArgs } from './run.js';
diff --git a/packages/evals/src/cli/run.ts b/packages/evals/src/cli/run.ts
index 084f1c5e..d1a86ebb 100644
--- a/packages/evals/src/cli/run.ts
+++ b/packages/evals/src/cli/run.ts
@@ -15,6 +15,7 @@
  *                 gemini-* → gemini-cli, gpt-* → codex, else copilot).
  *   --tools       Tools to inject for agent mode: skills, mcp (default: none). Case-insensitive.
  *   --workers     Parallel workers (default: 4)
+ *   --runs        Number of times to run each job — median score is used (default: 1)
  *   --output      JSON output path (default: scores-<mode>.json)
  *   --keep-workspace   (agent mode) Keep temp workspace after run
  *   --braintrust       Log results to Braintrust experiment (requires BRAINTRUST_API_KEY)
@@ -50,9 +51,9 @@ import {
 import type { EvalConfig, EvalDefinition, JobResult, AgentType, Mode } from '@a0/evals-core';
 
 import { parseRunConfig, extractConfigPath } from './config.js';
-import { spawnEval, mergeIntoOutput } from './subprocess-runner.js';
+import { spawnEval, collectFromTempFiles } from './subprocess-runner.js';
 import { DEFAULT_AGENT_TYPE } from './constants.js';
-import { resolveOutputPath, mergeResults, loadResults, saveResults } from '../persistence/index.js';
+import { resolveOutputPath, mergeResults, loadResults, saveResults, aggregateRuns } from '../persistence/index.js';
 import { runBaseline } from '../runners/baseline.js';
 import { score } from '../scorer.js';
 import { ClaudeCodeRunner } from '../runners/claude-code/runner.js';
@@ -356,6 +357,7 @@ export async function runCli(): Promise<void> {
     tools,
     evalIds,
     workers,
+    runs,
     outputPath: outputOverride,
     keepWorkspace,
     braintrust,
@@ -408,6 +410,9 @@ export async function runCli(): Promise<void> {
 
   const jobs = buildJobList(registry, models, modes, tools, agentType);
 
+  // Expand each job by the requested run count, tagging each with its run index.
+  const expandedJobs = jobs.flatMap((job) => Array.from({ length: runs }, (_, runIdx) => ({ job, runIdx })));
+
   // Ensure Docker image exists before dispatching subprocesses — avoids N parallel builds.
   if (sandbox && modes.includes('agent')) {
     const { ensureDockerImage } = await import('../sandbox/docker.js');
@@ -415,7 +420,7 @@ export async function runCli(): Promise<void> {
   }
 
   // ── Subprocess-per-job parallelism ──────────────────────────────────────────
-  if (jobs.length > 1) {
+  if (expandedJobs.length > 1) {
     const selfPath = join(__dirname, 'bin.js');
     const outputPath = resolveOutputPath(frameworkRoot, modes, outputOverride);
 
@@ -424,11 +429,15 @@ export async function runCli(): Promise<void> {
     const tempFiles: string[] = [];
     const subLimit = pLimit(workers);
     const settled = await Promise.allSettled(
-      jobs.map(([evalCfg, model, mode, jobTools, jobAgentType]) =>
+      expandedJobs.map(({ job: [evalCfg, model, mode, jobTools, jobAgentType], runIdx }) =>
         subLimit(async () => {
           const toolsSuffix = jobTools.length > 0 ? `-${jobTools.join('+')}` : '';
           const safeModel = model.replace(/[^a-zA-Z0-9.-]/g, '_');
-          const tempFile = join(frameworkRoot, `scores-tmp-${evalCfg.id}-${safeModel}-${mode}${toolsSuffix}.json`);
+          const runSuffix = runs > 1 ? `-r${runIdx}` : '';
+          const tempFile = join(
+            frameworkRoot,
+            `scores-tmp-${evalCfg.id}-${safeModel}-${mode}${toolsSuffix}${runSuffix}.json`,
+          );
           tempFiles.push(tempFile);
           const jobArgs = [
             ...baseArgs,
@@ -450,7 +459,11 @@ export async function runCli(): Promise<void> {
       logger.error(`  [Subprocess] ${(f as PromiseRejectedResult).reason}`);
     }
 
-    const merged = mergeIntoOutput(tempFiles, outputPath);
+    const allFresh = collectFromTempFiles(tempFiles) as unknown as JobResult[];
+    const toSave = runs > 1 ? aggregateRuns(allFresh) : allFresh;
+    const existing = loadResults(outputPath);
+    const merged = mergeResults(existing, toSave);
+    saveResults(outputPath, merged);
     logger.info(`\n[Output] Results saved to: ${outputPath}`);
 
     const hasErrors = failures.length > 0 || merged.some((r) => r.status === 'error');
diff --git a/packages/evals/src/cli/subprocess-runner.ts b/packages/evals/src/cli/subprocess-runner.ts
index e7a4024a..ace58a76 100644
--- a/packages/evals/src/cli/subprocess-runner.ts
+++ b/packages/evals/src/cli/subprocess-runner.ts
@@ -22,26 +22,35 @@ export function spawnEval(selfPath: string, evalId: string, args: string[]): Pro
 }
 
 /**
- * Reads each temp file, merges results with the existing final output (deduplicating
- * by eval_id|model|mode|tools), writes the merged array to finalOutputPath, and
- * deletes the temp files.
+ * Reads each temp file, collects all results into a flat array, and deletes the
+ * temp files. Does not deduplicate or merge with any existing output.
  */
-export function mergeIntoOutput(tempFiles: string[], finalOutputPath: string): Record<string, unknown>[] {
-  const key = (r: Record<string, unknown>) =>
-    `${r.eval_id}|${r.model}|${r.mode}|${((r.tools as string[]) ?? []).join(',')}`;
-
-  const fresh: Record<string, unknown>[] = [];
+export function collectFromTempFiles(tempFiles: string[]): Record<string, unknown>[] {
+  const results: Record<string, unknown>[] = [];
   for (const f of tempFiles) {
     if (existsSync(f)) {
       try {
         const loaded = JSON.parse(readFileSync(f, 'utf-8')) as unknown;
-        if (Array.isArray(loaded)) fresh.push(...(loaded as Record<string, unknown>[]));
+        if (Array.isArray(loaded)) results.push(...(loaded as Record<string, unknown>[]));
       } catch {
         /* ignore corrupt temp file */
       }
       rmSync(f, { force: true });
     }
   }
+  return results;
+}
+
+/**
+ * Reads each temp file, merges results with the existing final output (deduplicating
+ * by eval_id|model|mode|tools), writes the merged array to finalOutputPath, and
+ * deletes the temp files.
+ */
+export function mergeIntoOutput(tempFiles: string[], finalOutputPath: string): Record<string, unknown>[] {
+  const key = (r: Record<string, unknown>) =>
+    `${r.eval_id}|${r.model}|${r.mode}|${((r.tools as string[]) ?? []).join(',')}`;
+
+  const fresh = collectFromTempFiles(tempFiles);
 
   const newKeys = new Set(fresh.map(key));
   let existing: Record<string, unknown>[] = [];
diff --git a/packages/evals/src/cli/validators.ts b/packages/evals/src/cli/validators.ts
index b1bcbf6b..83cd5f78 100644
--- a/packages/evals/src/cli/validators.ts
+++ b/packages/evals/src/cli/validators.ts
@@ -103,6 +103,16 @@ export function validateTools(toolsArg: string): string[] {
   return tools;
 }
 
+/** Parses and validates the `--runs` count. Defaults to 1. */
+export function validateRuns(raw: string | undefined): number {
+  const runs = parseInt(raw ?? '1', 10);
+  if (!Number.isInteger(runs) || runs < 1) {
+    logger.error(`Invalid --runs value: ${JSON.stringify(raw)}. Must be a positive integer.`);
+    process.exit(1);
+  }
+  return runs;
+}
+
 /** Parses and validates the `--workers` count. Defaults to 4. */
 export function validateWorkers(raw: string | undefined): number {
   const workers = parseInt(raw ?? '4', 10);
diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts
index 3dc6de42..689cfe63 100644
--- a/packages/evals/src/index.ts
+++ b/packages/evals/src/index.ts
@@ -38,6 +38,7 @@ export {
   type RunConfig,
   spawnEval,
   mergeIntoOutput,
+  collectFromTempFiles,
   runCli,
   runJob,
   buildJobList,
diff --git a/packages/evals/src/persistence/index.ts b/packages/evals/src/persistence/index.ts
index d5899ce7..2b1fadec 100644
--- a/packages/evals/src/persistence/index.ts
+++ b/packages/evals/src/persistence/index.ts
@@ -1 +1 @@
-export { resultKey, mergeResults, loadResults, saveResults, resolveOutputPath } from './results.js';
+export { resultKey, mergeResults, loadResults, saveResults, resolveOutputPath, aggregateRuns } from './results.js';
diff --git a/packages/evals/src/persistence/results.ts b/packages/evals/src/persistence/results.ts
index 54bdedf4..b4cc2343 100644
--- a/packages/evals/src/persistence/results.ts
+++ b/packages/evals/src/persistence/results.ts
@@ -4,7 +4,16 @@
 
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { resolve, relative, isAbsolute } from 'node:path';
-import { ALL_MODES, type Mode, type JobResult } from '@a0/evals-core';
+import {
+  ALL_MODES,
+  type Mode,
+  type JobResult,
+  type AgentJobResult,
+  type BaselineJobResult,
+  type ErrorJobResult,
+  type DimensionSummary,
+} from '@a0/evals-core';
+import { scoreToGrade } from '../scorer.js';
 
 /**
  * Returns a stable string key that uniquely identifies a job within a results file.
@@ -110,3 +119,103 @@ export function resolveOutputPath(frameworkRoot: string, modes: string[], overri
   }
   return resolved;
 }
+
+/** Returns the median of a sorted numeric array. */
+function median(values: number[]): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 !== 0 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
+}
+
+function medianAgentResult(group: AgentJobResult[]): AgentJobResult {
+  const sorted = [...group].sort((a, b) => a.overall_score - b.overall_score);
+  const mid = Math.floor(sorted.length / 2);
+  const rep = sorted[mid]!;
+
+  const medianOverallScore = median(group.map((r) => r.overall_score));
+
+  const dimensions: DimensionSummary[] = rep.dimensions.map((dim, i) => {
+    const score = median(group.map((r) => r.dimensions[i]!.score));
+    return { ...dim, score, weighted: score * dim.weight, grade: scoreToGrade(score) };
+  });
+
+  return {
+    ...rep,
+    overall_score: medianOverallScore,
+    overall_grade: scoreToGrade(medianOverallScore),
+    grader_pass_rate: median(group.map((r) => r.grader_pass_rate)),
+    wall_time: median(group.map((r) => r.wall_time)),
+    active_time: median(group.map((r) => r.active_time)),
+    tool_calls: Math.round(median(group.map((r) => r.tool_calls))),
+    interruptions: Math.round(median(group.map((r) => r.interruptions))),
+    tokens: group.reduce((sum, r) => sum + r.tokens, 0),
+    cost_usd: group.reduce((sum, r) => sum + r.cost_usd, 0),
+    judge_cost_usd: group.reduce((sum, r) => sum + r.judge_cost_usd, 0),
+    total_cost_usd: group.reduce((sum, r) => sum + r.total_cost_usd, 0),
+    dimensions,
+    run_count: group.length,
+    runs: group,
+  };
+}
+
+function medianBaselineResult(group: BaselineJobResult[]): BaselineJobResult {
+  const sorted = [...group].sort((a, b) => a.grader_pass_rate - b.grader_pass_rate);
+  const mid = Math.floor(sorted.length / 2);
+  const rep = sorted[mid]!;
+
+  return {
+    ...rep,
+    grader_pass_rate: median(group.map((r) => r.grader_pass_rate)),
+    graders_passed: Math.round(median(group.map((r) => r.graders_passed))),
+    wall_time: median(group.map((r) => r.wall_time)),
+    tokens: group.reduce((sum, r) => sum + r.tokens, 0),
+    cost_usd: group.reduce((sum, r) => sum + r.cost_usd, 0),
+    judge_cost_usd: group.reduce((sum, r) => sum + r.judge_cost_usd, 0),
+    total_cost_usd: group.reduce((sum, r) => sum + r.total_cost_usd, 0),
+    run_count: group.length,
+    runs: group,
+  };
+}
+
+/**
+ * Aggregates multiple runs of the same job into a single result using median scoring.
+ *
+ * Groups results by `resultKey` (eval_id|model|mode|tools). For each group:
+ * - Size 1: passed through unchanged.
+ * - All errors: the last error result is kept as-is.
+ * - Otherwise: error results are dropped and the non-errors are aggregated.
+ *   Scores are medianed; costs and tokens are summed. The raw runs are
+ *   embedded in the `runs` field and `run_count` is set to the group size.
+ */
+export function aggregateRuns(results: JobResult[]): JobResult[] {
+  const groups = new Map<string, JobResult[]>();
+  for (const r of results) {
+    const key = resultKey(r);
+    const group = groups.get(key) ?? [];
+    group.push(r);
+    groups.set(key, group);
+  }
+
+  const aggregated: JobResult[] = [];
+  for (const [, group] of groups) {
+    if (group.length === 1) {
+      aggregated.push(group[0]!);
+      continue;
+    }
+
+    const nonErrors = group.filter((r) => r.status !== 'error');
+    if (nonErrors.length === 0) {
+      // All errored — keep last error result
+      aggregated.push(group[group.length - 1]!);
+      continue;
+    }
+
+    const rep = nonErrors[0]!;
+    if (rep.mode === 'agent') {
+      aggregated.push(medianAgentResult(nonErrors as AgentJobResult[]));
+    } else {
+      aggregated.push(medianBaselineResult(nonErrors as BaselineJobResult[]));
+    }
+  }
+  return aggregated;
+}
diff --git a/packages/evals/tests/cli-config.test.ts b/packages/evals/tests/cli-config.test.ts
index 821d1e58..6d1cc382 100644
--- a/packages/evals/tests/cli-config.test.ts
+++ b/packages/evals/tests/cli-config.test.ts
@@ -91,6 +91,11 @@ describe('defaults', () => {
     expect(config.workers).toBe(4);
   });
 
+  it('sets runs to 1', () => {
+    const config = parse();
+    expect(config.runs).toBe(1);
+  });
+
   it('sets tools to an empty array', () => {
     const config = parse();
     expect(config.tools).toEqual([]);
@@ -343,3 +348,32 @@ describe('extractConfigPath', () => {
     expect(extractConfigPath(argv('--config=/abs/path/eval.config.js'))).toBe('/abs/path/eval.config.js');
   });
 });
+
+// ── Runs validation ───────────────────────────────────────────────────────────
+
+describe('--runs', () => {
+  it('parses a valid positive integer', () => {
+    expect(parse('--runs', '3').runs).toBe(3);
+  });
+
+  it('accepts 1 as the minimum valid value', () => {
+    expect(parse('--runs', '1').runs).toBe(1);
+  });
+
+  it('exits for 0', () => {
+    expect(() => parse('--runs', '0')).toThrow('process.exit(1)');
+  });
+
+  it('exits for a negative number', () => {
+    expect(() => parse('--runs', '-1')).toThrow('process.exit(1)');
+  });
+
+  it('exits for a non-numeric string', () => {
+    expect(() => parse('--runs', 'many')).toThrow('process.exit(1)');
+  });
+
+  it('prints the invalid value in the error message', () => {
+    expect(() => parse('--runs', 'many')).toThrow();
+    expect(console.error).toHaveBeenCalledWith(expect.stringContaining('many'));
+  });
+});
diff --git a/packages/evals/tests/persistence.test.ts b/packages/evals/tests/persistence.test.ts
index ce514c5c..2f6268b0 100644
--- a/packages/evals/tests/persistence.test.ts
+++ b/packages/evals/tests/persistence.test.ts
@@ -5,8 +5,16 @@
 import { describe, it, expect } from 'vitest';
 import { join } from 'node:path';
 import { readFileSync, writeFileSync } from 'node:fs';
-import { resultKey, mergeResults, loadResults, saveResults, resolveOutputPath } from '../src/persistence/index.js';
-import type { AgentJobResult, BaselineJobResult, ErrorJobResult } from '@a0/evals-core';
+import {
+  resultKey,
+  mergeResults,
+  loadResults,
+  saveResults,
+  resolveOutputPath,
+  aggregateRuns,
+  findDroppedErrors,
+} from '../src/persistence/index.js';
+import type { AgentJobResult, BaselineJobResult, DimensionSummary, ErrorJobResult } from '@a0/evals-core';
 import { makeTmpDir } from './tmp.js';
 
 const tmpDir = makeTmpDir('persistence_test_');
@@ -391,3 +399,141 @@ describe('resolveOutputPath', () => {
     expect(resolveOutputPath('/root', ['baseline'], 'out/scores.json')).toBe('/root/out/scores.json');
   });
 });
+
+// ── aggregateRuns ─────────────────────────────────────────────────────────────
+
+function makeDimension(overrides: Partial<DimensionSummary> = {}): DimensionSummary {
+  return { name: 'Correctness', score: 80, grade: 'B', weight: 0.25, weighted: 20, ...overrides };
+}
+
+describe('aggregateRuns', () => {
+  it('passes a single result through unchanged', () => {
+    const r = makeBaseline();
+    expect(aggregateRuns([r])).toEqual([r]);
+  });
+
+  it('returns an empty array for empty input', () => {
+    expect(aggregateRuns([])).toEqual([]);
+  });
+
+  it('sets run_count to the number of runs', () => {
+    const runs = [makeBaseline({ grader_pass_rate: 0.6 }), makeBaseline({ grader_pass_rate: 0.8 })];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.run_count).toBe(2);
+  });
+
+  it('embeds raw runs in the runs[] field', () => {
+    const runs = [makeBaseline({ grader_pass_rate: 0.6 }), makeBaseline({ grader_pass_rate: 0.8 })];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.runs).toHaveLength(2);
+  });
+
+  // ── Baseline aggregation ───────────────────────────────────────────────────
+
+  it('uses median grader_pass_rate for two baseline runs', () => {
+    const runs = [makeBaseline({ grader_pass_rate: 0.6 }), makeBaseline({ grader_pass_rate: 0.8 })];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.grader_pass_rate).toBeCloseTo(0.7);
+  });
+
+  it('uses the exact middle value for an odd number of baseline runs', () => {
+    const runs = [
+      makeBaseline({ grader_pass_rate: 0.5 }),
+      makeBaseline({ grader_pass_rate: 0.7 }),
+      makeBaseline({ grader_pass_rate: 0.9 }),
+    ];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.grader_pass_rate).toBeCloseTo(0.7);
+  });
+
+  it('sums cost_usd across baseline runs', () => {
+    const runs = [makeBaseline({ cost_usd: 0.01 }), makeBaseline({ cost_usd: 0.02 })];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.cost_usd).toBeCloseTo(0.03);
+  });
+
+  it('sums tokens across baseline runs', () => {
+    const runs = [makeBaseline({ tokens: 100 }), makeBaseline({ tokens: 200 })];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.tokens).toBe(300);
+  });
+
+  // ── Agent aggregation ──────────────────────────────────────────────────────
+
+  it('uses median overall_score for two agent runs', () => {
+    const runs = [makeAgent({ overall_score: 60 }), makeAgent({ overall_score: 80 })];
+    const [result] = aggregateRuns(runs) as AgentJobResult[];
+    expect(result.overall_score).toBeCloseTo(70);
+  });
+
+  it('uses the exact middle overall_score for three agent runs', () => {
+    const runs = [makeAgent({ overall_score: 50 }), makeAgent({ overall_score: 70 }), makeAgent({ overall_score: 90 })];
+    const [result] = aggregateRuns(runs) as AgentJobResult[];
+    expect(result.overall_score).toBeCloseTo(70);
+  });
+
+  it('derives overall_grade from the median overall_score', () => {
+    // median is 92 → grade A
+    const runs = [makeAgent({ overall_score: 88 }), makeAgent({ overall_score: 92 }), makeAgent({ overall_score: 96 })];
+    const [result] = aggregateRuns(runs) as AgentJobResult[];
+    expect(result.overall_grade).toBe('A');
+  });
+
+  it('medianes per-dimension scores', () => {
+    const dim = (score: number) => makeDimension({ score, weighted: score * 0.25 });
+    const runs = [
+      makeAgent({ overall_score: 60, dimensions: [dim(60)] }),
+      makeAgent({ overall_score: 80, dimensions: [dim(80)] }),
+    ];
+    const [result] = aggregateRuns(runs) as AgentJobResult[];
+    expect(result.dimensions[0]?.score).toBeCloseTo(70);
+  });
+
+  it('sums cost_usd across agent runs', () => {
+    const runs = [makeAgent({ cost_usd: 0.05 }), makeAgent({ cost_usd: 0.1 })];
+    const [result] = aggregateRuns(runs) as AgentJobResult[];
+    expect(result.cost_usd).toBeCloseTo(0.15);
+  });
+
+  it('sums tokens across agent runs', () => {
+    const runs = [makeAgent({ tokens: 500 }), makeAgent({ tokens: 1000 })];
+    const [result] = aggregateRuns(runs) as AgentJobResult[];
+    expect(result.tokens).toBe(1500);
+  });
+
+  // ── Error handling ─────────────────────────────────────────────────────────
+
+  it('drops error results when at least one run succeeded', () => {
+    const ok = makeAgent({ overall_score: 70 });
+    const err = makeError();
+    const [result] = aggregateRuns([ok, err]) as AgentJobResult[];
+    expect(result.status).toBe('success');
+  });
+
+  it('keeps the last error result when all runs in a group errored', () => {
+    const err1 = makeError({ error: 'first' });
+    const err2 = makeError({ error: 'second' });
+    const [result] = aggregateRuns([err1, err2]);
+    expect(result.status).toBe('error');
+    expect((result as ErrorJobResult).error).toBe('second');
+  });
+
+  // ── Multiple keys ──────────────────────────────────────────────────────────
+
+  it('aggregates different keys independently', () => {
+    const reactRuns = [
+      makeBaseline({ eval_id: 'react_quickstart', grader_pass_rate: 0.6 }),
+      makeBaseline({ eval_id: 'react_quickstart', grader_pass_rate: 0.8 }),
+    ];
+    const nextRuns = [
+      makeBaseline({ eval_id: 'nextjs_quickstart', grader_pass_rate: 0.5 }),
+      makeBaseline({ eval_id: 'nextjs_quickstart', grader_pass_rate: 0.9 }),
+    ];
+    const results = aggregateRuns([...reactRuns, ...nextRuns]) as BaselineJobResult[];
+    expect(results).toHaveLength(2);
+    const react = results.find((r) => r.eval_id === 'react_quickstart')!;
+    const next = results.find((r) => r.eval_id === 'nextjs_quickstart')!;
+    expect(react.grader_pass_rate).toBeCloseTo(0.7);
+    expect(next.grader_pass_rate).toBeCloseTo(0.7);
+  });
+});

From b9c807166136a7eb62c7186f5b219b0080b58b0b Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 09:05:32 +0530
Subject: [PATCH 02/10] fix(eval): address coderabbit review comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Check pre-aggregation allFresh for hasErrors so partial run failures
  are not silently swallowed when aggregateRuns drops error results
- Derive grader_pass_rate from graders_passed/graders_total to keep
  both fields consistent after aggregation
- Add Vitest tests for collectFromTempFiles (7 cases)
- Fix typo: medianes → medians in test description
---
 packages/evals/src/cli/run.ts                 |  2 +-
 packages/evals/src/persistence/results.ts     |  2 +-
 packages/evals/tests/persistence.test.ts      | 43 +++++++---
 .../evals/tests/subprocess-runner.test.ts     | 81 +++++++++++++++++++
 4 files changed, 113 insertions(+), 15 deletions(-)
 create mode 100644 packages/evals/tests/subprocess-runner.test.ts

diff --git a/packages/evals/src/cli/run.ts b/packages/evals/src/cli/run.ts
index d1a86ebb..d645534d 100644
--- a/packages/evals/src/cli/run.ts
+++ b/packages/evals/src/cli/run.ts
@@ -466,7 +466,7 @@ export async function runCli(): Promise<void> {
     saveResults(outputPath, merged);
     logger.info(`\n[Output] Results saved to: ${outputPath}`);
 
-    const hasErrors = failures.length > 0 || merged.some((r) => r.status === 'error');
+    const hasErrors = failures.length > 0 || allFresh.some((r) => r.status === 'error');
     if (hasErrors) {
       process.exit(1);
     }
diff --git a/packages/evals/src/persistence/results.ts b/packages/evals/src/persistence/results.ts
index b4cc2343..4edb02cf 100644
--- a/packages/evals/src/persistence/results.ts
+++ b/packages/evals/src/persistence/results.ts
@@ -165,8 +165,8 @@ function medianBaselineResult(group: BaselineJobResult[]): BaselineJobResult {
 
   return {
     ...rep,
-    grader_pass_rate: median(group.map((r) => r.grader_pass_rate)),
     graders_passed: Math.round(median(group.map((r) => r.graders_passed))),
+    grader_pass_rate: Math.round(median(group.map((r) => r.graders_passed))) / rep.graders_total,
     wall_time: median(group.map((r) => r.wall_time)),
     tokens: group.reduce((sum, r) => sum + r.tokens, 0),
     cost_usd: group.reduce((sum, r) => sum + r.cost_usd, 0),
diff --git a/packages/evals/tests/persistence.test.ts b/packages/evals/tests/persistence.test.ts
index 2f6268b0..f04fceb1 100644
--- a/packages/evals/tests/persistence.test.ts
+++ b/packages/evals/tests/persistence.test.ts
@@ -430,20 +430,36 @@ describe('aggregateRuns', () => {
 
   // ── Baseline aggregation ───────────────────────────────────────────────────
 
-  it('uses median grader_pass_rate for two baseline runs', () => {
-    const runs = [makeBaseline({ grader_pass_rate: 0.6 }), makeBaseline({ grader_pass_rate: 0.8 })];
+  it('derives grader_pass_rate from median graders_passed for two runs', () => {
+    // graders_passed median = (3+4)/2 = 3.5 → rounds to 4; rate = 4/5 = 0.8
+    const runs = [
+      makeBaseline({ graders_passed: 3, graders_total: 5, grader_pass_rate: 0.6 }),
+      makeBaseline({ graders_passed: 4, graders_total: 5, grader_pass_rate: 0.8 }),
+    ];
+    const [result] = aggregateRuns(runs) as BaselineJobResult[];
+    expect(result.graders_passed).toBe(4);
+    expect(result.grader_pass_rate).toBeCloseTo(0.8);
+  });
+
+  it('grader_pass_rate and graders_passed are consistent after aggregation', () => {
+    const runs = [
+      makeBaseline({ graders_passed: 3, graders_total: 5, grader_pass_rate: 0.6 }),
+      makeBaseline({ graders_passed: 4, graders_total: 5, grader_pass_rate: 0.8 }),
+    ];
     const [result] = aggregateRuns(runs) as BaselineJobResult[];
-    expect(result.grader_pass_rate).toBeCloseTo(0.7);
+    expect(result.grader_pass_rate).toBeCloseTo(result.graders_passed / result.graders_total);
   });
 
-  it('uses the exact middle value for an odd number of baseline runs', () => {
+  it('uses the exact middle graders_passed for an odd number of baseline runs', () => {
+    // median graders_passed = 3; rate = 3/4 = 0.75
     const runs = [
-      makeBaseline({ grader_pass_rate: 0.5 }),
-      makeBaseline({ grader_pass_rate: 0.7 }),
-      makeBaseline({ grader_pass_rate: 0.9 }),
+      makeBaseline({ graders_passed: 2, graders_total: 4, grader_pass_rate: 0.5 }),
+      makeBaseline({ graders_passed: 3, graders_total: 4, grader_pass_rate: 0.75 }),
+      makeBaseline({ graders_passed: 4, graders_total: 4, grader_pass_rate: 1.0 }),
     ];
     const [result] = aggregateRuns(runs) as BaselineJobResult[];
-    expect(result.grader_pass_rate).toBeCloseTo(0.7);
+    expect(result.graders_passed).toBe(3);
+    expect(result.grader_pass_rate).toBeCloseTo(0.75);
   });
 
   it('sums cost_usd across baseline runs', () => {
@@ -479,7 +495,7 @@ describe('aggregateRuns', () => {
     expect(result.overall_grade).toBe('A');
   });
 
-  it('medianes per-dimension scores', () => {
+  it('medians per-dimension scores', () => {
     const dim = (score: number) => makeDimension({ score, weighted: score * 0.25 });
     const runs = [
       makeAgent({ overall_score: 60, dimensions: [dim(60)] }),
@@ -521,13 +537,14 @@ describe('aggregateRuns', () => {
   // ── Multiple keys ──────────────────────────────────────────────────────────
 
   it('aggregates different keys independently', () => {
+    // graders_passed medians: react → (6+8)/2=7 → 7/10=0.7; next → (5+9)/2=7 → 7/10=0.7
     const reactRuns = [
-      makeBaseline({ eval_id: 'react_quickstart', grader_pass_rate: 0.6 }),
-      makeBaseline({ eval_id: 'react_quickstart', grader_pass_rate: 0.8 }),
+      makeBaseline({ eval_id: 'react_quickstart', graders_passed: 6, graders_total: 10, grader_pass_rate: 0.6 }),
+      makeBaseline({ eval_id: 'react_quickstart', graders_passed: 8, graders_total: 10, grader_pass_rate: 0.8 }),
     ];
     const nextRuns = [
-      makeBaseline({ eval_id: 'nextjs_quickstart', grader_pass_rate: 0.5 }),
-      makeBaseline({ eval_id: 'nextjs_quickstart', grader_pass_rate: 0.9 }),
+      makeBaseline({ eval_id: 'nextjs_quickstart', graders_passed: 5, graders_total: 10, grader_pass_rate: 0.5 }),
+      makeBaseline({ eval_id: 'nextjs_quickstart', graders_passed: 9, graders_total: 10, grader_pass_rate: 0.9 }),
     ];
     const results = aggregateRuns([...reactRuns, ...nextRuns]) as BaselineJobResult[];
     expect(results).toHaveLength(2);
diff --git a/packages/evals/tests/subprocess-runner.test.ts b/packages/evals/tests/subprocess-runner.test.ts
new file mode 100644
index 00000000..e7445106
--- /dev/null
+++ b/packages/evals/tests/subprocess-runner.test.ts
@@ -0,0 +1,81 @@
+/**
+ * Tests for src/cli/subprocess-runner.ts
+ */
+
+import { describe, it, expect } from 'vitest';
+import { join } from 'node:path';
+import { writeFileSync, existsSync } from 'node:fs';
+import { collectFromTempFiles } from '../src/cli/subprocess-runner.js';
+import { makeTmpDir } from './tmp.js';
+
+const tmpDir = makeTmpDir('subprocess_runner_test_');
+
+describe('collectFromTempFiles', () => {
+  it('returns an empty array when given no files', () => {
+    expect(collectFromTempFiles([])).toEqual([]);
+  });
+
+  it('skips files that do not exist', () => {
+    const result = collectFromTempFiles(['/does/not/exist.json']);
+    expect(result).toEqual([]);
+  });
+
+  it('flattens results from a single valid temp file', () => {
+    const dir = tmpDir();
+    const f = join(dir, 'tmp.json');
+    const records = [{ eval_id: 'react_quickstart', model: 'gpt-5.4', mode: 'baseline' }];
+    writeFileSync(f, JSON.stringify(records), 'utf-8');
+
+    expect(collectFromTempFiles([f])).toEqual(records);
+  });
+
+  it('flattens and concatenates results from multiple temp files', () => {
+    const dir = tmpDir();
+    const f1 = join(dir, 'tmp1.json');
+    const f2 = join(dir, 'tmp2.json');
+    const r1 = { eval_id: 'eval_a', model: 'gpt-5.4', mode: 'baseline' };
+    const r2 = { eval_id: 'eval_b', model: 'gpt-5.4', mode: 'agent' };
+    writeFileSync(f1, JSON.stringify([r1]), 'utf-8');
+    writeFileSync(f2, JSON.stringify([r2]), 'utf-8');
+
+    expect(collectFromTempFiles([f1, f2])).toEqual([r1, r2]);
+  });
+
+  it('deletes each temp file after reading it', () => {
+    const dir = tmpDir();
+    const f = join(dir, 'tmp.json');
+    writeFileSync(f, JSON.stringify([{ eval_id: 'x', model: 'm', mode: 'baseline' }]), 'utf-8');
+
+    collectFromTempFiles([f]);
+    expect(existsSync(f)).toBe(false);
+  });
+
+  it('ignores corrupt JSON and still deletes the file', () => {
+    const dir = tmpDir();
+    const f = join(dir, 'corrupt.json');
+    writeFileSync(f, '{ not valid json', 'utf-8');
+
+    expect(collectFromTempFiles([f])).toEqual([]);
+    expect(existsSync(f)).toBe(false);
+  });
+
+  it('skips non-array JSON payloads', () => {
+    const dir = tmpDir();
+    const f = join(dir, 'object.json');
+    writeFileSync(f, JSON.stringify({ eval_id: 'x', model: 'm', mode: 'baseline' }), 'utf-8');
+
+    expect(collectFromTempFiles([f])).toEqual([]);
+  });
+
+  it('handles a mix of valid, missing, and corrupt files', () => {
+    const dir = tmpDir();
+    const valid = join(dir, 'valid.json');
+    const corrupt = join(dir, 'corrupt.json');
+    const record = { eval_id: 'react_quickstart', model: 'gpt-5.4', mode: 'baseline' };
+    writeFileSync(valid, JSON.stringify([record]), 'utf-8');
+    writeFileSync(corrupt, '!!!', 'utf-8');
+
+    const result = collectFromTempFiles([valid, '/missing.json', corrupt]);
+    expect(result).toEqual([record]);
+  });
+});

From a84c1227a5b668947ef317702d3ddb6ad35382be Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 09:11:02 +0530
Subject: [PATCH 03/10] revert: restore merged check for hasErrors
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The hasErrors change (allFresh vs merged) needs further discussion —
reverting until the product decision on partial-run failure semantics
is resolved.
---
 packages/evals/src/cli/run.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/evals/src/cli/run.ts b/packages/evals/src/cli/run.ts
index d645534d..d1a86ebb 100644
--- a/packages/evals/src/cli/run.ts
+++ b/packages/evals/src/cli/run.ts
@@ -466,7 +466,7 @@ export async function runCli(): Promise<void> {
     saveResults(outputPath, merged);
     logger.info(`\n[Output] Results saved to: ${outputPath}`);
 
-    const hasErrors = failures.length > 0 || allFresh.some((r) => r.status === 'error');
+    const hasErrors = failures.length > 0 || merged.some((r) => r.status === 'error');
     if (hasErrors) {
       process.exit(1);
     }

From fc1cccf8d56fd6142c4fc22e47131ffc6fcf961b Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 09:14:55 +0530
Subject: [PATCH 04/10] fix(eval): normalize tools in mergeIntoOutput key to
 match resultKey

Sort and deduplicate tools in the local key function so it stays
consistent with the shared resultKey helper in persistence.
---
 packages/evals/src/cli/subprocess-runner.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/evals/src/cli/subprocess-runner.ts b/packages/evals/src/cli/subprocess-runner.ts
index ace58a76..156e9c17 100644
--- a/packages/evals/src/cli/subprocess-runner.ts
+++ b/packages/evals/src/cli/subprocess-runner.ts
@@ -47,8 +47,11 @@ export function collectFromTempFiles(tempFiles: string[]): Record<string, unknow
  * deletes the temp files.
  */
 export function mergeIntoOutput(tempFiles: string[], finalOutputPath: string): Record<string, unknown>[] {
-  const key = (r: Record<string, unknown>) =>
-    `${r.eval_id}|${r.model}|${r.mode}|${((r.tools as string[]) ?? []).join(',')}`;
+  const key = (r: Record<string, unknown>) => {
+    const rawTools = Array.isArray(r.tools) ? (r.tools as string[]) : [];
+    const normalised = Array.from(new Set(rawTools)).sort();
+    return `${r.eval_id}|${r.model}|${r.mode}|${normalised.join(',')}`;
+  };
 
   const fresh = collectFromTempFiles(tempFiles);
 

From 3fb2d9ae297a0357ea9938d861a7e61adda4d0b1 Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 09:46:19 +0530
Subject: [PATCH 05/10] docs(eval): add docstrings to medianAgentResult and
 medianBaselineResult
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses CodeRabbit pre-merge check — docstring coverage was at 77.78%
(below the required 80% threshold). Both internal aggregation helpers were
missing JSDoc comments.
---
 packages/evals/src/persistence/results.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/evals/src/persistence/results.ts b/packages/evals/src/persistence/results.ts
index 4edb02cf..86508dd3 100644
--- a/packages/evals/src/persistence/results.ts
+++ b/packages/evals/src/persistence/results.ts
@@ -127,6 +127,7 @@ function median(values: number[]): number {
   return sorted.length % 2 !== 0 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
 }
 
+/** Aggregates a group of agent results into a single representative result using median scores and summed costs. */
 function medianAgentResult(group: AgentJobResult[]): AgentJobResult {
   const sorted = [...group].sort((a, b) => a.overall_score - b.overall_score);
   const mid = Math.floor(sorted.length / 2);
@@ -158,6 +159,7 @@ function medianAgentResult(group: AgentJobResult[]): AgentJobResult {
   };
 }
 
+/** Aggregates a group of baseline results into a single representative result using median scores and summed costs. */
 function medianBaselineResult(group: BaselineJobResult[]): BaselineJobResult {
   const sorted = [...group].sort((a, b) => a.grader_pass_rate - b.grader_pass_rate);
   const mid = Math.floor(sorted.length / 2);

From 66139d85cfc27c0b3a076ab2e045cec735c03c0e Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 14:58:05 +0530
Subject: [PATCH 06/10] fix(eval): reject float values for --runs; document
 --runs in AGENTS.md

---
 AGENTS.md                               | 1 +
 packages/evals/src/cli/validators.ts    | 5 +++--
 packages/evals/tests/cli-config.test.ts | 4 ++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index 4be79b76..4d7ea8e3 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -438,6 +438,7 @@ npm run report
 | `--mode <mode>`              | `baseline`, `agent`, `all`                      | `baseline`           | `all` expands to both                                    |
 | `--tools <tools>`            | `skills`, `mcp`, or comma-separated             | none                 | Only applies to agent mode                               |
 | `--agent-type <type>`        | `claude-code`, `copilot`, `gemini-cli`, `codex` | auto-routed by model | Overrides auto-routing                                   |
+| `--runs <n>`                 | number                                          | 1                    | Repeat each job N times; median score is used            |
 | `--workers <n>`              | number                                          | 4                    | Parallel job limit                                       |
 | `--output <path>`            | file path                                       | auto-named           | JSON results output                                      |
 | `--keep-workspace`           | flag                                            | off                  | Don't delete temp workspace after run                    |
diff --git a/packages/evals/src/cli/validators.ts b/packages/evals/src/cli/validators.ts
index 83cd5f78..c6842c61 100644
--- a/packages/evals/src/cli/validators.ts
+++ b/packages/evals/src/cli/validators.ts
@@ -105,8 +105,9 @@ export function validateTools(toolsArg: string): string[] {
 
 /** Parses and validates the `--runs` count. Defaults to 1. */
 export function validateRuns(raw: string | undefined): number {
-  const runs = parseInt(raw ?? '1', 10);
-  if (!Number.isInteger(runs) || runs < 1) {
+  const trimmed = raw?.trim();
+  const runs = parseInt(trimmed ?? '1', 10);
+  if (!Number.isInteger(runs) || runs < 1 || (trimmed !== undefined && String(runs) !== trimmed)) {
     logger.error(`Invalid --runs value: ${JSON.stringify(raw)}. Must be a positive integer.`);
     process.exit(1);
   }
diff --git a/packages/evals/tests/cli-config.test.ts b/packages/evals/tests/cli-config.test.ts
index 6d1cc382..a30cc0c6 100644
--- a/packages/evals/tests/cli-config.test.ts
+++ b/packages/evals/tests/cli-config.test.ts
@@ -372,6 +372,10 @@ describe('--runs', () => {
     expect(() => parse('--runs', 'many')).toThrow('process.exit(1)');
   });
 
+  it('exits for a float value', () => {
+    expect(() => parse('--runs', '2.9')).toThrow('process.exit(1)');
+  });
+
   it('prints the invalid value in the error message', () => {
     expect(() => parse('--runs', 'many')).toThrow();
     expect(console.error).toHaveBeenCalledWith(expect.stringContaining('many'));

From 858a7ce28a841b7143db590d0f7ee0d98b20ab31 Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Tue, 14 Jul 2026 15:25:40 +0530
Subject: [PATCH 07/10] fix(eval): warn on partial run failures excluded from
 aggregation

---
 packages/evals/src/cli/run.ts             | 16 +++++++++-
 packages/evals/src/persistence/index.ts   | 10 +++++-
 packages/evals/src/persistence/results.ts | 11 +++++++
 packages/evals/tests/persistence.test.ts  | 37 +++++++++++++++++++++++
 4 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/packages/evals/src/cli/run.ts b/packages/evals/src/cli/run.ts
index d1a86ebb..b079a61b 100644
--- a/packages/evals/src/cli/run.ts
+++ b/packages/evals/src/cli/run.ts
@@ -53,7 +53,14 @@ import type { EvalConfig, EvalDefinition, JobResult, AgentType, Mode } from '@a0
 import { parseRunConfig, extractConfigPath } from './config.js';
 import { spawnEval, collectFromTempFiles } from './subprocess-runner.js';
 import { DEFAULT_AGENT_TYPE } from './constants.js';
-import { resolveOutputPath, mergeResults, loadResults, saveResults, aggregateRuns } from '../persistence/index.js';
+import {
+  resolveOutputPath,
+  mergeResults,
+  loadResults,
+  saveResults,
+  aggregateRuns,
+  findDroppedErrors,
+} from '../persistence/index.js';
 import { runBaseline } from '../runners/baseline.js';
 import { score } from '../scorer.js';
 import { ClaudeCodeRunner } from '../runners/claude-code/runner.js';
@@ -460,6 +467,13 @@ export async function runCli(): Promise<void> {
     }
 
     const allFresh = collectFromTempFiles(tempFiles) as unknown as JobResult[];
+
+    if (runs > 1) {
+      for (const r of findDroppedErrors(allFresh)) {
+        logger.warn(`[Warning] run excluded from aggregation: ${r.eval_id} ${r.model} ${r.mode} — ${r.error}`);
+      }
+    }
+
     const toSave = runs > 1 ? aggregateRuns(allFresh) : allFresh;
     const existing = loadResults(outputPath);
     const merged = mergeResults(existing, toSave);
diff --git a/packages/evals/src/persistence/index.ts b/packages/evals/src/persistence/index.ts
index 2b1fadec..f66a5582 100644
--- a/packages/evals/src/persistence/index.ts
+++ b/packages/evals/src/persistence/index.ts
@@ -1 +1,9 @@
-export { resultKey, mergeResults, loadResults, saveResults, resolveOutputPath, aggregateRuns } from './results.js';
+export {
+  resultKey,
+  mergeResults,
+  loadResults,
+  saveResults,
+  resolveOutputPath,
+  aggregateRuns,
+  findDroppedErrors,
+} from './results.js';
diff --git a/packages/evals/src/persistence/results.ts b/packages/evals/src/persistence/results.ts
index 86508dd3..3291d074 100644
--- a/packages/evals/src/persistence/results.ts
+++ b/packages/evals/src/persistence/results.ts
@@ -221,3 +221,14 @@ export function aggregateRuns(results: JobResult[]): JobResult[] {
   }
   return aggregated;
 }
+
+/**
+ * Returns error results from `results` that will be dropped by `aggregateRuns` —
+ * i.e. errors whose job key also has at least one successful run in the same batch.
+ */
+export function findDroppedErrors(results: JobResult[]): ErrorJobResult[] {
+  return results.filter(
+    (r): r is ErrorJobResult =>
+      r.status === 'error' && results.some((s) => s.status !== 'error' && resultKey(s) === resultKey(r)),
+  );
+}
diff --git a/packages/evals/tests/persistence.test.ts b/packages/evals/tests/persistence.test.ts
index f04fceb1..b3484f62 100644
--- a/packages/evals/tests/persistence.test.ts
+++ b/packages/evals/tests/persistence.test.ts
@@ -554,3 +554,40 @@ describe('aggregateRuns', () => {
     expect(next.grader_pass_rate).toBeCloseTo(0.7);
   });
 });
+
+// ── findDroppedErrors ─────────────────────────────────────────────────────────
+
+describe('findDroppedErrors', () => {
+  it('returns an empty array when there are no errors', () => {
+    expect(findDroppedErrors([makeBaseline(), makeBaseline()])).toEqual([]);
+  });
+
+  it('returns an empty array when all runs errored (nothing is dropped)', () => {
+    expect(findDroppedErrors([makeError(), makeError()])).toEqual([]);
+  });
+
+  it('returns the error when its job key also has a successful run', () => {
+    // makeError defaults to mode: agent, tools: [] — makeAgent must match
+    const ok = makeAgent({ tools: [] });
+    const err = makeError();
+    const result = findDroppedErrors([ok, err]);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toBe(err);
+  });
+
+  it('returns multiple errors when several runs for the same job failed', () => {
+    const ok = makeAgent({ tools: [] });
+    const err1 = makeError({ error: 'first' });
+    const err2 = makeError({ error: 'second' });
+    expect(findDroppedErrors([ok, err1, err2])).toHaveLength(2);
+  });
+
+  it('does not return errors from an all-error group', () => {
+    const okGroup = makeAgent({ eval_id: 'react_quickstart', tools: [] });
+    const droppedErr = makeError({ eval_id: 'react_quickstart' });
+    const allErrGroup = makeError({ eval_id: 'nextjs_quickstart' });
+    const result = findDroppedErrors([okGroup, droppedErr, allErrGroup]);
+    expect(result).toHaveLength(1);
+    expect(result[0]?.eval_id).toBe('react_quickstart');
+  });
+});

From 1824f0543280ddcca3ca4384e31c214145b88065 Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Wed, 22 Jul 2026 18:45:02 +0530
Subject: [PATCH 08/10] fix(eval): strip --runs from subprocess args to prevent
 infinite recursion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When --runs N was passed, buildSubprocessArgs forwarded it to every child
process. Each child saw runs > 1, took the subprocess path, and spawned N
more children — each of which did the same, causing exponential process
explosion until the runner was OOM-killed (exit 137).

Fix: add '--runs' to VALUE_FLAGS so it is stripped before forwarding.
Subprocesses run a single job once; run-count expansion is the parent's job.

Adds a regression test that would have caught this.
---
 packages/evals/src/cli/run.ts    | 2 +-
 packages/evals/tests/run.test.ts | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/evals/src/cli/run.ts b/packages/evals/src/cli/run.ts
index b079a61b..961fd881 100644
--- a/packages/evals/src/cli/run.ts
+++ b/packages/evals/src/cli/run.ts
@@ -261,7 +261,7 @@ function printSummary(results: JobResult[], elapsed: number): void {
  * exactly one (eval × model × mode × tools) job without re-expanding them.
  */
 export function buildSubprocessArgs(argv: string[] = process.argv.slice(2)): string[] {
-  const VALUE_FLAGS = new Set(['--eval', '--output', '--model', '--mode', '--tools', '--agent-type']);
+  const VALUE_FLAGS = new Set(['--eval', '--output', '--model', '--mode', '--tools', '--agent-type', '--runs']);
   const stripped: string[] = [];
   for (let i = 0; i < argv.length; i++) {
     const arg = argv[i];
diff --git a/packages/evals/tests/run.test.ts b/packages/evals/tests/run.test.ts
index 4e840b75..08b0070c 100644
--- a/packages/evals/tests/run.test.ts
+++ b/packages/evals/tests/run.test.ts
@@ -150,6 +150,10 @@ describe('buildSubprocessArgs', () => {
     expect(buildSubprocessArgs(['--agent-type', 'claude-code', '--workers', '4'])).toEqual(['--workers', '4']);
   });
 
+  it('strips --runs and its value to prevent subprocess recursion', () => {
+    expect(buildSubprocessArgs(['--runs', '3', '--workers', '4'])).toEqual(['--workers', '4']);
+  });
+
   it('strips multiple per-job flags and keeps the rest', () => {
     const argv = [
       '--eval',

From 6953935e7778c861279cdb56d99022695e1bc63a Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Wed, 22 Jul 2026 23:00:09 +0530
Subject: [PATCH 09/10] fix(eval): cap --runs at MAX_RUNS=10 to prevent
 unbounded subprocess expansion

Without an upper bound, --runs 1000 would expand to 1000 subprocesses.
Adds a named MAX_RUNS constant (10), rejects values above it through the
existing invalid-value path, and adds boundary tests for the max and
first-over-max values.
---
 packages/evals/src/cli/validators.ts    | 7 +++++++
 packages/evals/tests/cli-config.test.ts | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/packages/evals/src/cli/validators.ts b/packages/evals/src/cli/validators.ts
index c6842c61..629b6cb9 100644
--- a/packages/evals/src/cli/validators.ts
+++ b/packages/evals/src/cli/validators.ts
@@ -103,6 +103,9 @@ export function validateTools(toolsArg: string): string[] {
   return tools;
 }
 
+/** Maximum number of runs allowed per job. Prevents unbounded subprocess expansion. */
+export const MAX_RUNS = 10;
+
 /** Parses and validates the `--runs` count. Defaults to 1. */
 export function validateRuns(raw: string | undefined): number {
   const trimmed = raw?.trim();
@@ -111,6 +114,10 @@ export function validateRuns(raw: string | undefined): number {
     logger.error(`Invalid --runs value: ${JSON.stringify(raw)}. Must be a positive integer.`);
     process.exit(1);
   }
+  if (runs > MAX_RUNS) {
+    logger.error(`Invalid --runs value: ${runs}. Must be between 1 and ${MAX_RUNS}.`);
+    process.exit(1);
+  }
   return runs;
 }
 
diff --git a/packages/evals/tests/cli-config.test.ts b/packages/evals/tests/cli-config.test.ts
index a30cc0c6..b03393ed 100644
--- a/packages/evals/tests/cli-config.test.ts
+++ b/packages/evals/tests/cli-config.test.ts
@@ -12,6 +12,7 @@ import {
   KNOWN_AGENT_TYPES,
   KNOWN_WORKING_MODELS,
 } from '../src/index.js';
+import { MAX_RUNS } from '../src/cli/validators.js';
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
 
@@ -380,4 +381,12 @@ describe('--runs', () => {
     expect(() => parse('--runs', 'many')).toThrow();
     expect(console.error).toHaveBeenCalledWith(expect.stringContaining('many'));
   });
+
+  it(`accepts ${MAX_RUNS} as the maximum valid value`, () => {
+    expect(parse('--runs', String(MAX_RUNS)).runs).toBe(MAX_RUNS);
+  });
+
+  it(`exits for ${MAX_RUNS + 1} (one above the maximum)`, () => {
+    expect(() => parse('--runs', String(MAX_RUNS + 1))).toThrow('process.exit(1)');
+  });
 });

From 361a067391ecae16ec06980ea0b14c24cf125e30 Mon Sep 17 00:00:00 2001
From: Yogesh Chaudhary <yogesh.chaudhary@okta.com>
Date: Thu, 23 Jul 2026 08:40:55 +0530
Subject: [PATCH 10/10] fix(eval): address coderabbit review comments on
 aggregation correctness
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add missing judge_cost_usd and total_cost_usd to makeBaseline,
  makeAgent, and makeError test fixtures so they satisfy the required
  type contract.
- Guard dimensions[i] access in medianAgentResult against runs that
  report fewer dimensions than the representative, falling back to the
  representative's own score instead of throwing a TypeError.
- Clarify run_count/runs JSDoc in BaselineJobResult and AgentJobResult:
  both fields are set whenever --runs N > 1, including when partial
  failures reduce successful runs to 1 (contradicting the old "absent
  when run_count is 1" wording).
- Add regression test: 1 success + 1 error → run_count: 1, runs: [ok].
---
 packages/evals-core/src/types/results.ts  |  8 ++++----
 packages/evals/src/persistence/results.ts |  5 +++--
 packages/evals/tests/persistence.test.ts  | 17 +++++++++++++++++
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/packages/evals-core/src/types/results.ts b/packages/evals-core/src/types/results.ts
index c90a8239..cba3279d 100644
--- a/packages/evals-core/src/types/results.ts
+++ b/packages/evals-core/src/types/results.ts
@@ -80,9 +80,9 @@ export interface BaselineJobResult {
   error: string;
   /** Per-grader pass/fail detail. */
   graders: GraderSummary[];
-  /** Number of runs aggregated into this result (absent when run_count is 1). */
+  /** Number of successful runs aggregated. Set whenever `--runs N` was > 1, including when partial failures reduce successful runs to 1. */
   run_count?: number;
-  /** Raw individual run results when run_count > 1. */
+  /** Raw results for each successful run. Present whenever `run_count` is set. */
   runs?: BaselineJobResult[];
 }
 
@@ -145,9 +145,9 @@ export interface AgentJobResult {
   turn_metrics: TurnMetricEntry[];
   /** Structured recommendations for improving graders, skills, MCP, and efficiency. Present only when skills or MCP tools are enabled. */
   recommendations?: Recommendations;
-  /** Number of runs aggregated into this result (absent when run_count is 1). */
+  /** Number of successful runs aggregated. Set whenever `--runs N` was > 1, including when partial failures reduce successful runs to 1. */
   run_count?: number;
-  /** Raw individual run results when run_count > 1. */
+  /** Raw results for each successful run. Present whenever `run_count` is set. */
   runs?: AgentJobResult[];
 }
 
diff --git a/packages/evals/src/persistence/results.ts b/packages/evals/src/persistence/results.ts
index 3291d074..5c38fd90 100644
--- a/packages/evals/src/persistence/results.ts
+++ b/packages/evals/src/persistence/results.ts
@@ -136,7 +136,8 @@ function medianAgentResult(group: AgentJobResult[]): AgentJobResult {
   const medianOverallScore = median(group.map((r) => r.overall_score));
 
   const dimensions: DimensionSummary[] = rep.dimensions.map((dim, i) => {
-    const score = median(group.map((r) => r.dimensions[i]!.score));
+    const scores = group.map((r) => r.dimensions[i]?.score).filter((s): s is number => typeof s === 'number');
+    const score = scores.length > 0 ? median(scores) : dim.score;
     return { ...dim, score, weighted: score * dim.weight, grade: scoreToGrade(score) };
   });
 
@@ -187,7 +188,7 @@ function medianBaselineResult(group: BaselineJobResult[]): BaselineJobResult {
  * - All errors: the last error result is kept as-is.
  * - Otherwise: error results are dropped and the non-errors are aggregated.
  *   Scores are medianed; costs and tokens are summed. The raw runs are
- *   embedded in the `runs` field and `run_count` is set to the group size.
+ *   embedded in the `runs` field and `run_count` is set to the number of successful runs.
  */
 export function aggregateRuns(results: JobResult[]): JobResult[] {
   const groups = new Map<string, JobResult[]>();
diff --git a/packages/evals/tests/persistence.test.ts b/packages/evals/tests/persistence.test.ts
index b3484f62..1ae39d92 100644
--- a/packages/evals/tests/persistence.test.ts
+++ b/packages/evals/tests/persistence.test.ts
@@ -37,6 +37,8 @@ function makeBaseline(overrides: Partial<BaselineJobResult> = {}): BaselineJobRe
     wall_time: 1.0,
     tokens: 100,
     cost_usd: 0.01,
+    judge_cost_usd: 0,
+    total_cost_usd: 0.01,
     error: '',
     graders: [],
     ...overrides,
@@ -63,6 +65,8 @@ function makeAgent(overrides: Partial<AgentJobResult> = {}): AgentJobResult {
     interruptions: 0,
     tokens: 500,
     cost_usd: 0.05,
+    judge_cost_usd: 0,
+    total_cost_usd: 0.05,
     dimensions: [],
     graders: [],
     session_trace: [],
@@ -83,6 +87,8 @@ function makeError(overrides: Partial<ErrorJobResult> = {}): ErrorJobResult {
     wall_time: 0,
     tokens: 0,
     cost_usd: 0,
+    judge_cost_usd: 0,
+    total_cost_usd: 0,
     ...overrides,
   };
 }
@@ -526,6 +532,17 @@ describe('aggregateRuns', () => {
     expect(result.status).toBe('success');
   });
 
+  it('sets run_count and runs on partial failure (1 success + 1 error)', () => {
+    // When 2 runs are requested but 1 errors, run_count reflects the successful
+    // runs count (1), and runs contains only the successful result.
+    const ok = makeAgent({ overall_score: 70, tools: [] });
+    const err = makeError();
+    const [result] = aggregateRuns([ok, err]) as AgentJobResult[];
+    expect(result.run_count).toBe(1);
+    expect(result.runs).toHaveLength(1);
+    expect(result.runs![0]!.status).toBe('success');
+  });
+
   it('keeps the last error result when all runs in a group errored', () => {
     const err1 = makeError({ error: 'first' });
     const err2 = makeError({ error: 'second' });