feat(eval): add --runs N flag with median scoring

AGENTS.md

@@ -438,6 +438,7 @@ npm run report
 | `--mode <mode>`              | `baseline`, `agent`, `all`                      | `baseline`           | `all` expands to both                                    |
 | `--tools <tools>`            | `skills`, `mcp`, or comma-separated             | none                 | Only applies to agent mode                               |
 | `--agent-type <type>`        | `claude-code`, `copilot`, `gemini-cli`, `codex` | auto-routed by model | Overrides auto-routing                                   |
+| `--runs <n>`                 | number                                          | 1                    | Repeat each job N times; median score is used            |
 | `--workers <n>`              | number                                          | 4                    | Parallel job limit                                       |
 | `--output <path>`            | file path                                       | auto-named           | JSON results output                                      |
 | `--keep-workspace`           | flag                                            | off                  | Don't delete temp workspace after run                    |

packages/evals-core/src/types/results.ts

@@ -80,6 +80,10 @@ export interface BaselineJobResult {
   error: string;
   /** Per-grader pass/fail detail. */
   graders: GraderSummary[];
+  /** Number of successful runs aggregated. Set whenever `--runs N` was > 1, including when partial failures reduce successful runs to 1. */
+  run_count?: number;
+  /** Raw results for each successful run. Present whenever `run_count` is set. */
+  runs?: BaselineJobResult[];
 }
 
 /**
@@ -141,6 +145,10 @@ export interface AgentJobResult {
   turn_metrics: TurnMetricEntry[];
   /** Structured recommendations for improving graders, skills, MCP, and efficiency. Present only when skills or MCP tools are enabled. */
   recommendations?: Recommendations;
+  /** Number of successful runs aggregated. Set whenever `--runs N` was > 1, including when partial failures reduce successful runs to 1. */
+  run_count?: number;
+  /** Raw results for each successful run. Present whenever `run_count` is set. */
+  runs?: AgentJobResult[];
 }
 
 /**

packages/evals/src/cli/config.ts

@@ -21,6 +21,7 @@ import {
   validateEvalIds,
   validateTools,
   validateWorkers,
+  validateRuns,
   validateAgentType,
 } from './validators.js';
 
@@ -36,6 +37,8 @@ export interface RunConfig {
   evalIds: string[];
   /** Maximum number of concurrent jobs. */
   workers: number;
+  /** Number of times to run each job — the median score is used as the final result. */
+  runs: number;
   /** Caller-supplied `--output` path, or `undefined` to use the default name. */
   outputPath: string | undefined;
   /** When `true`, agent workspaces are not deleted after the run. */
@@ -122,6 +125,7 @@ export function parseRunConfig(argv: string[], options: ParseRunConfigOptions =
       `Agent runner for agent mode: ${KNOWN_AGENT_TYPES.join(' | ')}. Auto-routed by model prefix when omitted (claude-* → claude-code, gemini-* → gemini-cli, gpt-* → codex, else ${DEFAULT_AGENT_TYPE}).`,
     )
     .option('--workers <n>', 'Parallel workers (default: 4)')
+    .option('--runs <n>', 'Number of times to run each job — median score is used (default: 1)')
     .option('--output <path>', 'JSON output path')
     .option('--keep-workspace', '(agent mode) Keep temp workspace after run', false)
     .option('--braintrust', 'Log results to Braintrust experiment', false)
@@ -144,6 +148,7 @@ export function parseRunConfig(argv: string[], options: ParseRunConfigOptions =
     : (opts.eval as string[]);
   const tools = validateTools(opts.tools as string);
   const workers = validateWorkers(opts.workers as string | undefined);
+  const runs = validateRuns(opts.runs as string | undefined);
   const agentType = validateAgentType(opts.agentType as string | undefined);
 
   return {
@@ -152,6 +157,7 @@ export function parseRunConfig(argv: string[], options: ParseRunConfigOptions =
     tools,
     evalIds,
     workers,
+    runs,
     outputPath: opts.output as string | undefined,
     keepWorkspace: opts.keepWorkspace as boolean,
     braintrust: opts.braintrust as boolean,

packages/evals/src/cli/index.ts

@@ -10,6 +10,6 @@ export {
 
 export { parseRunConfig, extractConfigPath, type RunConfig } from './config.js';
 
-export { spawnEval, mergeIntoOutput } from './subprocess-runner.js';
+export { spawnEval, mergeIntoOutput, collectFromTempFiles } from './subprocess-runner.js';
 
 export { runCli, runJob, buildJobList, buildSubprocessArgs } from './run.js';

packages/evals/src/cli/run.ts

@@ -15,6 +15,7 @@
  *                 gemini-* → gemini-cli, gpt-* → codex, else copilot).
  *   --tools       Tools to inject for agent mode: skills, mcp (default: none). Case-insensitive.
  *   --workers     Parallel workers (default: 4)
+ *   --runs        Number of times to run each job — median score is used (default: 1)
  *   --output      JSON output path (default: scores-<mode>.json)
  *   --keep-workspace   (agent mode) Keep temp workspace after run
  *   --braintrust       Log results to Braintrust experiment (requires BRAINTRUST_API_KEY)
@@ -50,9 +51,16 @@ import {
 import type { EvalConfig, EvalDefinition, JobResult, AgentType, Mode } from '@a0/evals-core';
 
 import { parseRunConfig, extractConfigPath } from './config.js';
-import { spawnEval, mergeIntoOutput } from './subprocess-runner.js';
+import { spawnEval, collectFromTempFiles } from './subprocess-runner.js';
 import { DEFAULT_AGENT_TYPE } from './constants.js';
-import { resolveOutputPath, mergeResults, loadResults, saveResults } from '../persistence/index.js';
+import {
+  resolveOutputPath,
+  mergeResults,
+  loadResults,
+  saveResults,
+  aggregateRuns,
+  findDroppedErrors,
+} from '../persistence/index.js';
 import { runBaseline } from '../runners/baseline.js';
 import { score } from '../scorer.js';
 import { ClaudeCodeRunner } from '../runners/claude-code/runner.js';
@@ -253,7 +261,7 @@ function printSummary(results: JobResult[], elapsed: number): void {
  * exactly one (eval × model × mode × tools) job without re-expanding them.
  */
 export function buildSubprocessArgs(argv: string[] = process.argv.slice(2)): string[] {
-  const VALUE_FLAGS = new Set(['--eval', '--output', '--model', '--mode', '--tools', '--agent-type']);
+  const VALUE_FLAGS = new Set(['--eval', '--output', '--model', '--mode', '--tools', '--agent-type', '--runs']);
   const stripped: string[] = [];
   for (let i = 0; i < argv.length; i++) {
     const arg = argv[i];
@@ -356,6 +364,7 @@ export async function runCli(): Promise<void> {
     tools,
     evalIds,
     workers,
+    runs,
     outputPath: outputOverride,
     keepWorkspace,
     braintrust,
@@ -408,14 +417,17 @@ export async function runCli(): Promise<void> {
 
   const jobs = buildJobList(registry, models, modes, tools, agentType);
 
+  // Expand each job by the requested run count, tagging each with its run index.
+  const expandedJobs = jobs.flatMap((job) => Array.from({ length: runs }, (_, runIdx) => ({ job, runIdx })));
+
   // Ensure Docker image exists before dispatching subprocesses — avoids N parallel builds.
   if (sandbox && modes.includes('agent')) {
     const { ensureDockerImage } = await import('../sandbox/docker.js');
     await ensureDockerImage();
   }
 
   // ── Subprocess-per-job parallelism ──────────────────────────────────────────
-  if (jobs.length > 1) {
+  if (expandedJobs.length > 1) {
     const selfPath = join(__dirname, 'bin.js');
     const outputPath = resolveOutputPath(frameworkRoot, modes, outputOverride);
 
@@ -424,11 +436,15 @@ export async function runCli(): Promise<void> {
     const tempFiles: string[] = [];
     const subLimit = pLimit(workers);
     const settled = await Promise.allSettled(
-      jobs.map(([evalCfg, model, mode, jobTools, jobAgentType]) =>
+      expandedJobs.map(({ job: [evalCfg, model, mode, jobTools, jobAgentType], runIdx }) =>
         subLimit(async () => {
           const toolsSuffix = jobTools.length > 0 ? `-${jobTools.join('+')}` : '';
           const safeModel = model.replace(/[^a-zA-Z0-9.-]/g, '_');
-          const tempFile = join(frameworkRoot, `scores-tmp-${evalCfg.id}-${safeModel}-${mode}${toolsSuffix}.json`);
+          const runSuffix = runs > 1 ? `-r${runIdx}` : '';
+          const tempFile = join(
+            frameworkRoot,
+            `scores-tmp-${evalCfg.id}-${safeModel}-${mode}${toolsSuffix}${runSuffix}.json`,
+          );
           tempFiles.push(tempFile);
           const jobArgs = [
             ...baseArgs,
@@ -450,7 +466,18 @@ export async function runCli(): Promise<void> {
       logger.error(`  [Subprocess] ${(f as PromiseRejectedResult).reason}`);
     }
 
-    const merged = mergeIntoOutput(tempFiles, outputPath);
+    const allFresh = collectFromTempFiles(tempFiles) as unknown as JobResult[];
+
+    if (runs > 1) {
+      for (const r of findDroppedErrors(allFresh)) {
+        logger.warn(`[Warning] run excluded from aggregation: ${r.eval_id} ${r.model} ${r.mode} — ${r.error}`);
+      }
+    }
+
+    const toSave = runs > 1 ? aggregateRuns(allFresh) : allFresh;
+    const existing = loadResults(outputPath);
+    const merged = mergeResults(existing, toSave);
+    saveResults(outputPath, merged);
     logger.info(`\n[Output] Results saved to: ${outputPath}`);
 
     const hasErrors = failures.length > 0 || merged.some((r) => r.status === 'error');

packages/evals/src/cli/subprocess-runner.ts

@@ -22,26 +22,38 @@ export function spawnEval(selfPath: string, evalId: string, args: string[]): Pro
 }
 
 /**
- * Reads each temp file, merges results with the existing final output (deduplicating
- * by eval_id|model|mode|tools), writes the merged array to finalOutputPath, and
- * deletes the temp files.
+ * Reads each temp file, collects all results into a flat array, and deletes the
+ * temp files. Does not deduplicate or merge with any existing output.
  */
-export function mergeIntoOutput(tempFiles: string[], finalOutputPath: string): Record<string, unknown>[] {
-  const key = (r: Record<string, unknown>) =>
-    `${r.eval_id}|${r.model}|${r.mode}|${((r.tools as string[]) ?? []).join(',')}`;
-
-  const fresh: Record<string, unknown>[] = [];
+export function collectFromTempFiles(tempFiles: string[]): Record<string, unknown>[] {
+  const results: Record<string, unknown>[] = [];
   for (const f of tempFiles) {
     if (existsSync(f)) {
       try {
         const loaded = JSON.parse(readFileSync(f, 'utf-8')) as unknown;
-        if (Array.isArray(loaded)) fresh.push(...(loaded as Record<string, unknown>[]));
+        if (Array.isArray(loaded)) results.push(...(loaded as Record<string, unknown>[]));
       } catch {
         /* ignore corrupt temp file */
       }
       rmSync(f, { force: true });
     }
   }
+  return results;
+}
+
+/**
+ * Reads each temp file, merges results with the existing final output (deduplicating
+ * by eval_id|model|mode|tools), writes the merged array to finalOutputPath, and
+ * deletes the temp files.
+ */
+export function mergeIntoOutput(tempFiles: string[], finalOutputPath: string): Record<string, unknown>[] {
+  const key = (r: Record<string, unknown>) => {
+    const rawTools = Array.isArray(r.tools) ? (r.tools as string[]) : [];
+    const normalised = Array.from(new Set(rawTools)).sort();
+    return `${r.eval_id}|${r.model}|${r.mode}|${normalised.join(',')}`;
+  };
+
+  const fresh = collectFromTempFiles(tempFiles);
 
   const newKeys = new Set(fresh.map(key));
   let existing: Record<string, unknown>[] = [];

packages/evals/src/cli/validators.ts

@@ -103,6 +103,24 @@ export function validateTools(toolsArg: string): string[] {
   return tools;
 }
 
+/** Maximum number of runs allowed per job. Prevents unbounded subprocess expansion. */
+export const MAX_RUNS = 10;
+
+/** Parses and validates the `--runs` count. Defaults to 1. */
+export function validateRuns(raw: string | undefined): number {
+  const trimmed = raw?.trim();
+  const runs = parseInt(trimmed ?? '1', 10);
+  if (!Number.isInteger(runs) || runs < 1 || (trimmed !== undefined && String(runs) !== trimmed)) {
+    logger.error(`Invalid --runs value: ${JSON.stringify(raw)}. Must be a positive integer.`);
+    process.exit(1);
+  }
+  if (runs > MAX_RUNS) {
+    logger.error(`Invalid --runs value: ${runs}. Must be between 1 and ${MAX_RUNS}.`);
+    process.exit(1);
+  }
+  return runs;
+}
+
 /** Parses and validates the `--workers` count. Defaults to 4. */
 export function validateWorkers(raw: string | undefined): number {
   const workers = parseInt(raw ?? '4', 10);

packages/evals/src/index.ts

@@ -38,6 +38,7 @@ export {
   type RunConfig,
   spawnEval,
   mergeIntoOutput,
+  collectFromTempFiles,
   runCli,
   runJob,
   buildJobList,

packages/evals/src/persistence/index.ts

@@ -1 +1,9 @@
-export { resultKey, mergeResults, loadResults, saveResults, resolveOutputPath } from './results.js';
+export {
+  resultKey,
+  mergeResults,
+  loadResults,
+  saveResults,
+  resolveOutputPath,
+  aggregateRuns,
+  findDroppedErrors,
+} from './results.js';