diff --git a/app/lib/runner/agentic_run_orchestrator.dart b/app/lib/runner/agentic_run_orchestrator.dart index 43e27fe..7ad5d58 100644 --- a/app/lib/runner/agentic_run_orchestrator.dart +++ b/app/lib/runner/agentic_run_orchestrator.dart @@ -4,7 +4,6 @@ import 'dart:io'; import 'package:crypto/crypto.dart'; import 'package:dart_arena/agent/agent_harness.dart'; import 'package:dart_arena/agent/agent_run_result.dart'; -import 'package:dart_arena/analytics/result_primitives.dart'; import 'package:dart_arena/core/benchmark_task.dart'; import 'package:dart_arena/core/evaluation_context.dart'; import 'package:dart_arena/core/evaluation_result.dart'; @@ -19,6 +18,7 @@ import 'package:dart_arena/core/workspace_path.dart'; import 'package:dart_arena/evaluators/evaluator.dart'; import 'package:dart_arena/runner/evaluator_resource_limits.dart'; import 'package:dart_arena/runner/generated_code_sandbox.dart'; +import 'package:dart_arena/runner/objective_evaluation.dart'; import 'package:dart_arena/runner/workdir_manager.dart'; import 'package:path/path.dart' as p; @@ -333,35 +333,24 @@ class AgenticRunOrchestrator { ); } else { resultProvenance['gradingMode'] = 'clean_replay'; - for (final evaluator in evaluators) { - final blocked = blockedEvaluationFor( - evaluatorId: evaluator.id, - previousResults: evaluations, - ); - if (blocked != null) { - evaluations.add(blocked); - continue; - } - evaluations.add( - await evaluator.evaluate( - EvaluationContext( - workDir: gradingWorkspace, - response: response, - task: task, - previousResults: evaluations, - deniedEnvironmentKeys: workdirManager.deniedEnvironmentKeys, - generatedCodeSandbox: generatedCodeSandbox, - ), - ), - ); - } + await runObjectiveEvaluators( + evaluators: evaluators, + evaluations: evaluations, + contextFor: (previousResults) => EvaluationContext( + workDir: gradingWorkspace, + response: response, + task: task, + previousResults: previousResults, + deniedEnvironmentKeys: workdirManager.deniedEnvironmentKeys, + generatedCodeSandbox: generatedCodeSandbox, + ), + ); } cancellationCheck?.call(); - final aggregateScore = aggregate(evaluations, weights); - final primitives = determineResultPrimitives( + final outcome = finalizeObjectiveEvaluation( evaluations: evaluations, - aggregateScore: aggregateScore, + weights: weights, response: response, ); @@ -371,15 +360,15 @@ class AgenticRunOrchestrator { modelId: modelId, taskId: task.id, response: response, - evaluations: evaluations, - aggregateScore: aggregateScore, + evaluations: outcome.evaluations, + aggregateScore: outcome.aggregateScore, completedAt: now(), trialIndex: trialIndex, taskVersion: task.version, benchmarkTrack: task.track.name, harnessId: harness.id, - primaryPass: primitives.primaryPass, - failureTag: primitives.failureTag, + primaryPass: outcome.primaryPass, + failureTag: outcome.failureTag, patchText: patchText, trajectoryLogPath: agentResult.trajectoryLogPath, planId: planId, @@ -434,10 +423,9 @@ class AgenticRunOrchestrator { rationale: rationale, details: {'error': error.toString()}, ); - final aggregateScore = aggregate([evaluation], weights); - final primitives = determineResultPrimitives( + final outcome = finalizeObjectiveEvaluation( evaluations: [evaluation], - aggregateScore: aggregateScore, + weights: weights, response: response, ); return TaskRunResult( @@ -446,15 +434,15 @@ class AgenticRunOrchestrator { modelId: modelId, taskId: task.id, response: response, - evaluations: [evaluation], - aggregateScore: aggregateScore, + evaluations: outcome.evaluations, + aggregateScore: outcome.aggregateScore, completedAt: now(), trialIndex: trialIndex, taskVersion: task.version, benchmarkTrack: task.track.name, harnessId: harnessId, - primaryPass: primitives.primaryPass, - failureTag: primitives.failureTag, + primaryPass: outcome.primaryPass, + failureTag: outcome.failureTag, planId: planId, provenance: provenance, ); @@ -481,29 +469,21 @@ class AgenticRunOrchestrator { completionTokens: null, latency: Duration.zero, ); - final evaluations = [ - environmentFailureEvaluation( + final evaluations = blockEvaluatorsForHardFailure( + evaluations: [], + evaluators: applyTaskResourceLimitsToEvaluators( + task.evaluatorsFor(evaluatorConfig), + task, + ), + failure: environmentFailureEvaluation( rationale: rationale, stderr: error.toString(), phase: phase, ), - ]; - for (final evaluator in applyTaskResourceLimitsToEvaluators( - task.evaluatorsFor(evaluatorConfig), - task, - )) { - evaluations.add( - blockedEvaluationFor( - evaluatorId: evaluator.id, - previousResults: evaluations, - blockAllDownstream: true, - )!, - ); - } - final aggregateScore = aggregate(evaluations, weights); - final primitives = determineResultPrimitives( + ); + final outcome = finalizeObjectiveEvaluation( evaluations: evaluations, - aggregateScore: aggregateScore, + weights: weights, response: response, ); return TaskRunResult( @@ -512,15 +492,15 @@ class AgenticRunOrchestrator { modelId: modelId, taskId: task.id, response: response, - evaluations: evaluations, - aggregateScore: aggregateScore, + evaluations: outcome.evaluations, + aggregateScore: outcome.aggregateScore, completedAt: now(), trialIndex: trialIndex, taskVersion: task.version, benchmarkTrack: task.track.name, harnessId: harnessId, - primaryPass: primitives.primaryPass, - failureTag: primitives.failureTag, + primaryPass: outcome.primaryPass, + failureTag: outcome.failureTag, planId: planId, provenance: provenance, ); @@ -614,24 +594,15 @@ class AgenticRunOrchestrator { required PrepareFailed failure, required String phase, }) { - if (!hasHardDownstreamBlocker(evaluations)) { - evaluations.add( - environmentFailureEvaluation( - rationale: 'prepare failed', - stderr: failure.stderr, - phase: phase, - ), - ); - } - for (final evaluator in evaluators) { - evaluations.add( - blockedEvaluationFor( - evaluatorId: evaluator.id, - previousResults: evaluations, - blockAllDownstream: true, - )!, - ); - } + blockEvaluatorsForHardFailure( + evaluations: evaluations, + evaluators: evaluators, + failure: environmentFailureEvaluation( + rationale: 'prepare failed', + stderr: failure.stderr, + phase: phase, + ), + ); } EvaluationResult _harnessEvaluation(AgentRunResult result) { diff --git a/app/lib/runner/codegen_task_executor.dart b/app/lib/runner/codegen_task_executor.dart index e02788c..7004b3d 100644 --- a/app/lib/runner/codegen_task_executor.dart +++ b/app/lib/runner/codegen_task_executor.dart @@ -1,6 +1,5 @@ import 'dart:async'; -import 'package:dart_arena/analytics/result_primitives.dart'; import 'package:dart_arena/core/benchmark_task.dart'; import 'package:dart_arena/core/code_extractor.dart'; import 'package:dart_arena/core/evaluation_context.dart'; @@ -8,12 +7,12 @@ import 'package:dart_arena/core/evaluation_result.dart'; import 'package:dart_arena/core/evaluator_blocking.dart'; import 'package:dart_arena/core/evaluator_config.dart'; import 'package:dart_arena/core/model_response.dart'; -import 'package:dart_arena/core/scoring.dart'; import 'package:dart_arena/core/task_run_result.dart'; import 'package:dart_arena/providers/model_provider.dart'; import 'package:dart_arena/providers/model_stream_event.dart'; import 'package:dart_arena/runner/evaluator_resource_limits.dart'; import 'package:dart_arena/runner/generated_code_sandbox.dart'; +import 'package:dart_arena/runner/objective_evaluation.dart'; import 'package:dart_arena/runner/prompts/plan_aware_prompt.dart'; import 'package:dart_arena/runner/workdir_manager.dart'; @@ -155,52 +154,34 @@ class CodegenTaskExecutor { final evaluations = []; if (prepResult is PrepareFailed) { - evaluations.add( - environmentFailureEvaluation( + blockEvaluatorsForHardFailure( + evaluations: evaluations, + evaluators: evaluators, + failure: environmentFailureEvaluation( rationale: 'prepare failed', stderr: prepResult.stderr, ), ); - for (final evaluator in evaluators) { - evaluations.add( - blockedEvaluationFor( - evaluatorId: evaluator.id, - previousResults: evaluations, - blockAllDownstream: true, - )!, - ); - } } else { - for (final evaluator in evaluators) { - cancellationCheck?.call(); - final blocked = blockedEvaluationFor( - evaluatorId: evaluator.id, - previousResults: evaluations, - ); - if (blocked != null) { - evaluations.add(blocked); - continue; - } - final result = await evaluator.evaluate( - EvaluationContext( - workDir: dir, - response: responseWithCode, - task: task, - previousResults: evaluations, - deniedEnvironmentKeys: workdirManager.deniedEnvironmentKeys, - generatedCodeSandbox: generatedCodeSandbox, - ), - ); - cancellationCheck?.call(); - evaluations.add(result); - } + await runObjectiveEvaluators( + evaluators: evaluators, + evaluations: evaluations, + contextFor: (previousResults) => EvaluationContext( + workDir: dir, + response: responseWithCode, + task: task, + previousResults: previousResults, + deniedEnvironmentKeys: workdirManager.deniedEnvironmentKeys, + generatedCodeSandbox: generatedCodeSandbox, + ), + cancellationCheck: cancellationCheck, + ); } cancellationCheck?.call(); - final aggregateScore = aggregate(evaluations, weights); - final primitives = determineResultPrimitives( + final outcome = finalizeObjectiveEvaluation( evaluations: evaluations, - aggregateScore: aggregateScore, + weights: weights, response: responseWithCode, ); @@ -210,14 +191,14 @@ class CodegenTaskExecutor { modelId: modelId, taskId: task.id, response: responseWithCode, - evaluations: evaluations, - aggregateScore: aggregateScore, + evaluations: outcome.evaluations, + aggregateScore: outcome.aggregateScore, completedAt: now(), trialIndex: trialIndex, taskVersion: task.version, benchmarkTrack: task.track.name, - primaryPass: primitives.primaryPass, - failureTag: primitives.failureTag, + primaryPass: outcome.primaryPass, + failureTag: outcome.failureTag, planId: planId, ); } diff --git a/app/lib/runner/objective_evaluation.dart b/app/lib/runner/objective_evaluation.dart new file mode 100644 index 0000000..af543b4 --- /dev/null +++ b/app/lib/runner/objective_evaluation.dart @@ -0,0 +1,111 @@ +import 'package:dart_arena/analytics/result_primitives.dart'; +import 'package:dart_arena/core/evaluation_context.dart'; +import 'package:dart_arena/core/evaluation_result.dart'; +import 'package:dart_arena/core/evaluator_blocking.dart'; +import 'package:dart_arena/core/model_response.dart'; +import 'package:dart_arena/core/scoring.dart'; +import 'package:dart_arena/evaluators/evaluator.dart'; + +/// Builds the [EvaluationContext] each evaluator runs with, given the +/// ordered results observed so far. Track-specific preparation (sandbox, +/// process, patch replay, harness, workdir policy) stays with the caller; +/// this module only owns the ordered choreography around it. +typedef ObjectiveEvaluationContextBuilder = + EvaluationContext Function(List previousResults); + +/// Runs [evaluators] in order against [evaluations], inserting a blocked +/// placeholder (via [blockedEvaluationFor]) instead of invoking an evaluator +/// once a hard blocker (environment/harness failure, or a compile failure +/// for runtime evaluators) has been observed. Each evaluator sees every +/// prior result, including blocked placeholders, as `previousResults`. +/// +/// [evaluations] is mutated in place and returned so callers can seed it +/// with results that must be visible to the first evaluator (for example an +/// agent-harness result or a patch-capture failure). +Future> runObjectiveEvaluators({ + required Iterable evaluators, + required List evaluations, + required ObjectiveEvaluationContextBuilder contextFor, + void Function()? cancellationCheck, +}) async { + for (final evaluator in evaluators) { + cancellationCheck?.call(); + final blocked = blockedEvaluationFor( + evaluatorId: evaluator.id, + previousResults: evaluations, + ); + if (blocked != null) { + evaluations.add(blocked); + continue; + } + final result = await evaluator.evaluate(contextFor(evaluations)); + cancellationCheck?.call(); + evaluations.add(result); + } + return evaluations; +} + +/// Records [failure] as a hard blocker and blocks every remaining +/// [evaluators] from running, inserting a blocked placeholder for each of +/// them. If [evaluations] already carries a hard downstream blocker (see +/// [hasHardDownstreamBlocker]), [failure] is not appended again; the +/// blocked placeholders still record the earlier blocker as their cause. +/// +/// [evaluations] is mutated in place and returned. +List blockEvaluatorsForHardFailure({ + required List evaluations, + required Iterable evaluators, + required EvaluationResult failure, +}) { + if (!hasHardDownstreamBlocker(evaluations)) { + evaluations.add(failure); + } + for (final evaluator in evaluators) { + evaluations.add( + blockedEvaluationFor( + evaluatorId: evaluator.id, + previousResults: evaluations, + blockAllDownstream: true, + )!, + ); + } + return evaluations; +} + +/// The aggregate score and primary-pass/failure-tag primitives derived from +/// a completed objective evaluation. +class ObjectiveEvaluationOutcome { + const ObjectiveEvaluationOutcome({ + required this.evaluations, + required this.aggregateScore, + required this.primaryPass, + required this.failureTag, + }); + + final List evaluations; + final double aggregateScore; + final bool primaryPass; + final String failureTag; +} + +/// Derives the aggregate score and primary-pass/failure-tag primitives for +/// [evaluations], using [weights] for aggregation and [response] (when +/// available) for output-presence failure tagging. +ObjectiveEvaluationOutcome finalizeObjectiveEvaluation({ + required List evaluations, + required Map weights, + ModelResponse? response, +}) { + final aggregateScore = aggregate(evaluations, weights); + final primitives = determineResultPrimitives( + evaluations: evaluations, + aggregateScore: aggregateScore, + response: response, + ); + return ObjectiveEvaluationOutcome( + evaluations: evaluations, + aggregateScore: aggregateScore, + primaryPass: primitives.primaryPass, + failureTag: primitives.failureTag, + ); +} diff --git a/app/lib/runner/task_qa_runner.dart b/app/lib/runner/task_qa_runner.dart index 401af76..d41d65f 100644 --- a/app/lib/runner/task_qa_runner.dart +++ b/app/lib/runner/task_qa_runner.dart @@ -5,7 +5,6 @@ import 'package:crypto/crypto.dart'; import 'package:dart_arena/core/benchmark_task.dart'; import 'package:dart_arena/core/evaluation_context.dart'; import 'package:dart_arena/core/evaluation_result.dart'; -import 'package:dart_arena/core/evaluator_blocking.dart'; import 'package:dart_arena/core/evaluator_config.dart'; import 'package:dart_arena/core/model_response.dart'; import 'package:dart_arena/core/reference_solution.dart'; @@ -15,6 +14,7 @@ import 'package:dart_arena/evaluators/hidden_test_evaluator.dart'; import 'package:dart_arena/evaluators/test_evaluator.dart'; import 'package:dart_arena/runner/evaluator_resource_limits.dart'; import 'package:dart_arena/runner/generated_code_sandbox.dart'; +import 'package:dart_arena/runner/objective_evaluation.dart'; import 'package:dart_arena/runner/prompt_safety.dart'; import 'package:dart_arena/runner/prompts/plan_aware_prompt.dart'; import 'package:dart_arena/runner/resource_enforcement_policy.dart'; @@ -804,38 +804,26 @@ class TaskQaRunner { BenchmarkTask task, Directory workDir, Iterable evaluators, - ) async { - final results = []; - for (final evaluator in evaluators.map((e) => _qaEvaluator(task, e))) { - final blocked = blockedEvaluationFor( - evaluatorId: evaluator.id, - previousResults: results, - ); - if (blocked != null) { - results.add(blocked); - continue; - } - results.add( - await evaluator.evaluate( - EvaluationContext( - workDir: workDir, - response: const ModelResponse( - rawText: '', - extractedCode: null, - promptTokens: null, - completionTokens: null, - latency: Duration.zero, - ), - task: task, - previousResults: results, - deniedEnvironmentKeys: workdirManager.deniedEnvironmentKeys, - allowReentrantFlutterTool: allowReentrantFlutterTool, - generatedCodeSandbox: generatedCodeSandbox, - ), + ) { + return runObjectiveEvaluators( + evaluators: evaluators.map((e) => _qaEvaluator(task, e)), + evaluations: [], + contextFor: (previousResults) => EvaluationContext( + workDir: workDir, + response: const ModelResponse( + rawText: '', + extractedCode: null, + promptTokens: null, + completionTokens: null, + latency: Duration.zero, ), - ); - } - return results; + task: task, + previousResults: previousResults, + deniedEnvironmentKeys: workdirManager.deniedEnvironmentKeys, + allowReentrantFlutterTool: allowReentrantFlutterTool, + generatedCodeSandbox: generatedCodeSandbox, + ), + ); } Evaluator _qaEvaluator(BenchmarkTask task, Evaluator evaluator) { diff --git a/app/test/runner/objective_evaluation_test.dart b/app/test/runner/objective_evaluation_test.dart new file mode 100644 index 0000000..190ce6b --- /dev/null +++ b/app/test/runner/objective_evaluation_test.dart @@ -0,0 +1,289 @@ +import 'dart:io'; + +import 'package:dart_arena/core/benchmark_task.dart'; +import 'package:dart_arena/core/category.dart'; +import 'package:dart_arena/core/evaluation_context.dart'; +import 'package:dart_arena/core/evaluation_result.dart'; +import 'package:dart_arena/core/evaluator_blocking.dart'; +import 'package:dart_arena/core/evaluator_config.dart'; +import 'package:dart_arena/core/model_response.dart'; +import 'package:dart_arena/evaluators/evaluator.dart'; +import 'package:dart_arena/runner/objective_evaluation.dart'; +import 'package:test/test.dart'; + +void main() { + group('runObjectiveEvaluators', () { + test('runs evaluators in order, threading prior results', () async { + final evaluatorA = _FakeEvaluator('a', result: _pass('a')); + final evaluatorB = _FakeEvaluator('b', result: _pass('b')); + final evaluatorC = _FakeEvaluator('c', result: _pass('c')); + + final evaluations = await runObjectiveEvaluators( + evaluators: [evaluatorA, evaluatorB, evaluatorC], + evaluations: [], + contextFor: _contextFor, + ); + + expect(evaluations.map((e) => e.evaluatorId), ['a', 'b', 'c']); + expect(evaluatorA.observedPreviousResults, isEmpty); + expect(evaluatorB.observedPreviousResults!.map((e) => e.evaluatorId), [ + 'a', + ]); + expect(evaluatorC.observedPreviousResults!.map((e) => e.evaluatorId), [ + 'a', + 'b', + ]); + }); + + test('gives every evaluator an immutable snapshot, including blocked ' + 'placeholders inserted for prior evaluators', () async { + final compile = _FakeEvaluator( + 'compile', + result: const EvaluationResult( + evaluatorId: 'compile', + passed: false, + score: 0.0, + rationale: 'synthetic compile failure', + ), + ); + // 'test' and 'sample_hidden' are objective/runtime evaluator ids, so + // the compile failure above blocks them without invoking evaluate. + final publicTest = _FakeEvaluator('test', result: _pass('test')); + final hiddenTest = _FakeEvaluator( + 'sample_hidden', + result: _pass('sample_hidden'), + ); + // 'llm_judge' is not an objective evaluator id, so it still runs and + // observes the blocked placeholders ahead of it. + final judge = _FakeEvaluator('llm_judge', result: _pass('llm_judge')); + + final evaluations = await runObjectiveEvaluators( + evaluators: [compile, publicTest, hiddenTest, judge], + evaluations: [], + contextFor: _contextFor, + ); + + expect(publicTest.calls, 0); + expect(hiddenTest.calls, 0); + expect(judge.calls, 1); + + final snapshot = judge.observedPreviousResults!; + expect(snapshot.map((e) => e.evaluatorId), [ + 'compile', + 'test', + 'sample_hidden', + ]); + expect(snapshot[1].details[blockedByDetailKey], 'compile'); + expect(snapshot[2].details[blockedByDetailKey], 'compile'); + + // The snapshot is a defensive copy: mutating the live evaluations + // list afterwards must not retroactively change what the evaluator + // observed. + evaluations.add(_pass('after')); + expect(snapshot.length, 3); + expect(() => snapshot.add(_pass('mutate')), throwsUnsupportedError); + }); + + test( + 'cancellation before an evaluator aborts without invoking it', + () async { + final evaluatorA = _FakeEvaluator('a', result: _pass('a')); + final evaluatorB = _FakeEvaluator('b', result: _pass('b')); + final evaluatorC = _FakeEvaluator('c', result: _pass('c')); + final evaluations = []; + var checkCalls = 0; + + await expectLater( + () => runObjectiveEvaluators( + evaluators: [evaluatorA, evaluatorB, evaluatorC], + evaluations: evaluations, + contextFor: _contextFor, + cancellationCheck: () { + checkCalls++; + // Call 1: before A. Call 2: after A. Call 3: before B - abort. + if (checkCalls == 3) throw _CancelledException(); + }, + ), + throwsA(isA<_CancelledException>()), + ); + + expect(evaluatorA.calls, 1); + expect(evaluatorB.calls, 0); + expect(evaluatorC.calls, 0); + expect(evaluations.map((e) => e.evaluatorId), ['a']); + }, + ); + + test( + 'cancellation after an evaluator discards its unrecorded result', + () async { + final evaluatorA = _FakeEvaluator('a', result: _pass('a')); + final evaluatorB = _FakeEvaluator('b', result: _pass('b')); + final evaluations = []; + var checkCalls = 0; + + await expectLater( + () => runObjectiveEvaluators( + evaluators: [evaluatorA, evaluatorB], + evaluations: evaluations, + contextFor: _contextFor, + cancellationCheck: () { + checkCalls++; + // Call 1: before A. Call 2: after A - abort before A is added. + if (checkCalls == 2) throw _CancelledException(); + }, + ), + throwsA(isA<_CancelledException>()), + ); + + expect(evaluatorA.calls, 1); + expect(evaluatorB.calls, 0); + // A ran but its result was never appended: the after-check aborts + // before the `evaluations.add(result)` line. + expect(evaluations, isEmpty); + }, + ); + }); + + group('blockEvaluatorsForHardFailure', () { + test('dedups a repeated hard failure and keeps blocked placeholders ' + 'referencing the earlier blocker', () { + final evaluations = []; + final firstFailure = environmentFailureEvaluation( + rationale: 'first blocker', + stderr: 'boom once', + ); + + blockEvaluatorsForHardFailure( + evaluations: evaluations, + evaluators: [_FakeEvaluator('a', result: _pass('a'))], + failure: firstFailure, + ); + + final secondFailure = environmentFailureEvaluation( + rationale: 'second blocker', + stderr: 'boom twice', + ); + blockEvaluatorsForHardFailure( + evaluations: evaluations, + evaluators: [_FakeEvaluator('b', result: _pass('b'))], + failure: secondFailure, + ); + + // Only one 'environment' failure is recorded; the second call is a + // no-op for the failure itself because a hard blocker already exists. + expect( + evaluations.where((e) => e.evaluatorId == 'environment').length, + 1, + ); + expect( + evaluations + .singleWhere((e) => e.evaluatorId == 'environment') + .rationale, + 'first blocker', + ); + + final blockedA = evaluations.singleWhere((e) => e.evaluatorId == 'a'); + final blockedB = evaluations.singleWhere((e) => e.evaluatorId == 'b'); + for (final blocked in [blockedA, blockedB]) { + expect(blocked.details[blockedByDetailKey], 'environment'); + expect(blocked.details[blockedByRationaleDetailKey], 'first blocker'); + } + }); + }); + + group('finalizeObjectiveEvaluation', () { + test('derives aggregate/primaryPass/failureTag for a passing run', () { + final outcome = finalizeObjectiveEvaluation( + evaluations: [_pass('compile'), _pass('analyze'), _pass('test')], + weights: const {'compile': 0.5, 'analyze': 0.5, 'test': 1.0}, + ); + + expect(outcome.aggregateScore, 1.0); + expect(outcome.primaryPass, isTrue); + expect(outcome.failureTag, 'pass'); + }); + + test( + 'derives aggregate/primaryPass/failureTag for a public test failure', + () { + final outcome = finalizeObjectiveEvaluation( + evaluations: [ + _pass('compile'), + _pass('analyze'), + const EvaluationResult( + evaluatorId: 'test', + passed: false, + score: 0.0, + ), + ], + weights: const {'compile': 0.5, 'analyze': 0.5, 'test': 1.0}, + ); + + expect(outcome.aggregateScore, 0.5); + expect(outcome.primaryPass, isFalse); + expect(outcome.failureTag, 'public_tests_failed'); + }, + ); + }); +} + +EvaluationResult _pass(String id) => + EvaluationResult(evaluatorId: id, passed: true, score: 1.0); + +EvaluationContext _contextFor(List previousResults) { + return EvaluationContext( + workDir: Directory('unused'), + response: const ModelResponse( + rawText: 'code', + extractedCode: 'code', + promptTokens: null, + completionTokens: null, + latency: Duration.zero, + ), + task: _FakeTask(), + previousResults: previousResults, + ); +} + +class _CancelledException implements Exception {} + +class _FakeEvaluator implements Evaluator { + _FakeEvaluator(this.id, {required this.result}); + + @override + final String id; + final EvaluationResult result; + var calls = 0; + List? observedPreviousResults; + + @override + Future evaluate(EvaluationContext ctx) async { + calls++; + observedPreviousResults = ctx.previousResults; + return result; + } +} + +class _FakeTask extends BenchmarkTask { + @override + String get id => 'fake.objective_evaluation'; + + @override + Category get category => Category.bugFix; + + @override + String get prompt => 'unused'; + + @override + Map get fixtures => const {}; + + @override + String get generatedCodePath => 'lib/answer.dart'; + + @override + String? get judgeRubric => null; + + @override + List evaluatorsFor(EvaluatorConfig config) => const []; +}