Add 'passed' field to EvalMetadata and update evaluation logic to determine if the evaluation passed based on the threshold of success.

Author: Dylan Huang

eval_protocol/models.py

@@ -212,6 +212,7 @@ class EvalMetadata(BaseModel):
     num_runs: int = Field(..., description="Number of times the evaluation was repeated")
     aggregation_method: str = Field(..., description="Method used to aggregate scores across runs")
     threshold_of_success: Optional[float] = Field(None, description="Threshold score for test success")
+    passed: Optional[bool] = Field(None, description="Whether the evaluation passed based on the threshold")
 
 
 class EvaluationRow(BaseModel):

eval_protocol/pytest/evaluation_test.py

@@ -275,6 +275,14 @@ def wrapper_body(**kwargs):
                             )
                         all_results.extend(results)
 
+                scores = [r.evaluation_result.score for r in all_results if r.evaluation_result]
+                agg_score = aggregate(scores, aggregation_method)
+
+                # Determine if the evaluation passed based on threshold
+                passed = None
+                if threshold_of_success is not None:
+                    passed = agg_score >= threshold_of_success
+
                 # Create eval metadata with test function info and current commit hash
                 eval_metadata = EvalMetadata(
                     name=test_func.__name__,
@@ -284,16 +292,16 @@ def wrapper_body(**kwargs):
                     num_runs=num_runs,
                     aggregation_method=aggregation_method,
                     threshold_of_success=threshold_of_success,
+                    passed=passed,
                 )
 
                 # Add metadata to all results before logging
                 for r in all_results:
                     r.eval_metadata = eval_metadata
                     default_logger.log(r)
 
-                scores = [r.evaluation_result.score for r in all_results if r.evaluation_result]
-                agg_score = aggregate(scores, aggregation_method)
-                if threshold_of_success is not None:
+                # Check threshold after logging
+                if threshold_of_success is not None and not passed:
                     assert (
                         agg_score >= threshold_of_success
                     ), f"Aggregated score {agg_score:.3f} below threshold {threshold_of_success}"

vite-app/src/types/eval-protocol.ts

@@ -81,7 +81,8 @@ export const EvalMetadataSchema = z.object({
   status: z.enum(['running', 'finished', 'error']).default('running').describe('Status of the evaluation'),
   num_runs: z.number().int().describe('Number of times the evaluation was repeated'),
   aggregation_method: z.string().describe('Method used to aggregate scores across runs'),
-  threshold_of_success: z.number().optional().describe('Threshold score for test success')
+  threshold_of_success: z.number().optional().describe('Threshold score for test success'),
+  passed: z.boolean().optional().describe('Whether the evaluation passed based on the threshold')
 });
 
 export const EvaluationRowSchema = z.object({