|
5 | 5 | and comparing the output against expected results in a pointwise manner. |
6 | 6 | """ |
7 | 7 |
|
8 | | -import logging |
9 | | -import time |
10 | 8 | from typing import Any, Dict, List |
11 | 9 |
|
12 | 10 | from eval_protocol.models import EvaluateResult, EvaluationRow, Message |
13 | 11 | from eval_protocol.pytest import default_single_turn_rollout_processor, evaluation_test |
14 | 12 | from eval_protocol.rewards.code_execution import execute_python_code, extract_code_blocks |
15 | 13 |
|
16 | | -logger = logging.getLogger(__name__) |
17 | | - |
18 | 14 |
|
19 | 15 | def coding_dataset_to_evaluation_row(data: List[Dict[str, Any]]) -> List[EvaluationRow]: |
20 | 16 | """ |
@@ -43,22 +39,18 @@ def test_coding_code_evaluation(row: EvaluationRow) -> EvaluationRow: |
43 | 39 | """ |
44 | 40 | Evaluation function that tests code correctness by executing it locally. |
45 | 41 |
|
46 | | -
|
47 | 42 | This function: |
48 | 43 | 1. Extracts Python code from the assistant's response |
49 | 44 | 2. Executes the code locally with timeout=10 |
50 | 45 | 3. Compares the output to ground_truth |
51 | 46 | 4. Returns a score of 1.0 if output matches, 0.0 otherwise |
52 | 47 |
|
53 | | -
|
54 | 48 | Args: |
55 | 49 | row: EvaluationRow containing the conversation messages and expected_output in ground_truth |
56 | 50 |
|
57 | | -
|
58 | 51 | Returns: |
59 | 52 | EvaluationRow with the evaluation result |
60 | 53 | """ |
61 | | - logger.info(f"STARTING TO EVALUATE ROW: {row.input_metadata.row_id} at time {time.time()}") |
62 | 54 | # Check if we have an assistant response |
63 | 55 | if len(row.messages) < 2 or row.messages[-1].role != "assistant": |
64 | 56 | row.evaluation_result = EvaluateResult(score=0.0, reason="No assistant response found") |
|
0 commit comments