Update

Author: xzrderek

eval_protocol/benchmarks/suites/gpqa.py

@@ -8,11 +8,11 @@
 
 from eval_protocol.benchmarks.registry import export_benchmark
 from eval_protocol.models import EvaluateResult, EvaluationRow, Message, MetricResult
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
 from eval_protocol.pytest.default_single_turn_rollout_process import (
     SingleTurnRolloutProcessor,
 )
 from eval_protocol.pytest.evaluation_test import evaluation_test
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import RolloutProcessorConfig
 
 SYSTEM_PROMPT = (
@@ -63,7 +63,7 @@ def _strip_gt_messages(msgs: List[Message]) -> List[Message]:
     return [m for m in msgs if not (m.role == "system" and (m.content or "").startswith("__GT__:"))]
 
 
-class GPQAStripGTRolloutProcessor(BaseRolloutProcessor):
+class GPQAStripGTRolloutProcessor(RolloutProcessor):
     """Preprocess rows to set ground_truth and remove __GT__ messages, then delegate to SingleTurnRolloutProcessor."""
 
     def __init__(self):

eval_protocol/pytest/__init__.py

@@ -1,16 +1,18 @@
 from .default_agent_rollout_processor import AgentRolloutProcessor
-from .default_base_rollout_process import BaseRolloutProcessor
 from .default_dataset_adapter import default_dataset_adapter
 from .default_mcp_gym_rollout_processor import MCPGymRolloutProcessor
+from .default_no_op_rollout_processor import NoOpRolloutProcessor
 from .default_single_turn_rollout_process import SingleTurnRolloutProcessor
 from .evaluation_test import evaluation_test
+from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
 
 __all__ = [
     "AgentRolloutProcessor",
     "MCPGymRolloutProcessor",
-    "BaseRolloutProcessor",
+    "RolloutProcessor",
     "SingleTurnRolloutProcessor",
+    "NoOpRolloutProcessor",
     "default_dataset_adapter",
     "RolloutProcessorConfig",
     "evaluation_test",

eval_protocol/pytest/default_agent_rollout_processor.py

@@ -13,7 +13,7 @@
 from eval_protocol.mcp.execution.policy import LiteLLMPolicy
 from eval_protocol.mcp.mcp_multi_client import MCPMultiClient
 from eval_protocol.models import EvaluationRow, Message
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import Dataset, RolloutProcessorConfig
 
 logger = logging.getLogger(__name__)
@@ -116,7 +116,7 @@ def _get_content_from_tool_result(self, tool_result: CallToolResult) -> List[Tex
         return tool_result.content
 
 
-class AgentRolloutProcessor(BaseRolloutProcessor):
+class AgentRolloutProcessor(RolloutProcessor):
     """Agent rollout processor for tool-calling agents."""
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:

eval_protocol/pytest/default_mcp_gym_rollout_processor.py

@@ -6,15 +6,13 @@
 import subprocess
 import time
 from pathlib import Path
-from typing import Any, AsyncIterator, Dict, List, Optional
+from typing import List, Optional
 
 import eval_protocol as ep
-from eval_protocol.models import EvaluationRow, Message
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
+from eval_protocol.models import EvaluationRow
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import RolloutProcessorConfig
 
-CURRENT_RUN_STATE: Dict[str, Any] = {}
-
 
 class MCPServerManager:
     """Manages MCP server lifecycle for testing."""
@@ -195,7 +193,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         return False  # Don't suppress exceptions
 
 
-class MCPGymRolloutProcessor(BaseRolloutProcessor):
+class MCPGymRolloutProcessor(RolloutProcessor):
     """
     Rollout processor for tau bench environments.
 

…l/pytest/default_base_rollout_process.py …ytest/default_no_op_rollout_processor.pyeval_protocol/pytest/default_base_rollout_process.py renamed to eval_protocol/pytest/default_no_op_rollout_processor.py eval_protocol/pytest/default_base_rollout_process.py renamed to eval_protocol/pytest/default_no_op_rollout_processor.py

@@ -2,18 +2,16 @@
 from typing import List
 
 from eval_protocol.models import EvaluationRow
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import RolloutProcessorConfig
 
 
-class BaseRolloutProcessor:
+class NoOpRolloutProcessor(RolloutProcessor):
     """
-    Base rollout processor - minimal implementation that all others inherit from.
+    No-op rollout processor that passes input dataset through unchanged.
 
-    This is the Strategy pattern base class. It provides:
-    1. __call__(rows, config) -> tasks (the main interface)
-    2. cleanup() -> None (resource cleanup)
-
-    All other processors inherit from this and override as needed.
+    Simply returns the input rows as completed tasks. This is useful for testing
+    or when you want to handle rollout processing manually.
     """
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
@@ -26,6 +24,4 @@ async def return_row(row: EvaluationRow) -> EvaluationRow:
         tasks = [asyncio.create_task(return_row(row)) for row in rows]
         return tasks
 
-    def cleanup(self) -> None:
-        """No-op cleanup - override in subclasses if needed."""
-        pass
+    # Inherits cleanup() from RolloutProcessor - no override needed

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -2,21 +2,20 @@
 import logging
 import os
 import time
-from typing import AsyncIterator, List
+from typing import List
 
-import litellm
 from litellm import acompletion
 from openai.types.chat.chat_completion_message import ChatCompletionMessageToolCall
 
 from eval_protocol.dataset_logger import default_logger
 from eval_protocol.models import EvaluationRow, Message
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import RolloutProcessorConfig
 
 logger = logging.getLogger(__name__)
 
 
-class SingleTurnRolloutProcessor(BaseRolloutProcessor):
+class SingleTurnRolloutProcessor(RolloutProcessor):
     """Single turn rollout processor for direct LLM calls."""
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:

eval_protocol/pytest/evaluation_test.py

@@ -24,8 +24,9 @@
     InputMetadata,
     Message,
 )
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
 from eval_protocol.pytest.default_dataset_adapter import default_dataset_adapter
+from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import (
     Dataset,
     DatasetPathParam,
@@ -61,7 +62,7 @@ def evaluation_test(  # noqa: C901
     input_messages: Optional[List[InputMessagesParam]] = None,
     input_dataset: Optional[List[DatasetPathParam]] = None,
     dataset_adapter: Callable[[List[Dict[str, Any]]], Dataset] = default_dataset_adapter,
-    rollout_processor: BaseRolloutProcessor = BaseRolloutProcessor(),
+    rollout_processor: RolloutProcessor = NoOpRolloutProcessor(),
     evaluation_test_kwargs: Optional[List[EvaluationInputParam]] = None,
     rollout_processor_kwargs: Optional[RolloutProcessorInputParam] = None,
     aggregation_method: AggregationMethod = "mean",
@@ -749,7 +750,7 @@ def run_evaluation_test_direct(
     input_dataset: Optional[List[DatasetPathParam]] = None,
     dataset_adapter: Callable[[List[Dict[str, Any]]], Dataset] = default_dataset_adapter,
     completion_params: Optional[CompletionParams] = None,
-    rollout_processor: BaseRolloutProcessor = BaseRolloutProcessor(),
+    rollout_processor: RolloutProcessor = NoOpRolloutProcessor(),
     rollout_processor_kwargs: Optional[RolloutProcessorInputParam] = None,
     aggregation_method: AggregationMethod = "mean",
     passed_threshold: Optional[Union[EvaluationThreshold, float]] = None,

eval_protocol/pytest/rollout_processor.py

@@ -0,0 +1,21 @@
+import asyncio
+from abc import ABC, abstractmethod
+from typing import List
+
+from eval_protocol.models import EvaluationRow
+from eval_protocol.pytest.types import RolloutProcessorConfig
+
+
+class RolloutProcessor(ABC):
+    """
+    Abstract base class for all rollout processor strategies.
+    """
+
+    @abstractmethod
+    def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
+        """Process evaluation rows and return async tasks. Must be implemented by subclasses."""
+        pass
+
+    def cleanup(self) -> None:
+        """Cleanup resources. Override in subclasses if cleanup is needed."""
+        pass

eval_protocol/pytest/utils.py

@@ -7,7 +7,7 @@
 
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
 from eval_protocol.models import EvalMetadata, EvaluationRow
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
+from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import (
     CompletionParams,
     DatasetPathParam,
@@ -234,7 +234,7 @@ def generate_parameter_combinations(
 
 
 async def rollout_processor_with_retry(
-    rollout_processor: BaseRolloutProcessor,
+    rollout_processor: RolloutProcessor,
     fresh_dataset: List[EvaluationRow],
     config: RolloutProcessorConfig,
     max_retry: int,

tests/pytest/test_pytest_ensure_logging.py

@@ -5,7 +5,7 @@
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
 from eval_protocol.dataset_logger.sqlite_evaluation_row_store import SqliteEvaluationRowStore
 from eval_protocol.models import EvaluationRow
-from eval_protocol.pytest.default_base_rollout_process import BaseRolloutProcessor
+from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 from tests.pytest.test_markdown_highlighting import markdown_dataset_to_evaluation_row
 
 
@@ -34,7 +34,7 @@ async def test_ensure_logging(monkeypatch):
         ],
         completion_params=[{"temperature": 0.0, "model": "dummy/local-model"}],
         dataset_adapter=markdown_dataset_to_evaluation_row,
-        rollout_processor=BaseRolloutProcessor(),
+        rollout_processor=NoOpRolloutProcessor(),
         mode="pointwise",
         combine_datasets=False,
         num_runs=2,