update

Author: mayinghan

eval_protocol/mcp/execution/manager.py

@@ -171,7 +171,7 @@ async def _execute_with_semaphore(idx):
             if trajectory.terminated:
                 if trajectory.termination_reason == TerminationReason.ERROR:
                     evaluation_rows[idx].rollout_status.status = "error"
-                    evaluation_rows[idx].rollout_status.reason= trajectory.control_plane_summary.get(
+                    evaluation_rows[idx].rollout_status.reason = trajectory.control_plane_summary.get(
                         "error_message", None
                     )
                 else:
@@ -362,6 +362,12 @@ async def _execute_rollout(
                             if recording_mode:
                                 policy.log_conversation_state_for_playback(rollout_idx, step - 1, conversation_history)
 
+                        if env_end:
+                            # if the env marks the end of the rollout, break the tool call loop
+                            # but set the termination reason later after the final policy call
+                            trajectory.terminated = True
+                            break
+
                         if step >= steps:
                             trajectory.terminated = True
                             trajectory.termination_reason = TerminationReason.MAX_STEPS
@@ -395,9 +401,16 @@ async def _execute_rollout(
                     if recording_mode:
                         policy.log_conversation_state_for_playback(rollout_idx, step - 1, conversation_history)
 
-                # update control plane summary if the env marks end
+                # if the env marks end, update control plane summary and do one last policy call, then break the agent loop
+                # this is to ensure each turn ends with an assistant message, which will align with the actual agentic llm behavior
                 if env_end:
-                    # Add final control plane summary
+                    _, usage_stats, finish_reason = await policy(tool_schema, rollout_idx, conversation_history)
+                    if usage_stats:
+                        trajectory.usage["prompt_tokens"] += usage_stats.prompt_tokens
+                        trajectory.usage["completion_tokens"] += usage_stats.completion_tokens
+                        trajectory.usage["total_tokens"] += usage_stats.total_tokens
+                    trajectory.terminated = True
+                    trajectory.termination_reason = TerminationReason.from_str(finish_reason)
                     trajectory.control_plane_summary.update(
                         {
                             "total_reward": trajectory.total_reward,

eval_protocol/types/types.py

@@ -1,8 +1,9 @@
+from contextlib import AsyncExitStack
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Dict, List, Optional
+
 from mcp.client.session import ClientSession
-from contextlib import AsyncExitStack
 
 
 class TerminationReason(str, Enum):
@@ -14,6 +15,7 @@ class TerminationReason(str, Enum):
     ERROR: Trajectory ends because of an error
     STOP: Trajectory ends by the policy (mapped to llm response stop reason "stop")
     LENGTH: Trajectory ends by the policy (mapped to llm response stop reason "length")
+    TOOL_CALLS: Trajectory ends by the policy with a hanging tool call response (mapped to llm response stop reason "tool_calls")
     """
 
     MAX_STEPS = "max_steps"
@@ -22,6 +24,7 @@ class TerminationReason(str, Enum):
     ERROR = "error"
     STOP = "stop"
     LENGTH = "length"
+    TOOL_CALLS = "tool_calls"
 
     @classmethod
     def from_str(cls, value: str) -> "TerminationReason":
@@ -37,6 +40,8 @@ def from_str(cls, value: str) -> "TerminationReason":
             return cls.USER_STOP
         elif value == "error":
             return cls.ERROR
+        elif value == "tool_calls":
+            return cls.TOOL_CALLS
         else:
             raise ValueError(f"Invalid termination reason: {value}")