diff --git a/README.md b/README.md
index 69185488bb..0dccba4ed3 100644
--- a/README.md
+++ b/README.md
@@ -317,8 +317,10 @@ The Dataset column links to publicly available datasets (e.g., on HuggingFace).
| Swe Agents | | - | - | ✓ | ✓ | Apache 2.0 | swebench_multi_tools.yaml | - |
| Swe Agents | | - | - | ✓ | ✓ | Apache 2.0 | swebench_openhands.yaml | - |
| Swe Agents | | - | - | ✓ | ✓ | Apache 2.0 | swebench_openhands_training.yaml | - |
-| Swe Agents | coding | SWE-bench driven by the opencode agent framework for RL training. | Train software engineering capabilities on SWE tasks using opencode rollouts. | ✓ | - | Apache 2.0 | swebench_opencode_training.yaml | - |
-| Swe Agents | coding | SWE-bench driven by the opencode agent framework. | Eval software engineering capabilities on SWE-bench using opencode. | ✓ | ✓ | Apache 2.0 | swebench_opencode.yaml | - |
+| Swe Agents | coding | SWE-bench driven by the opencode agent framework for RL training. | Train software engineering capabilities on SWE tasks using opencode rollouts. | ✓ | - | Apache 2.0 | swebench_nv_opencode_training.yaml | - |
+| Swe Agents | coding | SWE-bench driven by the opencode agent framework. | Eval software engineering capabilities on SWE-bench using opencode. | ✓ | ✓ | Apache 2.0 | swebench_nv_opencode.yaml | - |
+| Swe Agents | coding | SWE-bench driven by upstream opencode for RL training. | Train software engineering capabilities on SWE tasks using upstream opencode rollouts. | ✓ | - | Apache 2.0 | swebench_opencode_training.yaml | - |
+| Swe Agents | coding | SWE-bench driven by upstream opencode. | Eval software engineering capabilities on SWE-bench using upstream opencode. | ✓ | ✓ | Apache 2.0 | swebench_opencode.yaml | - |
| Swe Pivot | agent | SWE pivot verifier for PivotRL on coding agent trajectories | Improve coding agent fix-design decisions | ✓ | ✓ | Apache 2.0 | swe_pivot.yaml | - |
| Swerl Gen | coding | Running sandboxed evaluation for SWE-style tasks (either patch generation or reproduction test generation) | Improve SWE capabilities useful for benchmarks like SWE-bench | ✓ | ✓ | Apache 2.0 | swerl_gen.yaml | - |
| Swerl Llm Judge | coding | SWE-style multiple-choice LLM-judge tasks scored via ... choice. | Improve SWE capabilities useful for benchmarks like SWE-bench | ✓ | ✓ | MIT | swerl_llm_judge.yaml | - |
diff --git a/nemo_gym/switchyard_trace.py b/nemo_gym/switchyard_trace.py
index 6bbe41c917..0981d5bb45 100644
--- a/nemo_gym/switchyard_trace.py
+++ b/nemo_gym/switchyard_trace.py
@@ -26,6 +26,7 @@
masks the sample rather than emitting partially annotated training data.
"""
+import json
import math
from typing import Any, Dict, List, NamedTuple
@@ -106,9 +107,7 @@ def reconstruct_switchyard_rollout(envelope: Any, session_id: str, converter: Re
for i, record in enumerate(records):
_validate_record(record, i, session_id, record_uuids)
if record["prompt_token_ids"][: len(token_history)] != token_history:
- raise SwitchyardTraceError(
- f"record {i} prompt_token_ids do not extend the prior prompt+generation tokens"
- )
+ raise SwitchyardTraceError(f"record {i} prompt_token_ids do not extend the prior prompt+generation tokens")
token_history = record["prompt_token_ids"] + record["generation_token_ids"]
if record["model"] != model:
raise SwitchyardTraceError(f"record {i} model {record['model']!r} != session model {model!r}")
@@ -123,7 +122,11 @@ def reconstruct_switchyard_rollout(envelope: Any, session_id: str, converter: Re
if i == 0:
new_context = prompt
else:
- if prompt[: len(assembled)] != assembled:
+ # Compare with tool-call arguments canonicalized so a client that
+ # re-serializes JSON differently across turns (compact vs. pretty)
+ # does not fail the extension check. The reconstructed items keep the
+ # original arguments — this normalization is comparison-only.
+ if _canonicalize_tool_args(prompt[: len(assembled)]) != _canonicalize_tool_args(assembled):
raise SwitchyardTraceError(f"record {i} prompt does not extend the reconstructed history")
new_context = prompt[len(assembled) :]
if not new_context:
@@ -255,6 +258,34 @@ def _content_to_text(content: Any, i: int) -> str:
raise SwitchyardTraceError(f"record {i} contains unsupported content of type {type(content).__name__}")
+def _canonicalize_tool_args(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """Copy of *messages* with assistant tool-call ``arguments`` canonicalized to
+ sorted-key JSON, used only for the history-extension equality check.
+
+ A client may re-serialize the same tool-call JSON differently across turns
+ (compact vs. pretty-printed); that whitespace difference must not fail the
+ extension check. The reconstructed training items keep the original arguments
+ — this normalization never reaches them.
+ """
+ out: List[Dict[str, Any]] = []
+ for message in messages:
+ tool_calls = message.get("tool_calls")
+ if not tool_calls:
+ out.append(message)
+ continue
+ canonical_calls = []
+ for call in tool_calls:
+ function = call["function"]
+ arguments = function["arguments"]
+ try:
+ arguments = json.dumps(json.loads(arguments), sort_keys=True)
+ except (json.JSONDecodeError, TypeError):
+ pass # not valid JSON — compare as-is
+ canonical_calls.append({**call, "function": {**function, "arguments": arguments}})
+ out.append({**message, "tool_calls": canonical_calls})
+ return out
+
+
def _normalize_tool_calls(tool_calls: Any, i: int) -> List[Dict[str, Any]]:
if not isinstance(tool_calls, list):
raise SwitchyardTraceError(f"record {i} contains non-list tool_calls")
diff --git a/responses_api_agents/swe_agents/README.md b/responses_api_agents/swe_agents/README.md
index 75d3ec7929..179a38d143 100644
--- a/responses_api_agents/swe_agents/README.md
+++ b/responses_api_agents/swe_agents/README.md
@@ -409,8 +409,10 @@ Bundled YAML configs:
- `configs/swebench_openhands_training.yaml` — same shape as above but tuned for training.
- `configs/swe_agent_config.yaml` — alternative SWE-agent path (uses `agent_tools_file`).
- `configs/swebench_multi_tools.yaml` — full 15-way prompt × agent-class × tool-name bundle (OpenHands).
-- `configs/swebench_opencode.yaml` — opencode harness (`agent_framework: opencode`), pinned fork + commit, no prompt overrides.
-- `configs/swebench_opencode_training.yaml` — opencode harness tuned for training.
+- `configs/swebench_opencode.yaml` — upstream opencode harness (`opencode_source: opencode`), routed through Switchyard, no prompt overrides.
+- `configs/swebench_opencode_training.yaml` — upstream opencode harness tuned for training.
+- `configs/swebench_nv_opencode.yaml` — NeMo-Gym opencode fork (`opencode_source: nv-opencode`), pinned repo + commit. Deprecated; prefer the upstream configs above.
+- `configs/swebench_nv_opencode_training.yaml` — fork harness tuned for training. Deprecated; prefer the upstream config above.
- `configs/swebench_opencode_multi_tools.yaml` — opencode harness with the 11-way `prompts/opencode_harness/*.txt` prompt bundle. See [Prompt overrides](#opencode-integration).
- `configs/swebench_opencode_no_instruction.yaml` / `configs/swebench_opencode_empty.yaml` — opencode ablation baselines (minimal / no methodology instructions).
- `configs/swebench_deepswe.yaml` / `configs/swebench_denovoswe.yaml` — the `deepswe` / `denovoswe` datasets (⚠️ WIP, see [Supported datasets and harnesses](#supported-datasets-and-harnesses)).
diff --git a/responses_api_agents/swe_agents/app.py b/responses_api_agents/swe_agents/app.py
index e52dc9e8eb..92ad74f1cb 100644
--- a/responses_api_agents/swe_agents/app.py
+++ b/responses_api_agents/swe_agents/app.py
@@ -24,6 +24,7 @@
import shlex
import shutil
import signal
+import socket
import sys
import time
import uuid
@@ -65,7 +66,7 @@
)
from nemo_gym.profiling import Profiler
from nemo_gym.server_utils import get_first_server_config_dict, get_response_json, raise_for_status, request
-from nemo_gym.switchyard_trace import SwitchyardTrace, reconstruct_switchyard_rollout
+from nemo_gym.switchyard_trace import SwitchyardTrace, SwitchyardTraceError, reconstruct_switchyard_rollout
from responses_api_models.vllm_model.app import VLLMConverter, split_responses_input_output_items
@@ -107,6 +108,13 @@ class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig):
"fork at swe_openhands_setup/. 'opencode' uses the opencode fork at swe_opencode_setup/ via "
"its bench/ entry point.",
)
+ opencode_source: Literal["opencode", "nv-opencode"] = Field(
+ default="opencode",
+ description="For agent_framework='opencode', which opencode to run. 'opencode' (default) runs "
+ "upstream opencode headless (`opencode run`) with an opencode.json provider pointed at Switchyard "
+ "— no fork, so opencode's native X-Session-Id is captured. 'nv-opencode' uses the pinned fork's "
+ "bench entry point.",
+ )
agent_config: Optional[str] = Field(default=None, description="Path to agent configuration file")
agent_tools_file: Optional[str] = Field(
default=None, description="Path to JSON file containing tool definitions in OpenAI format (for SWE-agent)"
@@ -192,6 +200,26 @@ class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig):
),
)
+ switchyard_spawn_routing_profile: Optional[str] = Field(
+ default=None,
+ description=(
+ "Path to a Switchyard routing-profiles YAML. When set, one dedicated token-capture Switchyard "
+ "instance is spawned per agent run on a free port (stateful token injection requires every "
+ "call of a session to reach the same process) and torn down after trace retrieval in run(). "
+ "Records land under the run's persistent_dir, so retrieval and /v1/sessions listing are "
+ "scoped per run. Mutually exclusive with switchyard_base_url. The `switchyard` CLI must be "
+ "on PATH. Instances are reaped in run(); calling responses() directly leaks the instance."
+ ),
+ )
+ switchyard_spawn_host: Optional[str] = Field(
+ default=None,
+ description=(
+ "Host advertised to agent containers for spawned Switchyard instances (the instance binds "
+ "0.0.0.0). Defaults to this node's hostname, which peer nodes resolve on Slurm clusters. "
+ "Set explicitly when containers must use a specific routable IP."
+ ),
+ )
+
openhands_should_log: bool = False
debug: bool = False
@@ -266,6 +294,9 @@ class SWEBenchWrapperInstanceConfig(SWEBenchWrapperServerConfig, SWEBenchWrapper
agent_command: Optional[ExecuteContainerCommandArgs] = None
agent_apptainer_command_str: Optional[str] = None
agent_script: Optional[str] = None
+ # Served model context length, fetched from Switchyard's /v1/models before the
+ # agent command is built; None leaves the opencode provider entry limit-less.
+ opencode_context_len: Optional[int] = None
# GRPO related fields
mask_sample: bool = False
@@ -1518,6 +1549,29 @@ def postprocess_after_run(self, report_file: Path) -> None:
report_path.write_text(json.dumps(report, indent=2))
+def _validate_switchyard_config(config: SWEBenchWrapperConfig) -> None:
+ """Fail fast on malformed Switchyard settings (URL shape, spawn-mode conflicts)."""
+ if config.switchyard_base_url:
+ _parse_switchyard_base_url(config.switchyard_base_url)
+ if config.switchyard_spawn_routing_profile:
+ if config.switchyard_base_url:
+ raise ValueError(
+ "switchyard_spawn_routing_profile and switchyard_base_url are mutually exclusive: "
+ "spawn mode assigns each run its own instance URL"
+ )
+ if not Path(config.switchyard_spawn_routing_profile).is_file():
+ raise ValueError(
+ f"switchyard_spawn_routing_profile does not exist: {config.switchyard_spawn_routing_profile!r}"
+ )
+
+
+def _find_free_port() -> int:
+ """An OS-assigned free TCP port (small bind-to-use race, acceptable per run)."""
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+ sock.bind(("", 0))
+ return int(sock.getsockname()[1])
+
+
def _parse_switchyard_base_url(switchyard_base_url: str) -> Tuple[str, int]:
"""Host/port of the Switchyard proxy.
@@ -1975,10 +2029,89 @@ def _render_opencode_user_message(
)
+# Container path for the system-prompt instruction file we hand opencode via the
+# config's ``instructions`` list. Kept outside the workspace so it never shows up
+# in the ``git diff HEAD`` that becomes the task patch.
+_OPENCODE_INSTRUCTIONS_PATH = "/tmp/switchyard_opencode_instructions.md"
+
+# Steer the model away from opencode's `task` tool crash: upstream opencode
+# hard-fails when the model supplies a `task_id` that does not begin with `ses`
+# (a synchronous SessionID.make throw defeats its graceful fallback — see
+# sst/opencode task tool). Smaller models routinely invent one; omitting it lets
+# opencode spawn a fresh subagent session, which Switchyard captures cleanly.
+_OPENCODE_TASK_ID_GUIDANCE = (
+ "When you use the `task` tool to start a new subagent, do not set the "
+ "`task_id` parameter — omit it so a fresh subagent session is created. "
+ "Only pass `task_id` to resume a subagent session you started earlier in "
+ "this conversation; a valid session id begins with `ses`. Never invent or "
+ "guess a `task_id` value."
+)
+
+
+def _opencode_switchyard_config(
+ switchyard_base_url: str, model: str, context_len: Optional[int] = None
+) -> Dict[str, Any]:
+ """opencode.json for upstream opencode routed through Switchyard (no fork).
+
+ A custom ``@ai-sdk/openai-compatible`` provider — whose id does NOT start with
+ ``opencode`` — points at Switchyard. That id prefix is load-bearing: upstream
+ opencode only emits its native ``X-Session-Id`` / ``x-parent-session-id``
+ correlation headers for non-``opencode`` providers, and those headers are how
+ Switchyard captures the session tree. Set as the default model so opencode does
+ not fall back to its hosted free models. ``instructions`` appends the task-tool
+ guidance to opencode's system prompt (see ``_OPENCODE_TASK_ID_GUIDANCE``).
+
+ ``context_len`` (the served model's max_model_len) populates the model's
+ ``limit`` metadata: opencode budgets its ``max_tokens`` from ``limit.output``
+ (falling back to a flat 32000 without it) and enables compaction only when
+ ``limit.context`` is set. Output is reserved as ``min(32000, context // 4)``
+ so compaction (triggered at ``context - output``) keeps most of the window.
+ """
+ model_entry: Dict[str, Any] = {"name": model}
+ if context_len is not None:
+ model_entry["limit"] = {
+ "context": context_len,
+ "output": min(32000, context_len // 4),
+ }
+ return {
+ "$schema": "https://opencode.ai/config.json",
+ "provider": {
+ "switchyard": {
+ "npm": "@ai-sdk/openai-compatible",
+ "name": "Switchyard",
+ "options": {"baseURL": f"{switchyard_base_url.rstrip('/')}/v1", "apiKey": "dummy"},
+ "models": {model: model_entry},
+ }
+ },
+ "model": f"switchyard/{model}",
+ "instructions": [_OPENCODE_INSTRUCTIONS_PATH],
+ "autoupdate": False,
+ }
+
+
+def _filter_title_gen_records(envelope: dict) -> dict:
+ """Strip the title-generation record opencode fires before the agent loop.
+
+ opencode calls the model once with system='You are a title generator' to
+ name each session. Removing it keeps reconstruction to agent turns only.
+ """
+ filtered = [
+ r
+ for r in envelope.get("completions", [])
+ if not any(
+ msg.get("role") == "system" and "title generator" in str(msg.get("content", "")).lower()
+ for msg in r.get("messages", [])
+ )
+ ]
+ return {**envelope, "completions": filtered}
+
+
class OpenCodeHarnessProcessor(BaseDatasetHarnessProcessor):
"""Drives the opencode fork; mirrors OpenHandsHarnessProcessor."""
def setup(self) -> Path:
+ if self.config.opencode_source == "opencode":
+ return self._setup_upstream()
setup_dir = self.parent_dir / "swe_opencode_setup"
with self._setup_directory_lock(setup_dir, "opencode"):
@@ -2009,7 +2142,28 @@ def setup(self) -> Path:
self._run_setup_command(command)
return setup_dir
+ def _setup_upstream(self) -> Path:
+ """Install upstream opencode-ai via bun; no fork clone."""
+ setup_dir = self.parent_dir / "swe_upstream_opencode_setup"
+ with file_lock(setup_dir, "upstream opencode setup"):
+ bun_dir = setup_dir / "bun"
+ install_dir = setup_dir / "opencode"
+ opencode_bin = install_dir / "node_modules" / ".bin" / "opencode"
+ if opencode_bin.exists() and (bun_dir / "bin" / "bun").exists():
+ print(f"upstream opencode already installed at {setup_dir}", flush=True)
+ return setup_dir
+ print(f"Installing upstream opencode-ai at {setup_dir}...", flush=True)
+ setup_dir.mkdir(parents=True, exist_ok=True)
+ install_dir.mkdir(parents=True, exist_ok=True)
+ # The env assignment must ride the pipeline stage that runs the
+ # installer: `VAR=x cmd1 | cmd2` binds VAR to cmd1 only.
+ self._run_setup_command(f"curl -fsSL https://bun.sh/install | BUN_INSTALL={bun_dir} bash")
+ self._run_setup_command(f"cd {install_dir} && PATH={bun_dir}/bin:$PATH bun add opencode-ai")
+ return setup_dir
+
def get_run_command(self) -> ExecuteContainerCommandArgs:
+ if self.config.opencode_source == "opencode":
+ return self._get_upstream_run_command()
data_point = self.config.problem_info
agent_run_id = self.config.agent_run_id
@@ -2195,6 +2349,157 @@ def get_run_command(self) -> ExecuteContainerCommandArgs:
timeout=self.config.swebench_agent_timeout + 60,
)
+ def _get_upstream_run_command(self) -> ExecuteContainerCommandArgs:
+ """Run command for opencode_source='opencode': upstream stock opencode via opencode.json."""
+ data_point = self.config.problem_info
+ agent_run_id = self.config.agent_run_id
+ instance_id = data_point["instance_id"]
+ eval_dir_in_opencode = self.config.eval_dir_in_openhands
+
+ assert self.config.opencode_setup_dir is not None, (
+ "opencode setup directory is not set; opencode_source='opencode' requires _setup_upstream() to have run."
+ )
+ assert self.config.switchyard_base_url is not None, (
+ "opencode_source='opencode' requires switchyard_base_url — Switchyard captures the session."
+ )
+
+ try:
+ model_server_cfg = get_first_server_config_dict(get_global_config_dict(), self.config.model_server_name)
+ default_model_name = (
+ getattr(model_server_cfg, "openai_model", None) or getattr(model_server_cfg, "model", None) or ""
+ )
+ except Exception as e:
+ raise RuntimeError(
+ f"Could not resolve model server '{self.config.model_server_name}' for upstream opencode: {e}"
+ )
+
+ effective_model = self.config.body.model or default_model_name
+ workspace_path = _resolve_opencode_workspace_path(data_point)
+ user_message = _render_opencode_user_message(
+ data_point,
+ workspace_path,
+ template_override_path=self.config.resolved_user_prompt_template,
+ )
+ user_message_host_path = self.config.persistent_dir / f"user_message_{agent_run_id}.txt"
+ user_message_host_path.write_text(user_message)
+
+ opencode_cfg_json = json.dumps(
+ _opencode_switchyard_config(
+ self.config.switchyard_base_url, effective_model, self.config.opencode_context_len
+ )
+ )
+
+ # Output goes to the same eval-dir layout as the fork so _openhands_dir_copy_from_host works unchanged.
+ output_dir_in_container = f"/opencode_setup/opencode/{eval_dir_in_opencode}/{instance_id}/bench_run"
+ output_jsonl_in_container = f"{output_dir_in_container}/output.jsonl"
+
+ dataset_name = str(data_point.get("dataset_name", ""))
+ if "SWE-Gym" in dataset_name:
+ conda_activate_cmd = (
+ "{ deactivate >/dev/null 2>&1 || true; unset VIRTUAL_ENV; "
+ "if [ -d /opt/miniconda3 ]; then "
+ ". /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed || true; "
+ "fi; } && "
+ )
+ elif "R2E-Gym" in dataset_name:
+ conda_activate_cmd = (
+ "{ deactivate >/dev/null 2>&1 || true; unset VIRTUAL_ENV; "
+ "if [ -f /testbed/.venv/bin/activate ]; then "
+ ". /testbed/.venv/bin/activate || true; "
+ "fi; } && "
+ )
+ elif dataset_name in ("nv-internal-1", "swe-bench-ext") or "SWE-rebench-V2" in dataset_name:
+ conda_activate_cmd = ""
+ else:
+ conda_activate_cmd = (
+ "if [ -d /opt/miniconda3 ]; then "
+ ". /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed || true; "
+ "fi && "
+ )
+
+ baseline_fix = _extract_instance_dict(data_point).get("baseline_fix", "")
+ baseline_fix_cmd = f"{{ {baseline_fix} >/tmp/baseline_fix.log 2>&1 || true; }} && " if baseline_fix else ""
+
+ # args dict encoded as JSON so the extraction Python never sees shell metacharacters
+ py_args_json = json.dumps(
+ {
+ "workspace": workspace_path,
+ "instance_id": instance_id,
+ "model": effective_model,
+ "output_dir": output_dir_in_container,
+ "output_jsonl": output_jsonl_in_container,
+ }
+ )
+
+ agent_main_cmd = (
+ "mkdir -p /tmp/ && "
+ "export PATH=/opencode_setup/bun/bin:$PATH && "
+ f'date +"%s.%N" > {self.config.generation_apptainer_spinup_timestamp_mounted_fpath} && '
+ f"export NEMO_GYM_METRICS_FPATH={self.config.base_mounted_dir}/nemo_gym_metrics.json && "
+ f"export NEMO_GYM_CONFIG_DICT={self.config.ng_global_config_dict_str} && "
+ f"export NEMO_GYM_MODEL_SERVER_NAME={self.config.model_server_name} && "
+ "export OPENCODE_DISABLE_MODELS_FETCH=1 && "
+ f"export SWITCHYARD_BASE_URL={shlex.quote(self.config.switchyard_base_url)} && "
+ "mkdir -p /root/.cache/opencode && "
+ "echo '{}' >/root/.cache/opencode/models.json && "
+ f"{conda_activate_cmd}"
+ f"{baseline_fix_cmd}"
+ f"cd {shlex.quote(workspace_path)} && "
+ f"echo {shlex.quote(opencode_cfg_json)} > opencode.json && "
+ f"echo {shlex.quote(_OPENCODE_TASK_ID_GUIDANCE)} > {_OPENCODE_INSTRUCTIONS_PATH} && "
+ "_OC_EXIT=0 && "
+ f"timeout {self.config.swebench_agent_timeout} "
+ "/opencode_setup/opencode/node_modules/.bin/opencode run "
+ f"--model {shlex.quote(f'switchyard/{effective_model}')} "
+ '"$(cat /opencode_setup/opencode/user_message.txt)" '
+ "|| _OC_EXIT=$?"
+ )
+
+ # Post-run extraction: git diff → output.jsonl in bench format.
+ # Uses _OC_ARGS JSON so no shell metacharacter issues in paths/ids.
+ extract_py = (
+ "import json,os,subprocess;"
+ "a=json.loads(os.environ['_OC_ARGS']);"
+ "r=subprocess.run(['git','-C',a['workspace'],'diff','HEAD'],capture_output=True,text=True,errors='replace');"
+ "patch=r.stdout.strip() or None;"
+ "ec=int(os.environ.get('_OC_EXIT','0'));"
+ "os.makedirs(a['output_dir'],exist_ok=True);"
+ "f=open(a['output_jsonl'],'w');"
+ "json.dump({'instance_id':a['instance_id'],'test_result':{'git_patch':patch},"
+ "'metadata':{'llm_config':{'model':a['model']}},"
+ "'error':None if ec==0 else f'opencode exit {ec}','metrics':None},f);"
+ "f.close()"
+ )
+
+ agent_script_name = f"agent_script_{agent_run_id}.sh"
+ agent_script_path = self.config.persistent_dir / agent_script_name
+ with open(agent_script_path, "w") as f:
+ f.write("#!/bin/bash\nset -e\n")
+ f.write(agent_main_cmd)
+ f.write(f"\nexport _OC_ARGS={shlex.quote(py_args_json)}\n")
+ f.write(f"python3 -c {shlex.quote(extract_py)}\n")
+ f.flush()
+ os.fsync(f.fileno())
+
+ agent_timeout_seconds = self.config.swebench_agent_timeout
+ opencode_cmd = (
+ f"timeout --signal=TERM --kill-after=30 {agent_timeout_seconds} "
+ f"bash /trajectories_mount/{agent_script_name}"
+ )
+
+ search_path = os.path.join(
+ self.config.opencode_setup_dir / "opencode" / eval_dir_in_opencode,
+ "**",
+ "output.jsonl",
+ )
+
+ return ExecuteContainerCommandArgs(
+ command=opencode_cmd,
+ expected_file_pattern=search_path,
+ mode="agent",
+ timeout=self.config.swebench_agent_timeout + 60,
+ )
+
########################################
# START Ray worker logic
@@ -2832,9 +3137,17 @@ class SWEBenchWrapper(SimpleResponsesAPIAgent):
########################################
def model_post_init(self, context: Any) -> None:
- # Fail fast on a malformed Switchyard URL rather than masking every sample.
- if self.config.switchyard_base_url:
- _parse_switchyard_base_url(self.config.switchyard_base_url)
+ # Fail fast on a malformed Switchyard config rather than masking every sample.
+ _validate_switchyard_config(self.config)
+ # Spawned per-run Switchyard instances, keyed by agent_run_id; reaped in run().
+ self._switchyard_procs: Dict[str, Process] = {}
+ # Round-robin counter for distributing spawns across generation backends.
+ self._switchyard_spawn_count: int = 0
+ # Limit concurrent Switchyard spawns: 1024 simultaneous spawns on one node cause
+ # port exhaustion (ClientOSError) and uvicorn connection saturation (ServerDisconnectedError).
+ # Semaphore is released once Switchyard is up (TCP-ready), not when the rollout finishes,
+ # so all agents still run concurrently — only the startup burst is staggered.
+ self._switchyard_spawn_sem: asyncio.Semaphore = asyncio.Semaphore(64)
run_session_id = f"{int(time.time() * 1000)}_{str(uuid.uuid4())[:8]}"
workspace_root = Path(__file__).parent
@@ -3185,20 +3498,32 @@ def _build_apptainer_command(
opencode_dir = f"{params.opencode_setup_dir}/opencode"
bun_dir = f"{params.opencode_setup_dir}/bun"
(Path(opencode_dir) / "evaluation" / "oh").mkdir(parents=True, exist_ok=True)
- # opencode reads SQLite migrations from `/../../migration`
- # (packages/opencode/src/storage/db.ts) → /opencode_setup/migration.
- mount_args.extend(
- [
- f"--mount type=bind,src={opencode_dir},dst=/opencode_setup/opencode,ro",
- f"--mount type=bind,src={opencode_dir},dst={opencode_dir},ro",
- f"--mount type=bind,src={opencode_dir}/evaluation/oh,dst=/opencode_setup/opencode/evaluation/oh",
- f"--mount type=bind,src={opencode_dir}/evaluation/oh,dst={opencode_dir}/evaluation/oh",
- f"--mount type=bind,src={bun_dir},dst=/opencode_setup/bun,ro",
- f"--mount type=bind,src={bun_dir},dst={bun_dir},ro",
- f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl",
- f"--mount type=bind,src={opencode_dir}/packages/opencode/migration,dst=/opencode_setup/migration,ro",
- ]
- )
+ if params.opencode_source == "opencode":
+ # Upstream stock opencode: no fork repo structure, no migration mount.
+ mount_args.extend(
+ [
+ f"--mount type=bind,src={opencode_dir},dst=/opencode_setup/opencode,ro",
+ f"--mount type=bind,src={opencode_dir}/evaluation/oh,dst=/opencode_setup/opencode/evaluation/oh",
+ f"--mount type=bind,src={bun_dir},dst=/opencode_setup/bun,ro",
+ f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl",
+ ]
+ )
+ else:
+ # nv-opencode fork: existing mounts (unchanged).
+ # opencode reads SQLite migrations from `/../../migration`
+ # (packages/opencode/src/storage/db.ts) → /opencode_setup/migration.
+ mount_args.extend(
+ [
+ f"--mount type=bind,src={opencode_dir},dst=/opencode_setup/opencode,ro",
+ f"--mount type=bind,src={opencode_dir},dst={opencode_dir},ro",
+ f"--mount type=bind,src={opencode_dir}/evaluation/oh,dst=/opencode_setup/opencode/evaluation/oh",
+ f"--mount type=bind,src={opencode_dir}/evaluation/oh,dst={opencode_dir}/evaluation/oh",
+ f"--mount type=bind,src={bun_dir},dst=/opencode_setup/bun,ro",
+ f"--mount type=bind,src={bun_dir},dst={bun_dir},ro",
+ f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl",
+ f"--mount type=bind,src={opencode_dir}/packages/opencode/migration,dst=/opencode_setup/migration,ro",
+ ]
+ )
user_message_host = params.persistent_dir / f"user_message_{params.agent_run_id}.txt"
mount_args.append(
f"--mount type=bind,src={user_message_host},dst=/opencode_setup/opencode/user_message.txt,ro"
@@ -3479,7 +3804,7 @@ def _setup_params(
# retries. Golden-patch verification makes no policy calls, so no session.
switchyard_session_id = None
if (
- self.config.switchyard_base_url
+ (self.config.switchyard_base_url or self.config.switchyard_spawn_routing_profile)
and self.config.agent_framework == "openhands"
and not self.config.verify_golden_patch
):
@@ -3600,6 +3925,31 @@ def _setup_params(
params.eval_command = dataset_processor.get_run_command()
params.eval_apptainer_command_str = self._build_apptainer_command(params, params.eval_command)
+ # Switchyard-routed runs defer the build to responses(): the script
+ # embeds the Switchyard routing, and spawn mode's instance URL plus the
+ # served model's context length are only known there.
+ if not self._agent_command_deferred(params):
+ self._build_agent_command(params)
+
+ return params, dataset_processor
+
+ def _agent_command_deferred(self, params: SWEBenchWrapperInstanceConfig) -> bool:
+ """Whether ``responses`` builds the agent command instead of ``_setup_params``."""
+ if self._switchyard_spawn_needed(params):
+ return True
+ return bool(
+ self.config.switchyard_base_url
+ and self.config.agent_framework == "opencode"
+ and self.config.opencode_source == "opencode"
+ )
+
+ def _build_agent_command(self, params: SWEBenchWrapperInstanceConfig) -> None:
+ """Build the agent script and command from the current params.
+
+ Called at the end of ``_setup_params``, except for Switchyard-routed
+ runs, where ``responses`` calls it after the instance URL and the
+ served model's context length are known.
+ """
if self.config.agent_framework == "opencode":
params.agent_command = OpenCodeHarnessProcessor(config=params).get_run_command()
else:
@@ -3607,11 +3957,22 @@ def _setup_params(
params.agent_apptainer_command_str = self._build_apptainer_command(params, params.agent_command)
params.agent_script = params.agent_script_path.read_text()
- return params, dataset_processor
-
async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse:
params, dataset_processor = self._setup_params(body)
+ if self._switchyard_spawn_needed(params):
+ # Semaphore limits concurrent spawns to avoid port exhaustion and
+ # uvicorn connection saturation on the single Gym node. Released once
+ # Switchyard is up; agents then run concurrently without restriction.
+ async with self._switchyard_spawn_sem:
+ params.switchyard_base_url = await self._spawn_switchyard(params)
+ if self._agent_command_deferred(params):
+ if params.switchyard_base_url and self.config.agent_framework == "opencode":
+ params.opencode_context_len = await self._fetch_switchyard_max_model_len(
+ params.switchyard_base_url, params.model_server_name
+ )
+ self._build_agent_command(params)
+
with (params.eval_private_dir / "params.json").open("w") as f:
f.write(params.model_dump_json(indent=4))
@@ -3624,6 +3985,9 @@ async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()
print(f"Hit an exception in {self.config.name}! See {traceback_file} for more details", file=sys.stderr)
+ # The agent never ran to completion; run() may never see this
+ # instance, so reap its Switchyard instance here.
+ await self._teardown_switchyard(params.agent_run_id)
raise e
async def _inner_responses(
@@ -3775,28 +4139,47 @@ async def run(self, body: BaseRunRequest) -> SWEBenchVerifyResponse:
instance_config = SWEBenchWrapperInstanceConfig.model_validate_json(metadata["instance_config"])
switchyard_trace_error = None
- if instance_config.switchyard_base_url and instance_config.switchyard_session_id:
- try:
- trace = await self._retrieve_switchyard_trace(instance_config)
- switchyard_input = [item.model_dump() for item in trace.input_items]
- switchyard_tools = [tool.model_dump() for tool in trace.tools]
- responses_create_params["input"] = switchyard_input
- responses_create_params["tools"] = switchyard_tools
- response.output = trace.output_items
- response.metadata = {
- "switchyard_source": "switchyard",
- "switchyard_session_id": instance_config.switchyard_session_id,
- "switchyard_record_uuids": json.dumps(trace.record_uuids),
- "switchyard_model": trace.model,
- }
- except Exception as e:
- # Fail closed for training, open for diagnostics: keep the
- # OpenHands-derived rollout/patch/reward, but mask the sample
- # rather than emit a partially token-annotated trajectory.
- instance_config.mask_sample = True
- switchyard_trace_error = (
- f"session {instance_config.switchyard_session_id}: {type(e).__name__}: {e}"
- )
+ try:
+ if instance_config.switchyard_base_url and instance_config.switchyard_session_id:
+ try:
+ trace = await self._retrieve_switchyard_trace(
+ instance_config.switchyard_base_url, instance_config.switchyard_session_id
+ )
+ self._apply_switchyard_trace(
+ responses_create_params, response, trace, instance_config.switchyard_session_id
+ )
+ except Exception as e:
+ # Fail closed for training, open for diagnostics: keep the
+ # OpenHands-derived rollout/patch/reward, but mask the sample
+ # rather than emit a partially token-annotated trajectory.
+ instance_config.mask_sample = True
+ switchyard_trace_error = (
+ f"session {instance_config.switchyard_session_id}: {type(e).__name__}: {e}"
+ )
+ elif (
+ instance_config.switchyard_base_url
+ and instance_config.agent_framework == "opencode"
+ and instance_config.opencode_source == "opencode"
+ ):
+ # Upstream opencode mints its own session id per (sub)agent, so
+ # there is no Gym-minted switchyard_session_id: discover the session
+ # tree from
+ # Switchyard (no harness logs), then reconstruct root -> main rollout
+ # and subagents -> token-annotated subagent_trajectories (replacing
+ # the text ones on success).
+ try:
+ sessions = await self._list_switchyard_sessions(instance_config.switchyard_base_url)
+ root_trace, root_id, subagent_trajectories = await self._reconstruct_switchyard_sessions(
+ instance_config.switchyard_base_url, sessions, filter_title_gen=True
+ )
+ self._apply_switchyard_trace(responses_create_params, response, root_trace, root_id)
+ except Exception as e:
+ instance_config.mask_sample = True
+ switchyard_trace_error = f"opencode sessions: {type(e).__name__}: {e}"
+ finally:
+ # The run's dedicated Switchyard instance (spawn mode) has served
+ # its records; reap it regardless of retrieval outcome.
+ await self._teardown_switchyard(instance_config.agent_run_id)
return SWEBenchVerifyResponse(
responses_create_params=responses_create_params,
@@ -3809,21 +4192,223 @@ async def run(self, body: BaseRunRequest) -> SWEBenchVerifyResponse:
switchyard_trace_error=switchyard_trace_error,
)
- async def _retrieve_switchyard_trace(self, instance_config: SWEBenchWrapperInstanceConfig) -> SwitchyardTrace:
- """Fetch this run's captured completions from Switchyard and rebuild the rollout.
+ def _switchyard_spawn_needed(self, params: SWEBenchWrapperInstanceConfig) -> bool:
+ """Whether this run gets its own Switchyard instance.
+
+ Spawn for exactly the runs whose calls are captured: OpenHands runs
+ with a minted session id, and upstream-opencode runs (which mint their
+ own ids). Golden-patch verification makes no policy calls.
+ """
+ if not self.config.switchyard_spawn_routing_profile or self.config.verify_golden_patch:
+ return False
+ if params.switchyard_session_id:
+ return True
+ return self.config.agent_framework == "opencode" and self.config.opencode_source == "opencode"
+
+ async def _spawn_switchyard(self, params: SWEBenchWrapperInstanceConfig) -> str:
+ """Spawn this run's Switchyard instance and return its base URL when ready.
+
+ The instance binds 0.0.0.0 on a free port and is advertised via
+ ``switchyard_spawn_host`` (default: this node's hostname) so agent
+ containers on peer nodes can reach it. Records go under the run's
+ ``persistent_dir`` so retrieval is scoped per run. Registered under
+ ``agent_run_id`` and reaped by :meth:`_teardown_switchyard`.
+ """
+ port = _find_free_port()
+ host = self.config.switchyard_spawn_host or socket.gethostname()
+ base_url = f"http://{host}:{port}"
+ rl_log_dir = params.persistent_dir / "switchyard_traces"
+ rl_log_dir.mkdir(parents=True, exist_ok=True)
+ log_path = params.persistent_dir / "switchyard.log"
+
+ # Distribute spawns across all generation backends round-robin so no
+ # single vLLM server becomes a bottleneck when many rollouts run in parallel.
+ _cfg = OmegaConf.create(shlex.split(params.ng_global_config_dict_str)[0])
+ _urls = [u for u in (_cfg.get("policy_base_url") or []) if u]
+ _backend_url = _urls[self._switchyard_spawn_count % len(_urls)] if _urls else None
+ self._switchyard_spawn_count += 1
+ spawn_env = {**os.environ, "SWITCHYARD_VLLM_BASE_URL": _backend_url} if _backend_url else None
+
+ with log_path.open("w") as log_file:
+ process = await asyncio.create_subprocess_exec(
+ "switchyard",
+ "--routing-profiles",
+ str(self.config.switchyard_spawn_routing_profile),
+ "--enable-rl-logging",
+ "--rl-log-dir",
+ str(rl_log_dir),
+ "--",
+ "serve",
+ "--port",
+ str(port),
+ stdout=log_file,
+ stderr=log_file,
+ env=spawn_env,
+ )
+ self._switchyard_procs[params.agent_run_id] = process
+ try:
+ await self._wait_switchyard_ready(port, process)
+ except Exception:
+ await self._teardown_switchyard(params.agent_run_id)
+ raise
+ return base_url
+
+ async def _wait_switchyard_ready(self, port: int, process: Process, timeout_s: float = 60.0) -> None:
+ """Wait until the spawned Switchyard's port accepts TCP connections.
+
+ Uses a lightweight TCP socket probe instead of HTTP polling — no aiohttp
+ overhead, no connection pool pressure. With 1024 concurrent spawns on one
+ node, HTTP polling caused ClientOSError and uvicorn connection saturation.
+ """
+ deadline = time.monotonic() + timeout_s
+ while time.monotonic() < deadline:
+ if process.returncode is not None:
+ raise RuntimeError(
+ f"spawned Switchyard exited with code {process.returncode} before ready "
+ "(see the run's switchyard.log)"
+ )
+ try:
+ reader, writer = await asyncio.wait_for(
+ asyncio.open_connection("127.0.0.1", port), timeout=1.0
+ )
+ writer.close()
+ await writer.wait_closed()
+ return
+ except (ConnectionRefusedError, OSError, asyncio.TimeoutError):
+ pass
+ await asyncio.sleep(0.2)
+ raise RuntimeError(f"spawned Switchyard not ready after {timeout_s}s")
+
+ async def _teardown_switchyard(self, agent_run_id: str) -> None:
+ """Reap the run's spawned Switchyard instance, if any. Idempotent."""
+ process = self._switchyard_procs.pop(agent_run_id, None)
+ if process is None or process.returncode is not None:
+ return
+ process.terminate()
+ try:
+ await asyncio.wait_for(process.wait(), timeout=10)
+ except asyncio.TimeoutError:
+ process.kill()
+ await process.wait()
+
+ async def _fetch_switchyard_max_model_len(self, base_url: str, route_id: str) -> Optional[int]:
+ """The served model's context length from Switchyard's ``/v1/models``, or ``None``.
+
+ Switchyard surfaces the engine's ``max_model_len`` on token-capture
+ routes. Absence (older Switchyard, engine unreachable at proxy start)
+ degrades gracefully: the opencode provider entry ships without limits,
+ exactly today's behavior.
+ """
+ try:
+ response = await request(
+ "GET", f"{base_url.rstrip('/')}/v1/models", timeout=ClientTimeout(total=30)
+ )
+ await raise_for_status(response)
+ entries = (await get_response_json(response)).get("data") or []
+ except Exception as e:
+ print(f"WARNING: could not fetch max_model_len from Switchyard: {e}", flush=True)
+ return None
+ for entry in entries:
+ if isinstance(entry, dict) and entry.get("id") == route_id:
+ value = entry.get("max_model_len")
+ if isinstance(value, int) and not isinstance(value, bool):
+ return value
+ print(f"WARNING: Switchyard /v1/models has no max_model_len for route {route_id!r}", flush=True)
+ return None
+
+ async def _list_switchyard_sessions(self, base_url: str) -> List[Dict[str, Any]]:
+ """Session ids (with parent links) Switchyard captured for this run.
+
+ Harnesses that mint their own session ids (e.g. opencode), so Gym
+ discovers them from Switchyard rather than any harness log. Returns
+ ``[{"session_id", "parent_session_id"}, ...]``. Requires a per-run
+ ``--rl-log-dir`` so the list is scoped to this rollout.
+ """
+ url = f"{base_url.rstrip('/')}/v1/sessions"
+ response = await request("GET", url, timeout=ClientTimeout(total=60))
+ await raise_for_status(response)
+ envelope = await get_response_json(response)
+ return envelope.get("sessions", [])
+
+ async def _retrieve_switchyard_trace(
+ self, base_url: str, session_id: str, filter_title_gen: bool = False
+ ) -> SwitchyardTrace:
+ """Fetch one session's captured completions from Switchyard and rebuild the rollout.
+
+ Harness-neutral: keyed only on ``(base_url, session_id)`` so both the
+ single-session (OpenHands) and per-subagent (OpenCode) paths reuse it.
No polling is needed: the agent has exited and Switchyard durably writes
each record before returning that call's model response. The GET is
idempotent, so the helper's transport retries are safe.
"""
- url = (
- f"{instance_config.switchyard_base_url.rstrip('/')}"
- f"/v1/sessions/{instance_config.switchyard_session_id}/completions"
- )
+ url = f"{base_url.rstrip('/')}/v1/sessions/{session_id}/completions"
response = await request("GET", url, timeout=ClientTimeout(total=60))
await raise_for_status(response)
envelope = await get_response_json(response)
- return reconstruct_switchyard_rollout(envelope, instance_config.switchyard_session_id, self._vllm_converter)
+ if filter_title_gen:
+ envelope = _filter_title_gen_records(envelope)
+ return reconstruct_switchyard_rollout(envelope, session_id, self._vllm_converter)
+
+ @staticmethod
+ def _apply_switchyard_trace(
+ responses_create_params: dict,
+ response: NeMoGymResponse,
+ trace: SwitchyardTrace,
+ session_id: str,
+ ) -> None:
+ """Replace the rollout's input/tools/output with a reconstructed Switchyard trace.
+
+ Switchyard is the source of truth for the token-annotated messages; the
+ harness supplies only the outcome/reward.
+ """
+ responses_create_params["input"] = [item.model_dump() for item in trace.input_items]
+ responses_create_params["tools"] = [tool.model_dump() for tool in trace.tools]
+ response.output = trace.output_items
+ response.metadata = {
+ "switchyard_source": "switchyard",
+ "switchyard_session_id": session_id,
+ "switchyard_record_uuids": json.dumps(trace.record_uuids),
+ "switchyard_model": trace.model,
+ }
+
+ async def _reconstruct_switchyard_sessions(
+ self, base_url: str, sessions: List[Dict[str, Any]], filter_title_gen: bool = False
+ ) -> Tuple[SwitchyardTrace, str, List[Dict[str, Any]]]:
+ """Retrieve + reconstruct every session in an OpenCode session tree.
+
+ ``sessions`` is ``[{"session_id", "parent_session_id"}, ...]`` — OpenCode
+ mints a distinct session id per (sub)agent and Switchyard captures each
+ separately, so each is an independent linear chain reconstructed by
+ Layer 1. Returns ``(root_trace, root_session_id, subagent_entries)``: the
+ parent-less root becomes the main rollout; each subagent becomes a
+ token-annotated entry tagged with its parent. Fail-closed — any missing
+ root, ambiguous tree, or reconstruction failure raises so the caller masks
+ the whole sample rather than emit partial training data.
+ """
+ roots = [s for s in sessions if not s.get("parent_session_id")]
+ if len(roots) != 1:
+ raise SwitchyardTraceError(f"expected exactly one root session, got {len(roots)}")
+ root_id = str(roots[0]["session_id"])
+ root_trace = await self._retrieve_switchyard_trace(base_url, root_id, filter_title_gen=filter_title_gen)
+
+ subagents: List[Dict[str, Any]] = []
+ for entry in sessions:
+ if not entry.get("parent_session_id"):
+ continue
+ session_id = str(entry["session_id"])
+ trace = await self._retrieve_switchyard_trace(base_url, session_id, filter_title_gen=filter_title_gen)
+ subagents.append(
+ {
+ "session_id": session_id,
+ "parent_session_id": entry["parent_session_id"],
+ "input": [item.model_dump() for item in trace.input_items],
+ "output": [item.model_dump() for item in trace.output_items],
+ "model": trace.model,
+ "record_uuids": trace.record_uuids,
+ }
+ )
+ return root_trace, root_id, subagents
if __name__ == "__main__":
diff --git a/responses_api_agents/swe_agents/configs/swebench_nv_opencode.yaml b/responses_api_agents/swe_agents/configs/swebench_nv_opencode.yaml
new file mode 100644
index 0000000000..de3eca2dc5
--- /dev/null
+++ b/responses_api_agents/swe_agents/configs/swebench_nv_opencode.yaml
@@ -0,0 +1,69 @@
+# SWE-bench wrapper configuration for the NeMo-Gym opencode fork (nv-opencode).
+swe_agents:
+ responses_api_agents:
+ swe_agents: &swe_agents_config
+ entrypoint: app.py
+ domain: coding
+ description: SWE-bench driven by the opencode agent framework.
+ value: Eval software engineering capabilities on SWE-bench using opencode.
+
+ # Agent framework configuration
+ agent_framework: opencode
+ opencode_source: nv-opencode # pinned NeMo-Gym fork (bench entry point)
+ agent_max_turns: 100
+ # Pinned NeMo-Gym opencode fork (adds bench/cli.ts + the nemo-gym LanguageModelV3 provider).
+ agent_framework_repo: https://github.com/sdevare-nv/nv-opencode.git
+ agent_framework_commit: sdd/dev
+
+ # Container configuration (same SIFs as the openhands path)
+ container_formatter: ???
+ container_folder_path: null
+ swebench_agent_timeout: 1800
+ swebench_tests_timeout: 900
+ apptainer_memory_limit_mb: 32768
+ command_exec_timeout: 300
+ opencode_subagents_enabled: true
+
+ dataset_path: ???
+
+ # Optional model server reference
+ model_server:
+ name: policy_model
+ type: responses_api_models
+
+ datasets:
+ # Training dataset
+ - name: train
+ type: train
+ jsonl_fpath: responses_api_agents/swe_agents/data/swegym_for_sweagent_and_openhands.jsonl
+ gitlab_identifier:
+ dataset_name: swegym_for_sweagent_and_openhands
+ version: 0.0.2
+ artifact_fpath: swegym-converted.jsonl
+ license: Apache 2.0
+ # Validation dataset
+ - name: validation
+ type: validation
+ jsonl_fpath: responses_api_agents/swe_agents/data/swebench_verified_for_sweagent_and_openhands.jsonl
+ gitlab_identifier:
+ dataset_name: swebench_verified_for_sweagent_and_openhands
+ version: 0.0.1
+ artifact_fpath: swebench_verified_for_sweagent_and_openhands.jsonl
+ license: TBD
+ # Example dataset for quick testing
+ - name: example
+ type: example
+ jsonl_fpath: responses_api_agents/swe_agents/data/example.jsonl
+
+# Alias copies so input rows whose `agent_ref.name` is `swe_agents_val` or
+# `swe_agents_train` (baked in by upstream dataset prep, matching the keys
+# `swebench_multi_tools.yaml` exposes) resolve against this same config.
+swe_agents_val:
+ responses_api_agents:
+ swe_agents:
+ <<: *swe_agents_config
+
+swe_agents_train:
+ responses_api_agents:
+ swe_agents:
+ <<: *swe_agents_config
diff --git a/responses_api_agents/swe_agents/configs/swebench_nv_opencode_training.yaml b/responses_api_agents/swe_agents/configs/swebench_nv_opencode_training.yaml
new file mode 100644
index 0000000000..7e86aa1de4
--- /dev/null
+++ b/responses_api_agents/swe_agents/configs/swebench_nv_opencode_training.yaml
@@ -0,0 +1,57 @@
+# SWE-bench wrapper configuration for the NeMo-Gym opencode fork (nv-opencode), RL training.
+
+swe_agents_train:
+ responses_api_agents:
+ swe_agents: &swe_agents_config
+ entrypoint: app.py
+ domain: coding
+ description: SWE-bench driven by the opencode agent framework for RL training.
+ value: Train software engineering capabilities on SWE tasks using opencode rollouts.
+
+ # Agent framework configuration
+ agent_framework: opencode
+ opencode_source: nv-opencode # pinned NeMo-Gym fork (bench entry point)
+ agent_max_turns: 100
+ # Pinned NeMo-Gym opencode fork (adds bench/cli.ts + the nemo-gym LanguageModelV3 provider).
+ agent_framework_repo: https://github.com/sdevare-nv/nv-opencode.git
+ agent_framework_commit: sdd/dev
+
+ # Container configuration (same SIFs as the openhands path)
+ container_formatter: ???
+ container_folder_path: null
+ swebench_agent_timeout: 1800
+ swebench_tests_timeout: 900
+ apptainer_memory_limit_mb: 65536
+ command_exec_timeout: 300
+
+ dataset_path: ???
+
+ agent_prompt_overrides:
+ - user_prompt_template: responses_api_agents/swe_agents/prompts/opencode_harness/user_prompt.txt
+ agent_cls: OpenCodeAgent
+ diversify_tool_names: false
+
+ # Enable opencode's `task` tool (subagent sessions)
+ opencode_subagents_enabled: true
+
+ model_server:
+ name: policy_model
+ type: responses_api_models
+
+ datasets:
+ - name: train
+ type: train
+ jsonl_fpath: responses_api_agents/swe_agents/data/swegym_for_sweagent_and_openhands.jsonl
+ gitlab_identifier:
+ dataset_name: swegym_for_sweagent_and_openhands
+ version: 0.0.2
+ artifact_fpath: swegym-converted.jsonl
+ license: Apache 2.0
+ - name: example
+ type: example
+ jsonl_fpath: responses_api_agents/swe_agents/data/example.jsonl
+
+swe_agents_val:
+ responses_api_agents:
+ swe_agents:
+ <<: *swe_agents_config
diff --git a/responses_api_agents/swe_agents/configs/swebench_opencode.yaml b/responses_api_agents/swe_agents/configs/swebench_opencode.yaml
index c4298fb898..e006a81ccc 100644
--- a/responses_api_agents/swe_agents/configs/swebench_opencode.yaml
+++ b/responses_api_agents/swe_agents/configs/swebench_opencode.yaml
@@ -1,18 +1,19 @@
-# SWE-bench wrapper configuration for opencode
+# SWE-bench wrapper configuration for upstream opencode.
+# Runs `opencode run` headless with an opencode.json provider pointed at Switchyard
+# (no fork, no harness modification). Supply the Switchyard endpoint at run time via
+# `switchyard_base_url` or `switchyard_spawn_routing_profile` — token capture needs it.
swe_agents:
responses_api_agents:
swe_agents: &swe_agents_config
entrypoint: app.py
domain: coding
- description: SWE-bench driven by the opencode agent framework.
- value: Eval software engineering capabilities on SWE-bench using opencode.
+ description: SWE-bench driven by upstream opencode.
+ value: Eval software engineering capabilities on SWE-bench using upstream opencode.
# Agent framework configuration
agent_framework: opencode
+ opencode_source: opencode # upstream opencode-ai (installed via bun); no fork
agent_max_turns: 100
- # Pinned NeMo-Gym opencode fork (adds bench/cli.ts + the nemo-gym LanguageModelV3 provider).
- agent_framework_repo: https://github.com/sdevare-nv/nv-opencode.git
- agent_framework_commit: sdd/dev
# Container configuration (same SIFs as the openhands path)
container_formatter: ???
diff --git a/responses_api_agents/swe_agents/configs/swebench_opencode_training.yaml b/responses_api_agents/swe_agents/configs/swebench_opencode_training.yaml
index d724f97b6a..12d57cfd69 100644
--- a/responses_api_agents/swe_agents/configs/swebench_opencode_training.yaml
+++ b/responses_api_agents/swe_agents/configs/swebench_opencode_training.yaml
@@ -1,19 +1,20 @@
-# SWE-bench wrapper configuration for opencode RL training.
+# SWE-bench wrapper configuration for upstream opencode RL training.
+# Runs `opencode run` headless with an opencode.json provider pointed at Switchyard
+# (no fork, no harness modification). Supply the Switchyard endpoint at run time via
+# `switchyard_base_url` or `switchyard_spawn_routing_profile` — token capture needs it.
swe_agents_train:
responses_api_agents:
swe_agents: &swe_agents_config
entrypoint: app.py
domain: coding
- description: SWE-bench driven by the opencode agent framework for RL training.
- value: Train software engineering capabilities on SWE tasks using opencode rollouts.
+ description: SWE-bench driven by upstream opencode for RL training.
+ value: Train software engineering capabilities on SWE tasks using upstream opencode rollouts.
# Agent framework configuration
agent_framework: opencode
+ opencode_source: opencode # upstream opencode-ai (installed via bun); no fork
agent_max_turns: 100
- # Pinned NeMo-Gym opencode fork (adds bench/cli.ts + the nemo-gym LanguageModelV3 provider).
- agent_framework_repo: https://github.com/sdevare-nv/nv-opencode.git
- agent_framework_commit: sdd/dev
# Container configuration (same SIFs as the openhands path)
container_formatter: ???
diff --git a/responses_api_agents/swe_agents/tests/test_app.py b/responses_api_agents/swe_agents/tests/test_app.py
index fd39f56b5d..a4c03a2827 100644
--- a/responses_api_agents/swe_agents/tests/test_app.py
+++ b/responses_api_agents/swe_agents/tests/test_app.py
@@ -56,6 +56,7 @@
SWEBenchWrapperServerConfig,
SWERebenchDatasetProcessor,
_extract_instance_dict,
+ _filter_title_gen_records,
_parse_switchyard_base_url,
_render_opencode_user_message,
_resolve_opencode_workspace_path,
@@ -1105,6 +1106,7 @@ def _opencode_config(self, tmpdir, **overrides) -> SWEBenchWrapperInstanceConfig
return _make_instance_config(
tmpdir,
agent_framework="opencode",
+ opencode_source="nv-opencode", # fork run-command path (default is now upstream 'opencode')
opencode_setup_dir=opencode_setup_dir,
agent_framework_repo="https://example.invalid/opencode.git",
agent_framework_commit="deadbeef",
@@ -2428,6 +2430,124 @@ def test_setup_params_no_switchyard_for_golden_patch(self, monkeypatch) -> None:
assert params.switchyard_session_id is None
assert "proxy_x_session_id" not in params.agent_script
+ def test_validate_switchyard_config_spawn_conflicts(self, tmp_path: Path) -> None:
+ profile = tmp_path / "profile.yaml"
+ profile.write_text("routes: {}\n")
+ config = _minimal_server_config()
+
+ config.switchyard_spawn_routing_profile = str(profile)
+ swe_app._validate_switchyard_config(config) # valid: spawn only
+
+ config.switchyard_base_url = "http://switchyard:4000"
+ with pytest.raises(ValueError, match="mutually exclusive"):
+ swe_app._validate_switchyard_config(config)
+
+ config.switchyard_base_url = None
+ config.switchyard_spawn_routing_profile = str(tmp_path / "missing.yaml")
+ with pytest.raises(ValueError, match="does not exist"):
+ swe_app._validate_switchyard_config(config)
+
+ def test_setup_params_spawn_mode_mints_session_and_rebuild(self, monkeypatch, tmp_path: Path) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ (Path(tmpdir) / "django__django-12345.sif").touch()
+ wrapper.config.container_formatter = [str(Path(tmpdir) / "{instance_id}.sif")]
+ self._setup_oh_dirs(wrapper)
+ profile = tmp_path / "profile.yaml"
+ profile.write_text("routes: {}\n")
+ wrapper.config.switchyard_spawn_routing_profile = str(profile)
+ wrapper._swe_bench_wrapper_server_config.ng_global_config_dict_str = _ng_config_dict_str()
+
+ params, _ = wrapper._setup_params(self._switchyard_body())
+ # A session is minted even though the instance URL is not known yet.
+ assert params.switchyard_session_id is not None
+ assert params.switchyard_base_url is None
+ assert wrapper._switchyard_spawn_needed(params)
+
+ # Spawn mode assigns the URL after setup and rebuilds the script.
+ params.switchyard_base_url = "http://spawnhost:12345"
+ wrapper._build_agent_command(params)
+ assert "host: spawnhost" in params.agent_script
+ assert f'proxy_x_session_id = "{params.switchyard_session_id}"' in params.agent_script
+
+ def test_spawn_not_needed_for_golden_patch(self, monkeypatch, tmp_path: Path) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ (Path(tmpdir) / "django__django-12345.sif").touch()
+ wrapper.config.container_formatter = [str(Path(tmpdir) / "{instance_id}.sif")]
+ self._setup_oh_dirs(wrapper)
+ profile = tmp_path / "profile.yaml"
+ profile.write_text("routes: {}\n")
+ wrapper.config.switchyard_spawn_routing_profile = str(profile)
+ wrapper.config.verify_golden_patch = True
+ wrapper._swe_bench_wrapper_server_config.ng_global_config_dict_str = _ng_config_dict_str()
+
+ params, _ = wrapper._setup_params(self._switchyard_body())
+ assert not wrapper._switchyard_spawn_needed(params)
+
+
+class _FakeSwitchyardProc:
+ def __init__(self, returncode=None):
+ self.returncode = returncode
+ self.terminated = False
+
+ def terminate(self):
+ self.terminated = True
+ self.returncode = 0
+
+ def kill(self):
+ self.returncode = -9
+
+ async def wait(self):
+ return self.returncode
+
+
+class TestSwitchyardSpawnLifecycle:
+ def _spawn_wrapper(self, monkeypatch, tmp_path: Path) -> SWEBenchWrapper:
+ wrapper = _create_wrapper(monkeypatch)
+ profile = tmp_path / "profile.yaml"
+ profile.write_text("routes: {}\n")
+ wrapper.config.switchyard_spawn_routing_profile = str(profile)
+ wrapper.config.switchyard_spawn_host = "spawnhost"
+ return wrapper
+
+ @pytest.mark.asyncio
+ async def test_spawn_and_teardown(self, monkeypatch, tmp_path: Path) -> None:
+ wrapper = self._spawn_wrapper(monkeypatch, tmp_path)
+ params = _make_instance_config(str(tmp_path), agent_run_id="run-1")
+ proc = _FakeSwitchyardProc()
+ spawn_mock = AsyncMock(return_value=proc)
+ monkeypatch.setattr(swe_app.asyncio, "create_subprocess_exec", spawn_mock)
+ monkeypatch.setattr(swe_app, "request", AsyncMock(return_value=MagicMock(status=200)))
+
+ base_url = await wrapper._spawn_switchyard(params)
+
+ assert base_url.startswith("http://spawnhost:")
+ assert wrapper._switchyard_procs["run-1"] is proc
+ argv = spawn_mock.call_args.args
+ assert argv[0] == "switchyard"
+ assert "--enable-rl-logging" in argv
+ assert str(tmp_path / "persistent" / "switchyard_traces") in argv
+ assert (tmp_path / "persistent" / "switchyard_traces").is_dir()
+
+ await wrapper._teardown_switchyard("run-1")
+ assert proc.terminated
+ assert wrapper._switchyard_procs == {}
+ # Idempotent.
+ await wrapper._teardown_switchyard("run-1")
+
+ @pytest.mark.asyncio
+ async def test_spawn_readiness_failure_reaps(self, monkeypatch, tmp_path: Path) -> None:
+ wrapper = self._spawn_wrapper(monkeypatch, tmp_path)
+ params = _make_instance_config(str(tmp_path), agent_run_id="run-2")
+ proc = _FakeSwitchyardProc(returncode=1) # exits before ready
+ monkeypatch.setattr(swe_app.asyncio, "create_subprocess_exec", AsyncMock(return_value=proc))
+ monkeypatch.setattr(swe_app, "request", AsyncMock(side_effect=ConnectionError))
+
+ with pytest.raises(RuntimeError, match="exited with code 1"):
+ await wrapper._spawn_switchyard(params)
+ assert wrapper._switchyard_procs == {}
+
class TestSWEBenchWrapperResponses:
def _setup_oh_dirs(self, wrapper):
@@ -2765,7 +2885,9 @@ async def test_retrieve_switchyard_trace_url(self, monkeypatch) -> None:
reconstruct_mock = MagicMock(return_value="trace")
monkeypatch.setattr(swe_app, "reconstruct_switchyard_rollout", reconstruct_mock)
- result = await wrapper._retrieve_switchyard_trace(instance_config)
+ result = await wrapper._retrieve_switchyard_trace(
+ instance_config.switchyard_base_url, instance_config.switchyard_session_id
+ )
assert result == "trace"
args, kwargs = request_mock.await_args
@@ -2773,6 +2895,137 @@ async def test_retrieve_switchyard_trace_url(self, monkeypatch) -> None:
assert kwargs["timeout"].total == 60
reconstruct_mock.assert_called_once_with(envelope, "0123456789abcdef0123456789abcdef", wrapper._vllm_converter)
+ async def test_list_switchyard_sessions_url_and_parse(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ envelope = {
+ "schema_version": 1,
+ "sessions": [
+ {"session_id": "root", "parent_session_id": None},
+ {"session_id": "sub", "parent_session_id": "root"},
+ ],
+ }
+ request_mock = AsyncMock(return_value=MagicMock())
+ monkeypatch.setattr(swe_app, "request", request_mock)
+ monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock())
+ monkeypatch.setattr(swe_app, "get_response_json", AsyncMock(return_value=envelope))
+
+ result = await wrapper._list_switchyard_sessions("http://switchyard:4000/")
+
+ assert result == envelope["sessions"]
+ args, _ = request_mock.await_args
+ assert args == ("GET", "http://switchyard:4000/v1/sessions")
+
+ @staticmethod
+ def _fake_trace(model: str) -> SwitchyardTrace:
+ item = MagicMock()
+ item.model_dump.return_value = {"model": model}
+ return SwitchyardTrace(input_items=[item], output_items=[item], tools=[], record_uuids=["u"], model=model)
+
+ async def test_reconstruct_switchyard_sessions_tree(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ # trace.model echoes the session id so we can assert per-session retrieval.
+ monkeypatch.setattr(
+ wrapper,
+ "_retrieve_switchyard_trace",
+ AsyncMock(side_effect=lambda base, sid, **kw: self._fake_trace(sid)),
+ )
+ sessions = [
+ {"session_id": "root", "parent_session_id": None},
+ {"session_id": "sub1", "parent_session_id": "root"},
+ {"session_id": "sub2", "parent_session_id": "root"},
+ ]
+ root_trace, root_id, subs = await wrapper._reconstruct_switchyard_sessions("http://sw", sessions)
+
+ assert root_id == "root" and root_trace.model == "root"
+ assert [s["session_id"] for s in subs] == ["sub1", "sub2"]
+ assert all(s["parent_session_id"] == "root" for s in subs)
+ assert subs[0]["model"] == "sub1" and subs[0]["output"] == [{"model": "sub1"}]
+
+ async def test_reconstruct_switchyard_sessions_single_root(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ monkeypatch.setattr(
+ wrapper,
+ "_retrieve_switchyard_trace",
+ AsyncMock(side_effect=lambda base, sid, **kw: self._fake_trace(sid)),
+ )
+ root_trace, root_id, subs = await wrapper._reconstruct_switchyard_sessions(
+ "http://sw", [{"session_id": "only", "parent_session_id": None}]
+ )
+ assert root_id == "only" and subs == []
+
+ async def test_reconstruct_switchyard_sessions_requires_single_root(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ monkeypatch.setattr(wrapper, "_retrieve_switchyard_trace", AsyncMock())
+ with pytest.raises(SwitchyardTraceError):
+ await wrapper._reconstruct_switchyard_sessions(
+ "http://sw",
+ [
+ {"session_id": "a", "parent_session_id": None},
+ {"session_id": "b", "parent_session_id": None},
+ ],
+ )
+
+
+def test_opencode_switchyard_config_points_at_switchyard() -> None:
+ cfg = swe_app._opencode_switchyard_config("http://switchyard:4000/", "policy-model")
+ assert cfg["model"] == "switchyard/policy-model"
+ provider_id = next(iter(cfg["provider"]))
+ # Load-bearing: a non-"opencode" provider id is what makes upstream opencode
+ # emit its native X-Session-Id correlation headers.
+ assert not provider_id.startswith("opencode")
+ provider = cfg["provider"][provider_id]
+ assert provider["npm"] == "@ai-sdk/openai-compatible"
+ assert provider["options"]["baseURL"] == "http://switchyard:4000/v1"
+ assert "policy-model" in provider["models"]
+ # Task-tool guidance is appended to opencode's system prompt via instructions.
+ assert cfg["instructions"] == [swe_app._OPENCODE_INSTRUCTIONS_PATH]
+ # Without a known context length, the model entry ships no limit metadata.
+ assert "limit" not in provider["models"]["policy-model"]
+
+
+def test_opencode_switchyard_config_sets_model_limits() -> None:
+ cfg = swe_app._opencode_switchyard_config(
+ "http://switchyard:4000", "policy-model", context_len=32768
+ )
+ entry = cfg["provider"]["switchyard"]["models"]["policy-model"]
+ # context from the engine; output reserved as min(32000, context // 4) so
+ # opencode's compaction threshold (context - output) keeps most of the window.
+ assert entry["limit"] == {"context": 32768, "output": 8192}
+
+ large = swe_app._opencode_switchyard_config(
+ "http://switchyard:4000", "policy-model", context_len=131072
+ )
+ assert large["provider"]["switchyard"]["models"]["policy-model"]["limit"]["output"] == 32000
+
+
+class TestFetchSwitchyardMaxModelLen:
+ def _wrapper(self, monkeypatch) -> SWEBenchWrapper:
+ return _create_wrapper(monkeypatch)
+
+ @pytest.mark.asyncio
+ async def test_reads_route_entry(self, monkeypatch) -> None:
+ wrapper = self._wrapper(monkeypatch)
+ payload = {"data": [{"id": "other", "max_model_len": 1}, {"id": "policy-model", "max_model_len": 32768}]}
+ response = MagicMock(status=200)
+ monkeypatch.setattr(swe_app, "request", AsyncMock(return_value=response))
+ monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock())
+ monkeypatch.setattr(swe_app, "get_response_json", AsyncMock(return_value=payload))
+ assert await wrapper._fetch_switchyard_max_model_len("http://sy:4000", "policy-model") == 32768
+
+ @pytest.mark.asyncio
+ async def test_missing_route_returns_none(self, monkeypatch) -> None:
+ wrapper = self._wrapper(monkeypatch)
+ monkeypatch.setattr(swe_app, "request", AsyncMock(return_value=MagicMock(status=200)))
+ monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock())
+ monkeypatch.setattr(swe_app, "get_response_json", AsyncMock(return_value={"data": []}))
+ assert await wrapper._fetch_switchyard_max_model_len("http://sy:4000", "policy-model") is None
+
+ @pytest.mark.asyncio
+ async def test_fetch_failure_returns_none(self, monkeypatch) -> None:
+ wrapper = self._wrapper(monkeypatch)
+ monkeypatch.setattr(swe_app, "request", AsyncMock(side_effect=ConnectionError))
+ assert await wrapper._fetch_switchyard_max_model_len("http://sy:4000", "policy-model") is None
+
########################################
# _load_rebench_log_parsers tests
@@ -2804,3 +3057,471 @@ def test_loads_from_lib_agent_dir(self) -> None:
mod = _load_rebench_log_parsers(rebench_dir)
assert "lib_test" in mod.NAME_TO_PARSER
+
+
+########################################
+# _filter_title_gen_records tests
+########################################
+
+
+class TestFilterTitleGenRecords:
+ def _record(self, system_content: str = "", user_content: str = "fix it") -> dict:
+ # Matches the Switchyard token_capture_response_processor record shape:
+ # messages = request_msgs + assistant_turn (flat list, no "request" wrapper)
+ msgs = []
+ if system_content:
+ msgs.append({"role": "system", "content": system_content})
+ msgs.append({"role": "user", "content": user_content})
+ msgs.append({"role": "assistant", "content": "ok"})
+ return {"messages": msgs}
+
+ def test_strips_title_gen_record(self) -> None:
+ envelope = {
+ "completions": [
+ self._record("You are a title generator"),
+ self._record("You are a coding assistant"),
+ ]
+ }
+ result = _filter_title_gen_records(envelope)
+ assert len(result["completions"]) == 1
+ assert result["completions"][0]["messages"][0]["content"] == "You are a coding assistant"
+
+ def test_case_insensitive(self) -> None:
+ envelope = {"completions": [self._record("YOU ARE A TITLE GENERATOR")]}
+ assert _filter_title_gen_records(envelope)["completions"] == []
+
+ def test_preserves_non_title_gen_records(self) -> None:
+ envelope = {"completions": [self._record("You are a helpful assistant"), self._record("", "task")]}
+ assert len(_filter_title_gen_records(envelope)["completions"]) == 2
+
+ def test_empty_completions(self) -> None:
+ assert _filter_title_gen_records({"completions": []}) == {"completions": []}
+
+ def test_missing_completions_key(self) -> None:
+ assert _filter_title_gen_records({})["completions"] == []
+
+ def test_other_envelope_keys_preserved(self) -> None:
+ envelope = {"schema_version": 1, "completions": [self._record("You are a title generator")]}
+ result = _filter_title_gen_records(envelope)
+ assert result["schema_version"] == 1
+ assert result["completions"] == []
+
+
+########################################
+# Upstream opencode setup + run-command tests
+########################################
+
+
+class TestUpstreamOpenCodeSetup:
+ def _upstream_config(self, tmpdir, **overrides) -> SWEBenchWrapperInstanceConfig:
+ opencode_setup_dir = Path(tmpdir) / "upstream_opencode_setup"
+ opencode_setup_dir.mkdir(parents=True, exist_ok=True)
+ return _make_instance_config(
+ tmpdir,
+ agent_framework="opencode",
+ opencode_source="opencode",
+ opencode_setup_dir=opencode_setup_dir,
+ switchyard_base_url="http://switchyard:4000",
+ **overrides,
+ )
+
+ def test_setup_upstream_skips_if_already_installed(self, monkeypatch) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ setup_dir = Path(tmpdir) / "parent" / "swe_upstream_opencode_setup"
+ bun_bin = setup_dir / "bun" / "bin" / "bun"
+ opencode_bin = setup_dir / "opencode" / "node_modules" / ".bin" / "opencode"
+ bun_bin.parent.mkdir(parents=True, exist_ok=True)
+ opencode_bin.parent.mkdir(parents=True, exist_ok=True)
+ bun_bin.touch()
+ opencode_bin.touch()
+
+ run_cmd = MagicMock()
+ monkeypatch.setattr(BaseDatasetHarnessProcessor, "_run_setup_command", run_cmd)
+
+ with patch.object(
+ BaseDatasetHarnessProcessor,
+ "parent_dir",
+ new_callable=lambda: property(lambda self: Path(tmpdir) / "parent"),
+ ):
+ processor = OpenCodeHarnessProcessor(config=config)
+ result = processor._setup_upstream()
+
+ run_cmd.assert_not_called()
+ assert result == Path(tmpdir) / "parent" / "swe_upstream_opencode_setup"
+
+ def test_setup_upstream_runs_install_commands(self, monkeypatch) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ run_cmd = MagicMock()
+ monkeypatch.setattr(BaseDatasetHarnessProcessor, "_run_setup_command", run_cmd)
+
+ with patch.object(
+ BaseDatasetHarnessProcessor,
+ "parent_dir",
+ new_callable=lambda: property(lambda self: Path(tmpdir) / "parent"),
+ ):
+ (Path(tmpdir) / "parent" / "swe_upstream_opencode_setup" / "opencode").mkdir(
+ parents=True, exist_ok=True
+ )
+ processor = OpenCodeHarnessProcessor(config=config)
+ processor._setup_upstream()
+
+ assert run_cmd.call_count == 2
+ calls = [c.args[0] for c in run_cmd.call_args_list]
+ assert any("bun.sh/install" in c for c in calls)
+ assert any("bun add opencode-ai" in c for c in calls)
+
+ def test_setup_dispatches_to_upstream_for_opencode_source(self, monkeypatch) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ upstream_mock = MagicMock(return_value=Path(tmpdir) / "result")
+ monkeypatch.setattr(OpenCodeHarnessProcessor, "_setup_upstream", upstream_mock)
+ result = OpenCodeHarnessProcessor(config=config).setup()
+ upstream_mock.assert_called_once()
+ assert result == Path(tmpdir) / "result"
+
+
+class TestUpstreamOpenCodeRunCommand:
+ @pytest.fixture
+ def _stub_model_server(self, monkeypatch):
+ def _fake(_global, name):
+ return type("Cfg", (), {"host": "sw-host", "port": 4000, "model": "policy-model"})()
+
+ monkeypatch.setattr(swe_app, "get_first_server_config_dict", _fake)
+ monkeypatch.setattr(swe_app, "get_global_config_dict", MagicMock(return_value={}))
+
+ def _upstream_config(self, tmpdir, **overrides) -> SWEBenchWrapperInstanceConfig:
+ opencode_setup_dir = Path(tmpdir) / "upstream_opencode_setup"
+ opencode_setup_dir.mkdir(parents=True, exist_ok=True)
+ return _make_instance_config(
+ tmpdir,
+ agent_framework="opencode",
+ opencode_source="opencode",
+ opencode_setup_dir=opencode_setup_dir,
+ switchyard_base_url="http://switchyard:4000",
+ ng_global_config_dict_str=_ng_config_dict_str(),
+ **overrides,
+ )
+
+ def _read_agent_script(self, config) -> str:
+ return (config.persistent_dir / f"agent_script_{config.agent_run_id}.sh").read_text()
+
+ def test_get_run_command_dispatches_for_opencode_source(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ upstream_mock = MagicMock(
+ return_value=ExecuteContainerCommandArgs(
+ command="echo ok", expected_file_pattern="/tmp/*.jsonl", mode="agent", timeout=100
+ )
+ )
+ with patch.object(OpenCodeHarnessProcessor, "_get_upstream_run_command", upstream_mock):
+ result = OpenCodeHarnessProcessor(config=config).get_run_command()
+ upstream_mock.assert_called_once()
+ assert result.command == "echo ok"
+
+ def test_upstream_run_command_basic(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ result = OpenCodeHarnessProcessor(config=config)._get_upstream_run_command()
+ assert isinstance(result, ExecuteContainerCommandArgs)
+ assert result.mode == "agent"
+ assert "timeout" in result.command
+ assert "output.jsonl" in result.expected_file_pattern
+ assert str(config.opencode_setup_dir) in result.expected_file_pattern
+
+ def test_upstream_agent_script_contents(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ OpenCodeHarnessProcessor(config=config)._get_upstream_run_command()
+ script = self._read_agent_script(config)
+ assert "OPENCODE_DISABLE_MODELS_FETCH=1" in script
+ assert "SWITCHYARD_BASE_URL" in script
+ assert "switchyard:4000" in script
+ assert "opencode.json" in script
+ assert "opencode run" in script
+ assert "--model" in script
+ assert "switchyard/test-model" in script # body.model takes priority over server default
+ assert "_OC_ARGS" in script
+ assert "python3 -c" in script
+ assert "git_patch" in script
+
+ def test_upstream_writes_user_message_file(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ OpenCodeHarnessProcessor(config=config)._get_upstream_run_command()
+ user_msg_path = config.persistent_dir / f"user_message_{config.agent_run_id}.txt"
+ assert user_msg_path.exists()
+ assert "Fix bug" in user_msg_path.read_text()
+
+ def test_upstream_opencode_json_is_valid(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = self._upstream_config(tmpdir)
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ OpenCodeHarnessProcessor(config=config)._get_upstream_run_command()
+ script = self._read_agent_script(config)
+ # Verify opencode.json is written with key Switchyard provider fields.
+ assert "opencode.json" in script
+ assert '"model": "switchyard/test-model"' in script
+ assert '"autoupdate": false' in script
+ assert '"baseURL": "http://switchyard:4000/v1"' in script
+ assert '"npm": "@ai-sdk/openai-compatible"' in script
+ # The task-tool guidance file is written and referenced as an instruction.
+ assert swe_app._OPENCODE_INSTRUCTIONS_PATH in script
+ assert "do not set the `task_id` parameter" in script
+
+ def test_upstream_requires_switchyard_base_url(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = _make_instance_config(
+ tmpdir,
+ agent_framework="opencode",
+ opencode_source="opencode",
+ opencode_setup_dir=Path(tmpdir) / "setup",
+ switchyard_base_url=None,
+ ng_global_config_dict_str=_ng_config_dict_str(),
+ )
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ (Path(tmpdir) / "setup").mkdir(parents=True, exist_ok=True)
+ with pytest.raises(AssertionError, match="switchyard_base_url"):
+ OpenCodeHarnessProcessor(config=config)._get_upstream_run_command()
+
+ def test_upstream_requires_opencode_setup_dir(self, _stub_model_server) -> None:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ config = _make_instance_config(
+ tmpdir,
+ agent_framework="opencode",
+ opencode_source="opencode",
+ opencode_setup_dir=None,
+ switchyard_base_url="http://switchyard:4000",
+ ng_global_config_dict_str=_ng_config_dict_str(),
+ )
+ config.persistent_dir.mkdir(parents=True, exist_ok=True)
+ with pytest.raises(AssertionError, match="opencode setup directory"):
+ OpenCodeHarnessProcessor(config=config)._get_upstream_run_command()
+
+
+########################################
+# _retrieve_switchyard_trace filter_title_gen tests
+########################################
+
+
+class TestRetrieveSwitchyardTraceFilterTitleGen:
+ # Envelopes use the real Switchyard shape: completions list, messages flat on each record.
+ @pytest.mark.asyncio
+ async def test_filter_false_passes_envelope_unchanged(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ envelope = {"completions": [{"messages": [{"role": "system", "content": "You are a title generator"}]}]}
+ monkeypatch.setattr(swe_app, "request", AsyncMock(return_value=MagicMock()))
+ monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock())
+ monkeypatch.setattr(swe_app, "get_response_json", AsyncMock(return_value=envelope))
+ reconstruct = MagicMock(return_value="trace")
+ monkeypatch.setattr(swe_app, "reconstruct_switchyard_rollout", reconstruct)
+
+ await wrapper._retrieve_switchyard_trace("http://sw:4000", "ses_1", filter_title_gen=False)
+ assert len(reconstruct.call_args[0][0]["completions"]) == 1
+
+ @pytest.mark.asyncio
+ async def test_filter_true_strips_title_gen(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ envelope = {
+ "completions": [
+ {"messages": [{"role": "system", "content": "You are a title generator"}]},
+ {"messages": [{"role": "system", "content": "You are a coding assistant"}]},
+ ]
+ }
+ monkeypatch.setattr(swe_app, "request", AsyncMock(return_value=MagicMock()))
+ monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock())
+ monkeypatch.setattr(swe_app, "get_response_json", AsyncMock(return_value=envelope))
+ reconstruct = MagicMock(return_value="trace")
+ monkeypatch.setattr(swe_app, "reconstruct_switchyard_rollout", reconstruct)
+
+ await wrapper._retrieve_switchyard_trace("http://sw:4000", "ses_1", filter_title_gen=True)
+ called_env = reconstruct.call_args[0][0]
+ assert len(called_env["completions"]) == 1
+ assert called_env["completions"][0]["messages"][0]["content"] == "You are a coding assistant"
+
+
+########################################
+# _reconstruct_switchyard_sessions filter_title_gen threading
+########################################
+
+
+class TestReconstructFilterTitleGenThreading:
+ @staticmethod
+ def _fake_trace(sid: str):
+ item = MagicMock()
+ item.model_dump.return_value = {"sid": sid}
+ from nemo_gym.switchyard_trace import SwitchyardTrace
+
+ return SwitchyardTrace(input_items=[item], output_items=[item], tools=[], record_uuids=[], model=sid)
+
+ @pytest.mark.asyncio
+ async def test_filter_true_threaded_to_retrieve(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ retrieve = AsyncMock(side_effect=lambda base, sid, filter_title_gen=False: self._fake_trace(sid))
+ monkeypatch.setattr(wrapper, "_retrieve_switchyard_trace", retrieve)
+ sessions = [
+ {"session_id": "root", "parent_session_id": None},
+ {"session_id": "sub", "parent_session_id": "root"},
+ ]
+ await wrapper._reconstruct_switchyard_sessions("http://sw", sessions, filter_title_gen=True)
+ for call in retrieve.await_args_list:
+ assert call.kwargs.get("filter_title_gen") is True
+
+ @pytest.mark.asyncio
+ async def test_filter_defaults_false(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ retrieve = AsyncMock(side_effect=lambda base, sid, filter_title_gen=False: self._fake_trace(sid))
+ monkeypatch.setattr(wrapper, "_retrieve_switchyard_trace", retrieve)
+ sessions = [{"session_id": "only", "parent_session_id": None}]
+ await wrapper._reconstruct_switchyard_sessions("http://sw", sessions)
+ for call in retrieve.await_args_list:
+ assert call.kwargs.get("filter_title_gen") is False
+
+
+########################################
+# run() gate passes filter_title_gen=True
+########################################
+
+
+class TestRunOpencodeGateFilterTitleGen:
+ @staticmethod
+ def _opencode_response(**overrides) -> NeMoGymResponse:
+ return NeMoGymResponse(
+ id="swebench-test",
+ created_at=123,
+ model="test-model",
+ object="response",
+ output=[],
+ parallel_tool_calls=True,
+ tool_choice="auto",
+ tools=[],
+ metadata={
+ "input": "[]",
+ "metrics": json.dumps({"resolved": True}),
+ "instance_config": _make_instance_config(
+ tempfile.mkdtemp(),
+ agent_framework="opencode",
+ opencode_source="opencode",
+ switchyard_base_url="http://switchyard:4000",
+ **overrides,
+ ).model_dump_json(),
+ },
+ )
+
+ @staticmethod
+ def _run_body():
+ from nemo_gym.base_resources_server import BaseRunRequest
+
+ return BaseRunRequest(
+ responses_create_params=NeMoGymResponseCreateParamsNonStreaming(
+ model="test-model",
+ input=[],
+ metadata={
+ "problem_statement": "Fix",
+ "instance_id": "test-1",
+ "base_commit": "abc",
+ "dataset_name": "SWE-bench",
+ "split": "test",
+ "instance_dict": "{}",
+ },
+ )
+ )
+
+ @pytest.mark.asyncio
+ async def test_run_calls_reconstruct_with_filter_title_gen_true(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ from nemo_gym.switchyard_trace import SwitchyardTrace
+
+ fake_trace = SwitchyardTrace(input_items=[], output_items=[], tools=[], record_uuids=[], model="m")
+ sessions_mock = AsyncMock(return_value=[{"session_id": "root", "parent_session_id": None}])
+ reconstruct_mock = AsyncMock(return_value=(fake_trace, "root", []))
+ monkeypatch.setattr(wrapper, "_list_switchyard_sessions", sessions_mock)
+ monkeypatch.setattr(wrapper, "_reconstruct_switchyard_sessions", reconstruct_mock)
+
+ with patch.object(
+ SWEBenchWrapper, "responses", new_callable=AsyncMock, return_value=self._opencode_response()
+ ):
+ await wrapper.run(self._run_body())
+
+ reconstruct_mock.assert_awaited_once()
+ assert reconstruct_mock.call_args.kwargs.get("filter_title_gen") is True
+
+ @pytest.mark.asyncio
+ async def test_run_masks_on_reconstruct_failure(self, monkeypatch) -> None:
+ from nemo_gym.switchyard_trace import SwitchyardTraceError
+
+ wrapper = _create_wrapper(monkeypatch)
+ monkeypatch.setattr(wrapper, "_list_switchyard_sessions", AsyncMock(return_value=[]))
+ monkeypatch.setattr(
+ wrapper,
+ "_reconstruct_switchyard_sessions",
+ AsyncMock(side_effect=SwitchyardTraceError("no root")),
+ )
+ with patch.object(
+ SWEBenchWrapper, "responses", new_callable=AsyncMock, return_value=self._opencode_response()
+ ):
+ result = await wrapper.run(self._run_body())
+ assert result.mask_sample is True
+ assert "opencode sessions" in result.switchyard_trace_error
+
+
+########################################
+# _build_apptainer_command upstream mount tests
+########################################
+
+
+class TestBuildApptainerCommandUpstreamOpencode:
+ def _setup_upstream_dirs(self, params: SWEBenchWrapperInstanceConfig) -> None:
+ opencode_dir = Path(str(params.opencode_setup_dir)) / "opencode"
+ bun_dir = Path(str(params.opencode_setup_dir)) / "bun"
+ (opencode_dir / "evaluation" / "oh").mkdir(parents=True, exist_ok=True)
+ bun_dir.mkdir(parents=True, exist_ok=True)
+ (params.persistent_dir / f"user_message_{params.agent_run_id}.txt").write_text("task")
+
+ def test_upstream_agent_omits_migration_mount(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ opencode_setup_dir = Path(tmpdir) / "upstream_setup"
+ params = _make_instance_config(
+ tmpdir,
+ agent_framework="opencode",
+ opencode_source="opencode",
+ opencode_setup_dir=opencode_setup_dir,
+ switchyard_base_url="http://switchyard:4000",
+ )
+ params.persistent_dir.mkdir(parents=True, exist_ok=True)
+ self._setup_upstream_dirs(params)
+ cmd_args = ExecuteContainerCommandArgs(
+ command="x", expected_file_pattern="/tmp/*.jsonl", mode="agent", timeout=300
+ )
+ result = wrapper._build_apptainer_command(params, cmd_args)
+ assert "/opencode_setup/bun" in result
+ assert "/opencode_setup/opencode" in result
+ assert "migration" not in result
+
+ def test_fork_path_retains_migration_mount(self, monkeypatch) -> None:
+ wrapper = _create_wrapper(monkeypatch)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ opencode_setup_dir = Path(tmpdir) / "fork_setup"
+ params = _make_instance_config(
+ tmpdir,
+ agent_framework="opencode",
+ opencode_source="nv-opencode",
+ opencode_setup_dir=opencode_setup_dir,
+ )
+ params.persistent_dir.mkdir(parents=True, exist_ok=True)
+ opencode_dir = opencode_setup_dir / "opencode"
+ (opencode_dir / "evaluation" / "oh").mkdir(parents=True, exist_ok=True)
+ (opencode_dir / "packages" / "opencode" / "migration").mkdir(parents=True, exist_ok=True)
+ (opencode_setup_dir / "bun").mkdir(parents=True, exist_ok=True)
+ (params.persistent_dir / f"user_message_{params.agent_run_id}.txt").write_text("x")
+ cmd_args = ExecuteContainerCommandArgs(
+ command="x", expected_file_pattern="/tmp/*.jsonl", mode="agent", timeout=300
+ )
+ result = wrapper._build_apptainer_command(params, cmd_args)
+ assert "migration" in result
diff --git a/tests/unit_tests/test_switchyard_trace.py b/tests/unit_tests/test_switchyard_trace.py
index 0057a879a2..3d013fb4ce 100644
--- a/tests/unit_tests/test_switchyard_trace.py
+++ b/tests/unit_tests/test_switchyard_trace.py
@@ -126,6 +126,48 @@ def test_two_call_tool_trajectory(self) -> None:
assert trace.tools[0].name == "str_replace_editor"
assert trace.tools[0].description == "Edit files"
assert trace.tools[0].parameters == TOOL_SCHEMA
+
+ def test_reformatted_tool_args_tolerated_and_original_kept(self) -> None:
+ # The model emits pretty-printed JSON args; the client re-sends them compact
+ # in the next turn's history. The extension check must tolerate that
+ # (canonicalized comparison), and the reconstructed item must keep the
+ # ORIGINAL arguments (canonicalization is comparison-only).
+ pretty = '{\n "path": "a.py"\n}'
+ compact = '{"path": "a.py"}'
+ first = _record(
+ 0,
+ [
+ {"role": "system", "content": "You are a SWE agent."},
+ {"role": "user", "content": "Fix the bug."},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {"id": "call_1", "type": "function", "function": {"name": "str_replace_editor", "arguments": pretty}}
+ ],
+ },
+ ],
+ TRIPLE_0,
+ )
+ second = _record(
+ 1,
+ [
+ {"role": "system", "content": "You are a SWE agent."},
+ {"role": "user", "content": "Fix the bug."},
+ {
+ "role": "assistant",
+ "tool_calls": [
+ {"id": "call_1", "type": "function", "function": {"name": "str_replace_editor", "arguments": compact}}
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": "edited ok"},
+ {"role": "assistant", "content": "All fixed."},
+ ],
+ TRIPLE_1,
+ )
+
+ trace = _reconstruct(_envelope([first, second])) # tolerated — does not raise
+
+ assert trace.output_items[0].arguments == pretty # original kept, not canonicalized
assert trace.tools[0].type == "function"
def test_single_record_session(self) -> None: