From 47de40c16a13c08ee5976c19831037cd517654ac Mon Sep 17 00:00:00 2001 From: Chathurangi Shyalika Date: Mon, 13 Jul 2026 16:05:30 -0400 Subject: [PATCH] Keep OpenCode scenario workspaces empty Signed-off-by: Chathurangi Shyalika --- docs/opencode-agent.md | 12 +- src/agent/opencode_agent/cli.py | 4 +- src/agent/opencode_agent/runner.py | 71 ++++++-- src/agent/opencode_agent/tests/test_runner.py | 108 ++++++++++-- src/benchmark/scenario_suite_runner.py | 155 +----------------- .../tests/test_scenario_suite_runner.py | 106 ++++++------ 6 files changed, 207 insertions(+), 249 deletions(-) diff --git a/docs/opencode-agent.md b/docs/opencode-agent.md index 7d5d98f1..b425d380 100644 --- a/docs/opencode-agent.md +++ b/docs/opencode-agent.md @@ -164,9 +164,11 @@ For scenario `401`, this creates a workspace such as: traces/opencode_workspaces/opencode_agent_401 ``` -OpenCode is instructed to use the current working directory as the run workspace -when file or bash access is enabled. It should write scripts, temporary files, -intermediate data, and final artifacts there. +Each scenario run starts with an empty per-run workspace directory. The +benchmark loader fills CouchDB from the scenario repository, but the OpenCode +process itself does not get the scenario's raw `shared/` files staged into the +workspace. This keeps the CLI run focused on MCP/CouchDB access instead of +reading the source files directly. > **Safety note.** `--opencode-allow-bash` is not a hard OS-level sandbox. A > shell or Python process can still attempt to access files outside the workspace. @@ -339,7 +341,7 @@ Scenario-suite OpenCode flags: | Flag | Description | | ---- | ----------- | -| `--opencode-workspace-root PATH` | Root directory for per-run OpenCode workspaces. | +| `--opencode-workspace-root PATH` | Root directory for empty per-run OpenCode workspaces. | | `--opencode-allow-files` | Pass `--allow-files` to `opencode-agent`. | | `--opencode-allow-bash` | Pass `--allow-bash` to `opencode-agent`. | | `--opencode-allow-edit` | Pass `--allow-edit` to `opencode-agent`. | @@ -435,4 +437,4 @@ uv run python -m benchmark.scenario_suite_runner \ documentation. No MCP servers are modified. The runner generates OpenCode MCP/provider config -at runtime. \ No newline at end of file +at runtime. diff --git a/src/agent/opencode_agent/cli.py b/src/agent/opencode_agent/cli.py index ddd91206..7a32e728 100644 --- a/src/agent/opencode_agent/cli.py +++ b/src/agent/opencode_agent/cli.py @@ -35,7 +35,7 @@ def _build_parser() -> argparse.ArgumentParser: LITELLM_API_KEY LiteLLM router key for litellm_proxy/* models LITELLM_BASE_URL LiteLLM OpenAI-compatible base URL TOKENROUTER_API_KEY TokenRouter key for tokenrouter/* models - TOKENROUTER_BASE_URL TokenRouter OpenAI-compatible base URL + TOKENROUTER_BASE_URL TokenRouter base URL; Claude models use Anthropic protocol examples: opencode-agent "What assets are at site MAIN?" @@ -159,4 +159,4 @@ def main() -> None: if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/src/agent/opencode_agent/runner.py b/src/agent/opencode_agent/runner.py index 9df56973..797c9dae 100644 --- a/src/agent/opencode_agent/runner.py +++ b/src/agent/opencode_agent/runner.py @@ -60,6 +60,11 @@ class OpenCodeTrajectory(Trajectory): stderr: str = "" +def _needs_reasoning_effort_none(provider_id: str, model_name: str) -> bool: + """Whether OpenCode should disable reasoning_effort for this router model.""" + return provider_id == "tokenrouter" and model_name.startswith("openai/gpt-5") + + def _build_mcp_config( server_paths: dict[str, Path | str], *, @@ -115,8 +120,9 @@ def _resolve_opencode_model_and_provider( """Translate AssetOpsBench router model IDs into OpenCode config. OpenCode wants ``provider/model``. For AssetOpsBench router prefixes such - as ``litellm_proxy/`` and ``tokenrouter/``, declare a custom - OpenAI-compatible provider and register the requested model explicitly. + as ``litellm_proxy/`` and ``tokenrouter/``, declare a custom provider and + register the requested model explicitly. TokenRouter Claude models need the + Anthropic protocol so OpenCode preserves native Anthropic message handling. """ creds = resolve_router_creds(model_id, strict=True) if creds is None: @@ -126,18 +132,26 @@ def _resolve_opencode_model_and_provider( provider_name = "TokenRouter" if provider_id == "tokenrouter" else "LiteLLM Proxy" model_name = resolve_model(model_id) opencode_model = f"{provider_id}/{model_name}" + provider_npm = ( + "@ai-sdk/anthropic" + if provider_id == "tokenrouter" and model_name.startswith("anthropic/") + else "@ai-sdk/openai-compatible" + ) + model_config: dict[str, Any] = {"name": model_name} + if _needs_reasoning_effort_none(provider_id, model_name): + # TokenRouter rejects function tools for OpenAI GPT-5 models on + # chat/completions unless reasoning_effort is explicitly disabled. + model_config["options"] = {"reasoningEffort": "none"} provider = { provider_id: { - "npm": "@ai-sdk/openai-compatible", + "npm": provider_npm, "name": provider_name, "options": { "baseURL": creds.base_url, "apiKey": "{env:ASSETOPSBENCH_OPENCODE_API_KEY}", }, "models": { - model_name: { - "name": model_name, - } + model_name: model_config, }, } } @@ -249,6 +263,35 @@ def _json_events(stdout: str) -> tuple[list[dict[str, Any]], list[str]]: return events, plain_lines +def _opencode_error_message(events: list[dict[str, Any]]) -> str | None: + """Return a concise message for fatal OpenCode error events, if present.""" + for event in events: + if event.get("type") != "error" and not event.get("error"): + continue + + error = event.get("error") + if isinstance(error, dict): + data = error.get("data") if isinstance(error.get("data"), dict) else {} + message = ( + data.get("message") + or error.get("message") + or data.get("responseBody") + or json.dumps(error, default=str) + ) + name = error.get("name") + status = data.get("statusCode") + prefix = "OpenCode error" + if name: + prefix += f" {name}" + if status: + prefix += f" ({status})" + return f"{prefix}: {message}" + + return f"OpenCode error event: {json.dumps(event, default=str)}" + + return None + + def _candidate_part(event: dict[str, Any]) -> dict[str, Any] | None: """Find the message/tool part inside common OpenCode event shapes.""" for key in ("part", "messagePart"): @@ -395,7 +438,9 @@ def _merge_text(existing: str, new: str) -> str: return existing + new -def _is_step_finish(event: dict[str, Any], part: dict[str, Any], part_type: str) -> bool: +def _is_step_finish( + event: dict[str, Any], part: dict[str, Any], part_type: str +) -> bool: """True for OpenCode step-finish boundaries.""" event_type = str(event.get("type") or "").lower() return ("step" in part_type and "finish" in part_type) or ( @@ -543,10 +588,9 @@ def _close_step(input_tokens: int, output_tokens: int) -> None: output_tokens=step["output_tokens"], ) ) - if ( - sum(step["input_tokens"] + step["output_tokens"] for step in steps) == 0 - and (total_input or total_output) - ): + if sum( + step["input_tokens"] + step["output_tokens"] for step in steps + ) == 0 and (total_input or total_output): trajectory.turns[-1].input_tokens = total_input trajectory.turns[-1].output_tokens = total_output trajectory.turns[-1].duration_ms = duration_ms @@ -692,6 +736,9 @@ async def run(self, question: str) -> AgentResult: duration_ms = (time.perf_counter() - run_started) * 1000 events, plain_lines = _json_events(stdout) + error_message = _opencode_error_message(events) + if error_message: + raise RuntimeError(error_message) answer, trajectory = _build_trajectory_from_events( events, plain_lines, @@ -717,4 +764,4 @@ async def run(self, question: str) -> AgentResult: answer=answer, trajectory=trajectory, ) - return AgentResult(question=question, answer=answer, trajectory=trajectory) + return AgentResult(question=question, answer=answer, trajectory=trajectory) \ No newline at end of file diff --git a/src/agent/opencode_agent/tests/test_runner.py b/src/agent/opencode_agent/tests/test_runner.py index f6c8e6a3..80fd77f4 100644 --- a/src/agent/opencode_agent/tests/test_runner.py +++ b/src/agent/opencode_agent/tests/test_runner.py @@ -13,6 +13,8 @@ _build_permissions, _build_trajectory_from_events, _json_events, + _needs_reasoning_effort_none, + _opencode_error_message, _resolve_run_dir, _resolve_opencode_model_and_provider, ) @@ -108,19 +110,66 @@ def test_resolve_litellm_model(monkeypatch): assert model == "litellm-proxy/azure/gpt-5.4" assert provider["litellm-proxy"]["npm"] == "@ai-sdk/openai-compatible" assert provider["litellm-proxy"]["options"]["baseURL"] == "http://localhost:4000" - assert provider["litellm-proxy"]["models"]["azure/gpt-5.4"]["name"] == "azure/gpt-5.4" - assert env["ASSETOPSBENCH_OPENCODE_API_KEY"] == "sk-test" # pragma: allowlist secret + assert ( + provider["litellm-proxy"]["models"]["azure/gpt-5.4"]["name"] == "azure/gpt-5.4" + ) + assert ( + env["ASSETOPSBENCH_OPENCODE_API_KEY"] == "sk-test" + ) # pragma: allowlist secret def test_resolve_tokenrouter_model(monkeypatch): monkeypatch.setenv("TOKENROUTER_BASE_URL", "https://router.example/v1") monkeypatch.setenv("TOKENROUTER_API_KEY", "tr-test") - model, provider, env = _resolve_opencode_model_and_provider("tokenrouter/MiniMax-M3") + model, provider, env = _resolve_opencode_model_and_provider( + "tokenrouter/MiniMax-M3" + ) assert model == "tokenrouter/MiniMax-M3" assert provider["tokenrouter"]["npm"] == "@ai-sdk/openai-compatible" assert provider["tokenrouter"]["options"]["baseURL"] == "https://router.example/v1" assert provider["tokenrouter"]["models"]["MiniMax-M3"]["name"] == "MiniMax-M3" - assert env["ASSETOPSBENCH_OPENCODE_API_KEY"] == "tr-test" # pragma: allowlist secret + assert ( + env["ASSETOPSBENCH_OPENCODE_API_KEY"] == "tr-test" + ) # pragma: allowlist secret + + +def test_resolve_tokenrouter_anthropic_model(monkeypatch): + monkeypatch.setenv("TOKENROUTER_BASE_URL", "https://router.example/v1") + monkeypatch.setenv("TOKENROUTER_API_KEY", "tr-test") + model, provider, env = _resolve_opencode_model_and_provider( + "tokenrouter/anthropic/claude-opus-4.8" + ) + assert model == "tokenrouter/anthropic/claude-opus-4.8" + assert provider["tokenrouter"]["npm"] == "@ai-sdk/anthropic" + assert provider["tokenrouter"]["options"]["baseURL"] == "https://router.example/v1" + assert ( + provider["tokenrouter"]["models"]["anthropic/claude-opus-4.8"]["name"] + == "anthropic/claude-opus-4.8" + ) + assert ( + env["ASSETOPSBENCH_OPENCODE_API_KEY"] == "tr-test" + ) # pragma: allowlist secret + + +def test_tokenrouter_gpt5_models_need_reasoning_effort_none(): + assert _needs_reasoning_effort_none("tokenrouter", "openai/gpt-5.5") + assert _needs_reasoning_effort_none("tokenrouter", "openai/gpt-5.6-sol") + assert not _needs_reasoning_effort_none("tokenrouter", "MiniMax-M3") + assert not _needs_reasoning_effort_none("litellm-proxy", "openai/gpt-5.6-sol") + + +def test_resolve_tokenrouter_gpt5_disables_reasoning_effort(monkeypatch): + monkeypatch.setenv("TOKENROUTER_BASE_URL", "https://router.example/v1") + monkeypatch.setenv("TOKENROUTER_API_KEY", "tr-test") + for model_id in ("tokenrouter/openai/gpt-5.5", "tokenrouter/openai/gpt-5.6-sol"): + model, provider, env = _resolve_opencode_model_and_provider(model_id) + model_name = model_id.removeprefix("tokenrouter/") + assert model == model_id + model_config = provider["tokenrouter"]["models"][model_name] + assert model_config["options"] == {"reasoningEffort": "none"} + assert ( + env["ASSETOPSBENCH_OPENCODE_API_KEY"] == "tr-test" + ) # pragma: allowlist secret def test_build_opencode_config_includes_agent_and_mcp(): @@ -144,6 +193,29 @@ def test_json_events_parses_ndjson_and_plain_lines(): assert plain == ["not-json"] +def test_opencode_error_message_extracts_api_error(): + message = _opencode_error_message( + [ + { + "type": "error", + "error": { + "name": "APIError", + "data": { + "statusCode": 400, + "message": "Function tools are not supported.", + }, + }, + } + ] + ) + + assert message == "OpenCode error APIError (400): Function tools are not supported." + + +def test_opencode_error_message_ignores_null_error_field(): + assert _opencode_error_message([{"type": "step_finish", "error": None}]) is None + + def test_build_trajectory_from_text_and_tool_parts(): events = [ { @@ -239,13 +311,14 @@ def test_runner_workspace_mode(tmp_path): assert runner._run_dir == workspace.resolve() assert runner._config["agent"]["assetops"]["permission"]["read"] == "allow" + def _text_part(part_id, text, *, message_id=None, part_type="text"): part = {"id": part_id, "type": part_type, "text": text} if message_id is not None: part["messageID"] = message_id return {"type": "message.part.updated", "properties": {"part": part}} - - + + def _tool_part(part_id, tool, *, input=None, output=None): return { "type": "message.part.updated", @@ -259,8 +332,8 @@ def _tool_part(part_id, tool, *, input=None, output=None): } }, } - - + + def _step_finish(input_tokens, output_tokens): return { "type": "step_finish", @@ -269,8 +342,8 @@ def _step_finish(input_tokens, output_tokens): "tokens": {"input": input_tokens, "output": output_tokens}, }, } - - + + def test_answer_excludes_intermediate_narration(): """Only the final assistant message is the answer, not scratch talk.""" events = [ @@ -286,8 +359,8 @@ def test_answer_excludes_intermediate_narration(): assert len(trajectory.all_tool_calls) == 1 assert trajectory.total_input_tokens == 110 assert trajectory.total_output_tokens == 13 - - + + def test_answer_excludes_reasoning_parts(): events = [ _text_part("r1", "(internal) suspect bearing wear", part_type="reasoning"), @@ -295,8 +368,8 @@ def test_answer_excludes_reasoning_parts(): ] answer, _ = _build_trajectory_from_events(events, []) assert answer == "The pump is healthy." - - + + def test_parallel_tool_calls_keep_their_own_outputs(): """Two tool calls emitted before their outputs must not be mismatched.""" events = [ @@ -308,8 +381,8 @@ def test_parallel_tool_calls_keep_their_own_outputs(): by_name = {tc.name: tc.output for tc in trajectory.all_tool_calls} assert by_name["get_temp"] == "temp=42" assert by_name["get_vibration"] == "vib=0.3" - - + + def test_text_deltas_are_reconstructed(): snapshot = [ _text_part("t1", "Chiller 6 "), @@ -320,5 +393,4 @@ def test_text_deltas_are_reconstructed(): _text_part("t1", "has 5 sensors."), ] assert _build_trajectory_from_events(snapshot, [])[0] == "Chiller 6 has 5 sensors." - assert _build_trajectory_from_events(delta, [])[0] == "Chiller 6 has 5 sensors." - + assert _build_trajectory_from_events(delta, [])[0] == "Chiller 6 has 5 sensors." \ No newline at end of file diff --git a/src/benchmark/scenario_suite_runner.py b/src/benchmark/scenario_suite_runner.py index 606b30c1..348a3bc4 100644 --- a/src/benchmark/scenario_suite_runner.py +++ b/src/benchmark/scenario_suite_runner.py @@ -26,7 +26,6 @@ from __future__ import annotations import argparse -import json import os import re import shutil @@ -50,7 +49,6 @@ class MethodConfig: model_id: str extra_args: tuple[str, ...] = () workspace_root: Path | None = None - stage_workspace: bool = False def model_dir_name(model_id: str) -> str: @@ -163,138 +161,6 @@ def validate_workspace_root_outside_repo(workspace_root: Path, label: str) -> No ) -def _iter_manifest_paths(value: object) -> list[str]: - """Return path-like strings from a scenario manifest value.""" - if isinstance(value, str): - return [value] - if isinstance(value, list): - paths: list[str] = [] - for item in value: - paths.extend(_iter_manifest_paths(item)) - return paths - if isinstance(value, dict): - paths = [] - for item in value.values(): - paths.extend(_iter_manifest_paths(item)) - return paths - return [] - - -def _resolve_manifest_data_path( - *, - scenario_root: Path, - scenario_dir: Path, - spec: str, -) -> Path | None: - """Resolve a manifest data path using the same search order as init_data.""" - raw = Path(spec).expanduser() - candidates = [raw] if raw.is_absolute() else [ - scenario_dir / raw, - scenario_root / raw, - ] - - for candidate in candidates: - try: - resolved = candidate.resolve(strict=True) - except FileNotFoundError: - continue - if resolved.is_file() and _is_relative_to(resolved, scenario_root): - return resolved - return None - - -def _copy_into_workspace( - *, - source: Path, - scenario_root: Path, - workspace_dir: Path, -) -> Path: - relative = source.resolve().relative_to(scenario_root.resolve()) - destination = workspace_dir / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) - return destination - - -def stage_scenario_workspace( - *, - scenario_root: Path, - scenario_id: str, - workspace_dir: Path, -) -> list[Path]: - """Create a clean run workspace with only allowed scenario input files. - - The staged workspace intentionally excludes ``groundtruth.txt`` and any - existing reports/traces. It includes ``question.txt``, ``manifest.json``, - and data files referenced by the manifest. - """ - scenario_root = scenario_root.expanduser().resolve() - scenario_dir = scenario_dir_for_id(scenario_root, scenario_id) - manifest_path = scenario_dir / "manifest.json" - question_path = scenario_dir / "question.txt" - - if not question_path.exists(): - raise FileNotFoundError( - f"Missing question file for scenario {scenario_id}: {question_path}" - ) - if not manifest_path.exists(): - raise FileNotFoundError( - f"Missing manifest file for scenario {scenario_id}: {manifest_path}" - ) - - workspace_dir = workspace_dir.expanduser().resolve() - if workspace_dir.exists(): - shutil.rmtree(workspace_dir) - workspace_dir.mkdir(parents=True, exist_ok=True) - - copied = [ - _copy_into_workspace( - source=question_path, - scenario_root=scenario_root, - workspace_dir=workspace_dir, - ), - _copy_into_workspace( - source=manifest_path, - scenario_root=scenario_root, - workspace_dir=workspace_dir, - ), - ] - - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - for spec in sorted(set(_iter_manifest_paths(manifest))): - source = _resolve_manifest_data_path( - scenario_root=scenario_root, - scenario_dir=scenario_dir, - spec=spec, - ) - if source is None or source.name == "groundtruth.txt": - continue - copied.append( - _copy_into_workspace( - source=source, - scenario_root=scenario_root, - workspace_dir=workspace_dir, - ) - ) - - readme = workspace_dir / "README.md" - readme.write_text( - "\n".join( - [ - f"# AssetOpsBench scenario {scenario_id} workspace", - "", - "This workspace contains only staged scenario inputs.", - "Do not inspect files outside this directory.", - "Ground truth, reports, and previous trajectories are intentionally excluded.", - "", - ] - ), - encoding="utf-8", - ) - copied.append(readme) - return copied - - def reset_and_load_couchdb(scenario_id: str, scenario_root: Path, dry_run: bool) -> None: """Reset CouchDB and load the scenario-specific data from scenario_root.""" env = os.environ.copy() @@ -338,22 +204,9 @@ def run_agent_for_scenario( workspace_dir = method.workspace_root / run_id extra_args.extend(["--workspace-dir", str(workspace_dir)]) if not dry_run: - if method.stage_workspace: - if scenario_root is None: - raise ValueError( - "scenario_root is required when staging a run workspace" - ) - staged = stage_scenario_workspace( - scenario_root=scenario_root, - scenario_id=scenario_id, - workspace_dir=workspace_dir, - ) - print( - f"Staged {len(staged)} files for scenario {scenario_id} " - f"into {workspace_dir}" - ) - else: - workspace_dir.mkdir(parents=True, exist_ok=True) + if workspace_dir.exists(): + shutil.rmtree(workspace_dir) + workspace_dir.mkdir(parents=True, exist_ok=True) cmd = [ "uv", @@ -477,7 +330,6 @@ def build_methods(args: argparse.Namespace) -> dict[str, MethodConfig]: model_id=args.model_id, extra_args=tuple(opencode_extra_args), workspace_root=args.opencode_workspace_root, - stage_workspace=True, ), "gemini_cli_agent": MethodConfig( agent_name="gemini_cli_agent", @@ -769,7 +621,6 @@ def main() -> None: model_id=method.model_id, extra_args=method.extra_args, workspace_root=method_workspace_root, - stage_workspace=method.stage_workspace, ) if not args.dry_run: diff --git a/src/benchmark/tests/test_scenario_suite_runner.py b/src/benchmark/tests/test_scenario_suite_runner.py index 9bd9795f..bd8b4eae 100644 --- a/src/benchmark/tests/test_scenario_suite_runner.py +++ b/src/benchmark/tests/test_scenario_suite_runner.py @@ -54,51 +54,55 @@ def test_read_question_raises_when_missing(tmp_path: Path) -> None: mr.read_question(tmp_path, "11") -def test_stage_scenario_workspace_copies_inputs_without_groundtruth( - tmp_path: Path, +def test_run_agent_for_scenario_starts_with_empty_opencode_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - scenario_root = tmp_path / "scenarios_data" - scenario_dir = scenario_root / "scenario_1001" - shared_iot = scenario_root / "shared" / "iot" - shared_failure = scenario_root / "shared" / "failure_code" - scenario_dir.mkdir(parents=True) - shared_iot.mkdir(parents=True) - shared_failure.mkdir(parents=True) - - (scenario_dir / "question.txt").write_text("Find anomaly.", encoding="utf-8") - (scenario_dir / "groundtruth.txt").write_text( - '{"condition":"faulty"}', - encoding="utf-8", - ) - (scenario_dir / "manifest.json").write_text( - """ - { - "iot": "shared/iot/asset_data.json", - "asset": ["shared/iot/asset_registry.json"], - "failure_code": "shared/failure_code/failure_codes.csv" - } - """, - encoding="utf-8", + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + captured["kwargs"] = kwargs + + monkeypatch.setattr(mr.subprocess, "run", fake_run) + + workspace_root = tmp_path / "workspaces" + expected_workspace = workspace_root / "opencode_agent_1001" + expected_workspace.mkdir(parents=True) + (expected_workspace / "stale.txt").write_text("old data", encoding="utf-8") + + method = mr.MethodConfig( + agent_name="opencode_agent", + command="opencode-agent", + model_id="tokenrouter/MiniMax-M3", + extra_args=("--allow-files",), + workspace_root=workspace_root, ) - (shared_iot / "asset_data.json").write_text("[]", encoding="utf-8") - (shared_iot / "asset_registry.json").write_text("[]", encoding="utf-8") - (shared_failure / "failure_codes.csv").write_text("code,name\n", encoding="utf-8") - workspace = tmp_path / "workspace" - copied = mr.stage_scenario_workspace( - scenario_root=scenario_root, + mr.run_agent_for_scenario( + method=method, scenario_id="1001", - workspace_dir=workspace, + question="Find anomaly.", + trajectory_dir=tmp_path / "traj", + dry_run=False, ) - copied_relative = {path.relative_to(workspace) for path in copied} - assert Path("scenario_1001/question.txt") in copied_relative - assert Path("scenario_1001/manifest.json") in copied_relative - assert Path("shared/iot/asset_data.json") in copied_relative - assert Path("shared/iot/asset_registry.json") in copied_relative - assert Path("shared/failure_code/failure_codes.csv") in copied_relative - assert Path("README.md") in copied_relative - assert not (workspace / "scenario_1001" / "groundtruth.txt").exists() + assert expected_workspace.exists() + assert list(expected_workspace.iterdir()) == [] + assert captured["cmd"] == [ + "uv", + "run", + "opencode-agent", + "--model-id", + "tokenrouter/MiniMax-M3", + "--allow-files", + "--workspace-dir", + str(expected_workspace), + "--scenario-id", + "1001", + "--run-id", + "opencode_agent_1001", + "Find anomaly.", + ] def test_validate_workspace_root_rejects_repo_paths() -> None: @@ -217,7 +221,6 @@ def test_build_methods_opencode_workspace_options(tmp_path: Path) -> None: assert opencode.extra_args == ("--allow-files", "--allow-bash") assert opencode.workspace_root == tmp_path / "workspaces" - assert opencode.stage_workspace is True def test_build_methods_opencode_thinking_and_variant() -> None: @@ -440,7 +443,7 @@ def fake_run(cmd, **kwargs): ] -def test_run_agent_for_scenario_stages_opencode_workspace( +def test_run_agent_for_scenario_recreates_empty_opencode_workspace( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: captured = {} @@ -451,26 +454,12 @@ def fake_run(cmd, **kwargs): monkeypatch.setattr(mr.subprocess, "run", fake_run) - scenario_root = tmp_path / "scenarios_data" - scenario_dir = scenario_root / "scenario_1001" - shared_iot = scenario_root / "shared" / "iot" - scenario_dir.mkdir(parents=True) - shared_iot.mkdir(parents=True) - (scenario_dir / "question.txt").write_text("Find anomaly.", encoding="utf-8") - (scenario_dir / "groundtruth.txt").write_text("secret", encoding="utf-8") - (scenario_dir / "manifest.json").write_text( - '{"iot": "shared/iot/asset_data.json"}', - encoding="utf-8", - ) - (shared_iot / "asset_data.json").write_text("[]", encoding="utf-8") - method = mr.MethodConfig( agent_name="opencode_agent", command="opencode-agent", model_id="tokenrouter/MiniMax-M3", extra_args=("--allow-files",), workspace_root=tmp_path / "workspaces", - stage_workspace=True, ) mr.run_agent_for_scenario( @@ -479,14 +468,11 @@ def fake_run(cmd, **kwargs): question="Find anomaly.", trajectory_dir=tmp_path / "traj", dry_run=False, - scenario_root=scenario_root, ) expected_workspace = tmp_path / "workspaces" / "opencode_agent_1001" - assert (expected_workspace / "scenario_1001" / "question.txt").exists() - assert (expected_workspace / "scenario_1001" / "manifest.json").exists() - assert (expected_workspace / "shared" / "iot" / "asset_data.json").exists() - assert not (expected_workspace / "scenario_1001" / "groundtruth.txt").exists() + assert expected_workspace.exists() + assert list(expected_workspace.iterdir()) == [] assert "--workspace-dir" in captured["cmd"]