diff --git a/docs/opencode-agent.md b/docs/opencode-agent.md index 3cf86d3e..0499024d 100644 --- a/docs/opencode-agent.md +++ b/docs/opencode-agent.md @@ -164,9 +164,11 @@ For scenario `401`, this creates a workspace such as: traces/opencode_workspaces/opencode_agent_401 ``` -OpenCode is instructed to use the current working directory as the run workspace -when file or bash access is enabled. It should write scripts, temporary files, -intermediate data, and final artifacts there. +Each scenario run starts with an empty per-run workspace directory. The +benchmark loader fills CouchDB from the scenario repository, but the OpenCode +process itself does not get the scenario's raw `shared/` files staged into the +workspace. This keeps the CLI run focused on MCP/CouchDB access instead of +reading the source files directly. > **Safety note.** `--opencode-allow-bash` is not a hard OS-level sandbox. A > shell or Python process can still attempt to access files outside the workspace. @@ -351,7 +353,7 @@ Scenario-suite OpenCode flags: | Flag | Description | | ---- | ----------- | -| `--opencode-workspace-root PATH` | Root directory for per-run OpenCode workspaces. | +| `--opencode-workspace-root PATH` | Root directory for empty per-run OpenCode workspaces. | | `--opencode-allow-files` | Pass `--allow-files` to `opencode-agent`. | | `--opencode-allow-bash` | Pass `--allow-bash` to `opencode-agent`. | | `--opencode-allow-edit` | Pass `--allow-edit` to `opencode-agent`. | diff --git a/src/agent/opencode_agent/cli.py b/src/agent/opencode_agent/cli.py index ef4ca6ad..7a32e728 100644 --- a/src/agent/opencode_agent/cli.py +++ b/src/agent/opencode_agent/cli.py @@ -159,4 +159,4 @@ def main() -> None: if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/src/agent/opencode_agent/runner.py b/src/agent/opencode_agent/runner.py index a893fd67..797c9dae 100644 --- a/src/agent/opencode_agent/runner.py +++ b/src/agent/opencode_agent/runner.py @@ -764,4 +764,4 @@ async def run(self, question: str) -> AgentResult: answer=answer, trajectory=trajectory, ) - return AgentResult(question=question, answer=answer, trajectory=trajectory) + return AgentResult(question=question, answer=answer, trajectory=trajectory) \ No newline at end of file diff --git a/src/benchmark/scenario_suite_runner.py b/src/benchmark/scenario_suite_runner.py index 606b30c1..348a3bc4 100644 --- a/src/benchmark/scenario_suite_runner.py +++ b/src/benchmark/scenario_suite_runner.py @@ -26,7 +26,6 @@ from __future__ import annotations import argparse -import json import os import re import shutil @@ -50,7 +49,6 @@ class MethodConfig: model_id: str extra_args: tuple[str, ...] = () workspace_root: Path | None = None - stage_workspace: bool = False def model_dir_name(model_id: str) -> str: @@ -163,138 +161,6 @@ def validate_workspace_root_outside_repo(workspace_root: Path, label: str) -> No ) -def _iter_manifest_paths(value: object) -> list[str]: - """Return path-like strings from a scenario manifest value.""" - if isinstance(value, str): - return [value] - if isinstance(value, list): - paths: list[str] = [] - for item in value: - paths.extend(_iter_manifest_paths(item)) - return paths - if isinstance(value, dict): - paths = [] - for item in value.values(): - paths.extend(_iter_manifest_paths(item)) - return paths - return [] - - -def _resolve_manifest_data_path( - *, - scenario_root: Path, - scenario_dir: Path, - spec: str, -) -> Path | None: - """Resolve a manifest data path using the same search order as init_data.""" - raw = Path(spec).expanduser() - candidates = [raw] if raw.is_absolute() else [ - scenario_dir / raw, - scenario_root / raw, - ] - - for candidate in candidates: - try: - resolved = candidate.resolve(strict=True) - except FileNotFoundError: - continue - if resolved.is_file() and _is_relative_to(resolved, scenario_root): - return resolved - return None - - -def _copy_into_workspace( - *, - source: Path, - scenario_root: Path, - workspace_dir: Path, -) -> Path: - relative = source.resolve().relative_to(scenario_root.resolve()) - destination = workspace_dir / relative - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) - return destination - - -def stage_scenario_workspace( - *, - scenario_root: Path, - scenario_id: str, - workspace_dir: Path, -) -> list[Path]: - """Create a clean run workspace with only allowed scenario input files. - - The staged workspace intentionally excludes ``groundtruth.txt`` and any - existing reports/traces. It includes ``question.txt``, ``manifest.json``, - and data files referenced by the manifest. - """ - scenario_root = scenario_root.expanduser().resolve() - scenario_dir = scenario_dir_for_id(scenario_root, scenario_id) - manifest_path = scenario_dir / "manifest.json" - question_path = scenario_dir / "question.txt" - - if not question_path.exists(): - raise FileNotFoundError( - f"Missing question file for scenario {scenario_id}: {question_path}" - ) - if not manifest_path.exists(): - raise FileNotFoundError( - f"Missing manifest file for scenario {scenario_id}: {manifest_path}" - ) - - workspace_dir = workspace_dir.expanduser().resolve() - if workspace_dir.exists(): - shutil.rmtree(workspace_dir) - workspace_dir.mkdir(parents=True, exist_ok=True) - - copied = [ - _copy_into_workspace( - source=question_path, - scenario_root=scenario_root, - workspace_dir=workspace_dir, - ), - _copy_into_workspace( - source=manifest_path, - scenario_root=scenario_root, - workspace_dir=workspace_dir, - ), - ] - - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - for spec in sorted(set(_iter_manifest_paths(manifest))): - source = _resolve_manifest_data_path( - scenario_root=scenario_root, - scenario_dir=scenario_dir, - spec=spec, - ) - if source is None or source.name == "groundtruth.txt": - continue - copied.append( - _copy_into_workspace( - source=source, - scenario_root=scenario_root, - workspace_dir=workspace_dir, - ) - ) - - readme = workspace_dir / "README.md" - readme.write_text( - "\n".join( - [ - f"# AssetOpsBench scenario {scenario_id} workspace", - "", - "This workspace contains only staged scenario inputs.", - "Do not inspect files outside this directory.", - "Ground truth, reports, and previous trajectories are intentionally excluded.", - "", - ] - ), - encoding="utf-8", - ) - copied.append(readme) - return copied - - def reset_and_load_couchdb(scenario_id: str, scenario_root: Path, dry_run: bool) -> None: """Reset CouchDB and load the scenario-specific data from scenario_root.""" env = os.environ.copy() @@ -338,22 +204,9 @@ def run_agent_for_scenario( workspace_dir = method.workspace_root / run_id extra_args.extend(["--workspace-dir", str(workspace_dir)]) if not dry_run: - if method.stage_workspace: - if scenario_root is None: - raise ValueError( - "scenario_root is required when staging a run workspace" - ) - staged = stage_scenario_workspace( - scenario_root=scenario_root, - scenario_id=scenario_id, - workspace_dir=workspace_dir, - ) - print( - f"Staged {len(staged)} files for scenario {scenario_id} " - f"into {workspace_dir}" - ) - else: - workspace_dir.mkdir(parents=True, exist_ok=True) + if workspace_dir.exists(): + shutil.rmtree(workspace_dir) + workspace_dir.mkdir(parents=True, exist_ok=True) cmd = [ "uv", @@ -477,7 +330,6 @@ def build_methods(args: argparse.Namespace) -> dict[str, MethodConfig]: model_id=args.model_id, extra_args=tuple(opencode_extra_args), workspace_root=args.opencode_workspace_root, - stage_workspace=True, ), "gemini_cli_agent": MethodConfig( agent_name="gemini_cli_agent", @@ -769,7 +621,6 @@ def main() -> None: model_id=method.model_id, extra_args=method.extra_args, workspace_root=method_workspace_root, - stage_workspace=method.stage_workspace, ) if not args.dry_run: diff --git a/src/benchmark/tests/test_scenario_suite_runner.py b/src/benchmark/tests/test_scenario_suite_runner.py index 9bd9795f..bd8b4eae 100644 --- a/src/benchmark/tests/test_scenario_suite_runner.py +++ b/src/benchmark/tests/test_scenario_suite_runner.py @@ -54,51 +54,55 @@ def test_read_question_raises_when_missing(tmp_path: Path) -> None: mr.read_question(tmp_path, "11") -def test_stage_scenario_workspace_copies_inputs_without_groundtruth( - tmp_path: Path, +def test_run_agent_for_scenario_starts_with_empty_opencode_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - scenario_root = tmp_path / "scenarios_data" - scenario_dir = scenario_root / "scenario_1001" - shared_iot = scenario_root / "shared" / "iot" - shared_failure = scenario_root / "shared" / "failure_code" - scenario_dir.mkdir(parents=True) - shared_iot.mkdir(parents=True) - shared_failure.mkdir(parents=True) - - (scenario_dir / "question.txt").write_text("Find anomaly.", encoding="utf-8") - (scenario_dir / "groundtruth.txt").write_text( - '{"condition":"faulty"}', - encoding="utf-8", - ) - (scenario_dir / "manifest.json").write_text( - """ - { - "iot": "shared/iot/asset_data.json", - "asset": ["shared/iot/asset_registry.json"], - "failure_code": "shared/failure_code/failure_codes.csv" - } - """, - encoding="utf-8", + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + captured["kwargs"] = kwargs + + monkeypatch.setattr(mr.subprocess, "run", fake_run) + + workspace_root = tmp_path / "workspaces" + expected_workspace = workspace_root / "opencode_agent_1001" + expected_workspace.mkdir(parents=True) + (expected_workspace / "stale.txt").write_text("old data", encoding="utf-8") + + method = mr.MethodConfig( + agent_name="opencode_agent", + command="opencode-agent", + model_id="tokenrouter/MiniMax-M3", + extra_args=("--allow-files",), + workspace_root=workspace_root, ) - (shared_iot / "asset_data.json").write_text("[]", encoding="utf-8") - (shared_iot / "asset_registry.json").write_text("[]", encoding="utf-8") - (shared_failure / "failure_codes.csv").write_text("code,name\n", encoding="utf-8") - workspace = tmp_path / "workspace" - copied = mr.stage_scenario_workspace( - scenario_root=scenario_root, + mr.run_agent_for_scenario( + method=method, scenario_id="1001", - workspace_dir=workspace, + question="Find anomaly.", + trajectory_dir=tmp_path / "traj", + dry_run=False, ) - copied_relative = {path.relative_to(workspace) for path in copied} - assert Path("scenario_1001/question.txt") in copied_relative - assert Path("scenario_1001/manifest.json") in copied_relative - assert Path("shared/iot/asset_data.json") in copied_relative - assert Path("shared/iot/asset_registry.json") in copied_relative - assert Path("shared/failure_code/failure_codes.csv") in copied_relative - assert Path("README.md") in copied_relative - assert not (workspace / "scenario_1001" / "groundtruth.txt").exists() + assert expected_workspace.exists() + assert list(expected_workspace.iterdir()) == [] + assert captured["cmd"] == [ + "uv", + "run", + "opencode-agent", + "--model-id", + "tokenrouter/MiniMax-M3", + "--allow-files", + "--workspace-dir", + str(expected_workspace), + "--scenario-id", + "1001", + "--run-id", + "opencode_agent_1001", + "Find anomaly.", + ] def test_validate_workspace_root_rejects_repo_paths() -> None: @@ -217,7 +221,6 @@ def test_build_methods_opencode_workspace_options(tmp_path: Path) -> None: assert opencode.extra_args == ("--allow-files", "--allow-bash") assert opencode.workspace_root == tmp_path / "workspaces" - assert opencode.stage_workspace is True def test_build_methods_opencode_thinking_and_variant() -> None: @@ -440,7 +443,7 @@ def fake_run(cmd, **kwargs): ] -def test_run_agent_for_scenario_stages_opencode_workspace( +def test_run_agent_for_scenario_recreates_empty_opencode_workspace( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: captured = {} @@ -451,26 +454,12 @@ def fake_run(cmd, **kwargs): monkeypatch.setattr(mr.subprocess, "run", fake_run) - scenario_root = tmp_path / "scenarios_data" - scenario_dir = scenario_root / "scenario_1001" - shared_iot = scenario_root / "shared" / "iot" - scenario_dir.mkdir(parents=True) - shared_iot.mkdir(parents=True) - (scenario_dir / "question.txt").write_text("Find anomaly.", encoding="utf-8") - (scenario_dir / "groundtruth.txt").write_text("secret", encoding="utf-8") - (scenario_dir / "manifest.json").write_text( - '{"iot": "shared/iot/asset_data.json"}', - encoding="utf-8", - ) - (shared_iot / "asset_data.json").write_text("[]", encoding="utf-8") - method = mr.MethodConfig( agent_name="opencode_agent", command="opencode-agent", model_id="tokenrouter/MiniMax-M3", extra_args=("--allow-files",), workspace_root=tmp_path / "workspaces", - stage_workspace=True, ) mr.run_agent_for_scenario( @@ -479,14 +468,11 @@ def fake_run(cmd, **kwargs): question="Find anomaly.", trajectory_dir=tmp_path / "traj", dry_run=False, - scenario_root=scenario_root, ) expected_workspace = tmp_path / "workspaces" / "opencode_agent_1001" - assert (expected_workspace / "scenario_1001" / "question.txt").exists() - assert (expected_workspace / "scenario_1001" / "manifest.json").exists() - assert (expected_workspace / "shared" / "iot" / "asset_data.json").exists() - assert not (expected_workspace / "scenario_1001" / "groundtruth.txt").exists() + assert expected_workspace.exists() + assert list(expected_workspace.iterdir()) == [] assert "--workspace-dir" in captured["cmd"]