diff --git a/envs/tbench2_env/README.md b/envs/tbench2_env/README.md index 62315da78..b36266883 100644 --- a/envs/tbench2_env/README.md +++ b/envs/tbench2_env/README.md @@ -164,6 +164,7 @@ env.step(Tbench2Action(action_type="view", session_id="sess1")) | `TB2_OUTPUT_DIR` | `/tmp/tbench2_env_runs` | Directory for session logs and cache | | `TB2_CACHE_DIR` | `$TB2_OUTPUT_DIR/repo_cache` | Where to extract TB2 repo | | `TB2_REPO_URL` | (GitHub main.zip) | Repo zip URL for auto-download | +| `TB2_WITHHOLD_TESTS` | `0` | `1`: `reset()` deletes the task's `tests/` and `solution/` from disk (tests kept in server memory, staged at `/tests` only while scoring). Set for RL training so the agent cannot read the expected outputs or the answer; leave off when `TB2_TASKS_DIR` is a working checkout you don't want modified | ## Reward diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index 33c917227..44a0fd0cc 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -8,8 +8,14 @@ from __future__ import annotations +import io import logging import os +import re +import shlex +import shutil +import tarfile +import threading import sys import urllib.request import zipfile @@ -36,6 +42,39 @@ _CAMEL_IMPORT_ERROR: Exception | None = None +# Official TB2 fixed paths: tests/test.sh assumes it runs from /tests and +# writes /logs/verifier/reward.txt. Module-level only so unit tests can +# redirect them off the real root filesystem. +_VERIFY_TESTS_DIR = "/tests" +_VERIFIER_LOG_DIR = "/logs/verifier" + + +def _hard_remove(path: str) -> None: + """Remove whatever is at *path* — file, symlink, or directory tree. + + shutil.rmtree refuses to remove a symlink and, with ignore_errors=True, + no-ops on one. That leaves an escape hatch: a root agent that symlinks a + fixed path (/tests, /logs/verifier) at a directory it controls would keep + that target alive across the "wipe", so a pre-planted conftest.py survives + into scoring. Unlink the symlink itself before falling back to rmtree. + """ + p = Path(path) + if p.is_symlink() or p.is_file(): + p.unlink(missing_ok=True) + else: + shutil.rmtree(path, ignore_errors=True) + + +def _extractall(tar: tarfile.TarFile, dest: str) -> None: + """extractall with the 3.12 data filter when available. The tarball is one + this server wrote from a trusted tests/ dir, so the filter is defence in + depth; on <3.12 (the ``filter`` kwarg predates it) fall back cleanly rather + than raising TypeError mid-verify (the package targets Python >=3.10).""" + try: + tar.extractall(dest, filter="data") + except TypeError: + tar.extractall(dest) + def _require_terminal_toolkit() -> Any: global _CAMEL_IMPORT_ERROR @@ -82,6 +121,38 @@ def _read_instruction(task_dir: Path) -> str: return "" +def _task_image_workdir(task_dir: Path) -> str: + """WORKDIR pinned by the task's own image (environment/Dockerfile). + + Empty string when the task ships no Dockerfile or no WORKDIR; multi-stage + Dockerfiles resolve to the last WORKDIR (the final stage's). + """ + dockerfile = task_dir / "environment" / "Dockerfile" + workdir = "" + if dockerfile.is_file(): + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + m = re.match(r"^\s*WORKDIR\s+(\S+)", line, re.IGNORECASE) + if m: + workdir = m.group(1) + return workdir + + +def _workdir_is_server_tree(workdir: str) -> bool: + """True when *workdir* contains this server's own code. + + A Dockerfile-parsed WORKDIR is only meaningful when the server runs + inside the task image (per-task sandboxes, where the server layer lives + elsewhere, e.g. /opt/envserver). The standard env-server image also + installs to /app — the most common task WORKDIR — so the path merely + existing is not enough: if the server's own code sits under it, it is + the server tree, not the task tree. + """ + try: + return Path(__file__).resolve().is_relative_to(Path(workdir).resolve()) + except OSError: + return True # unresolvable path: fail toward the task-dir fallback + + def _read_timeout(task_dir: Path, fallback: float) -> float: task_toml = task_dir / "task.toml" if not task_toml.exists(): @@ -103,15 +174,28 @@ def __init__( self, tasks_dir: str | None = None, output_dir: str | None = None, - command_timeout_s: float = 20.0, + command_timeout_s: float | None = None, safe_mode: bool = False, default_task_id: str | None = None, + withhold_tests: bool | None = None, ) -> None: super().__init__() self.tasks_dir = tasks_dir or os.getenv("TB2_TASKS_DIR", "") + # RL hygiene: the env server and the agent share one container + # filesystem, so tests/ and solution/ in the task dir are readable + # (and writable) by the agent. When enabled, reset() moves them out + # of reach — see _withhold_verifier_assets. + if withhold_tests is None: + withhold_tests = os.getenv("TB2_WITHHOLD_TESTS", "0") == "1" + self.withhold_tests = withhold_tests self.output_dir = Path( output_dir or os.getenv("TB2_OUTPUT_DIR", "/tmp/tbench2_env_runs") ) + # Overridable via TB2_COMMAND_TIMEOUT_S: create_app instantiates the + # environment with no arguments, and real TB2 agent commands / the + # canonical tests/test.sh routinely exceed the old 20s default. + if command_timeout_s is None: + command_timeout_s = float(os.getenv("TB2_COMMAND_TIMEOUT_S", "20.0")) self.command_timeout_s = command_timeout_s self.safe_mode = safe_mode self.default_task_id = default_task_id or os.getenv( @@ -122,6 +206,7 @@ def __init__( self._task_dir: Path | None = None self._terminal_toolkit = None self._instruction = "" + self._workdir = "" def reset( self, @@ -143,6 +228,8 @@ def reset( self._instruction = _read_instruction(task_dir) self._task_dir = task_dir + if self.withhold_tests: + self._withhold_verifier_assets(task_dir) trial_name = f"{resolved_task_id}.{episode_id or uuid4().hex}" session_logs_dir = ( @@ -150,13 +237,30 @@ def reset( ) session_logs_dir.mkdir(parents=True, exist_ok=True) + # Commands run in the task image's real WORKDIR — where agents and + # the canonical harness expect to operate, and not always /app + # (fix-git: /app/personal-site, prove-plus-comm: /workspace). + # TB2_TASK_WORKDIR overrides; otherwise the task's own Dockerfile is + # parsed, trusted only when the path isn't the server's own install + # tree (see _workdir_is_server_tree); plain local mode falls back to + # the task source dir. + workdir = os.getenv("TB2_TASK_WORKDIR", "").strip() + if not workdir: + workdir = _task_image_workdir(task_dir) + if workdir and _workdir_is_server_tree(workdir): + workdir = "" + if not (workdir and Path(workdir).is_dir()): + workdir = str(task_dir) + self._workdir = workdir + # No install_dependencies: evaluation runs the task's canonical + # tests/test.sh, which pins its own pytest toolchain via uvx (see + # _evaluate_canonical), so the toolkit venv does not need pytest. self._terminal_toolkit = TerminalToolkit( timeout=self.command_timeout_s, - working_directory=str(task_dir), + working_directory=workdir, use_docker_backend=False, session_logs_dir=session_logs_dir, safe_mode=self.safe_mode, - install_dependencies=["pytest"], ) self._state = Tbench2State( @@ -209,10 +313,14 @@ def step( try: if action.action_type == "exec": + # Pass the timeout explicitly: camel's shell_exec has its own + # per-call default (20s) and ignores the constructor timeout, + # which silently truncates any longer-running agent command. output = self._terminal_toolkit.shell_exec( command=action.command, block=action.block, id=session_id, + timeout=self.command_timeout_s, ) elif action.action_type == "write": self._ensure_session_id(action.session_id, action.action_type) @@ -306,19 +414,187 @@ def _ensure_session_id(self, session_id: str | None, action_type: str) -> None: if not session_id: raise ValueError(f"session_id is required for action_type='{action_type}'") + # tests/ tarballs withheld out of agent reach at reset(), keyed by task + # dir. Class-level: the server may construct one environment instance per + # session, and a later reset of the same task must still find the copy + # after the on-disk source is gone. In-process only — withhold mode + # therefore assumes a single server process owns the task filesystem for + # its lifetime (the intended per-task-sandbox deployment: one uvicorn + # worker, one ephemeral sandbox per episode). It is not safe with + # ``uvicorn --workers N`` sharing one on-disk checkout, where a sibling + # worker would see tests/ already deleted with no cache to restore from. + _WITHHELD_TESTS: dict[str, bytes] = {} + _WITHHOLD_LOCK = threading.Lock() + + def _withhold_verifier_assets(self, task_dir: Path) -> None: + """Move verifier assets out of the agent's reach before it can act. + + The task directory is agent-readable (same filesystem, root agent): + tests/test_outputs.py spells out the expected outputs and solution/ is + the literal answer — for RL training both are reward-hacking bait. + tests/ is read into server memory (the staging source for + _stage_tests_for_verify) and removed from disk; solution/ is simply + removed (nothing in this env reads it — it exists only for oracle + runs). Gated behind TB2_WITHHOLD_TESTS=1 because in plain local mode + task_dir may be a developer's working checkout, where deleting from + it would be hostile. + """ + key = str(task_dir) + with self._WITHHOLD_LOCK: + tests_dir = task_dir / "tests" + if tests_dir.is_dir(): + if key not in self._WITHHELD_TESTS: + # realpath so a symlinked tests/ caches the target's files, + # not just a link member that would restore empty (tar.add + # does not follow a top-level symlink). + tests_real = os.path.realpath(tests_dir) + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + tar.add(tests_real, arcname=".") + self._WITHHELD_TESTS[key] = buf.getvalue() + _hard_remove(str(tests_dir)) + _hard_remove(str(task_dir / "solution")) + + def _stage_tests_for_verify(self) -> bool: + """Stage a pristine tests/ copy at the official fixed path for exactly + the verify window. Caller must hold _CANONICAL_EVAL_LOCK. + + Both fixed paths are recreated from scratch (symlink included — see + _hard_remove) so nothing an agent may have pre-planted survives into + scoring: a /tests/conftest.py pytest would auto-load, or a symlinked + /tests or /logs/verifier redirecting the stage at a dir the agent + controls. When the source dir was withheld at reset, the staged copy + comes from server memory: a copy the agent never had filesystem + access to. + """ + _hard_remove(_VERIFY_TESTS_DIR) + _hard_remove(_VERIFIER_LOG_DIR) + Path(_VERIFIER_LOG_DIR).mkdir(parents=True, exist_ok=True) + + blob = self._WITHHELD_TESTS.get(str(self._task_dir)) + if blob is not None: + Path(_VERIFY_TESTS_DIR).mkdir(parents=True, exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(blob)) as tar: + _extractall(tar, _VERIFY_TESTS_DIR) + return True + tests_dir = Path(self._task_dir) / "tests" + if not tests_dir.is_dir(): + return False + shutil.copytree(tests_dir, _VERIFY_TESTS_DIR, symlinks=True) + return True + def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: if self._task_dir is None: raise RuntimeError("TB2 environment not initialized. Call reset() first.") if self._terminal_toolkit is None: raise RuntimeError("Terminal toolkit not initialized.") - _read_timeout(self._task_dir, fallback=900.0) # Validate timeout config - tests_dir = self._task_dir / "tests" - cmd = f"cd {self._task_dir} && python -m pytest -q {tests_dir} -rA; echo __TB2_EXIT_CODE__:$?" + # The task's own verifier budget (task.toml [verifier].timeout_sec) — + # heavy tests legitimately run minutes (circuit-fibsqrt declares 3600s). + verifier_timeout_s = _read_timeout(self._task_dir, fallback=900.0) + + with self._CANONICAL_EVAL_LOCK: + try: + # Staging happens inside the try: a stage that errors partway + # (I/O failure mid-extract) must be wiped like a completed + # one — step() reports scoring errors without ending the + # episode, so the still-live session must not find a partial + # tests/ copy left behind. + if not self._stage_tests_for_verify(): + return ( + f"no tests/ found for task {self._state.task_id}", + 0.0, + {"tests_passed": False, "error": "missing tests"}, + ) + if (Path(_VERIFY_TESTS_DIR) / "test.sh").is_file(): + return self._evaluate_canonical(verifier_timeout_s) + return self._evaluate_fallback(verifier_timeout_s) + finally: + # Verifier artifacts live on disk only for the verify window: + # don't leave them readable to a later episode / concurrent + # session sharing this filesystem — or to the agent itself if + # scoring times out / errors (done stays false, so it keeps + # its session). /logs/verifier holds reward.txt and the + # test.sh log, whose pytest -rA output can reveal expected + # values. + _hard_remove(_VERIFY_TESTS_DIR) + _hard_remove(_VERIFIER_LOG_DIR) + + # The canonical harness uses the official fixed paths (/tests, + # /logs/verifier/reward.txt), which are per-container, not per-session. + # Serialize staging + evaluation so concurrent sessions on one server + # cannot overwrite each other's staged tests or read another session's + # reward. + _CANONICAL_EVAL_LOCK = threading.Lock() + + def _evaluate_canonical( + self, timeout_s: float + ) -> tuple[str, float, dict[str, Any]]: + """Score via the task's OFFICIAL harness, exactly as the TB2 verifier does. + + /tests/test.sh (staged by _stage_tests_for_verify) pins its own pytest + toolchain (uvx: Python 3.13 + pytest 8.4.1 + ctrf), runs + tests/test_outputs.py from /tests against the task workdir, and writes + the binary result to /logs/verifier/reward.txt. Bare ``pytest tests/`` + (the fallback below) skips that toolchain pinning and mis-scores any + task whose tests need it, so the canonical harness is preferred + whenever the task ships it. + """ + # Same working dir the agent operated in (resolved during reset). + workdir = self._workdir or str(self._task_dir) + marker = "__TB2_REWARD__:" + # test.sh stdout goes under /logs/verifier (wiped with the verify + # window in _evaluate_task) rather than a fixed /tmp path that would + # outlive scoring: its pytest -rA output can spell out expected values. + # Its tail is read back into the returned output — the episode is over + # by then, and callers need the pytest diagnostics on failure; the + # reward marker line stays last for parsing. + cmd = ( + f"cd {shlex.quote(workdir)} && " + f"bash {_VERIFY_TESTS_DIR}/test.sh > {_VERIFIER_LOG_DIR}/testsh.log 2>&1; " + f"tail -c 20000 {_VERIFIER_LOG_DIR}/testsh.log 2>/dev/null; " + f"echo {marker}$(cat {_VERIFIER_LOG_DIR}/reward.txt 2>/dev/null)" + ) + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, + ) + + reward = 0.0 + for line in output.splitlines()[::-1]: + if marker in line: + raw = line.split(marker, 1)[1].strip() + try: + reward = float(raw) if raw else 0.0 + except ValueError: + reward = 0.0 + break + + info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} + return output, reward, info + + def _evaluate_fallback(self, timeout_s: float) -> tuple[str, float, dict[str, Any]]: + """For task dirs without the canonical harness (none of the 89 official + TB2 tasks — they all ship test.sh — but custom task dirs may only have + bare pytest tests). Runs pytest against the staged /tests copy; prefer + uvx so pytest comes with its own toolchain like the canonical harness + does. Verify from the same directory the agent worked in. + """ + fallback_cwd = self._workdir or str(self._task_dir) + cmd = ( + f"cd {shlex.quote(fallback_cwd)} && " + "if command -v uvx >/dev/null 2>&1; " + f"then uvx --with pytest==8.4.1 pytest -q {_VERIFY_TESTS_DIR} -rA; " + f"else python -m pytest -q {_VERIFY_TESTS_DIR} -rA; fi; " + "echo __TB2_EXIT_CODE__:$?" + ) output = self._terminal_toolkit.shell_exec( id="tb2-tests", command=cmd, block=True, + timeout=timeout_s, ) exit_code = 1 @@ -484,8 +760,14 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: # Copy task files into container using tar archive # This works in Docker-in-Docker because we read files from our - # filesystem and stream them to the container via the Docker API - self._copy_dir_to_container(task_dir, "/task") + # filesystem and stream them to the container via the Docker API. + # Verifier assets stay out: the agent works in /task and must not + # be able to read solution/ (the literal answer) or tests/ (the + # expected outputs) — tests are staged only at verify time by + # _evaluate_docker, like the official TB2 harness does. + self._copy_dir_to_container( + task_dir, "/task", exclude_top=("solution", "tests") + ) self._state = Tbench2State( episode_id=str(uuid4()), @@ -498,24 +780,28 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: except Exception as exc: raise RuntimeError(f"Failed to start container: {exc}") from exc - def _copy_dir_to_container(self, src_dir: Path, dest_path: str) -> None: + def _copy_dir_to_container( + self, src_dir: Path, dest_path: str, exclude_top: tuple[str, ...] = () + ) -> None: """Copy a directory into the container using tar archive. This method streams files via the Docker API, avoiding bind mount - issues in Docker-in-Docker scenarios. + issues in Docker-in-Docker scenarios. Top-level entries named in + *exclude_top* are skipped entirely. """ - import io - import tarfile - if self._container is None: raise RuntimeError("Container not started") - # Create tar archive in memory + # Create tar archive in memory (recursive=False: rglob already yields + # every descendant, so per-entry recursion would duplicate members — + # and would re-add excluded subtrees through their parents). tar_stream = io.BytesIO() with tarfile.open(fileobj=tar_stream, mode="w") as tar: for item in src_dir.rglob("*"): - arcname = str(item.relative_to(src_dir)) - tar.add(str(item), arcname=arcname) + rel = item.relative_to(src_dir) + if rel.parts and rel.parts[0] in exclude_top: + continue + tar.add(str(item), arcname=str(rel), recursive=False) tar_stream.seek(0) @@ -639,28 +925,71 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: raise RuntimeError("Container not started.") assert self._task_dir is not None, "Task directory not set" - # Run pytest in the container's /task directory - # Use exit code marker for consistency with local mode - cmd = "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" - - exit_code, output = self._container.exec_run( - cmd=f"bash -c '{cmd}'", - workdir="/task", - stdout=True, - stderr=True, + # Stage-at-verify, like the official TB2 harness: the initial /task + # copy excludes tests/, and this server sits OUTSIDE the container, so + # the copy staged here is one the agent never saw. rm -rf first so + # nothing an agent pre-planted at /task/tests (e.g. a conftest.py + # pytest would auto-load) survives into scoring. + tests_src = self._task_dir / "tests" + if not tests_src.is_dir(): + return ( + f"no tests/ found for task {self._state.task_id}", + 0.0, + {"tests_passed": False, "error": "missing tests"}, + ) + wipe_ec, wipe_out = self._exec_in_container( + "rm -rf /task/tests && mkdir -p /task/tests" ) - output_str = output.decode("utf-8", errors="replace") + if wipe_ec != 0: + # Fail closed: put_archive into a dir that survived the wipe would + # merge the staged tests into whatever the agent planted there. + raise RuntimeError( + f"could not reset /task/tests before verify: {wipe_out.strip()}" + ) + try: + self._copy_dir_to_container(tests_src, "/task/tests") - # Parse exit code from marker (same logic as local mode) - ec = 1 - marker = "__TB2_EXIT_CODE__" - for line in output_str.splitlines()[::-1]: - if marker in line: - try: - ec = int(line.split(":", 1)[1].strip()) - except Exception: - ec = 1 - break + # Run pytest in the container's /task directory + # Use exit code marker for consistency with local mode + cmd = ( + "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" + ) + + exit_code, output = self._container.exec_run( + cmd=f"bash -c '{cmd}'", + workdir="/task", + stdout=True, + stderr=True, + ) + output_str = output.decode("utf-8", errors="replace") + + # Parse exit code from marker (same logic as local mode) + ec = 1 + marker = "__TB2_EXIT_CODE__" + for line in output_str.splitlines()[::-1]: + if marker in line: + try: + ec = int(line.split(":", 1)[1].strip()) + except Exception: + ec = 1 + break + finally: + # Tests live in the container only for the verify window, matching + # the local-mode staging: don't leave /task/tests readable + # afterward — the agent keeps its session if scoring errored + # (step() reports the failure without ending the episode). + try: + rm_ec, rm_out = self._exec_in_container("rm -rf /task/tests") + if rm_ec != 0: + logging.warning( + "failed to remove staged /task/tests after verify: %s", + rm_out.strip(), + ) + except Exception: + logging.warning( + "failed to remove staged /task/tests after verify", + exc_info=True, + ) reward = 1.0 if ec == 0 else 0.0 info = {"tests_passed": ec == 0, "exit_code": ec} diff --git a/tests/envs/test_tbench2_env.py b/tests/envs/test_tbench2_env.py index 9e95cf6fb..f0afdb064 100644 --- a/tests/envs/test_tbench2_env.py +++ b/tests/envs/test_tbench2_env.py @@ -1,11 +1,20 @@ +import io +import logging import os import sys +import tarfile from pathlib import Path import pytest -# Add the project root to the path for envs imports -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +# Add the project root to the path for envs imports, and envs/ itself so the +# server module's Spaces-style absolute imports (``tbench2_env.models``) +# resolve when this file runs standalone (in the full suite another test +# happens to insert envs/ first — don't rely on collection order). +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +for _p in (_REPO_ROOT, os.path.join(_REPO_ROOT, "envs")): + if _p not in sys.path: + sys.path.insert(0, _p) try: import camel # noqa: F401 @@ -14,7 +23,10 @@ from envs.tbench2_env.models import Tbench2Action from envs.tbench2_env.server import tbench2_env_environment -from envs.tbench2_env.server.tbench2_env_environment import Tbench2Environment +from envs.tbench2_env.server.tbench2_env_environment import ( + Tbench2DockerEnvironment, + Tbench2Environment, +) class _FakeTerminalToolkit: @@ -50,6 +62,428 @@ def test_tbench2_reset_uses_default_task_id(monkeypatch, tmp_path: Path): assert "terminal task" in observation.instruction +def _make_env(monkeypatch, tmp_path: Path, task_id: str) -> Tbench2Environment: + monkeypatch.setattr( + tbench2_env_environment, + "_require_terminal_toolkit", + lambda: _FakeTerminalToolkit, + ) + return Tbench2Environment( + tasks_dir=str(tmp_path), + output_dir=str(tmp_path / "runs"), + default_task_id=task_id, + ) + + +def test_tbench2_workdir_from_task_dockerfile(monkeypatch, tmp_path: Path): + """A parsed WORKDIR that exists and isn't the server tree becomes the cwd.""" + task_dir = tmp_path / "demo-task" + (task_dir / "environment").mkdir(parents=True) + (task_dir / "instruction.md").write_text("do it\n") + image_workdir = tmp_path / "image-workdir" + image_workdir.mkdir() + (task_dir / "environment" / "Dockerfile").write_text( + f"FROM python:3.13\nWORKDIR {image_workdir}\n" + ) + + env = _make_env(monkeypatch, tmp_path, "demo-task") + env.reset() + + assert env._workdir == str(image_workdir) + + +def test_tbench2_workdir_rejects_server_tree(monkeypatch, tmp_path: Path): + """A parsed WORKDIR that is the server's own install tree (e.g. /app on + the standard server image) falls back to the task source dir.""" + server_tree = Path(tbench2_env_environment.__file__).resolve().parent + task_dir = tmp_path / "demo-task" + (task_dir / "environment").mkdir(parents=True) + (task_dir / "instruction.md").write_text("do it\n") + (task_dir / "environment" / "Dockerfile").write_text( + f"FROM python:3.13\nWORKDIR {server_tree}\n" + ) + + env = _make_env(monkeypatch, tmp_path, "demo-task") + env.reset() + + assert env._workdir == str(task_dir) + + +# --------------------------------------------------------------------------- +# Verifier-asset withholding (TB2_WITHHOLD_TESTS) and stage-at-verify +# --------------------------------------------------------------------------- + + +def _make_task_dir(root: Path, name: str = "my-task") -> Path: + task = root / name + (task / "tests").mkdir(parents=True) + (task / "tests" / "test.sh").write_text( + "#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n" + ) + (task / "tests" / "test_outputs.py").write_text("EXPECTED = 'secret42'\n") + (task / "solution").mkdir() + (task / "solution" / "solve.sh").write_text("echo the-answer\n") + (task / "task.toml").write_text('[environment]\ndocker_image = "debian:12"\n') + (task / "instruction.md").write_text("do the thing\n") + return task + + +@pytest.fixture() +def staged_paths(monkeypatch, tmp_path: Path): + """Redirect the official fixed paths (/tests, /logs/verifier) into tmp.""" + tests = tmp_path / "stage" / "tests" + logs = tmp_path / "stage" / "logs" / "verifier" + monkeypatch.setattr(tbench2_env_environment, "_VERIFY_TESTS_DIR", str(tests)) + monkeypatch.setattr(tbench2_env_environment, "_VERIFIER_LOG_DIR", str(logs)) + return tests, logs + + +@pytest.fixture(autouse=True) +def _fresh_withhold_cache(monkeypatch): + monkeypatch.setattr(Tbench2Environment, "_WITHHELD_TESTS", {}) + + +def test_withhold_removes_assets_and_caches_tests(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + + env._withhold_verifier_assets(task) + + assert not (task / "solution").exists() + assert not (task / "tests").exists() + assert (task / "task.toml").is_file() + assert (task / "instruction.md").is_file() + assert str(task) in Tbench2Environment._WITHHELD_TESTS + + # Second reset of the same task (dir already gone) must keep the cache. + env._withhold_verifier_assets(task) + assert str(task) in Tbench2Environment._WITHHELD_TESTS + + +def test_stage_from_memory_and_wipe_planted_files(tmp_path: Path, staged_paths): + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._withhold_verifier_assets(task) + + # An agent pre-plants a conftest.py (pytest would auto-load it) and a + # fake reward before verification. + stage_tests.mkdir(parents=True) + (stage_tests / "conftest.py").write_text("import sys; sys.exit(0)\n") + stage_logs.mkdir(parents=True) + (stage_logs / "reward.txt").write_text("1\n") + + assert env._stage_tests_for_verify() + + assert (stage_tests / "test.sh").is_file() + assert (stage_tests / "test_outputs.py").read_text() == "EXPECTED = 'secret42'\n" + assert not (stage_tests / "conftest.py").exists() + assert not (stage_logs / "reward.txt").exists() + assert stage_logs.is_dir() and not list(stage_logs.iterdir()) + + +def test_stage_from_disk_without_gate(tmp_path: Path, staged_paths): + stage_tests, _ = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + + assert env._stage_tests_for_verify() + + assert (stage_tests / "test_outputs.py").is_file() + # Without the gate the source checkout is never touched. + assert (task / "tests").is_dir() + assert (task / "solution").is_dir() + + +def test_stage_reports_missing_tests(tmp_path: Path, staged_paths): + task = tmp_path / "empty-task" + task.mkdir() + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + + assert not env._stage_tests_for_verify() + + +class _RecordingToolkit: + def __init__(self, output: str): + self.calls: list[dict] = [] + self.output = output + + def shell_exec(self, **kwargs): + self.calls.append(kwargs) + return self.output + + +def test_evaluate_canonical_from_withheld_copy(tmp_path: Path, staged_paths): + """Full evaluate flow: staging from memory + canonical test.sh scoring.""" + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._workdir = str(tmp_path) + env._terminal_toolkit = _RecordingToolkit(output="__TB2_REWARD__:1") + env._withhold_verifier_assets(task) + + output, reward, info = env._evaluate_task() + + assert reward == 1.0 + assert info["harness"] == "tests/test.sh" + # The verify command ran test.sh from the staged fixed path, and wrote its + # log under the wiped verifier dir — not a fixed /tmp path that outlives it. + command = env._terminal_toolkit.calls[0]["command"] + assert f"{stage_tests}/test.sh" in command + assert "/tmp/tb2_testsh.log" not in command + assert f"{stage_logs}/testsh.log" in command + # Both the staged tests and the verifier log/reward dir are gone once + # scoring returns. + assert not stage_tests.exists() + assert not stage_logs.exists() + + +def test_evaluate_without_tests_scores_zero(tmp_path: Path, staged_paths): + task = tmp_path / "empty-task" + task.mkdir() + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + env._terminal_toolkit = _RecordingToolkit(output="") + + output, reward, info = env._evaluate_task() + + assert reward == 0.0 + assert info["tests_passed"] is False + assert env._terminal_toolkit.calls == [] # nothing was run + + +def test_evaluate_wipes_partial_stage_on_error( + tmp_path: Path, staged_paths, monkeypatch +): + """A stage that errors partway must be wiped like a completed one — the + session survives a scoring error, so no partial tests/ copy may remain.""" + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._workdir = str(tmp_path) + env._terminal_toolkit = _RecordingToolkit(output="") + env._withhold_verifier_assets(task) + + def _explode(tar, dest): + (Path(dest) / "test_outputs.py").write_text("EXPECTED = 'secret42'\n") + raise OSError("disk full") + + monkeypatch.setattr(tbench2_env_environment, "_extractall", _explode) + + with pytest.raises(OSError, match="disk full"): + env._evaluate_task() + + assert not stage_tests.exists() + assert not stage_logs.exists() + + +def test_stage_wipes_symlinked_fixed_path(tmp_path: Path, staged_paths): + """A symlinked /tests must not let the official suite merge into a dir the + agent controls (rmtree silently no-ops on a symlink; _hard_remove doesn't).""" + stage_tests, _ = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._withhold_verifier_assets(task) + + evil = tmp_path / "agent_controlled" + evil.mkdir() + (evil / "conftest.py").write_text("import sys; sys.exit(0)\n") + stage_tests.parent.mkdir(parents=True, exist_ok=True) + stage_tests.symlink_to(evil, target_is_directory=True) + + assert env._stage_tests_for_verify() + + assert not stage_tests.is_symlink() + assert (stage_tests / "test.sh").is_file() + # The official suite never leaked into the agent's directory. + assert not (evil / "test.sh").exists() + assert (evil / "conftest.py").exists() # their dir left alone + + +def test_withhold_removes_symlinked_tests(tmp_path: Path): + """tests/ as a symlink is removed too — the link, not just its target.""" + task = tmp_path / "task" + task.mkdir() + real = tmp_path / "real_tests" + real.mkdir() + (real / "test_outputs.py").write_text("x = 1\n") + (task / "tests").symlink_to(real, target_is_directory=True) + + env = Tbench2Environment(withhold_tests=True) + env._withhold_verifier_assets(task) + + assert not (task / "tests").exists() # symlink gone + assert real.exists() # its target is outside the task dir; untouched + # The cache captured the target's real files, not a bare link member that + # would restore an empty suite (tar.add doesn't follow a top-level link). + import io + import tarfile + + blob = Tbench2Environment._WITHHELD_TESTS[str(task)] + with tarfile.open(fileobj=io.BytesIO(blob)) as tar: + assert any(n.endswith("test_outputs.py") for n in tar.getnames()) + + +# --------------------------------------------------------------------------- +# Docker mode: copy exclusions + stage-at-verify (mock container, no docker) +# --------------------------------------------------------------------------- + + +class _FakeContainer: + """Records exec/put_archive calls in one ordered event log.""" + + def __init__(self, exec_output: bytes = b"__TB2_EXIT_CODE__:0\n"): + self.events: list[tuple] = [] + self.exec_output = exec_output + + def put_archive(self, dest, data): + self.events.append(("put", dest, data)) + return True + + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + return 0, self.exec_output + + +def _tar_names(data: bytes) -> list[str]: + with tarfile.open(fileobj=io.BytesIO(data)) as tar: + return tar.getnames() + + +def test_docker_copy_excludes_verifier_assets(tmp_path: Path): + task = _make_task_dir(tmp_path) + (task / "environment").mkdir() + (task / "environment" / "Dockerfile").write_text("FROM debian:12\n") + + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._copy_dir_to_container(task, "/task", exclude_top=("solution", "tests")) + + ((kind, dest, data),) = container.events + assert (kind, dest) == ("put", "/task") + names = _tar_names(data) + assert "task.toml" in names + assert "environment/Dockerfile" in names + assert not any(n.startswith(("solution", "tests")) for n in names) + assert len(names) == len(set(names)) # recursive=False: no duplicates + + +def test_evaluate_docker_stages_tests_at_verify(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 1.0 + kinds = [e[0] for e in container.events] + # rm -rf the agent-writable staging dir BEFORE the fresh copy goes in, + # the copy lands before pytest runs, and tests are removed again after. + assert kinds == ["exec", "put", "exec", "exec"] + assert "rm -rf /task/tests" in container.events[0][1] + assert container.events[1][1] == "/task/tests" + assert "test.sh" in _tar_names(container.events[1][2]) + assert "pytest" in container.events[2][1] + assert "rm -rf /task/tests" in container.events[3][1] + + +def test_evaluate_docker_cleans_up_when_scoring_raises(tmp_path: Path): + """The staged /task/tests must not outlive a verify that errors out — + step() reports the failure without ending the episode, so the agent + keeps its session.""" + task = _make_task_dir(tmp_path) + + class _ExplodingContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + if "pytest" in cmd: + self.events.append(("exec", cmd)) + raise RuntimeError("docker daemon hiccup") + return super().exec_run(cmd, workdir=workdir, stdout=stdout, stderr=stderr) + + env = Tbench2DockerEnvironment() + container = _ExplodingContainer() + env._container = container + env._task_dir = task + + with pytest.raises(RuntimeError, match="hiccup"): + env._evaluate_docker() + + assert container.events[-1][0] == "exec" + assert "rm -rf /task/tests" in container.events[-1][1] + + +def test_evaluate_docker_fails_closed_when_prestage_wipe_fails(tmp_path: Path): + """If the pre-stage rm -rf fails, scoring must error out rather than + put_archive into a dir the agent still controls.""" + task = _make_task_dir(tmp_path) + + class _StubbornContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + if "rm -rf /task/tests" in cmd and "mkdir" in cmd: + return 1, b"rm: cannot remove '/task/tests': busy\n" + return 0, self.exec_output + + env = Tbench2DockerEnvironment() + container = _StubbornContainer() + env._container = container + env._task_dir = task + + with pytest.raises(RuntimeError, match="could not reset /task/tests"): + env._evaluate_docker() + + assert not any(e[0] == "put" for e in container.events) + + +def test_evaluate_docker_warns_when_cleanup_fails(tmp_path: Path, caplog): + """A post-verify rm -rf that exits nonzero must not stay silent.""" + task = _make_task_dir(tmp_path) + + class _LeakyContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + if "rm -rf /task/tests" in cmd and "mkdir" not in cmd: + return 1, b"rm: cannot remove '/task/tests': busy\n" + return 0, self.exec_output + + env = Tbench2DockerEnvironment() + container = _LeakyContainer() + env._container = container + env._task_dir = task + + with caplog.at_level(logging.WARNING): + output, reward, info = env._evaluate_docker() + + assert reward == 1.0 + assert any( + "failed to remove staged /task/tests" in record.message + for record in caplog.records + ) + + +def test_evaluate_docker_missing_tests_scores_zero(tmp_path: Path): + task = tmp_path / "no-tests-task" + task.mkdir() + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 0.0 + assert container.events == [] + + @pytest.mark.skipif(camel is None, reason="camel-ai not installed") @pytest.mark.skipif( os.environ.get("TB2_ENABLE_TESTS", "0") != "1",