From 85343bd445cd7ba64bd496d8705fbc85404bc205 Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Tue, 14 Jul 2026 21:03:54 -0700 Subject: [PATCH 1/2] fix(tbench2_env): canonical test.sh evaluate, real timeouts, native task env & workdir - _evaluate_task runs the task's official tests/test.sh (pinned uvx toolchain, /logs/verifier/reward.txt) when the task ships it; the bare pytest fallback (custom task dirs without the canonical harness) prefers uvx with a pinned pytest so it carries its own toolchain, and runs from the same workdir the agent worked in. - Pass timeouts through to every shell_exec call: camel's shell_exec has its own per-call default (20s) and ignores the constructor timeout, so agent commands and evaluations were silently truncated at 20s (circuit-fibsqrt's verifier declares 3600s and was cut mid-test). Agent execs use command_timeout_s (TB2_COMMAND_TIMEOUT_S-overridable); evaluate uses the task's own [verifier].timeout_sec budget. - Shell cwd resolves to the task image's own WORKDIR, parsed from the task's environment/Dockerfile (TB2_TASK_WORKDIR overrides): task images pin their own WORKDIR (fix-git: /app/personal-site, prove-plus-comm: /workspace) and both agents and oracle solutions assume commands run there. Falls back to the task source dir for plain local mode. - Drop install_dependencies=['pytest'] (evaluation carries its own pytest via uvx); with CAMEL_RUNTIME=true camel's .initial_env venv then stops hijacking PATH away from the task image's native python. - Serialize canonical evaluation under a lock so concurrent sessions on one server cannot clobber the official fixed verifier paths (/tests, /logs/verifier/reward.txt). --- .../server/tbench2_env_environment.py | 147 +++++++++++++++++- tests/envs/test_tbench2_env.py | 47 ++++++ 2 files changed, 189 insertions(+), 5 deletions(-) diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index 33c917227..28c9c033b 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -10,6 +10,9 @@ import logging import os +import re +import shlex +import threading import sys import urllib.request import zipfile @@ -82,6 +85,38 @@ def _read_instruction(task_dir: Path) -> str: return "" +def _task_image_workdir(task_dir: Path) -> str: + """WORKDIR pinned by the task's own image (environment/Dockerfile). + + Empty string when the task ships no Dockerfile or no WORKDIR; multi-stage + Dockerfiles resolve to the last WORKDIR (the final stage's). + """ + dockerfile = task_dir / "environment" / "Dockerfile" + workdir = "" + if dockerfile.is_file(): + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + m = re.match(r"^\s*WORKDIR\s+(\S+)", line, re.IGNORECASE) + if m: + workdir = m.group(1) + return workdir + + +def _workdir_is_server_tree(workdir: str) -> bool: + """True when *workdir* contains this server's own code. + + A Dockerfile-parsed WORKDIR is only meaningful when the server runs + inside the task image (per-task sandboxes, where the server layer lives + elsewhere, e.g. /opt/envserver). The standard env-server image also + installs to /app — the most common task WORKDIR — so the path merely + existing is not enough: if the server's own code sits under it, it is + the server tree, not the task tree. + """ + try: + return Path(__file__).resolve().is_relative_to(Path(workdir).resolve()) + except OSError: + return True # unresolvable path: fail toward the task-dir fallback + + def _read_timeout(task_dir: Path, fallback: float) -> float: task_toml = task_dir / "task.toml" if not task_toml.exists(): @@ -103,7 +138,7 @@ def __init__( self, tasks_dir: str | None = None, output_dir: str | None = None, - command_timeout_s: float = 20.0, + command_timeout_s: float | None = None, safe_mode: bool = False, default_task_id: str | None = None, ) -> None: @@ -112,6 +147,11 @@ def __init__( self.output_dir = Path( output_dir or os.getenv("TB2_OUTPUT_DIR", "/tmp/tbench2_env_runs") ) + # Overridable via TB2_COMMAND_TIMEOUT_S: create_app instantiates the + # environment with no arguments, and real TB2 agent commands / the + # canonical tests/test.sh routinely exceed the old 20s default. + if command_timeout_s is None: + command_timeout_s = float(os.getenv("TB2_COMMAND_TIMEOUT_S", "20.0")) self.command_timeout_s = command_timeout_s self.safe_mode = safe_mode self.default_task_id = default_task_id or os.getenv( @@ -122,6 +162,7 @@ def __init__( self._task_dir: Path | None = None self._terminal_toolkit = None self._instruction = "" + self._workdir = "" def reset( self, @@ -150,13 +191,30 @@ def reset( ) session_logs_dir.mkdir(parents=True, exist_ok=True) + # Commands run in the task image's real WORKDIR — where agents and + # the canonical harness expect to operate, and not always /app + # (fix-git: /app/personal-site, prove-plus-comm: /workspace). + # TB2_TASK_WORKDIR overrides; otherwise the task's own Dockerfile is + # parsed, trusted only when the path isn't the server's own install + # tree (see _workdir_is_server_tree); plain local mode falls back to + # the task source dir. + workdir = os.getenv("TB2_TASK_WORKDIR", "").strip() + if not workdir: + workdir = _task_image_workdir(task_dir) + if workdir and _workdir_is_server_tree(workdir): + workdir = "" + if not (workdir and Path(workdir).is_dir()): + workdir = str(task_dir) + self._workdir = workdir + # No install_dependencies: evaluation runs the task's canonical + # tests/test.sh, which pins its own pytest toolchain via uvx (see + # _evaluate_canonical), so the toolkit venv does not need pytest. self._terminal_toolkit = TerminalToolkit( timeout=self.command_timeout_s, - working_directory=str(task_dir), + working_directory=workdir, use_docker_backend=False, session_logs_dir=session_logs_dir, safe_mode=self.safe_mode, - install_dependencies=["pytest"], ) self._state = Tbench2State( @@ -209,10 +267,14 @@ def step( try: if action.action_type == "exec": + # Pass the timeout explicitly: camel's shell_exec has its own + # per-call default (20s) and ignores the constructor timeout, + # which silently truncates any longer-running agent command. output = self._terminal_toolkit.shell_exec( command=action.command, block=action.block, id=session_id, + timeout=self.command_timeout_s, ) elif action.action_type == "write": self._ensure_session_id(action.session_id, action.action_type) @@ -312,13 +374,32 @@ def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: if self._terminal_toolkit is None: raise RuntimeError("Terminal toolkit not initialized.") - _read_timeout(self._task_dir, fallback=900.0) # Validate timeout config + # The task's own verifier budget (task.toml [verifier].timeout_sec) — + # heavy tests legitimately run minutes (circuit-fibsqrt declares 3600s). + verifier_timeout_s = _read_timeout(self._task_dir, fallback=900.0) tests_dir = self._task_dir / "tests" - cmd = f"cd {self._task_dir} && python -m pytest -q {tests_dir} -rA; echo __TB2_EXIT_CODE__:$?" + if (tests_dir / "test.sh").is_file(): + return self._evaluate_canonical(tests_dir, verifier_timeout_s) + + # Fallback for tasks without the canonical harness (none of the 89 + # official TB2 tasks — they all ship test.sh — but custom task dirs + # may only have bare pytest tests). Prefer uvx so pytest comes with + # its own toolchain like the canonical harness does; fall back to a + # preinstalled pytest for environments without uv. + # Verify from the same directory the agent worked in. + fallback_cwd = self._workdir or str(self._task_dir) + cmd = ( + f"cd {shlex.quote(fallback_cwd)} && " + "if command -v uvx >/dev/null 2>&1; " + f"then uvx --with pytest==8.4.1 pytest -q {shlex.quote(str(tests_dir))} -rA; " + f"else python -m pytest -q {shlex.quote(str(tests_dir))} -rA; fi; " + "echo __TB2_EXIT_CODE__:$?" + ) output = self._terminal_toolkit.shell_exec( id="tb2-tests", command=cmd, block=True, + timeout=verifier_timeout_s, ) exit_code = 1 @@ -335,6 +416,62 @@ def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: info = {"tests_passed": exit_code == 0, "exit_code": exit_code} return output, reward, info + # The canonical harness uses the official fixed paths (/tests, + # /logs/verifier/reward.txt), which are per-container, not per-session. + # Serialize evaluations so concurrent sessions on one server cannot + # overwrite each other's staged tests or read another session's reward. + _CANONICAL_EVAL_LOCK = threading.Lock() + + def _evaluate_canonical( + self, tests_dir: Path, timeout_s: float + ) -> tuple[str, float, dict[str, Any]]: + """Score via the task's OFFICIAL harness, exactly as the TB2 verifier does. + + tests/test.sh pins its own pytest toolchain (uvx: Python 3.13 + + pytest 8.4.1 + ctrf), runs tests/test_outputs.py from /tests against + the task workdir, and writes the binary result to + /logs/verifier/reward.txt. Bare ``pytest tests/`` (the fallback above) + skips that toolchain pinning and mis-scores any task whose tests need + it, so the canonical harness is preferred whenever the task ships it. + """ + # Same working dir the agent operated in (resolved during reset). + workdir = self._workdir or str(self._task_dir) + marker = "__TB2_REWARD__:" + cmd = ( + # /tests is recreated from scratch — a prior task's files on the + # same server must not leak into pytest collection — and the + # reward file removed so a stale one can never be read back if + # test.sh fails to run. + "rm -rf /tests && mkdir -p /tests /logs/verifier && " + "rm -f /logs/verifier/reward.txt && " + f"cp -a {shlex.quote(str(tests_dir))}/. /tests/ && " + f"cd {shlex.quote(workdir)} && bash /tests/test.sh > /tmp/tb2_testsh.log 2>&1; " + # Surface the harness log tail so callers keep the pytest + # diagnostics; the reward marker line stays last for parsing. + "tail -c 20000 /tmp/tb2_testsh.log 2>/dev/null; " + f"echo {marker}$(cat /logs/verifier/reward.txt 2>/dev/null)" + ) + with self._CANONICAL_EVAL_LOCK: + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, + ) + + reward = 0.0 + for line in output.splitlines()[::-1]: + if marker in line: + raw = line.split(marker, 1)[1].strip() + try: + reward = float(raw) if raw else 0.0 + except ValueError: + reward = 0.0 + break + + info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} + return output, reward, info + class Tbench2DockerEnvironment( Environment[Tbench2Action, Tbench2Observation, Tbench2State] diff --git a/tests/envs/test_tbench2_env.py b/tests/envs/test_tbench2_env.py index 9e95cf6fb..6490d1d1a 100644 --- a/tests/envs/test_tbench2_env.py +++ b/tests/envs/test_tbench2_env.py @@ -50,6 +50,53 @@ def test_tbench2_reset_uses_default_task_id(monkeypatch, tmp_path: Path): assert "terminal task" in observation.instruction +def _make_env(monkeypatch, tmp_path: Path, task_id: str) -> Tbench2Environment: + monkeypatch.setattr( + tbench2_env_environment, + "_require_terminal_toolkit", + lambda: _FakeTerminalToolkit, + ) + return Tbench2Environment( + tasks_dir=str(tmp_path), + output_dir=str(tmp_path / "runs"), + default_task_id=task_id, + ) + + +def test_tbench2_workdir_from_task_dockerfile(monkeypatch, tmp_path: Path): + """A parsed WORKDIR that exists and isn't the server tree becomes the cwd.""" + task_dir = tmp_path / "demo-task" + (task_dir / "environment").mkdir(parents=True) + (task_dir / "instruction.md").write_text("do it\n") + image_workdir = tmp_path / "image-workdir" + image_workdir.mkdir() + (task_dir / "environment" / "Dockerfile").write_text( + f"FROM python:3.13\nWORKDIR {image_workdir}\n" + ) + + env = _make_env(monkeypatch, tmp_path, "demo-task") + env.reset() + + assert env._workdir == str(image_workdir) + + +def test_tbench2_workdir_rejects_server_tree(monkeypatch, tmp_path: Path): + """A parsed WORKDIR that is the server's own install tree (e.g. /app on + the standard server image) falls back to the task source dir.""" + server_tree = Path(tbench2_env_environment.__file__).resolve().parent + task_dir = tmp_path / "demo-task" + (task_dir / "environment").mkdir(parents=True) + (task_dir / "instruction.md").write_text("do it\n") + (task_dir / "environment" / "Dockerfile").write_text( + f"FROM python:3.13\nWORKDIR {server_tree}\n" + ) + + env = _make_env(monkeypatch, tmp_path, "demo-task") + env.reset() + + assert env._workdir == str(task_dir) + + @pytest.mark.skipif(camel is None, reason="camel-ai not installed") @pytest.mark.skipif( os.environ.get("TB2_ENABLE_TESTS", "0") != "1", From 356045c179a05ca3546ddfdc90642fa6d1c3a689 Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Thu, 16 Jul 2026 13:02:56 -0700 Subject: [PATCH 2/2] fix(tbench2_env): keep verifier assets out of the agent's reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both execution modes let the agent read (and tamper with) the assets the verifier scores against — tests/test_outputs.py spells out the expected outputs and solution/ is the literal answer. Fine for eval plumbing with a cooperative agent; for RL training it is a reward-hacking vector. Align with the official TB2 harness's model instead: the solution never ships, and tests exist in the agent's filesystem only during the verify window. - Local mode: with TB2_WITHHOLD_TESTS=1, reset() reads tests/ into server memory and deletes it (and solution/ — nothing in this env reads it; it exists only for oracle runs) from disk before the agent's first action. Gated off by default because in plain local mode TB2_TASKS_DIR may be a developer's working checkout; deployments that stage per-task copies (e.g. cloud sandboxes) should turn it on. The in-memory copy is process-local, so withhold mode assumes one server process owns the task filesystem for its lifetime (the per-task-sandbox deployment: one worker, one ephemeral sandbox per episode) — documented on the cache. - Evaluation stages a pristine tests/ copy at /tests for exactly the verify window — from server memory when withheld, i.e. a copy the agent never had filesystem access to — and removes it again once scoring returns, so a later episode / concurrent session sharing the filesystem can't read it. Both fixed paths (/tests, /logs/verifier) are recreated from scratch first, symlink included: the previous "mkdir -p && cp -a" merged into whatever already existed, and shutil.rmtree silently no-ops on a symlink, so an agent could pre-plant a /tests/conftest.py (pytest auto-loads it) or symlink a fixed path at a dir it controls and survive into scoring. _hard_remove unlinks the symlink itself. The in-memory tarball is restored with the 3.12 data filter when available, falling back cleanly on 3.10/3.11 (the package targets >=3.10). The bare-pytest fallback runs against the same staged copy. The canonical harness writes its test.sh stdout under /logs/verifier (not a fixed /tmp path) and the verify-window cleanup wipes /logs/verifier alongside /tests, so neither reward.txt nor the pytest -rA log — which can spell out expected values — outlives scoring for a timed-out episode or a later session. The withhold cache tars the realpath of tests/ so a symlinked tests/ caches its target's files rather than a bare link that would restore empty. - Docker mode gets true stage-at-verify: the initial /task copy excludes tests/ and solution/, and _evaluate_docker streams tests in from the host (this server sits outside the container) right before pytest runs, then removes them after scoring — in a finally, because docker-mode step() reports a scoring error without ending the episode, and the agent must not keep its session alongside a staged /task/tests that an errored verify left behind. Residual risk, shared with the official TB2 harness: verification necessarily runs inside the same container the agent controlled, and agent-started background processes must be left alive (many tasks are scored on services the agent set up), so a leftover watcher that copies the staged tests or rewrites the reward file during the verify window — or a root agent that tampers with container binaries — could still fake a pass. Task state (services, git state) is not portable, so verification cannot move to a container the agent never touched. tests/envs/test_tbench2_env.py covers the withhold round-trip, staging hygiene (pre-planted conftest.py and stale reward.txt wiped; symlinked fixed paths and a symlinked tests/ removed without following into the target; source checkout untouched without the gate), post-scoring cleanup, the full evaluate flow from the in-memory copy, and docker mode's copy exclusions, evaluate-time staging order, and cleanup when scoring itself raises (mock container — no docker needed). Also verified end-to-end on a per-task Daytona cloud sandbox (fix-git, gate enabled): the staged task dir carries no tests/solution, a filesystem-wide find turns up neither, pre-solution evaluate scores 0.0 via the canonical test.sh from the in-memory copy, and applying the oracle solution scores 1.0. A static scan of all 89 official tasks' tests/ trees (regular files and dirs only — no symlinks or special modes) confirms the tar round-trip is safe suite-wide. Co-Authored-By: Claude Fable 5 --- envs/tbench2_env/README.md | 1 + .../server/tbench2_env_environment.py | 382 ++++++++++++----- tests/envs/test_tbench2_env.py | 393 +++++++++++++++++- 3 files changed, 678 insertions(+), 98 deletions(-) diff --git a/envs/tbench2_env/README.md b/envs/tbench2_env/README.md index 62315da78..b36266883 100644 --- a/envs/tbench2_env/README.md +++ b/envs/tbench2_env/README.md @@ -164,6 +164,7 @@ env.step(Tbench2Action(action_type="view", session_id="sess1")) | `TB2_OUTPUT_DIR` | `/tmp/tbench2_env_runs` | Directory for session logs and cache | | `TB2_CACHE_DIR` | `$TB2_OUTPUT_DIR/repo_cache` | Where to extract TB2 repo | | `TB2_REPO_URL` | (GitHub main.zip) | Repo zip URL for auto-download | +| `TB2_WITHHOLD_TESTS` | `0` | `1`: `reset()` deletes the task's `tests/` and `solution/` from disk (tests kept in server memory, staged at `/tests` only while scoring). Set for RL training so the agent cannot read the expected outputs or the answer; leave off when `TB2_TASKS_DIR` is a working checkout you don't want modified | ## Reward diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index 28c9c033b..44a0fd0cc 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -8,10 +8,13 @@ from __future__ import annotations +import io import logging import os import re import shlex +import shutil +import tarfile import threading import sys import urllib.request @@ -39,6 +42,39 @@ _CAMEL_IMPORT_ERROR: Exception | None = None +# Official TB2 fixed paths: tests/test.sh assumes it runs from /tests and +# writes /logs/verifier/reward.txt. Module-level only so unit tests can +# redirect them off the real root filesystem. +_VERIFY_TESTS_DIR = "/tests" +_VERIFIER_LOG_DIR = "/logs/verifier" + + +def _hard_remove(path: str) -> None: + """Remove whatever is at *path* — file, symlink, or directory tree. + + shutil.rmtree refuses to remove a symlink and, with ignore_errors=True, + no-ops on one. That leaves an escape hatch: a root agent that symlinks a + fixed path (/tests, /logs/verifier) at a directory it controls would keep + that target alive across the "wipe", so a pre-planted conftest.py survives + into scoring. Unlink the symlink itself before falling back to rmtree. + """ + p = Path(path) + if p.is_symlink() or p.is_file(): + p.unlink(missing_ok=True) + else: + shutil.rmtree(path, ignore_errors=True) + + +def _extractall(tar: tarfile.TarFile, dest: str) -> None: + """extractall with the 3.12 data filter when available. The tarball is one + this server wrote from a trusted tests/ dir, so the filter is defence in + depth; on <3.12 (the ``filter`` kwarg predates it) fall back cleanly rather + than raising TypeError mid-verify (the package targets Python >=3.10).""" + try: + tar.extractall(dest, filter="data") + except TypeError: + tar.extractall(dest) + def _require_terminal_toolkit() -> Any: global _CAMEL_IMPORT_ERROR @@ -141,9 +177,17 @@ def __init__( command_timeout_s: float | None = None, safe_mode: bool = False, default_task_id: str | None = None, + withhold_tests: bool | None = None, ) -> None: super().__init__() self.tasks_dir = tasks_dir or os.getenv("TB2_TASKS_DIR", "") + # RL hygiene: the env server and the agent share one container + # filesystem, so tests/ and solution/ in the task dir are readable + # (and writable) by the agent. When enabled, reset() moves them out + # of reach — see _withhold_verifier_assets. + if withhold_tests is None: + withhold_tests = os.getenv("TB2_WITHHOLD_TESTS", "0") == "1" + self.withhold_tests = withhold_tests self.output_dir = Path( output_dir or os.getenv("TB2_OUTPUT_DIR", "/tmp/tbench2_env_runs") ) @@ -184,6 +228,8 @@ def reset( self._instruction = _read_instruction(task_dir) self._task_dir = task_dir + if self.withhold_tests: + self._withhold_verifier_assets(task_dir) trial_name = f"{resolved_task_id}.{episode_id or uuid4().hex}" session_logs_dir = ( @@ -368,6 +414,75 @@ def _ensure_session_id(self, session_id: str | None, action_type: str) -> None: if not session_id: raise ValueError(f"session_id is required for action_type='{action_type}'") + # tests/ tarballs withheld out of agent reach at reset(), keyed by task + # dir. Class-level: the server may construct one environment instance per + # session, and a later reset of the same task must still find the copy + # after the on-disk source is gone. In-process only — withhold mode + # therefore assumes a single server process owns the task filesystem for + # its lifetime (the intended per-task-sandbox deployment: one uvicorn + # worker, one ephemeral sandbox per episode). It is not safe with + # ``uvicorn --workers N`` sharing one on-disk checkout, where a sibling + # worker would see tests/ already deleted with no cache to restore from. + _WITHHELD_TESTS: dict[str, bytes] = {} + _WITHHOLD_LOCK = threading.Lock() + + def _withhold_verifier_assets(self, task_dir: Path) -> None: + """Move verifier assets out of the agent's reach before it can act. + + The task directory is agent-readable (same filesystem, root agent): + tests/test_outputs.py spells out the expected outputs and solution/ is + the literal answer — for RL training both are reward-hacking bait. + tests/ is read into server memory (the staging source for + _stage_tests_for_verify) and removed from disk; solution/ is simply + removed (nothing in this env reads it — it exists only for oracle + runs). Gated behind TB2_WITHHOLD_TESTS=1 because in plain local mode + task_dir may be a developer's working checkout, where deleting from + it would be hostile. + """ + key = str(task_dir) + with self._WITHHOLD_LOCK: + tests_dir = task_dir / "tests" + if tests_dir.is_dir(): + if key not in self._WITHHELD_TESTS: + # realpath so a symlinked tests/ caches the target's files, + # not just a link member that would restore empty (tar.add + # does not follow a top-level symlink). + tests_real = os.path.realpath(tests_dir) + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + tar.add(tests_real, arcname=".") + self._WITHHELD_TESTS[key] = buf.getvalue() + _hard_remove(str(tests_dir)) + _hard_remove(str(task_dir / "solution")) + + def _stage_tests_for_verify(self) -> bool: + """Stage a pristine tests/ copy at the official fixed path for exactly + the verify window. Caller must hold _CANONICAL_EVAL_LOCK. + + Both fixed paths are recreated from scratch (symlink included — see + _hard_remove) so nothing an agent may have pre-planted survives into + scoring: a /tests/conftest.py pytest would auto-load, or a symlinked + /tests or /logs/verifier redirecting the stage at a dir the agent + controls. When the source dir was withheld at reset, the staged copy + comes from server memory: a copy the agent never had filesystem + access to. + """ + _hard_remove(_VERIFY_TESTS_DIR) + _hard_remove(_VERIFIER_LOG_DIR) + Path(_VERIFIER_LOG_DIR).mkdir(parents=True, exist_ok=True) + + blob = self._WITHHELD_TESTS.get(str(self._task_dir)) + if blob is not None: + Path(_VERIFY_TESTS_DIR).mkdir(parents=True, exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(blob)) as tar: + _extractall(tar, _VERIFY_TESTS_DIR) + return True + tests_dir = Path(self._task_dir) / "tests" + if not tests_dir.is_dir(): + return False + shutil.copytree(tests_dir, _VERIFY_TESTS_DIR, symlinks=True) + return True + def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: if self._task_dir is None: raise RuntimeError("TB2 environment not initialized. Call reset() first.") @@ -377,87 +492,75 @@ def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: # The task's own verifier budget (task.toml [verifier].timeout_sec) — # heavy tests legitimately run minutes (circuit-fibsqrt declares 3600s). verifier_timeout_s = _read_timeout(self._task_dir, fallback=900.0) - tests_dir = self._task_dir / "tests" - if (tests_dir / "test.sh").is_file(): - return self._evaluate_canonical(tests_dir, verifier_timeout_s) - - # Fallback for tasks without the canonical harness (none of the 89 - # official TB2 tasks — they all ship test.sh — but custom task dirs - # may only have bare pytest tests). Prefer uvx so pytest comes with - # its own toolchain like the canonical harness does; fall back to a - # preinstalled pytest for environments without uv. - # Verify from the same directory the agent worked in. - fallback_cwd = self._workdir or str(self._task_dir) - cmd = ( - f"cd {shlex.quote(fallback_cwd)} && " - "if command -v uvx >/dev/null 2>&1; " - f"then uvx --with pytest==8.4.1 pytest -q {shlex.quote(str(tests_dir))} -rA; " - f"else python -m pytest -q {shlex.quote(str(tests_dir))} -rA; fi; " - "echo __TB2_EXIT_CODE__:$?" - ) - output = self._terminal_toolkit.shell_exec( - id="tb2-tests", - command=cmd, - block=True, - timeout=verifier_timeout_s, - ) - exit_code = 1 - marker = "__TB2_EXIT_CODE__" - for line in output.splitlines()[::-1]: - if marker in line: - try: - exit_code = int(line.split(":", 1)[1].strip()) - except Exception: - exit_code = 1 - break - - reward = 1.0 if exit_code == 0 else 0.0 - info = {"tests_passed": exit_code == 0, "exit_code": exit_code} - return output, reward, info + with self._CANONICAL_EVAL_LOCK: + try: + # Staging happens inside the try: a stage that errors partway + # (I/O failure mid-extract) must be wiped like a completed + # one — step() reports scoring errors without ending the + # episode, so the still-live session must not find a partial + # tests/ copy left behind. + if not self._stage_tests_for_verify(): + return ( + f"no tests/ found for task {self._state.task_id}", + 0.0, + {"tests_passed": False, "error": "missing tests"}, + ) + if (Path(_VERIFY_TESTS_DIR) / "test.sh").is_file(): + return self._evaluate_canonical(verifier_timeout_s) + return self._evaluate_fallback(verifier_timeout_s) + finally: + # Verifier artifacts live on disk only for the verify window: + # don't leave them readable to a later episode / concurrent + # session sharing this filesystem — or to the agent itself if + # scoring times out / errors (done stays false, so it keeps + # its session). /logs/verifier holds reward.txt and the + # test.sh log, whose pytest -rA output can reveal expected + # values. + _hard_remove(_VERIFY_TESTS_DIR) + _hard_remove(_VERIFIER_LOG_DIR) # The canonical harness uses the official fixed paths (/tests, # /logs/verifier/reward.txt), which are per-container, not per-session. - # Serialize evaluations so concurrent sessions on one server cannot - # overwrite each other's staged tests or read another session's reward. + # Serialize staging + evaluation so concurrent sessions on one server + # cannot overwrite each other's staged tests or read another session's + # reward. _CANONICAL_EVAL_LOCK = threading.Lock() def _evaluate_canonical( - self, tests_dir: Path, timeout_s: float + self, timeout_s: float ) -> tuple[str, float, dict[str, Any]]: """Score via the task's OFFICIAL harness, exactly as the TB2 verifier does. - tests/test.sh pins its own pytest toolchain (uvx: Python 3.13 + - pytest 8.4.1 + ctrf), runs tests/test_outputs.py from /tests against - the task workdir, and writes the binary result to - /logs/verifier/reward.txt. Bare ``pytest tests/`` (the fallback above) - skips that toolchain pinning and mis-scores any task whose tests need - it, so the canonical harness is preferred whenever the task ships it. + /tests/test.sh (staged by _stage_tests_for_verify) pins its own pytest + toolchain (uvx: Python 3.13 + pytest 8.4.1 + ctrf), runs + tests/test_outputs.py from /tests against the task workdir, and writes + the binary result to /logs/verifier/reward.txt. Bare ``pytest tests/`` + (the fallback below) skips that toolchain pinning and mis-scores any + task whose tests need it, so the canonical harness is preferred + whenever the task ships it. """ # Same working dir the agent operated in (resolved during reset). workdir = self._workdir or str(self._task_dir) marker = "__TB2_REWARD__:" + # test.sh stdout goes under /logs/verifier (wiped with the verify + # window in _evaluate_task) rather than a fixed /tmp path that would + # outlive scoring: its pytest -rA output can spell out expected values. + # Its tail is read back into the returned output — the episode is over + # by then, and callers need the pytest diagnostics on failure; the + # reward marker line stays last for parsing. cmd = ( - # /tests is recreated from scratch — a prior task's files on the - # same server must not leak into pytest collection — and the - # reward file removed so a stale one can never be read back if - # test.sh fails to run. - "rm -rf /tests && mkdir -p /tests /logs/verifier && " - "rm -f /logs/verifier/reward.txt && " - f"cp -a {shlex.quote(str(tests_dir))}/. /tests/ && " - f"cd {shlex.quote(workdir)} && bash /tests/test.sh > /tmp/tb2_testsh.log 2>&1; " - # Surface the harness log tail so callers keep the pytest - # diagnostics; the reward marker line stays last for parsing. - "tail -c 20000 /tmp/tb2_testsh.log 2>/dev/null; " - f"echo {marker}$(cat /logs/verifier/reward.txt 2>/dev/null)" + f"cd {shlex.quote(workdir)} && " + f"bash {_VERIFY_TESTS_DIR}/test.sh > {_VERIFIER_LOG_DIR}/testsh.log 2>&1; " + f"tail -c 20000 {_VERIFIER_LOG_DIR}/testsh.log 2>/dev/null; " + f"echo {marker}$(cat {_VERIFIER_LOG_DIR}/reward.txt 2>/dev/null)" + ) + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, ) - with self._CANONICAL_EVAL_LOCK: - output = self._terminal_toolkit.shell_exec( - id="tb2-tests", - command=cmd, - block=True, - timeout=timeout_s, - ) reward = 0.0 for line in output.splitlines()[::-1]: @@ -472,6 +575,42 @@ def _evaluate_canonical( info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} return output, reward, info + def _evaluate_fallback(self, timeout_s: float) -> tuple[str, float, dict[str, Any]]: + """For task dirs without the canonical harness (none of the 89 official + TB2 tasks — they all ship test.sh — but custom task dirs may only have + bare pytest tests). Runs pytest against the staged /tests copy; prefer + uvx so pytest comes with its own toolchain like the canonical harness + does. Verify from the same directory the agent worked in. + """ + fallback_cwd = self._workdir or str(self._task_dir) + cmd = ( + f"cd {shlex.quote(fallback_cwd)} && " + "if command -v uvx >/dev/null 2>&1; " + f"then uvx --with pytest==8.4.1 pytest -q {_VERIFY_TESTS_DIR} -rA; " + f"else python -m pytest -q {_VERIFY_TESTS_DIR} -rA; fi; " + "echo __TB2_EXIT_CODE__:$?" + ) + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, + ) + + exit_code = 1 + marker = "__TB2_EXIT_CODE__" + for line in output.splitlines()[::-1]: + if marker in line: + try: + exit_code = int(line.split(":", 1)[1].strip()) + except Exception: + exit_code = 1 + break + + reward = 1.0 if exit_code == 0 else 0.0 + info = {"tests_passed": exit_code == 0, "exit_code": exit_code} + return output, reward, info + class Tbench2DockerEnvironment( Environment[Tbench2Action, Tbench2Observation, Tbench2State] @@ -621,8 +760,14 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: # Copy task files into container using tar archive # This works in Docker-in-Docker because we read files from our - # filesystem and stream them to the container via the Docker API - self._copy_dir_to_container(task_dir, "/task") + # filesystem and stream them to the container via the Docker API. + # Verifier assets stay out: the agent works in /task and must not + # be able to read solution/ (the literal answer) or tests/ (the + # expected outputs) — tests are staged only at verify time by + # _evaluate_docker, like the official TB2 harness does. + self._copy_dir_to_container( + task_dir, "/task", exclude_top=("solution", "tests") + ) self._state = Tbench2State( episode_id=str(uuid4()), @@ -635,24 +780,28 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: except Exception as exc: raise RuntimeError(f"Failed to start container: {exc}") from exc - def _copy_dir_to_container(self, src_dir: Path, dest_path: str) -> None: + def _copy_dir_to_container( + self, src_dir: Path, dest_path: str, exclude_top: tuple[str, ...] = () + ) -> None: """Copy a directory into the container using tar archive. This method streams files via the Docker API, avoiding bind mount - issues in Docker-in-Docker scenarios. + issues in Docker-in-Docker scenarios. Top-level entries named in + *exclude_top* are skipped entirely. """ - import io - import tarfile - if self._container is None: raise RuntimeError("Container not started") - # Create tar archive in memory + # Create tar archive in memory (recursive=False: rglob already yields + # every descendant, so per-entry recursion would duplicate members — + # and would re-add excluded subtrees through their parents). tar_stream = io.BytesIO() with tarfile.open(fileobj=tar_stream, mode="w") as tar: for item in src_dir.rglob("*"): - arcname = str(item.relative_to(src_dir)) - tar.add(str(item), arcname=arcname) + rel = item.relative_to(src_dir) + if rel.parts and rel.parts[0] in exclude_top: + continue + tar.add(str(item), arcname=str(rel), recursive=False) tar_stream.seek(0) @@ -776,28 +925,71 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: raise RuntimeError("Container not started.") assert self._task_dir is not None, "Task directory not set" - # Run pytest in the container's /task directory - # Use exit code marker for consistency with local mode - cmd = "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" - - exit_code, output = self._container.exec_run( - cmd=f"bash -c '{cmd}'", - workdir="/task", - stdout=True, - stderr=True, + # Stage-at-verify, like the official TB2 harness: the initial /task + # copy excludes tests/, and this server sits OUTSIDE the container, so + # the copy staged here is one the agent never saw. rm -rf first so + # nothing an agent pre-planted at /task/tests (e.g. a conftest.py + # pytest would auto-load) survives into scoring. + tests_src = self._task_dir / "tests" + if not tests_src.is_dir(): + return ( + f"no tests/ found for task {self._state.task_id}", + 0.0, + {"tests_passed": False, "error": "missing tests"}, + ) + wipe_ec, wipe_out = self._exec_in_container( + "rm -rf /task/tests && mkdir -p /task/tests" ) - output_str = output.decode("utf-8", errors="replace") + if wipe_ec != 0: + # Fail closed: put_archive into a dir that survived the wipe would + # merge the staged tests into whatever the agent planted there. + raise RuntimeError( + f"could not reset /task/tests before verify: {wipe_out.strip()}" + ) + try: + self._copy_dir_to_container(tests_src, "/task/tests") - # Parse exit code from marker (same logic as local mode) - ec = 1 - marker = "__TB2_EXIT_CODE__" - for line in output_str.splitlines()[::-1]: - if marker in line: - try: - ec = int(line.split(":", 1)[1].strip()) - except Exception: - ec = 1 - break + # Run pytest in the container's /task directory + # Use exit code marker for consistency with local mode + cmd = ( + "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" + ) + + exit_code, output = self._container.exec_run( + cmd=f"bash -c '{cmd}'", + workdir="/task", + stdout=True, + stderr=True, + ) + output_str = output.decode("utf-8", errors="replace") + + # Parse exit code from marker (same logic as local mode) + ec = 1 + marker = "__TB2_EXIT_CODE__" + for line in output_str.splitlines()[::-1]: + if marker in line: + try: + ec = int(line.split(":", 1)[1].strip()) + except Exception: + ec = 1 + break + finally: + # Tests live in the container only for the verify window, matching + # the local-mode staging: don't leave /task/tests readable + # afterward — the agent keeps its session if scoring errored + # (step() reports the failure without ending the episode). + try: + rm_ec, rm_out = self._exec_in_container("rm -rf /task/tests") + if rm_ec != 0: + logging.warning( + "failed to remove staged /task/tests after verify: %s", + rm_out.strip(), + ) + except Exception: + logging.warning( + "failed to remove staged /task/tests after verify", + exc_info=True, + ) reward = 1.0 if ec == 0 else 0.0 info = {"tests_passed": ec == 0, "exit_code": ec} diff --git a/tests/envs/test_tbench2_env.py b/tests/envs/test_tbench2_env.py index 6490d1d1a..f0afdb064 100644 --- a/tests/envs/test_tbench2_env.py +++ b/tests/envs/test_tbench2_env.py @@ -1,11 +1,20 @@ +import io +import logging import os import sys +import tarfile from pathlib import Path import pytest -# Add the project root to the path for envs imports -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +# Add the project root to the path for envs imports, and envs/ itself so the +# server module's Spaces-style absolute imports (``tbench2_env.models``) +# resolve when this file runs standalone (in the full suite another test +# happens to insert envs/ first — don't rely on collection order). +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +for _p in (_REPO_ROOT, os.path.join(_REPO_ROOT, "envs")): + if _p not in sys.path: + sys.path.insert(0, _p) try: import camel # noqa: F401 @@ -14,7 +23,10 @@ from envs.tbench2_env.models import Tbench2Action from envs.tbench2_env.server import tbench2_env_environment -from envs.tbench2_env.server.tbench2_env_environment import Tbench2Environment +from envs.tbench2_env.server.tbench2_env_environment import ( + Tbench2DockerEnvironment, + Tbench2Environment, +) class _FakeTerminalToolkit: @@ -97,6 +109,381 @@ def test_tbench2_workdir_rejects_server_tree(monkeypatch, tmp_path: Path): assert env._workdir == str(task_dir) +# --------------------------------------------------------------------------- +# Verifier-asset withholding (TB2_WITHHOLD_TESTS) and stage-at-verify +# --------------------------------------------------------------------------- + + +def _make_task_dir(root: Path, name: str = "my-task") -> Path: + task = root / name + (task / "tests").mkdir(parents=True) + (task / "tests" / "test.sh").write_text( + "#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n" + ) + (task / "tests" / "test_outputs.py").write_text("EXPECTED = 'secret42'\n") + (task / "solution").mkdir() + (task / "solution" / "solve.sh").write_text("echo the-answer\n") + (task / "task.toml").write_text('[environment]\ndocker_image = "debian:12"\n') + (task / "instruction.md").write_text("do the thing\n") + return task + + +@pytest.fixture() +def staged_paths(monkeypatch, tmp_path: Path): + """Redirect the official fixed paths (/tests, /logs/verifier) into tmp.""" + tests = tmp_path / "stage" / "tests" + logs = tmp_path / "stage" / "logs" / "verifier" + monkeypatch.setattr(tbench2_env_environment, "_VERIFY_TESTS_DIR", str(tests)) + monkeypatch.setattr(tbench2_env_environment, "_VERIFIER_LOG_DIR", str(logs)) + return tests, logs + + +@pytest.fixture(autouse=True) +def _fresh_withhold_cache(monkeypatch): + monkeypatch.setattr(Tbench2Environment, "_WITHHELD_TESTS", {}) + + +def test_withhold_removes_assets_and_caches_tests(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + + env._withhold_verifier_assets(task) + + assert not (task / "solution").exists() + assert not (task / "tests").exists() + assert (task / "task.toml").is_file() + assert (task / "instruction.md").is_file() + assert str(task) in Tbench2Environment._WITHHELD_TESTS + + # Second reset of the same task (dir already gone) must keep the cache. + env._withhold_verifier_assets(task) + assert str(task) in Tbench2Environment._WITHHELD_TESTS + + +def test_stage_from_memory_and_wipe_planted_files(tmp_path: Path, staged_paths): + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._withhold_verifier_assets(task) + + # An agent pre-plants a conftest.py (pytest would auto-load it) and a + # fake reward before verification. + stage_tests.mkdir(parents=True) + (stage_tests / "conftest.py").write_text("import sys; sys.exit(0)\n") + stage_logs.mkdir(parents=True) + (stage_logs / "reward.txt").write_text("1\n") + + assert env._stage_tests_for_verify() + + assert (stage_tests / "test.sh").is_file() + assert (stage_tests / "test_outputs.py").read_text() == "EXPECTED = 'secret42'\n" + assert not (stage_tests / "conftest.py").exists() + assert not (stage_logs / "reward.txt").exists() + assert stage_logs.is_dir() and not list(stage_logs.iterdir()) + + +def test_stage_from_disk_without_gate(tmp_path: Path, staged_paths): + stage_tests, _ = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + + assert env._stage_tests_for_verify() + + assert (stage_tests / "test_outputs.py").is_file() + # Without the gate the source checkout is never touched. + assert (task / "tests").is_dir() + assert (task / "solution").is_dir() + + +def test_stage_reports_missing_tests(tmp_path: Path, staged_paths): + task = tmp_path / "empty-task" + task.mkdir() + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + + assert not env._stage_tests_for_verify() + + +class _RecordingToolkit: + def __init__(self, output: str): + self.calls: list[dict] = [] + self.output = output + + def shell_exec(self, **kwargs): + self.calls.append(kwargs) + return self.output + + +def test_evaluate_canonical_from_withheld_copy(tmp_path: Path, staged_paths): + """Full evaluate flow: staging from memory + canonical test.sh scoring.""" + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._workdir = str(tmp_path) + env._terminal_toolkit = _RecordingToolkit(output="__TB2_REWARD__:1") + env._withhold_verifier_assets(task) + + output, reward, info = env._evaluate_task() + + assert reward == 1.0 + assert info["harness"] == "tests/test.sh" + # The verify command ran test.sh from the staged fixed path, and wrote its + # log under the wiped verifier dir — not a fixed /tmp path that outlives it. + command = env._terminal_toolkit.calls[0]["command"] + assert f"{stage_tests}/test.sh" in command + assert "/tmp/tb2_testsh.log" not in command + assert f"{stage_logs}/testsh.log" in command + # Both the staged tests and the verifier log/reward dir are gone once + # scoring returns. + assert not stage_tests.exists() + assert not stage_logs.exists() + + +def test_evaluate_without_tests_scores_zero(tmp_path: Path, staged_paths): + task = tmp_path / "empty-task" + task.mkdir() + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + env._terminal_toolkit = _RecordingToolkit(output="") + + output, reward, info = env._evaluate_task() + + assert reward == 0.0 + assert info["tests_passed"] is False + assert env._terminal_toolkit.calls == [] # nothing was run + + +def test_evaluate_wipes_partial_stage_on_error( + tmp_path: Path, staged_paths, monkeypatch +): + """A stage that errors partway must be wiped like a completed one — the + session survives a scoring error, so no partial tests/ copy may remain.""" + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._workdir = str(tmp_path) + env._terminal_toolkit = _RecordingToolkit(output="") + env._withhold_verifier_assets(task) + + def _explode(tar, dest): + (Path(dest) / "test_outputs.py").write_text("EXPECTED = 'secret42'\n") + raise OSError("disk full") + + monkeypatch.setattr(tbench2_env_environment, "_extractall", _explode) + + with pytest.raises(OSError, match="disk full"): + env._evaluate_task() + + assert not stage_tests.exists() + assert not stage_logs.exists() + + +def test_stage_wipes_symlinked_fixed_path(tmp_path: Path, staged_paths): + """A symlinked /tests must not let the official suite merge into a dir the + agent controls (rmtree silently no-ops on a symlink; _hard_remove doesn't).""" + stage_tests, _ = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._withhold_verifier_assets(task) + + evil = tmp_path / "agent_controlled" + evil.mkdir() + (evil / "conftest.py").write_text("import sys; sys.exit(0)\n") + stage_tests.parent.mkdir(parents=True, exist_ok=True) + stage_tests.symlink_to(evil, target_is_directory=True) + + assert env._stage_tests_for_verify() + + assert not stage_tests.is_symlink() + assert (stage_tests / "test.sh").is_file() + # The official suite never leaked into the agent's directory. + assert not (evil / "test.sh").exists() + assert (evil / "conftest.py").exists() # their dir left alone + + +def test_withhold_removes_symlinked_tests(tmp_path: Path): + """tests/ as a symlink is removed too — the link, not just its target.""" + task = tmp_path / "task" + task.mkdir() + real = tmp_path / "real_tests" + real.mkdir() + (real / "test_outputs.py").write_text("x = 1\n") + (task / "tests").symlink_to(real, target_is_directory=True) + + env = Tbench2Environment(withhold_tests=True) + env._withhold_verifier_assets(task) + + assert not (task / "tests").exists() # symlink gone + assert real.exists() # its target is outside the task dir; untouched + # The cache captured the target's real files, not a bare link member that + # would restore an empty suite (tar.add doesn't follow a top-level link). + import io + import tarfile + + blob = Tbench2Environment._WITHHELD_TESTS[str(task)] + with tarfile.open(fileobj=io.BytesIO(blob)) as tar: + assert any(n.endswith("test_outputs.py") for n in tar.getnames()) + + +# --------------------------------------------------------------------------- +# Docker mode: copy exclusions + stage-at-verify (mock container, no docker) +# --------------------------------------------------------------------------- + + +class _FakeContainer: + """Records exec/put_archive calls in one ordered event log.""" + + def __init__(self, exec_output: bytes = b"__TB2_EXIT_CODE__:0\n"): + self.events: list[tuple] = [] + self.exec_output = exec_output + + def put_archive(self, dest, data): + self.events.append(("put", dest, data)) + return True + + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + return 0, self.exec_output + + +def _tar_names(data: bytes) -> list[str]: + with tarfile.open(fileobj=io.BytesIO(data)) as tar: + return tar.getnames() + + +def test_docker_copy_excludes_verifier_assets(tmp_path: Path): + task = _make_task_dir(tmp_path) + (task / "environment").mkdir() + (task / "environment" / "Dockerfile").write_text("FROM debian:12\n") + + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._copy_dir_to_container(task, "/task", exclude_top=("solution", "tests")) + + ((kind, dest, data),) = container.events + assert (kind, dest) == ("put", "/task") + names = _tar_names(data) + assert "task.toml" in names + assert "environment/Dockerfile" in names + assert not any(n.startswith(("solution", "tests")) for n in names) + assert len(names) == len(set(names)) # recursive=False: no duplicates + + +def test_evaluate_docker_stages_tests_at_verify(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 1.0 + kinds = [e[0] for e in container.events] + # rm -rf the agent-writable staging dir BEFORE the fresh copy goes in, + # the copy lands before pytest runs, and tests are removed again after. + assert kinds == ["exec", "put", "exec", "exec"] + assert "rm -rf /task/tests" in container.events[0][1] + assert container.events[1][1] == "/task/tests" + assert "test.sh" in _tar_names(container.events[1][2]) + assert "pytest" in container.events[2][1] + assert "rm -rf /task/tests" in container.events[3][1] + + +def test_evaluate_docker_cleans_up_when_scoring_raises(tmp_path: Path): + """The staged /task/tests must not outlive a verify that errors out — + step() reports the failure without ending the episode, so the agent + keeps its session.""" + task = _make_task_dir(tmp_path) + + class _ExplodingContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + if "pytest" in cmd: + self.events.append(("exec", cmd)) + raise RuntimeError("docker daemon hiccup") + return super().exec_run(cmd, workdir=workdir, stdout=stdout, stderr=stderr) + + env = Tbench2DockerEnvironment() + container = _ExplodingContainer() + env._container = container + env._task_dir = task + + with pytest.raises(RuntimeError, match="hiccup"): + env._evaluate_docker() + + assert container.events[-1][0] == "exec" + assert "rm -rf /task/tests" in container.events[-1][1] + + +def test_evaluate_docker_fails_closed_when_prestage_wipe_fails(tmp_path: Path): + """If the pre-stage rm -rf fails, scoring must error out rather than + put_archive into a dir the agent still controls.""" + task = _make_task_dir(tmp_path) + + class _StubbornContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + if "rm -rf /task/tests" in cmd and "mkdir" in cmd: + return 1, b"rm: cannot remove '/task/tests': busy\n" + return 0, self.exec_output + + env = Tbench2DockerEnvironment() + container = _StubbornContainer() + env._container = container + env._task_dir = task + + with pytest.raises(RuntimeError, match="could not reset /task/tests"): + env._evaluate_docker() + + assert not any(e[0] == "put" for e in container.events) + + +def test_evaluate_docker_warns_when_cleanup_fails(tmp_path: Path, caplog): + """A post-verify rm -rf that exits nonzero must not stay silent.""" + task = _make_task_dir(tmp_path) + + class _LeakyContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + if "rm -rf /task/tests" in cmd and "mkdir" not in cmd: + return 1, b"rm: cannot remove '/task/tests': busy\n" + return 0, self.exec_output + + env = Tbench2DockerEnvironment() + container = _LeakyContainer() + env._container = container + env._task_dir = task + + with caplog.at_level(logging.WARNING): + output, reward, info = env._evaluate_docker() + + assert reward == 1.0 + assert any( + "failed to remove staged /task/tests" in record.message + for record in caplog.records + ) + + +def test_evaluate_docker_missing_tests_scores_zero(tmp_path: Path): + task = tmp_path / "no-tests-task" + task.mkdir() + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 0.0 + assert container.events == [] + + @pytest.mark.skipif(camel is None, reason="camel-ai not installed") @pytest.mark.skipif( os.environ.get("TB2_ENABLE_TESTS", "0") != "1",