From 501a20c2606aff7966409e3fae7c7fcf963a208d Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Tue, 14 Jul 2026 21:03:54 -0700 Subject: [PATCH 1/3] fix(tbench2_env): canonical test.sh evaluate, real timeouts, native task env & workdir - _evaluate_task runs the task's official tests/test.sh (pinned uvx toolchain, /logs/verifier/reward.txt) when the task ships it; the bare pytest fallback (custom task dirs without the canonical harness) prefers uvx with a pinned pytest so it carries its own toolchain, and runs from the same workdir the agent worked in. - Pass timeouts through to every shell_exec call: camel's shell_exec has its own per-call default (20s) and ignores the constructor timeout, so agent commands and evaluations were silently truncated at 20s (circuit-fibsqrt's verifier declares 3600s and was cut mid-test). Agent execs use command_timeout_s (TB2_COMMAND_TIMEOUT_S-overridable); evaluate uses the task's own [verifier].timeout_sec budget. - Shell cwd resolves to the task image's own WORKDIR, parsed from the task's environment/Dockerfile (TB2_TASK_WORKDIR overrides): task images pin their own WORKDIR (fix-git: /app/personal-site, prove-plus-comm: /workspace) and both agents and oracle solutions assume commands run there. Falls back to the task source dir for plain local mode. - Drop install_dependencies=['pytest'] (evaluation carries its own pytest via uvx); with CAMEL_RUNTIME=true camel's .initial_env venv then stops hijacking PATH away from the task image's native python. - Serialize canonical evaluation under a lock so concurrent sessions on one server cannot clobber the official fixed verifier paths (/tests, /logs/verifier/reward.txt). --- .../server/tbench2_env_environment.py | 121 +++++++++++++++++- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index 33c917227..1af57c343 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -10,6 +10,9 @@ import logging import os +import re +import shlex +import threading import sys import urllib.request import zipfile @@ -82,6 +85,22 @@ def _read_instruction(task_dir: Path) -> str: return "" +def _task_image_workdir(task_dir: Path) -> str: + """WORKDIR pinned by the task's own image (environment/Dockerfile). + + Empty string when the task ships no Dockerfile or no WORKDIR; multi-stage + Dockerfiles resolve to the last WORKDIR (the final stage's). + """ + dockerfile = task_dir / "environment" / "Dockerfile" + workdir = "" + if dockerfile.is_file(): + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + m = re.match(r"^\s*WORKDIR\s+(\S+)", line, re.IGNORECASE) + if m: + workdir = m.group(1) + return workdir + + def _read_timeout(task_dir: Path, fallback: float) -> float: task_toml = task_dir / "task.toml" if not task_toml.exists(): @@ -103,7 +122,7 @@ def __init__( self, tasks_dir: str | None = None, output_dir: str | None = None, - command_timeout_s: float = 20.0, + command_timeout_s: float | None = None, safe_mode: bool = False, default_task_id: str | None = None, ) -> None: @@ -112,6 +131,11 @@ def __init__( self.output_dir = Path( output_dir or os.getenv("TB2_OUTPUT_DIR", "/tmp/tbench2_env_runs") ) + # Overridable via TB2_COMMAND_TIMEOUT_S: create_app instantiates the + # environment with no arguments, and real TB2 agent commands / the + # canonical tests/test.sh routinely exceed the old 20s default. + if command_timeout_s is None: + command_timeout_s = float(os.getenv("TB2_COMMAND_TIMEOUT_S", "20.0")) self.command_timeout_s = command_timeout_s self.safe_mode = safe_mode self.default_task_id = default_task_id or os.getenv( @@ -122,6 +146,7 @@ def __init__( self._task_dir: Path | None = None self._terminal_toolkit = None self._instruction = "" + self._workdir = "" def reset( self, @@ -150,13 +175,26 @@ def reset( ) session_logs_dir.mkdir(parents=True, exist_ok=True) + # Commands run in the task image's real WORKDIR — where agents and + # the canonical harness expect to operate, and not always /app + # (fix-git: /app/personal-site, prove-plus-comm: /workspace). + # TB2_TASK_WORKDIR overrides; otherwise the task's own Dockerfile is + # parsed; plain local mode falls back to the task source dir. + workdir = os.getenv("TB2_TASK_WORKDIR", "").strip() or _task_image_workdir( + task_dir + ) + if not (workdir and Path(workdir).is_dir()): + workdir = str(task_dir) + self._workdir = workdir + # No install_dependencies: evaluation runs the task's canonical + # tests/test.sh, which pins its own pytest toolchain via uvx (see + # _evaluate_canonical), so the toolkit venv does not need pytest. self._terminal_toolkit = TerminalToolkit( timeout=self.command_timeout_s, - working_directory=str(task_dir), + working_directory=workdir, use_docker_backend=False, session_logs_dir=session_logs_dir, safe_mode=self.safe_mode, - install_dependencies=["pytest"], ) self._state = Tbench2State( @@ -209,10 +247,14 @@ def step( try: if action.action_type == "exec": + # Pass the timeout explicitly: camel's shell_exec has its own + # per-call default (20s) and ignores the constructor timeout, + # which silently truncates any longer-running agent command. output = self._terminal_toolkit.shell_exec( command=action.command, block=action.block, id=session_id, + timeout=self.command_timeout_s, ) elif action.action_type == "write": self._ensure_session_id(action.session_id, action.action_type) @@ -312,13 +354,32 @@ def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: if self._terminal_toolkit is None: raise RuntimeError("Terminal toolkit not initialized.") - _read_timeout(self._task_dir, fallback=900.0) # Validate timeout config + # The task's own verifier budget (task.toml [verifier].timeout_sec) — + # heavy tests legitimately run minutes (circuit-fibsqrt declares 3600s). + verifier_timeout_s = _read_timeout(self._task_dir, fallback=900.0) tests_dir = self._task_dir / "tests" - cmd = f"cd {self._task_dir} && python -m pytest -q {tests_dir} -rA; echo __TB2_EXIT_CODE__:$?" + if (tests_dir / "test.sh").is_file(): + return self._evaluate_canonical(tests_dir, verifier_timeout_s) + + # Fallback for tasks without the canonical harness (none of the 89 + # official TB2 tasks — they all ship test.sh — but custom task dirs + # may only have bare pytest tests). Prefer uvx so pytest comes with + # its own toolchain like the canonical harness does; fall back to a + # preinstalled pytest for environments without uv. + # Verify from the same directory the agent worked in. + fallback_cwd = self._workdir or str(self._task_dir) + cmd = ( + f"cd {shlex.quote(fallback_cwd)} && " + "if command -v uvx >/dev/null 2>&1; " + f"then uvx --with pytest==8.4.1 pytest -q {tests_dir} -rA; " + f"else python -m pytest -q {tests_dir} -rA; fi; " + "echo __TB2_EXIT_CODE__:$?" + ) output = self._terminal_toolkit.shell_exec( id="tb2-tests", command=cmd, block=True, + timeout=verifier_timeout_s, ) exit_code = 1 @@ -335,6 +396,56 @@ def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: info = {"tests_passed": exit_code == 0, "exit_code": exit_code} return output, reward, info + # The canonical harness uses the official fixed paths (/tests, + # /logs/verifier/reward.txt), which are per-container, not per-session. + # Serialize evaluations so concurrent sessions on one server cannot + # overwrite each other's staged tests or read another session's reward. + _CANONICAL_EVAL_LOCK = threading.Lock() + + def _evaluate_canonical( + self, tests_dir: Path, timeout_s: float + ) -> tuple[str, float, dict[str, Any]]: + """Score via the task's OFFICIAL harness, exactly as the TB2 verifier does. + + tests/test.sh pins its own pytest toolchain (uvx: Python 3.13 + + pytest 8.4.1 + ctrf), runs tests/test_outputs.py from /tests against + the task workdir, and writes the binary result to + /logs/verifier/reward.txt. Bare ``pytest tests/`` (the fallback above) + skips that toolchain pinning and mis-scores any task whose tests need + it, so the canonical harness is preferred whenever the task ships it. + """ + # Same working dir the agent operated in (resolved during reset). + workdir = self._workdir or str(self._task_dir) + marker = "__TB2_REWARD__:" + cmd = ( + # rm the reward file first so a stale one can never be read back + # if test.sh fails to run. + "mkdir -p /tests /logs/verifier && rm -f /logs/verifier/reward.txt && " + f"cp -a {shlex.quote(str(tests_dir))}/. /tests/ && " + f"cd {shlex.quote(workdir)} && bash /tests/test.sh > /tmp/tb2_testsh.log 2>&1; " + f"echo {marker}$(cat /logs/verifier/reward.txt 2>/dev/null)" + ) + with self._CANONICAL_EVAL_LOCK: + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, + ) + + reward = 0.0 + for line in output.splitlines()[::-1]: + if marker in line: + raw = line.split(marker, 1)[1].strip() + try: + reward = float(raw) if raw else 0.0 + except ValueError: + reward = 0.0 + break + + info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} + return output, reward, info + class Tbench2DockerEnvironment( Environment[Tbench2Action, Tbench2Observation, Tbench2State] From d2b7a24549b133e346917ac16884ed2b069f3311 Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Thu, 16 Jul 2026 13:02:56 -0700 Subject: [PATCH 2/3] fix(tbench2_env): keep verifier assets out of the agent's reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both execution modes let the agent read (and tamper with) the assets the verifier scores against — tests/test_outputs.py spells out the expected outputs and solution/ is the literal answer. Fine for eval plumbing with a cooperative agent; for RL training it is a reward-hacking vector. Align with the official TB2 harness's model instead: the solution never ships, and tests exist in the agent's filesystem only during the verify window. - Local mode: with TB2_WITHHOLD_TESTS=1, reset() reads tests/ into server memory and deletes it (and solution/ — nothing in this env reads it; it exists only for oracle runs) from disk before the agent's first action. Gated off by default because in plain local mode TB2_TASKS_DIR may be a developer's working checkout; deployments that stage per-task copies (e.g. cloud sandboxes) should turn it on. The in-memory copy is process-local, so withhold mode assumes one server process owns the task filesystem for its lifetime (the per-task-sandbox deployment: one worker, one ephemeral sandbox per episode) — documented on the cache. - Evaluation stages a pristine tests/ copy at /tests for exactly the verify window — from server memory when withheld, i.e. a copy the agent never had filesystem access to — and removes it again once scoring returns, so a later episode / concurrent session sharing the filesystem can't read it. Both fixed paths (/tests, /logs/verifier) are recreated from scratch first, symlink included: the previous "mkdir -p && cp -a" merged into whatever already existed, and shutil.rmtree silently no-ops on a symlink, so an agent could pre-plant a /tests/conftest.py (pytest auto-loads it) or symlink a fixed path at a dir it controls and survive into scoring. _hard_remove unlinks the symlink itself. The in-memory tarball is restored with the 3.12 data filter when available, falling back cleanly on 3.10/3.11 (the package targets >=3.10). The bare-pytest fallback runs against the same staged copy. The canonical harness writes its test.sh stdout under /logs/verifier (not a fixed /tmp path) and the verify-window cleanup wipes /logs/verifier alongside /tests, so neither reward.txt nor the pytest -rA log — which can spell out expected values — outlives scoring for a timed-out episode or a later session. The withhold cache tars the realpath of tests/ so a symlinked tests/ caches its target's files rather than a bare link that would restore empty. - Docker mode gets true stage-at-verify: the initial /task copy excludes tests/ and solution/, and _evaluate_docker streams tests in from the host (this server sits outside the container) right before pytest runs, then removes them after scoring — in a finally, because docker-mode step() reports a scoring error without ending the episode, and the agent must not keep its session alongside a staged /task/tests that an errored verify left behind. Residual risk, shared with the official TB2 harness: verification necessarily runs inside the same container the agent controlled, and agent-started background processes must be left alive (many tasks are scored on services the agent set up), so a leftover watcher that copies the staged tests or rewrites the reward file during the verify window — or a root agent that tampers with container binaries — could still fake a pass. Task state (services, git state) is not portable, so verification cannot move to a container the agent never touched. tests/envs/test_tbench2_env.py covers the withhold round-trip, staging hygiene (pre-planted conftest.py and stale reward.txt wiped; symlinked fixed paths and a symlinked tests/ removed without following into the target; source checkout untouched without the gate), post-scoring cleanup, the full evaluate flow from the in-memory copy, and docker mode's copy exclusions, evaluate-time staging order, and cleanup when scoring itself raises (mock container — no docker needed). Also verified end-to-end on a per-task Daytona cloud sandbox (fix-git, gate enabled): the staged task dir carries no tests/solution, a filesystem-wide find turns up neither, pre-solution evaluate scores 0.0 via the canonical test.sh from the in-memory copy, and applying the oracle solution scores 1.0. A static scan of all 89 official tasks' tests/ trees (regular files and dirs only — no symlinks or special modes) confirms the tar round-trip is safe suite-wide. Co-Authored-By: Claude Fable 5 --- envs/tbench2_env/README.md | 1 + .../server/tbench2_env_environment.py | 354 +++++++++++++----- tests/envs/test_tbench2_env.py | 317 +++++++++++++++- 3 files changed, 580 insertions(+), 92 deletions(-) diff --git a/envs/tbench2_env/README.md b/envs/tbench2_env/README.md index 62315da78..b36266883 100644 --- a/envs/tbench2_env/README.md +++ b/envs/tbench2_env/README.md @@ -164,6 +164,7 @@ env.step(Tbench2Action(action_type="view", session_id="sess1")) | `TB2_OUTPUT_DIR` | `/tmp/tbench2_env_runs` | Directory for session logs and cache | | `TB2_CACHE_DIR` | `$TB2_OUTPUT_DIR/repo_cache` | Where to extract TB2 repo | | `TB2_REPO_URL` | (GitHub main.zip) | Repo zip URL for auto-download | +| `TB2_WITHHOLD_TESTS` | `0` | `1`: `reset()` deletes the task's `tests/` and `solution/` from disk (tests kept in server memory, staged at `/tests` only while scoring). Set for RL training so the agent cannot read the expected outputs or the answer; leave off when `TB2_TASKS_DIR` is a working checkout you don't want modified | ## Reward diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index 1af57c343..217cc4536 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -8,10 +8,13 @@ from __future__ import annotations +import io import logging import os import re import shlex +import shutil +import tarfile import threading import sys import urllib.request @@ -39,6 +42,39 @@ _CAMEL_IMPORT_ERROR: Exception | None = None +# Official TB2 fixed paths: tests/test.sh assumes it runs from /tests and +# writes /logs/verifier/reward.txt. Module-level only so unit tests can +# redirect them off the real root filesystem. +_VERIFY_TESTS_DIR = "/tests" +_VERIFIER_LOG_DIR = "/logs/verifier" + + +def _hard_remove(path: str) -> None: + """Remove whatever is at *path* — file, symlink, or directory tree. + + shutil.rmtree refuses to remove a symlink and, with ignore_errors=True, + no-ops on one. That leaves an escape hatch: a root agent that symlinks a + fixed path (/tests, /logs/verifier) at a directory it controls would keep + that target alive across the "wipe", so a pre-planted conftest.py survives + into scoring. Unlink the symlink itself before falling back to rmtree. + """ + p = Path(path) + if p.is_symlink() or p.is_file(): + p.unlink(missing_ok=True) + else: + shutil.rmtree(path, ignore_errors=True) + + +def _extractall(tar: tarfile.TarFile, dest: str) -> None: + """extractall with the 3.12 data filter when available. The tarball is one + this server wrote from a trusted tests/ dir, so the filter is defence in + depth; on <3.12 (the ``filter`` kwarg predates it) fall back cleanly rather + than raising TypeError mid-verify (the package targets Python >=3.10).""" + try: + tar.extractall(dest, filter="data") + except TypeError: + tar.extractall(dest) + def _require_terminal_toolkit() -> Any: global _CAMEL_IMPORT_ERROR @@ -125,9 +161,17 @@ def __init__( command_timeout_s: float | None = None, safe_mode: bool = False, default_task_id: str | None = None, + withhold_tests: bool | None = None, ) -> None: super().__init__() self.tasks_dir = tasks_dir or os.getenv("TB2_TASKS_DIR", "") + # RL hygiene: the env server and the agent share one container + # filesystem, so tests/ and solution/ in the task dir are readable + # (and writable) by the agent. When enabled, reset() moves them out + # of reach — see _withhold_verifier_assets. + if withhold_tests is None: + withhold_tests = os.getenv("TB2_WITHHOLD_TESTS", "0") == "1" + self.withhold_tests = withhold_tests self.output_dir = Path( output_dir or os.getenv("TB2_OUTPUT_DIR", "/tmp/tbench2_env_runs") ) @@ -168,6 +212,8 @@ def reset( self._instruction = _read_instruction(task_dir) self._task_dir = task_dir + if self.withhold_tests: + self._withhold_verifier_assets(task_dir) trial_name = f"{resolved_task_id}.{episode_id or uuid4().hex}" session_logs_dir = ( @@ -348,6 +394,75 @@ def _ensure_session_id(self, session_id: str | None, action_type: str) -> None: if not session_id: raise ValueError(f"session_id is required for action_type='{action_type}'") + # tests/ tarballs withheld out of agent reach at reset(), keyed by task + # dir. Class-level: the server may construct one environment instance per + # session, and a later reset of the same task must still find the copy + # after the on-disk source is gone. In-process only — withhold mode + # therefore assumes a single server process owns the task filesystem for + # its lifetime (the intended per-task-sandbox deployment: one uvicorn + # worker, one ephemeral sandbox per episode). It is not safe with + # ``uvicorn --workers N`` sharing one on-disk checkout, where a sibling + # worker would see tests/ already deleted with no cache to restore from. + _WITHHELD_TESTS: dict[str, bytes] = {} + _WITHHOLD_LOCK = threading.Lock() + + def _withhold_verifier_assets(self, task_dir: Path) -> None: + """Move verifier assets out of the agent's reach before it can act. + + The task directory is agent-readable (same filesystem, root agent): + tests/test_outputs.py spells out the expected outputs and solution/ is + the literal answer — for RL training both are reward-hacking bait. + tests/ is read into server memory (the staging source for + _stage_tests_for_verify) and removed from disk; solution/ is simply + removed (nothing in this env reads it — it exists only for oracle + runs). Gated behind TB2_WITHHOLD_TESTS=1 because in plain local mode + task_dir may be a developer's working checkout, where deleting from + it would be hostile. + """ + key = str(task_dir) + with self._WITHHOLD_LOCK: + tests_dir = task_dir / "tests" + if tests_dir.is_dir(): + if key not in self._WITHHELD_TESTS: + # realpath so a symlinked tests/ caches the target's files, + # not just a link member that would restore empty (tar.add + # does not follow a top-level symlink). + tests_real = os.path.realpath(tests_dir) + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + tar.add(tests_real, arcname=".") + self._WITHHELD_TESTS[key] = buf.getvalue() + _hard_remove(str(tests_dir)) + _hard_remove(str(task_dir / "solution")) + + def _stage_tests_for_verify(self) -> bool: + """Stage a pristine tests/ copy at the official fixed path for exactly + the verify window. Caller must hold _CANONICAL_EVAL_LOCK. + + Both fixed paths are recreated from scratch (symlink included — see + _hard_remove) so nothing an agent may have pre-planted survives into + scoring: a /tests/conftest.py pytest would auto-load, or a symlinked + /tests or /logs/verifier redirecting the stage at a dir the agent + controls. When the source dir was withheld at reset, the staged copy + comes from server memory: a copy the agent never had filesystem + access to. + """ + _hard_remove(_VERIFY_TESTS_DIR) + _hard_remove(_VERIFIER_LOG_DIR) + Path(_VERIFIER_LOG_DIR).mkdir(parents=True, exist_ok=True) + + blob = self._WITHHELD_TESTS.get(str(self._task_dir)) + if blob is not None: + Path(_VERIFY_TESTS_DIR).mkdir(parents=True, exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(blob)) as tar: + _extractall(tar, _VERIFY_TESTS_DIR) + return True + tests_dir = Path(self._task_dir) / "tests" + if not tests_dir.is_dir(): + return False + shutil.copytree(tests_dir, _VERIFY_TESTS_DIR, symlinks=True) + return True + def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: if self._task_dir is None: raise RuntimeError("TB2 environment not initialized. Call reset() first.") @@ -357,81 +472,66 @@ def _evaluate_task(self) -> tuple[str, float, dict[str, Any]]: # The task's own verifier budget (task.toml [verifier].timeout_sec) — # heavy tests legitimately run minutes (circuit-fibsqrt declares 3600s). verifier_timeout_s = _read_timeout(self._task_dir, fallback=900.0) - tests_dir = self._task_dir / "tests" - if (tests_dir / "test.sh").is_file(): - return self._evaluate_canonical(tests_dir, verifier_timeout_s) - - # Fallback for tasks without the canonical harness (none of the 89 - # official TB2 tasks — they all ship test.sh — but custom task dirs - # may only have bare pytest tests). Prefer uvx so pytest comes with - # its own toolchain like the canonical harness does; fall back to a - # preinstalled pytest for environments without uv. - # Verify from the same directory the agent worked in. - fallback_cwd = self._workdir or str(self._task_dir) - cmd = ( - f"cd {shlex.quote(fallback_cwd)} && " - "if command -v uvx >/dev/null 2>&1; " - f"then uvx --with pytest==8.4.1 pytest -q {tests_dir} -rA; " - f"else python -m pytest -q {tests_dir} -rA; fi; " - "echo __TB2_EXIT_CODE__:$?" - ) - output = self._terminal_toolkit.shell_exec( - id="tb2-tests", - command=cmd, - block=True, - timeout=verifier_timeout_s, - ) - - exit_code = 1 - marker = "__TB2_EXIT_CODE__" - for line in output.splitlines()[::-1]: - if marker in line: - try: - exit_code = int(line.split(":", 1)[1].strip()) - except Exception: - exit_code = 1 - break - reward = 1.0 if exit_code == 0 else 0.0 - info = {"tests_passed": exit_code == 0, "exit_code": exit_code} - return output, reward, info + with self._CANONICAL_EVAL_LOCK: + if not self._stage_tests_for_verify(): + return ( + f"no tests/ found for task {self._state.task_id}", + 0.0, + {"tests_passed": False, "error": "missing tests"}, + ) + try: + if (Path(_VERIFY_TESTS_DIR) / "test.sh").is_file(): + return self._evaluate_canonical(verifier_timeout_s) + return self._evaluate_fallback(verifier_timeout_s) + finally: + # Verifier artifacts live on disk only for the verify window: + # don't leave them readable to a later episode / concurrent + # session sharing this filesystem — or to the agent itself if + # scoring times out / errors (done stays false, so it keeps + # its session). /logs/verifier holds reward.txt and the + # test.sh log, whose pytest -rA output can reveal expected + # values. + _hard_remove(_VERIFY_TESTS_DIR) + _hard_remove(_VERIFIER_LOG_DIR) # The canonical harness uses the official fixed paths (/tests, # /logs/verifier/reward.txt), which are per-container, not per-session. - # Serialize evaluations so concurrent sessions on one server cannot - # overwrite each other's staged tests or read another session's reward. + # Serialize staging + evaluation so concurrent sessions on one server + # cannot overwrite each other's staged tests or read another session's + # reward. _CANONICAL_EVAL_LOCK = threading.Lock() def _evaluate_canonical( - self, tests_dir: Path, timeout_s: float + self, timeout_s: float ) -> tuple[str, float, dict[str, Any]]: """Score via the task's OFFICIAL harness, exactly as the TB2 verifier does. - tests/test.sh pins its own pytest toolchain (uvx: Python 3.13 + - pytest 8.4.1 + ctrf), runs tests/test_outputs.py from /tests against - the task workdir, and writes the binary result to - /logs/verifier/reward.txt. Bare ``pytest tests/`` (the fallback above) - skips that toolchain pinning and mis-scores any task whose tests need - it, so the canonical harness is preferred whenever the task ships it. + /tests/test.sh (staged by _stage_tests_for_verify) pins its own pytest + toolchain (uvx: Python 3.13 + pytest 8.4.1 + ctrf), runs + tests/test_outputs.py from /tests against the task workdir, and writes + the binary result to /logs/verifier/reward.txt. Bare ``pytest tests/`` + (the fallback below) skips that toolchain pinning and mis-scores any + task whose tests need it, so the canonical harness is preferred + whenever the task ships it. """ # Same working dir the agent operated in (resolved during reset). workdir = self._workdir or str(self._task_dir) marker = "__TB2_REWARD__:" + # test.sh stdout goes under /logs/verifier (wiped with the verify + # window in _evaluate_task) rather than a fixed /tmp path that would + # outlive scoring: its pytest -rA output can spell out expected values. cmd = ( - # rm the reward file first so a stale one can never be read back - # if test.sh fails to run. - "mkdir -p /tests /logs/verifier && rm -f /logs/verifier/reward.txt && " - f"cp -a {shlex.quote(str(tests_dir))}/. /tests/ && " - f"cd {shlex.quote(workdir)} && bash /tests/test.sh > /tmp/tb2_testsh.log 2>&1; " - f"echo {marker}$(cat /logs/verifier/reward.txt 2>/dev/null)" + f"cd {shlex.quote(workdir)} && " + f"bash {_VERIFY_TESTS_DIR}/test.sh > {_VERIFIER_LOG_DIR}/testsh.log 2>&1; " + f"echo {marker}$(cat {_VERIFIER_LOG_DIR}/reward.txt 2>/dev/null)" + ) + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, ) - with self._CANONICAL_EVAL_LOCK: - output = self._terminal_toolkit.shell_exec( - id="tb2-tests", - command=cmd, - block=True, - timeout=timeout_s, - ) reward = 0.0 for line in output.splitlines()[::-1]: @@ -446,6 +546,42 @@ def _evaluate_canonical( info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} return output, reward, info + def _evaluate_fallback(self, timeout_s: float) -> tuple[str, float, dict[str, Any]]: + """For task dirs without the canonical harness (none of the 89 official + TB2 tasks — they all ship test.sh — but custom task dirs may only have + bare pytest tests). Runs pytest against the staged /tests copy; prefer + uvx so pytest comes with its own toolchain like the canonical harness + does. Verify from the same directory the agent worked in. + """ + fallback_cwd = self._workdir or str(self._task_dir) + cmd = ( + f"cd {shlex.quote(fallback_cwd)} && " + "if command -v uvx >/dev/null 2>&1; " + f"then uvx --with pytest==8.4.1 pytest -q {_VERIFY_TESTS_DIR} -rA; " + f"else python -m pytest -q {_VERIFY_TESTS_DIR} -rA; fi; " + "echo __TB2_EXIT_CODE__:$?" + ) + output = self._terminal_toolkit.shell_exec( + id="tb2-tests", + command=cmd, + block=True, + timeout=timeout_s, + ) + + exit_code = 1 + marker = "__TB2_EXIT_CODE__" + for line in output.splitlines()[::-1]: + if marker in line: + try: + exit_code = int(line.split(":", 1)[1].strip()) + except Exception: + exit_code = 1 + break + + reward = 1.0 if exit_code == 0 else 0.0 + info = {"tests_passed": exit_code == 0, "exit_code": exit_code} + return output, reward, info + class Tbench2DockerEnvironment( Environment[Tbench2Action, Tbench2Observation, Tbench2State] @@ -595,8 +731,14 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: # Copy task files into container using tar archive # This works in Docker-in-Docker because we read files from our - # filesystem and stream them to the container via the Docker API - self._copy_dir_to_container(task_dir, "/task") + # filesystem and stream them to the container via the Docker API. + # Verifier assets stay out: the agent works in /task and must not + # be able to read solution/ (the literal answer) or tests/ (the + # expected outputs) — tests are staged only at verify time by + # _evaluate_docker, like the official TB2 harness does. + self._copy_dir_to_container( + task_dir, "/task", exclude_top=("solution", "tests") + ) self._state = Tbench2State( episode_id=str(uuid4()), @@ -609,24 +751,28 @@ def _start_container(self, task_dir: Path, trial_dir: Path) -> None: except Exception as exc: raise RuntimeError(f"Failed to start container: {exc}") from exc - def _copy_dir_to_container(self, src_dir: Path, dest_path: str) -> None: + def _copy_dir_to_container( + self, src_dir: Path, dest_path: str, exclude_top: tuple[str, ...] = () + ) -> None: """Copy a directory into the container using tar archive. This method streams files via the Docker API, avoiding bind mount - issues in Docker-in-Docker scenarios. + issues in Docker-in-Docker scenarios. Top-level entries named in + *exclude_top* are skipped entirely. """ - import io - import tarfile - if self._container is None: raise RuntimeError("Container not started") - # Create tar archive in memory + # Create tar archive in memory (recursive=False: rglob already yields + # every descendant, so per-entry recursion would duplicate members — + # and would re-add excluded subtrees through their parents). tar_stream = io.BytesIO() with tarfile.open(fileobj=tar_stream, mode="w") as tar: for item in src_dir.rglob("*"): - arcname = str(item.relative_to(src_dir)) - tar.add(str(item), arcname=arcname) + rel = item.relative_to(src_dir) + if rel.parts and rel.parts[0] in exclude_top: + continue + tar.add(str(item), arcname=str(rel), recursive=False) tar_stream.seek(0) @@ -750,28 +896,58 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: raise RuntimeError("Container not started.") assert self._task_dir is not None, "Task directory not set" - # Run pytest in the container's /task directory - # Use exit code marker for consistency with local mode - cmd = "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" + # Stage-at-verify, like the official TB2 harness: the initial /task + # copy excludes tests/, and this server sits OUTSIDE the container, so + # the copy staged here is one the agent never saw. rm -rf first so + # nothing an agent pre-planted at /task/tests (e.g. a conftest.py + # pytest would auto-load) survives into scoring. + tests_src = self._task_dir / "tests" + if not tests_src.is_dir(): + return ( + f"no tests/ found for task {self._state.task_id}", + 0.0, + {"tests_passed": False, "error": "missing tests"}, + ) + self._exec_in_container("rm -rf /task/tests && mkdir -p /task/tests") + try: + self._copy_dir_to_container(tests_src, "/task/tests") - exit_code, output = self._container.exec_run( - cmd=f"bash -c '{cmd}'", - workdir="/task", - stdout=True, - stderr=True, - ) - output_str = output.decode("utf-8", errors="replace") + # Run pytest in the container's /task directory + # Use exit code marker for consistency with local mode + cmd = ( + "cd /task && python -m pytest -q tests/ -rA; echo __TB2_EXIT_CODE__:$?" + ) - # Parse exit code from marker (same logic as local mode) - ec = 1 - marker = "__TB2_EXIT_CODE__" - for line in output_str.splitlines()[::-1]: - if marker in line: - try: - ec = int(line.split(":", 1)[1].strip()) - except Exception: - ec = 1 - break + exit_code, output = self._container.exec_run( + cmd=f"bash -c '{cmd}'", + workdir="/task", + stdout=True, + stderr=True, + ) + output_str = output.decode("utf-8", errors="replace") + + # Parse exit code from marker (same logic as local mode) + ec = 1 + marker = "__TB2_EXIT_CODE__" + for line in output_str.splitlines()[::-1]: + if marker in line: + try: + ec = int(line.split(":", 1)[1].strip()) + except Exception: + ec = 1 + break + finally: + # Tests live in the container only for the verify window, matching + # the local-mode staging: don't leave /task/tests readable + # afterward — the agent keeps its session if scoring errored + # (step() reports the failure without ending the episode). + try: + self._exec_in_container("rm -rf /task/tests") + except Exception: + logging.warning( + "failed to remove staged /task/tests after verify", + exc_info=True, + ) reward = 1.0 if ec == 0 else 0.0 info = {"tests_passed": ec == 0, "exit_code": ec} diff --git a/tests/envs/test_tbench2_env.py b/tests/envs/test_tbench2_env.py index 9e95cf6fb..1aa3d3cbd 100644 --- a/tests/envs/test_tbench2_env.py +++ b/tests/envs/test_tbench2_env.py @@ -1,11 +1,19 @@ +import io import os import sys +import tarfile from pathlib import Path import pytest -# Add the project root to the path for envs imports -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +# Add the project root to the path for envs imports, and envs/ itself so the +# server module's Spaces-style absolute imports (``tbench2_env.models``) +# resolve when this file runs standalone (in the full suite another test +# happens to insert envs/ first — don't rely on collection order). +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +for _p in (_REPO_ROOT, os.path.join(_REPO_ROOT, "envs")): + if _p not in sys.path: + sys.path.insert(0, _p) try: import camel # noqa: F401 @@ -14,7 +22,10 @@ from envs.tbench2_env.models import Tbench2Action from envs.tbench2_env.server import tbench2_env_environment -from envs.tbench2_env.server.tbench2_env_environment import Tbench2Environment +from envs.tbench2_env.server.tbench2_env_environment import ( + Tbench2DockerEnvironment, + Tbench2Environment, +) class _FakeTerminalToolkit: @@ -50,6 +61,306 @@ def test_tbench2_reset_uses_default_task_id(monkeypatch, tmp_path: Path): assert "terminal task" in observation.instruction +# --------------------------------------------------------------------------- +# Verifier-asset withholding (TB2_WITHHOLD_TESTS) and stage-at-verify +# --------------------------------------------------------------------------- + + +def _make_task_dir(root: Path, name: str = "my-task") -> Path: + task = root / name + (task / "tests").mkdir(parents=True) + (task / "tests" / "test.sh").write_text( + "#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n" + ) + (task / "tests" / "test_outputs.py").write_text("EXPECTED = 'secret42'\n") + (task / "solution").mkdir() + (task / "solution" / "solve.sh").write_text("echo the-answer\n") + (task / "task.toml").write_text('[environment]\ndocker_image = "debian:12"\n') + (task / "instruction.md").write_text("do the thing\n") + return task + + +@pytest.fixture() +def staged_paths(monkeypatch, tmp_path: Path): + """Redirect the official fixed paths (/tests, /logs/verifier) into tmp.""" + tests = tmp_path / "stage" / "tests" + logs = tmp_path / "stage" / "logs" / "verifier" + monkeypatch.setattr(tbench2_env_environment, "_VERIFY_TESTS_DIR", str(tests)) + monkeypatch.setattr(tbench2_env_environment, "_VERIFIER_LOG_DIR", str(logs)) + return tests, logs + + +@pytest.fixture(autouse=True) +def _fresh_withhold_cache(monkeypatch): + monkeypatch.setattr(Tbench2Environment, "_WITHHELD_TESTS", {}) + + +def test_withhold_removes_assets_and_caches_tests(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + + env._withhold_verifier_assets(task) + + assert not (task / "solution").exists() + assert not (task / "tests").exists() + assert (task / "task.toml").is_file() + assert (task / "instruction.md").is_file() + assert str(task) in Tbench2Environment._WITHHELD_TESTS + + # Second reset of the same task (dir already gone) must keep the cache. + env._withhold_verifier_assets(task) + assert str(task) in Tbench2Environment._WITHHELD_TESTS + + +def test_stage_from_memory_and_wipe_planted_files(tmp_path: Path, staged_paths): + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._withhold_verifier_assets(task) + + # An agent pre-plants a conftest.py (pytest would auto-load it) and a + # fake reward before verification. + stage_tests.mkdir(parents=True) + (stage_tests / "conftest.py").write_text("import sys; sys.exit(0)\n") + stage_logs.mkdir(parents=True) + (stage_logs / "reward.txt").write_text("1\n") + + assert env._stage_tests_for_verify() + + assert (stage_tests / "test.sh").is_file() + assert (stage_tests / "test_outputs.py").read_text() == "EXPECTED = 'secret42'\n" + assert not (stage_tests / "conftest.py").exists() + assert not (stage_logs / "reward.txt").exists() + assert stage_logs.is_dir() and not list(stage_logs.iterdir()) + + +def test_stage_from_disk_without_gate(tmp_path: Path, staged_paths): + stage_tests, _ = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + + assert env._stage_tests_for_verify() + + assert (stage_tests / "test_outputs.py").is_file() + # Without the gate the source checkout is never touched. + assert (task / "tests").is_dir() + assert (task / "solution").is_dir() + + +def test_stage_reports_missing_tests(tmp_path: Path, staged_paths): + task = tmp_path / "empty-task" + task.mkdir() + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + + assert not env._stage_tests_for_verify() + + +class _RecordingToolkit: + def __init__(self, output: str): + self.calls: list[dict] = [] + self.output = output + + def shell_exec(self, **kwargs): + self.calls.append(kwargs) + return self.output + + +def test_evaluate_canonical_from_withheld_copy(tmp_path: Path, staged_paths): + """Full evaluate flow: staging from memory + canonical test.sh scoring.""" + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._workdir = str(tmp_path) + env._terminal_toolkit = _RecordingToolkit(output="__TB2_REWARD__:1") + env._withhold_verifier_assets(task) + + output, reward, info = env._evaluate_task() + + assert reward == 1.0 + assert info["harness"] == "tests/test.sh" + # The verify command ran test.sh from the staged fixed path, and wrote its + # log under the wiped verifier dir — not a fixed /tmp path that outlives it. + command = env._terminal_toolkit.calls[0]["command"] + assert f"{stage_tests}/test.sh" in command + assert "/tmp/tb2_testsh.log" not in command + assert f"{stage_logs}/testsh.log" in command + # Both the staged tests and the verifier log/reward dir are gone once + # scoring returns. + assert not stage_tests.exists() + assert not stage_logs.exists() + + +def test_evaluate_without_tests_scores_zero(tmp_path: Path, staged_paths): + task = tmp_path / "empty-task" + task.mkdir() + env = Tbench2Environment(withhold_tests=False) + env._task_dir = task + env._terminal_toolkit = _RecordingToolkit(output="") + + output, reward, info = env._evaluate_task() + + assert reward == 0.0 + assert info["tests_passed"] is False + assert env._terminal_toolkit.calls == [] # nothing was run + + +def test_stage_wipes_symlinked_fixed_path(tmp_path: Path, staged_paths): + """A symlinked /tests must not let the official suite merge into a dir the + agent controls (rmtree silently no-ops on a symlink; _hard_remove doesn't).""" + stage_tests, _ = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._withhold_verifier_assets(task) + + evil = tmp_path / "agent_controlled" + evil.mkdir() + (evil / "conftest.py").write_text("import sys; sys.exit(0)\n") + stage_tests.parent.mkdir(parents=True, exist_ok=True) + stage_tests.symlink_to(evil, target_is_directory=True) + + assert env._stage_tests_for_verify() + + assert not stage_tests.is_symlink() + assert (stage_tests / "test.sh").is_file() + # The official suite never leaked into the agent's directory. + assert not (evil / "test.sh").exists() + assert (evil / "conftest.py").exists() # their dir left alone + + +def test_withhold_removes_symlinked_tests(tmp_path: Path): + """tests/ as a symlink is removed too — the link, not just its target.""" + task = tmp_path / "task" + task.mkdir() + real = tmp_path / "real_tests" + real.mkdir() + (real / "test_outputs.py").write_text("x = 1\n") + (task / "tests").symlink_to(real, target_is_directory=True) + + env = Tbench2Environment(withhold_tests=True) + env._withhold_verifier_assets(task) + + assert not (task / "tests").exists() # symlink gone + assert real.exists() # its target is outside the task dir; untouched + # The cache captured the target's real files, not a bare link member that + # would restore an empty suite (tar.add doesn't follow a top-level link). + import io + import tarfile + + blob = Tbench2Environment._WITHHELD_TESTS[str(task)] + with tarfile.open(fileobj=io.BytesIO(blob)) as tar: + assert any(n.endswith("test_outputs.py") for n in tar.getnames()) + + +# --------------------------------------------------------------------------- +# Docker mode: copy exclusions + stage-at-verify (mock container, no docker) +# --------------------------------------------------------------------------- + + +class _FakeContainer: + """Records exec/put_archive calls in one ordered event log.""" + + def __init__(self, exec_output: bytes = b"__TB2_EXIT_CODE__:0\n"): + self.events: list[tuple] = [] + self.exec_output = exec_output + + def put_archive(self, dest, data): + self.events.append(("put", dest, data)) + return True + + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + self.events.append(("exec", cmd)) + return 0, self.exec_output + + +def _tar_names(data: bytes) -> list[str]: + with tarfile.open(fileobj=io.BytesIO(data)) as tar: + return tar.getnames() + + +def test_docker_copy_excludes_verifier_assets(tmp_path: Path): + task = _make_task_dir(tmp_path) + (task / "environment").mkdir() + (task / "environment" / "Dockerfile").write_text("FROM debian:12\n") + + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._copy_dir_to_container(task, "/task", exclude_top=("solution", "tests")) + + ((kind, dest, data),) = container.events + assert (kind, dest) == ("put", "/task") + names = _tar_names(data) + assert "task.toml" in names + assert "environment/Dockerfile" in names + assert not any(n.startswith(("solution", "tests")) for n in names) + assert len(names) == len(set(names)) # recursive=False: no duplicates + + +def test_evaluate_docker_stages_tests_at_verify(tmp_path: Path): + task = _make_task_dir(tmp_path) + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 1.0 + kinds = [e[0] for e in container.events] + # rm -rf the agent-writable staging dir BEFORE the fresh copy goes in, + # the copy lands before pytest runs, and tests are removed again after. + assert kinds == ["exec", "put", "exec", "exec"] + assert "rm -rf /task/tests" in container.events[0][1] + assert container.events[1][1] == "/task/tests" + assert "test.sh" in _tar_names(container.events[1][2]) + assert "pytest" in container.events[2][1] + assert "rm -rf /task/tests" in container.events[3][1] + + +def test_evaluate_docker_cleans_up_when_scoring_raises(tmp_path: Path): + """The staged /task/tests must not outlive a verify that errors out — + step() reports the failure without ending the episode, so the agent + keeps its session.""" + task = _make_task_dir(tmp_path) + + class _ExplodingContainer(_FakeContainer): + def exec_run(self, cmd, workdir=None, stdout=True, stderr=True): + if "pytest" in cmd: + self.events.append(("exec", cmd)) + raise RuntimeError("docker daemon hiccup") + return super().exec_run(cmd, workdir=workdir, stdout=stdout, stderr=stderr) + + env = Tbench2DockerEnvironment() + container = _ExplodingContainer() + env._container = container + env._task_dir = task + + with pytest.raises(RuntimeError, match="hiccup"): + env._evaluate_docker() + + assert container.events[-1][0] == "exec" + assert "rm -rf /task/tests" in container.events[-1][1] + + +def test_evaluate_docker_missing_tests_scores_zero(tmp_path: Path): + task = tmp_path / "no-tests-task" + task.mkdir() + env = Tbench2DockerEnvironment() + container = _FakeContainer() + env._container = container + env._task_dir = task + + output, reward, info = env._evaluate_docker() + + assert reward == 0.0 + assert container.events == [] + + @pytest.mark.skipif(camel is None, reason="camel-ai not installed") @pytest.mark.skipif( os.environ.get("TB2_ENABLE_TESTS", "0") != "1", From eea94f004805c37c559bdaf3cae70660a15ee617 Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Thu, 16 Jul 2026 13:03:13 -0700 Subject: [PATCH 3/3] feat(tbench2_env): per-task sandbox image recipe + Daytona materialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TB2 is per-task-image (task.toml [environment].docker_image pins the official image the TB2 harness itself runs); a cloud sandbox serving this env must be built per task: official task image + this env's server layer, one layer, no DinD. - Provider-agnostic core: _server_layer_commands is plain shell; this checkout's env source is embedded into the build so local fixes ship without a released package. The task image's apt index is preserved (solutions/agents run bare 'apt install' against it). - Daytona materialization: create_task_sandbox (declarative per-episode create, no named-snapshot quota, repeat creates hit the build cache; every task dir is staged from the checkout's pinned-SHA GitHub tarball — one uniform, payload-free path; sandboxes carry an ownership label for safe cleanup) plus an optional named-snapshot bake CLI as a warm cache. make_daytona is public API: callers driving create_task_sandbox need a client configured the same way this module's own CLI is. - Verifier-asset hygiene: the staged task directory excludes solution/ at build time (tar --exclude anchored to the one task), and SERVER_CMD sets TB2_WITHHOLD_TESTS=1 so the server withholds tests/ at reset and stages them only at verify (the underlying env fix). tests/envs/test_tbench2_task_snapshots.py pins both. Validated by a full 89-task golden sweep (official solutions + official test.sh scoring): 0 infra errors; failures traced to upstream solution drift, not the environment. The verifier-asset hygiene was re-verified end-to-end on a per-task Daytona sandbox (fix-git): no tests/solution anywhere agent-readable, pre-solution evaluate 0.0, oracle solution 1.0. --- envs/tbench2_env/task_snapshots.py | 408 ++++++++++++++++++++++ tests/envs/test_tbench2_task_snapshots.py | 61 ++++ 2 files changed, 469 insertions(+) create mode 100644 envs/tbench2_env/task_snapshots.py create mode 100644 tests/envs/test_tbench2_task_snapshots.py diff --git a/envs/tbench2_env/task_snapshots.py b/envs/tbench2_env/task_snapshots.py new file mode 100644 index 000000000..902528835 --- /dev/null +++ b/envs/tbench2_env/task_snapshots.py @@ -0,0 +1,408 @@ +"""Per-task sandboxes for Terminal-Bench-2. + +TB2 is a per-task-image benchmark: every task pins its official runtime image +in ``task.toml`` (``[environment].docker_image`` — the exact image the TB2 +harness itself runs). A cloud sandbox serving this env must therefore be built +per task: **the official task image ⊕ this env's server layer**, one layer, no +DinD. This module owns that recipe. + +Provider-agnostic core: + ``_server_layer_commands(task_dir)`` shell commands that turn the official + task image into a combined task+env-server image: a uv-managed Python + venv at ``/opt/envserver`` running THIS checkout's tbench2_env (source + embedded into the build, so local fixes ship without a released + package), plus the task directory staged at ``/opt/tb2-tasks/`` + (downloaded from the tasks checkout's pinned-commit GitHub tarball) for + ``reset(task_id)`` via ``TB2_TASKS_DIR``. These are plain shell + commands — nothing Daytona-specific — so the same recipe can back a + Dockerfile or another provider's build. + ``server_cmd()`` starts the env server inside the sandbox. Sets + ``CAMEL_RUNTIME=true`` so camel's TerminalToolkit runs commands in the + task image's native environment instead of hijacking PATH with its own + Python 3.10 ``.initial_env`` venv (real TB2 agents see the image's + python), and ``TB2_COMMAND_TIMEOUT_S`` for realistic command budgets. + +Daytona materialization: + ``create_task_sandbox(...)`` per-episode declarative create straight from + the ``Image`` definition. Named snapshots count against an org-level + quota, so registering one per task may not scale to a full task suite; + the declarative path avoids the quota entirely, and repeat creates hit + Daytona's build cache (~1min after the first build). Daytona does not + run the image CMD, so this execs ``server_cmd()`` and waits for /health. + bake CLI (``python -m tbench2_env.task_snapshots ...``) optionally + pre-register named snapshots ```` as a warm cache. + +Verifier-asset hygiene (mirrors the official harness's stage-at-verify +model): ``solution/`` is excluded from the staged task directory at build +time (nothing in this env ever reads it — it exists only for oracle runs), +and ``SERVER_CMD`` sets ``TB2_WITHHOLD_TESTS=1`` so the server pulls +``tests/`` into process memory at ``reset()`` and deletes it from disk before +the agent's first action; a pristine copy is staged at ``/tests`` only for +the verify window. Residual risk, shared with the official TB2 harness: +verification necessarily runs inside the same container the agent controlled +(task state — services, git state — is not portable), so a root agent that +tampers with container binaries could still fake a pass. +""" + +import argparse +import base64 +import io +import os +import re +import shlex +import subprocess +import sys +import tarfile +import time +from pathlib import Path + +try: + import tomllib +except ImportError: # Python < 3.11 + import tomli as tomllib + +# Guard for the ONE payload that must be embedded into the build: this +# checkout's own env source (it exists nowhere downloadable until released). +# The hard ceiling is Daytona's Dockerfile parser: a single line may not +# exceed 65535 bytes ("dockerfile line greater than max allowed size", +# observed on real builds), and base64 inflates by 4/3; 45KB of tar.gz stays +# safely under that. Task directories are never embedded — they download from +# the pinned-SHA GitHub tarball instead (see _task_layer_command). +_MAX_INLINE_TAR_BYTES = 45_000 + +_ENV_SRC_DIR = Path(__file__).resolve().parent +# What `pip install ` needs from this checkout; everything else +# (uv.lock, caches, this file) stays out of the image. +_ENV_SRC_ITEMS = ( + "pyproject.toml", + "README.md", + "openenv.yaml", + "__init__.py", + "client.py", + "models.py", + "server", +) + +# The command to start the env server inside a task sandbox (Daytona does not +# run the image CMD). /opt/envserver and /opt/tb2-tasks are baked by +# _server_layer_commands. +SERVER_CMD = ( + "TB2_TASKS_DIR=/opt/tb2-tasks " + "TB2_DEFAULT_TASK_ID={default_task_id} " + "TB2_COMMAND_TIMEOUT_S={command_timeout_s} " + "TB2_WITHHOLD_TESTS=1 " + "CAMEL_RUNTIME=true " + "MAX_CONCURRENT_ENVS=1 " + "/opt/envserver/bin/python -m uvicorn tbench2_env.server.app:app " + "--host 0.0.0.0 --port 8000" +) + + +def server_cmd(command_timeout_s: int = 900, default_task_id: str = "") -> str: + # A per-task sandbox stages exactly one task, so make it the default: + # a reset() with no task_id resolves to the staged task rather than the + # env's built-in headless-terminal default (which isn't present here). + return SERVER_CMD.format( + command_timeout_s=command_timeout_s, + default_task_id=shlex.quote(default_task_id), + ) + + +def snapshot_name(prefix: str, task_id: str) -> str: + return prefix + re.sub(r"[^a-z0-9-]", "-", task_id.lower()) + + +def read_task_config(task_dir: Path) -> dict: + toml_path = task_dir / "task.toml" + if not toml_path.is_file(): + raise FileNotFoundError(f"{task_dir}: no task.toml") + return tomllib.loads(toml_path.read_text()) + + +def _tar_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: + name = Path(tarinfo.name).name + if name in {"__pycache__", ".initial_env"} or name.endswith((".pyc", ".egg-info")): + return None + return tarinfo + + +def _dir_tar_b64(paths: list[Path], arcnames: list[str], max_bytes: int) -> str: + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for path, arcname in zip(paths, arcnames): + tar.add(path, arcname=arcname, filter=_tar_filter) + raw = buf.getvalue() + if len(raw) > max_bytes: + raise ValueError( + f"embedded tar is {len(raw)} bytes (> {max_bytes}); " + "inline embedding not suitable." + ) + return base64.b64encode(raw).decode() + + +def _task_layer_command(task_dir: Path) -> str: + """Build command that stages the task dir at /opt/tb2-tasks/. + + One uniform path for every task: download the checkout's pinned-commit + GitHub tarball and extract just this task. Deterministic (the SHA pins + the content — note: the committed tree, not uncommitted local edits) and + payload-free, so build commands stay far from Daytona's 64KB + Dockerfile-line ceiling regardless of task-dir size. Requires the tasks + checkout to be a git clone with a GitHub origin. + """ + repo_root = task_dir.parent + sha = subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + remote = subprocess.run( + ["git", "-C", str(repo_root), "remote", "get-url", "origin"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + m = re.search(r"github\.com[:/]+([^/]+)/([^/.]+)", remote) + if not m: + raise ValueError( + f"{task_dir.name}: the tasks checkout's origin ({remote}) is not " + "a GitHub remote to download the task tarball from." + ) + owner, repo = m.group(1), m.group(2) + url = f"https://github.com/{owner}/{repo}/archive/{sha}.tar.gz" + # solution/ never enters the image: nothing in this env reads it (it + # exists only for oracle runs), and an agent must not be able to cat the + # answer out of the staged task directory. + prefix = f"{repo}-{sha}/{task_dir.name}" + return ( + f"mkdir -p /opt/tb2-tasks && curl -fsSL {url} | " + f"tar xz --strip-components=1 -C /opt/tb2-tasks " + f"--exclude='{prefix}/solution' --exclude='{prefix}/solution/*' '{prefix}'" + ) + + +def _env_src_tar_b64() -> str: + paths, arcnames = [], [] + for item in _ENV_SRC_ITEMS: + p = _ENV_SRC_DIR / item + if p.exists(): + paths.append(p) + arcnames.append(f"tbench2_env_src/{item}") + return _dir_tar_b64(paths, arcnames, _MAX_INLINE_TAR_BYTES) + + +def _resolve_docker_image(task_dir: Path, docker_image: str | None) -> str: + if docker_image: + return docker_image + docker_image = read_task_config(task_dir).get("environment", {}).get("docker_image") + if not docker_image: + raise ValueError( + f"{task_dir.name}: task.toml has no [environment].docker_image" + ) + return docker_image + + +def _server_layer_commands(task_dir: Path) -> list[str]: + """The provider-agnostic recipe: shell commands that turn the OFFICIAL task + image into a combined task+env-server image. Nothing here is + Daytona-specific — the same layers work for docker build, Modal, ACA, etc. + """ + return [ + # Server-layer OS deps. Task images are heterogeneous; assume + # debian-ish (all 89 current TB2 images are debian/ubuntu based). Do + # NOT rm /var/lib/apt/lists afterwards: the official task image's apt + # state is part of the task environment — solutions and agents run + # bare `apt install` relying on the index the task image baked in. + # (curl/ca-certificates/bash stay installed: debian-ish images almost + # always ship them anyway, and the official test.sh apt-installs curl + # itself at verify time — unlike uv below, no observed task behavior + # depends on their absence.) + "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y " + "--no-install-recommends curl ca-certificates bash", + # uv + its own managed Python: immune to whatever python (if any) + # the task base image ships. Installed OUTSIDE PATH (/opt/uv) so the + # agent's PATH lookup stays faithful to the official task image: no + # tool the image didn't ship resolves from PATH, and a task image's + # own uv (e.g. financial-document-processor ships uv 0.8.14 at /bin) + # is never shadowed. (Filesystem traces under /opt remain visible — + # a single-container sandbox cannot hide them from a root agent.) + "curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/opt/uv UV_NO_MODIFY_PATH=1 sh", + "/opt/uv/uv venv --python 3.12 /opt/envserver", + # THIS checkout's tbench2_env source (embedded), not a released + # package: local fixes (canonical evaluate, TB2_COMMAND_TIMEOUT_S) + # ship with the image. Deps (openenv, camel-ai, ...) come from PyPI. + f"mkdir -p /opt/src && echo {_env_src_tar_b64()} | base64 -d | tar xz -C /opt/src", + "/opt/uv/uv pip install --python /opt/envserver/bin/python /opt/src/tbench2_env_src uvicorn gradio", + # Task directory for reset(task_id) via TB2_TASKS_DIR (pinned-SHA + # GitHub tarball; see _task_layer_command). + _task_layer_command(task_dir), + ] + + +def build_task_image(task_dir: Path, docker_image: str | None = None): + """Daytona-declarative expression of the same recipe (same layers, so the + Daytona build cache is shared with the Dockerfile expression).""" + from daytona import Image + + task_dir = Path(task_dir) + base = _resolve_docker_image(task_dir, docker_image) + return ( + Image.base(base) + .run_commands(*_server_layer_commands(task_dir)) + # Daytona does not execute the image CMD; a long-lived entrypoint keeps + # the sandbox alive and the caller execs server_cmd() explicitly. + .entrypoint(["sleep", "infinity"]) + ) + + +def task_resources(task_dir: Path): + from daytona import Resources + + env_cfg = read_task_config(task_dir).get("environment", {}) + return Resources( + cpu=max(1, int(env_cfg.get("cpus", 1))), + memory=max(2, int(env_cfg.get("memory_mb", 2048)) // 1024), + disk=max(10, int(env_cfg.get("storage_mb", 10240)) // 1024), + ) + + +def wait_server_ready(base_url: str, timeout_s: float = 300.0) -> None: + import requests + + deadline = time.time() + timeout_s + last_err: Exception | None = None + while time.time() < deadline: + try: + if requests.get(f"{base_url}/health", timeout=5.0).status_code == 200: + return + except requests.RequestException as e: + last_err = e + time.sleep(2.0) + raise TimeoutError( + f"env server at {base_url} not ready in {timeout_s}s ({last_err})" + ) + + +def create_task_sandbox( + daytona, + task_dir: Path, + *, + command_timeout_s: int = 900, + create_timeout_s: float = 1800.0, + ready_timeout_s: float = 300.0, +): + """Create ONE per-episode sandbox for *task_dir*, declaratively (no named snapshot). + + Returns ``(sandbox, base_url)``. Caller must ``daytona.delete(sandbox)`` + when the episode ends. First create for a task pays the image build; + repeat creates hit Daytona's build cache. + """ + from daytona import CreateSandboxFromImageParams + + params = CreateSandboxFromImageParams( + image=build_task_image(task_dir), + resources=task_resources(task_dir), + auto_stop_interval=0, + # Ownership marker: lets sweep/cleanup tooling target exactly the + # sandboxes this recipe created (shared orgs run other workloads). + labels={"openenv-tbench2-task": task_dir.name}, + ) + sandbox = daytona.create(params, timeout=create_timeout_s) + try: + cmd = server_cmd(command_timeout_s, default_task_id=task_dir.name) + sandbox.process.exec( + f"nohup bash -c {shlex.quote(cmd)} > /tmp/openenv-server.log 2>&1 &" + " echo $! > /tmp/openenv-server.pid", + timeout=10, + ) + url = sandbox.create_signed_preview_url(8000, expires_in_seconds=86400).url + wait_server_ready(url, timeout_s=ready_timeout_s) + return sandbox, url + except Exception: + daytona.delete(sandbox) + raise + + +def make_daytona(): + """Daytona client from the env-var contract (DAYTONA_API_KEY, optional + DAYTONA_API_URL). Public: callers driving create_task_sandbox() need a + client configured the same way this module's own CLI is.""" + from daytona import Daytona, DaytonaConfig + + api_key = os.environ.get("DAYTONA_API_KEY") + if not api_key: + raise RuntimeError("DAYTONA_API_KEY not set") + return Daytona( + DaytonaConfig( + api_key=api_key, + api_url=os.getenv("DAYTONA_API_URL", "https://app.daytona.io/api"), + ) + ) + + +def bake(daytona, tasks_dir: Path, task_id: str, prefix: str, force: bool) -> None: + """Register the named snapshot ```` (optional warm cache).""" + from daytona import CreateSnapshotParams + + task_dir = tasks_dir / task_id + name = snapshot_name(prefix, task_id) + try: + existing = daytona.snapshot.get(name) + except Exception: + existing = None + if existing is not None: + if not force: + print( + f"[skip] {name} already exists (state={getattr(existing, 'state', '?')})" + ) + return + print(f"[force] deleting existing {name}") + daytona.snapshot.delete(existing) + + resources = task_resources(task_dir) + print( + f"[bake] {name} cpu={resources.cpu} mem={resources.memory}G disk={resources.disk}G" + ) + daytona.snapshot.create( + CreateSnapshotParams( + name=name, + image=build_task_image(task_dir), + resources=resources, + entrypoint=["sleep", "infinity"], + ), + on_logs=lambda line: print(f" | {line}", flush=True), + timeout=1800, + ) + print(f"[done] {name}") + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument( + "--tasks-dir", required=True, help="local terminal-bench-2 checkout" + ) + group = ap.add_mutually_exclusive_group(required=True) + group.add_argument("--tasks", help="comma-separated task_ids") + group.add_argument("--all", action="store_true", help="every dir with a task.toml") + ap.add_argument( + "--prefix", default="tb2-", help="snapshot name prefix (default: tb2-)" + ) + ap.add_argument("--force", action="store_true", help="recreate existing snapshots") + args = ap.parse_args() + + daytona = make_daytona() + tasks_dir = Path(args.tasks_dir).expanduser().resolve() + if args.all: + task_ids = sorted( + p.name for p in tasks_dir.iterdir() if (p / "task.toml").is_file() + ) + else: + task_ids = [t.strip() for t in args.tasks.split(",") if t.strip()] + + for task_id in task_ids: + bake(daytona, tasks_dir, task_id, args.prefix, args.force) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/envs/test_tbench2_task_snapshots.py b/tests/envs/test_tbench2_task_snapshots.py new file mode 100644 index 000000000..a6766db33 --- /dev/null +++ b/tests/envs/test_tbench2_task_snapshots.py @@ -0,0 +1,61 @@ +"""Tests for the per-task sandbox recipe's verifier-asset hygiene.""" + +import os +import subprocess +import sys +from pathlib import Path + +# Add the project root to the path for envs imports (and envs/ itself for the +# package's own absolute imports — see test_tbench2_env.py). +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +for _p in (_REPO_ROOT, os.path.join(_REPO_ROOT, "envs")): + if _p not in sys.path: + sys.path.insert(0, _p) + +from envs.tbench2_env import task_snapshots + + +def _make_tasks_repo(root: Path, task_name: str = "some-task") -> Path: + """A minimal tasks checkout: git repo with a GitHub origin and one task.""" + repo = root / "tb2repo" + task = repo / task_name + task.mkdir(parents=True) + (task / "task.toml").write_text('[environment]\ndocker_image = "debian:12"\n') + for cmd in ( + ["git", "init", "-q"], + ["git", "remote", "add", "origin", "https://github.com/acme/tb2tasks.git"], + ["git", "add", "-A"], + ["git", "-c", "user.email=t@e.st", "-c", "user.name=t", "commit", "-qm", "x"], + ): + subprocess.run(cmd, cwd=repo, check=True) + return task + + +def test_task_layer_excludes_solution(tmp_path: Path): + task = _make_tasks_repo(tmp_path) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=task.parent, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + cmd = task_snapshots._task_layer_command(task) + + # The exclusion is anchored to this one task's solution/ (dir + contents), + # not a loose pattern that could drop task files elsewhere. + assert f"--exclude='tb2tasks-{sha}/some-task/solution'" in cmd + assert f"--exclude='tb2tasks-{sha}/some-task/solution/*'" in cmd + assert cmd.rstrip().endswith(f"'tb2tasks-{sha}/some-task'") + + +def test_server_cmd_sets_withhold_gate(): + assert "TB2_WITHHOLD_TESTS=1" in task_snapshots.server_cmd() + + +def test_server_cmd_defaults_to_the_staged_task(): + # A per-task sandbox stages one task; a reset() with no task_id must land + # on it, not the env's built-in headless-terminal default. + cmd = task_snapshots.server_cmd(default_task_id="fix-git") + assert "TB2_DEFAULT_TASK_ID=fix-git " in cmd