Skip to content

Commit ef894f8

Browse files
Enforce the wall-clock budget on verify, not just act
A loop's max_seconds is a per-run ceiling, but the runner computed the remaining budget once before the act step and reused that same figure for verify. If act consumed most of the window, verify was still handed the pre-act budget and could run a second full window — overrunning max_seconds by up to 2x. verify now recomputes the time left from the shared deadline after act returns, so its timeout reflects what's actually left. _run_command also short-circuits a non-positive timeout instead of spawning a process only to kill it. 83 -> 86 passing (budget-recompute regression, short-circuit, _time_left).
1 parent b144273 commit ef894f8

2 files changed

Lines changed: 60 additions & 5 deletions

File tree

loopforge/runner.py

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,18 @@ def _read_context(loop: Loop, root: Path) -> str:
6363
return "\n\n".join(parts)
6464

6565

66+
def _time_left(deadline: float | None) -> float | None:
67+
"""Seconds remaining until the wall-clock budget deadline (None = no budget)."""
68+
if deadline is None:
69+
return None
70+
return deadline - time.monotonic()
71+
72+
6673
def _run_command(command: str, prompt: str | None, cwd: Path, timeout: float | None) -> tuple[int, str]:
6774
"""Run a command. {prompt} is replaced with a temp file path; otherwise prompt goes to stdin."""
75+
if timeout is not None and timeout <= 0:
76+
# the wall-clock budget is already spent — don't spawn a process just to kill it
77+
return 124, "timeout"
6878
stdin_text: str | None = None
6979
tmp: Path | None = None
7080
try:
@@ -240,20 +250,21 @@ def run(
240250
result.handback_reason = "budget-exceeded"
241251
break
242252

243-
remaining = deadline - time.monotonic() if deadline else None
244253
iteration_started = _dt.datetime.now(_dt.UTC).isoformat()
245-
rc, out = _run_command(act_cmd, context, workdir, remaining)
254+
rc, out = _run_command(act_cmd, context, workdir, _time_left(deadline))
246255

247256
# verify is independent: a `command` is a test/build (no prompt); a `reviewer_command` is a
248-
# second model that must SEE the act output to review it.
257+
# second model that must SEE the act output to review it. Recompute the time budget here —
258+
# act may have consumed most of it, and reusing the pre-act figure would let verify run the
259+
# full window again, overrunning max_seconds by up to 2x.
249260
verified: bool | None = None
250261
verify_rc: int | None = None
251262
verify_out = ""
252263
if verify_command:
253-
verify_rc, verify_out = _run_command(verify_command, None, workdir, remaining)
264+
verify_rc, verify_out = _run_command(verify_command, None, workdir, _time_left(deadline))
254265
verified = verify_rc == 0
255266
elif reviewer_command:
256-
verify_rc, verify_out = _run_command(reviewer_command, out, workdir, remaining)
267+
verify_rc, verify_out = _run_command(reviewer_command, out, workdir, _time_left(deadline))
257268
verified = verify_rc == 0
258269
if verified is not None:
259270
consecutive_failures = 0 if verified else consecutive_failures + 1

tests/test_runner.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,3 +167,47 @@ def test_max_iterations_override(tmp_path):
167167
result = run(p, max_iterations=3)
168168
assert result.iterations == 3
169169
assert result.handback_reason == "max-iterations"
170+
171+
172+
def test_run_command_short_circuits_when_budget_is_spent():
173+
# A non-positive remaining budget must NOT spawn the process — it returns a timeout immediately.
174+
# If it spawned, `echo` would return (0, "hi"); the guard makes it (124, "timeout").
175+
from loopforge.runner import _run_command
176+
rc, out = _run_command("echo hi", None, Path("."), timeout=-0.5)
177+
assert rc == 124
178+
assert out == "timeout"
179+
180+
181+
def test_time_left_counts_down_from_deadline():
182+
import time as _t
183+
184+
from loopforge.runner import _time_left
185+
assert _time_left(None) is None
186+
deadline = _t.monotonic() + 10
187+
left = _time_left(deadline)
188+
assert left is not None and 0 < left <= 10
189+
assert _time_left(_t.monotonic() - 1) < 0 # already past the deadline
190+
191+
192+
def test_verify_budget_is_recomputed_after_act(tmp_path, monkeypatch):
193+
# Regression: verify used to reuse the pre-act time budget, so it could run a full second window
194+
# after act already consumed max_seconds. With the fix, verify's timeout is recomputed from the
195+
# shared deadline AFTER act ran — so it is strictly less than act's, never the original window.
196+
import loopforge.runner as runner
197+
198+
p = make_loop(tmp_path, act="echo working", verify="echo verifying", max_iter=1) # default max_seconds
199+
200+
calls = []
201+
real = runner._run_command
202+
203+
def spy(command, prompt, cwd, timeout):
204+
calls.append((command, timeout))
205+
return real(command, prompt, cwd, timeout)
206+
207+
monkeypatch.setattr(runner, "_run_command", spy)
208+
runner.run(p)
209+
210+
act_timeout = next(to for cmd, to in calls if cmd == "echo working")
211+
verify_timeout = next(to for cmd, to in calls if cmd == "echo verifying")
212+
assert act_timeout is not None and verify_timeout is not None # deadline enforced on both
213+
assert verify_timeout < act_timeout # recomputed after act, so strictly less time remains

0 commit comments

Comments
 (0)