From ec564a34af1da6cad92e4a603371294ab7b34ca4 Mon Sep 17 00:00:00 2001 From: Kevin Simback <16635828+ksimback@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:46:15 -0700 Subject: [PATCH 1/2] Add RUNNER-CONTRACT.md and nine-scenario conformance suite ROADMAP Part 3 #6: make loop.resolved.json an interchange format instead of an internal detail, so ports (the Hermes signal) become allies under a contract rather than forks. RUNNER-CONTRACT.md is the normative v1 contract for runners: inputs, path safety, model invocation, gate/verdict semantics (including which configurations make judge criteria unreachable), caps and termination, fail-closed consent with the leak warning visible before the consent question, two-layer redaction (path-based non-send + content scrub with surfacing, host included, unscrubbable files reported), state and run-log obligations, and exit codes. conformance/check_runner.py proves a runner honors it: nine scenarios with self-contained deterministic fixtures (fake host/judge, no model CLIs) - happy path, judge-degrade, consent fail-closed, prompt redaction, host-prompt scrub, context non-send, cmd-output scrub, workspace escape refusal, revision cap. A .py runner runs under the harness interpreter; anything else executes directly. The reference templates/run-loop.py is held to the suite via a new unittest wrapper, so CI enforces the contract on every change (48 tests total). Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 14 + README.md | 16 ++ RUNNER-CONTRACT.md | 151 +++++++++++ conformance/check_runner.py | 506 ++++++++++++++++++++++++++++++++++++ tests/test_looper.py | 14 + 5 files changed, 701 insertions(+) create mode 100644 RUNNER-CONTRACT.md create mode 100644 conformance/check_runner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8248219..5ed4ad9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ separately via `version:` in `loop.yaml` (currently `1`). ## Unreleased +### Added — runner contract v1 + conformance suite +- `RUNNER-CONTRACT.md` — the normative contract for third-party runners + executing `loop.resolved.json` (spec version 1): inputs, path safety, + model invocation, gate/verdict semantics, caps and termination, + fail-closed consent, two-layer redaction with surfacing, state/log + obligations, exit codes. +- `conformance/check_runner.py` — nine-scenario conformance harness any + runner can be tested against (`python conformance/check_runner.py + path/to/runner`): happy path, judge-degrade, consent fail-closed, prompt + redaction, host-prompt scrub, context non-send, cmd-output scrub, + workspace escape refusal, revision cap. Self-contained deterministic + fixtures — no model CLIs needed. The reference `templates/run-loop.py` + is held to the suite in CI. + ### Fixed — redaction covers every send (runner) - **Host prompts are now scrubbed.** The host was the one recipient whose prompts never passed through the content scrub: flagged-file content that diff --git a/README.md b/README.md index 648b332..d5b21cb 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,22 @@ another terminal, or outside the LLM session: python3 ./looper-output/run-loop.py ``` +### Building your own runner + +`loop.resolved.json` is an interchange format, not just this repo's internal +detail — a Looper spec can be executed by runners other than the bundled +`run-loop.py` (other languages, other host platforms, other agent stacks). +[`RUNNER-CONTRACT.md`](RUNNER-CONTRACT.md) is the normative contract: what a +runner MUST do about gates, caps, consent, redaction, state, and exit codes. +Prove a runner honors it with the conformance suite: + +```bash +python conformance/check_runner.py path/to/your-runner +``` + +Nine scenarios, deterministic fixtures, no real model CLIs needed. The +reference runner is held to the same suite in CI. + For local development, this repository root is the skill root. Edit and test it here, then install or update the global skill by cloning or copying the repo to `$HOME/.claude/skills/looper` and copying `commands/looper.md` to diff --git a/RUNNER-CONTRACT.md b/RUNNER-CONTRACT.md new file mode 100644 index 0000000..6388bd4 --- /dev/null +++ b/RUNNER-CONTRACT.md @@ -0,0 +1,151 @@ +# Looper Runner Contract (v1) + +This document is the normative contract for **runners**: programs that execute +a compiled Looper spec (`loop.resolved.json`, spec `version: 1`). The bundled +`templates/run-loop.py` is the reference implementation; third-party runners +(other languages, other host platforms) that satisfy every MUST below — and +pass the conformance suite in `conformance/` — may honestly claim to run +Looper loops. + +"MUST" / "MUST NOT" / "SHOULD" are used as in RFC 2119. The contract is +versioned with the spec format: this file describes runners for spec +`version: 1` and the resolved-spec schema `schemas/loop.resolved.v1.schema.json`. + +## 1. Inputs + +- A runner MUST take a single `loop.resolved.json` as its input and MUST NOT + read or re-interpret `loop.yaml`. Compilation is the compiler's job; the + resolved spec is the whole interface. +- A runner MUST reject a spec whose required fields are missing with a clear + error rather than guessing defaults for: `goal.statement`, `gates.*`, + `loop_control.max_iterations`, `workspace.dir`, `host.invoke`. + +## 2. Paths and workspace + +- `workspace.dir` and every context-source `file` path MUST resolve inside + the directory containing the spec (the *loop directory*). A path that + escapes it (absolute, `..`, symlink tricks) MUST abort the run before any + filesystem writes. +- Artifacts live in the workspace: `plan.md`, `delivery-.md`, + `review--.md`, plus the configured `observability.state_file` and + `observability.run_log`. + +## 3. Model invocations + +- Every model call MUST use the member's `invoke` argv array verbatim — no + shell string interpolation — with the prompt on stdin, and MUST apply the + member's `timeout_sec`. +- Programmatic checks MUST run their `check` argv with `timeout_sec` and + evaluate `expect` as: `exit_zero` (return code 0), `exit_nonzero` + (non-zero), `stdout_contains` (the `contains` string appears in stdout). +- Model and check subprocess output MUST be decoded as UTF-8 (with + replacement on errors), not the platform locale codepage. + +## 4. Gates and verdicts + +- The plan gate MUST pass on `plan.md` before any delivery is produced; the + delivery gate MUST run against each `delivery-.md`. +- Criterion types: + - `programmatic` and `human` criteria are evaluated on every gate round. + - `judge` criteria are evaluated **only** by the gate's `verdict_source` + when `verdict_policy: revise_until_clean` names a judge member. A runner + MUST NOT silently skip judge criteria in other configurations — if a + gate's configuration makes a judge criterion unreachable, the runner + SHOULD surface that (the compiler's `lint` flags it statically). +- `revise_until_clean`: the gate passes only when no programmatic/human + failures remain AND the verdict source returns `pass`. A `human` verdict + source is an interactive approval. +- `fixed_passes`: the gate runs reviewer passes until `max_revisions` + review rounds have completed with no programmatic/human failures. The + synthetic "one more reviewer pass" marker MUST NOT feed the no-progress + detector or be reported as a real failure. +- Judge output MUST be parsed as a JSON object with `verdict` of `pass` or + `revise`. Unparseable or malformed output MUST degrade to `revise` with a + recorded warning — never to `pass`. +- On a failed round the runner MUST write the review artifact, request a + host revision, and stop the gate at `max_revisions`, marking the run + failed. + +## 5. Caps and termination + +- `loop_control.max_iterations` MUST bound delivery iterations; exhausting it + fails the run. +- The wall-clock budget (`loop_control.budget.wall_clock_min`) MUST be + enforced at least at step boundaries, and elapsed time MUST accumulate + across resumed runs (persisted in state). +- Token / USD budgets are advisory unless the runner has real accounting; a + runner MUST NOT claim to enforce them when it cannot measure them. +- No-progress: the runner MUST derive a signature from a gate round's + failures; when the same gate yields the same signature + `max_stalled_iterations` times consecutively, the configured action fires: + `stop` fails the **entire run** (not just the current artifact); + `human_checkpoint` asks for an interactive override. +- A run MUST end in exactly one terminal state: `passed`, `failed`, or + `blocked`. A crash or model failure MUST NOT leave state claiming + `running`. + +## 6. Consent (fail closed) + +- Before the first send to any council member not flagged `local: true`, the + runner MUST obtain interactive consent — unless every `privacy.egress` + entry targeting that member declares `consent: granted`. +- No matching egress entry means consent is REQUIRED, not waived. +- Granted consent MUST be recorded in state (timestamp, what is sent, active + redactions) so later runs and audits can see it. Refused consent MUST + block the run (`blocked`, non-zero exit) without invoking the member. +- The prompt content MUST be scrubbed (section 7) before consent is + requested, and the consent prompt SHOULD display any leak warnings + already recorded for that member — the consent decision is made with the + leak signal visible, not before it exists. + +## 7. Redaction (two layers) + +- **Layer 1 — path-based non-send.** Files matching the redaction globs (the + default set plus every `privacy.egress[].redact` pattern; matching must + cover nested paths, e.g. bare `.env` also matches `config/.env`) MUST NOT + be read into any prompt. A context source naming a flagged file yields a + `[redacted]` marker instead of content. +- **Layer 2 — content scrub with surfacing.** Prompts to **any** model — + the host as much as council members — and the output of `cmd` context + sources MUST be scrubbed against the content of flagged files + (full-content and per-line). When a scrub catches anything, the runner + MUST log a redaction event and record a state warning naming **every** + source file whose content matched and the destination. Scrubbing is + best-effort and errs toward over-redaction; surfacing it is what keeps + the contract honest. +- A flagged file the runner cannot content-scrub (too large, undecodable) + MUST be surfaced as a blind spot (log event + state warning) rather than + silently skipped — the operator must know the scrub cannot see it. + +## 8. State and run log + +- The state file MUST track at minimum: `status`, `iteration`, `consent`, + `warnings`, accumulated wall-clock, and timestamps; it is the resumable + snapshot. +- The run log MUST be append-only and record at minimum: run start, context + reads (with redaction status), host calls, programmatic check results, + judge verdicts, gate transitions, revisions, no-progress detections, and + every stop decision with its reason. + +## 9. Exit codes + +- `0` — the loop passed all gates. +- `1` — the loop ran and failed (caps, gate exhaustion, no-progress). +- `2` — the runner itself refused or errored (bad spec, bad paths, consent + refused, interactive input unavailable). + +## 10. Conformance + +Run the suite against your runner: + +```bash +python conformance/check_runner.py path/to/your-runner +``` + +The harness scaffolds fixture loops (deterministic fake host/judge scripts — +no real model CLIs needed), invokes your runner in each, and asserts the +observable obligations above: happy path, judge-degrade, consent fail-closed, +prompt redaction, host-prompt scrub, context non-send, cmd-output scrub, +workspace escape refusal, and revision-cap failure. Every scenario must pass +for a conformance claim. The reference runner is checked in CI against this +same suite. diff --git a/conformance/check_runner.py b/conformance/check_runner.py new file mode 100644 index 0000000..b405e5c --- /dev/null +++ b/conformance/check_runner.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +"""Looper runner conformance harness (contract v1). + +Checks that a runner honors the observable obligations in RUNNER-CONTRACT.md: +gates, caps, consent, redaction, state, and exit codes. Scenarios scaffold a +fixture loop in a temp directory (deterministic fake host/judge scripts - no +real model CLIs), compile it with the repo compiler, invoke the candidate +runner, and assert on what the contract makes observable: exit codes, state +file contents, artifacts, and what reached the fake judge's stdin. + +Usage: + python conformance/check_runner.py path/to/runner[.py] [--only NAME] + +A .py runner is invoked with this same Python interpreter; anything else is +executed directly. The resolved spec path is passed as the single argument, +with the loop directory as the working directory. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +from typing import Any, Callable, Optional + +ROOT = Path(__file__).resolve().parents[1] +COMPILER = ROOT / "scripts" / "looper.py" + +FAKE_HOST = ''' +import pathlib, sys +prompt = sys.stdin.read() +if len(sys.argv) > 1: + capture = pathlib.Path(sys.argv[1]) + previous = capture.read_text(encoding="utf-8") if capture.exists() else "" + capture.write_text(previous + "\\n---CALL---\\n" + prompt, encoding="utf-8") +if "Revise the artifact" in prompt: + print("Revised artifact") + print() + print("Owner: host") + print("No TBD") +elif "Draft plan.md" in prompt: + print("# Plan") + print() + print("Owner: host") + print("No TBD") +else: + print("# Delivery") + print() + print("Owner: host") + print("No TBD") +''' + +# Verdict-emitting judge: revises `revise_count` times, then passes. Records +# every prompt it receives so scenarios can assert what crossed the boundary. +FAKE_JUDGE = ''' +import json, pathlib, sys +state_path = pathlib.Path(sys.argv[1]) +revise_count = int(sys.argv[2]) +capture = pathlib.Path(sys.argv[3]) if len(sys.argv) > 3 else None +prompt = sys.stdin.read() +if capture is not None: + previous = capture.read_text(encoding="utf-8") if capture.exists() else "" + capture.write_text(previous + "\\n---CALL---\\n" + prompt, encoding="utf-8") +state = {"count": 0} +if state_path.exists(): + state = json.loads(state_path.read_text(encoding="utf-8")) +count = int(state.get("count", 0)) +state["count"] = count + 1 +state_path.write_text(json.dumps(state), encoding="utf-8") +verdict = ( + {"verdict": "revise", "blocking_issues": ["Add an explicit owner."], "confidence": 0.9, "notes": ""} + if count < revise_count + else {"verdict": "pass", "blocking_issues": [], "confidence": 0.9, "notes": ""} +) +print("```json") +print(json.dumps(verdict)) +print("```") +''' + +BAD_JUDGE = ''' +print("Looks pretty good, but this is not JSON.") +''' + +CHECK_CONTAINS = ''' +import pathlib, sys +path = pathlib.Path(sys.argv[1]) +needle = sys.argv[2] +if path.exists() and needle in path.read_text(encoding="utf-8"): + raise SystemExit(0) +print(f"{needle!r} not found in {path}", file=sys.stderr) +raise SystemExit(1) +''' + +SECRET = "CONFORMANCE-SECRET-VALUE-93af" + + +class Scenario: + def __init__(self, name: str, doc: str, fn: Callable[["Harness"], Optional[str]]): + self.name = name + self.doc = doc + self.fn = fn + + +class Harness: + def __init__(self, runner: Path, python: str): + self.runner = runner + self.python = python + + def runner_argv(self, spec: Path) -> list: + if self.runner.suffix == ".py": + return [self.python, str(self.runner), str(spec)] + return [str(self.runner), str(spec)] + + def scaffold( + self, + work: Path, + *, + judge_role: str = "judge", + judge_local: bool = True, + judge_script: str = "fake_judge", + judge_revise_count: int = 1, + egress_yaml: str = " egress: []", + extra_context: str = "", + max_revisions: int = 2, + ) -> Path: + """Write fixture scripts + loop.yaml into `work`, compile, return spec path.""" + fixtures = work / "fixtures" + fixtures.mkdir(parents=True, exist_ok=True) + (fixtures / "fake_host.py").write_text(textwrap.dedent(FAKE_HOST), encoding="utf-8") + (fixtures / "fake_judge.py").write_text(textwrap.dedent(FAKE_JUDGE), encoding="utf-8") + (fixtures / "bad_judge.py").write_text(textwrap.dedent(BAD_JUDGE), encoding="utf-8") + (fixtures / "check_contains.py").write_text(textwrap.dedent(CHECK_CONTAINS), encoding="utf-8") + (work / "inputs").mkdir(exist_ok=True) + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + + python = Path(self.python).as_posix() + judge_state = (work / "judge-state.json").as_posix() + capture = (work / "judge-capture.txt").as_posix() + if judge_script == "fake_judge": + judge_invoke = ( + f'["{python}", "{(fixtures / "fake_judge.py").as_posix()}", ' + f'"{judge_state}", "{judge_revise_count}", "{capture}"]' + ) + else: + judge_invoke = f'["{python}", "{(fixtures / "bad_judge.py").as_posix()}"]' + + loop_yaml = textwrap.dedent( + f""" + version: 1 + meta: + name: conformance-loop + description: Conformance fixture loop + author: conformance + created: 2026-07-07 + + goal: + statement: Produce a checked delivery. + context_sources: + - file: ./inputs/process-notes.md + __EXTRA_CONTEXT__ + definition_of_done: The delivery includes an owner and no TBD. + verification: + - id: has-owner + type: programmatic + check: ["{python}", "{(fixtures / 'check_contains.py').as_posix()}", "loop-workspace/delivery-1.md", "Owner:"] + expect: exit_zero + - id: covers-goal + type: judge + rubric: The artifact includes an owner and no unresolved TBD. + + host: + cli: fake-host + model: fixture + invoke: ["{python}", "{(fixtures / 'fake_host.py').as_posix()}", "{(work / 'host-capture.txt').as_posix()}"] + timeout_sec: 30 + + council: + - id: reviewer-1 + role: {judge_role} + cli: fake-judge + model: fixture + invoke: {judge_invoke} + timeout_sec: 30 + scope: [plan, delivery] + local: {str(judge_local).lower()} + + gates: + plan_gate: + when: after_plan + members: [reviewer-1] + verdict_policy: revise_until_clean + verdict_source: reviewer-1 + criteria: [covers-goal] + max_revisions: {max_revisions} + delivery_gate: + when: after_each_delivery + members: [reviewer-1] + verdict_policy: revise_until_clean + verdict_source: reviewer-1 + criteria: [has-owner, covers-goal] + max_revisions: {max_revisions} + + loop_control: + max_iterations: 2 + budget: + wall_clock_min: 5 + human_checkpoints: [] + stop_conditions: + - all deliveries pass their gate clean + - max_iterations reached + + privacy: + __EGRESS__ + + workspace: + dir: ./loop-workspace + layout: [plan.md, "delivery-{{n}}.md", "review-{{n}}.md", state.json] + """ + ).strip() + "\n" + # Sentinels are replaced after dedent so injected blocks supply their + # own indentation relative to the dedented document. + loop_yaml = loop_yaml.replace( + "\n__EXTRA_CONTEXT__", ("\n" + extra_context.rstrip()) if extra_context else "" + ) + loop_yaml = loop_yaml.replace("__EGRESS__", egress_yaml.strip("\n")) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = subprocess.run( + [self.python, str(COMPILER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + cwd=str(work), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False, + ) + if compiled.returncode != 0: + raise RuntimeError(f"fixture failed to compile: {compiled.stderr}") + return work / "loop.resolved.json" + + def run(self, work: Path, spec: Path, stdin: str = "") -> "subprocess.CompletedProcess[str]": + return subprocess.run( + self.runner_argv(spec), cwd=str(work), input=stdin, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + encoding="utf-8", errors="replace", timeout=300, check=False, + ) + + def state(self, work: Path) -> dict: + path = work / "loop-workspace" / "state.json" + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def expect(condition: Any, message: str) -> Optional[str]: + return None if condition else message + + +def check_all(*failures: Optional[str]) -> Optional[str]: + real = [item for item in failures if item] + return "; ".join(real) if real else None + + +# --- Scenarios ------------------------------------------------------------- + +def scenario_happy_path(h: Harness) -> Optional[str]: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_revise_count=1) + result = h.run(work, spec) + ws = work / "loop-workspace" + state = h.state(work) + return check_all( + expect(result.returncode == 0, f"expected exit 0, got {result.returncode}: {result.stderr}"), + expect((ws / "plan.md").exists(), "plan.md missing"), + expect((ws / "delivery-1.md").exists(), "delivery-1.md missing"), + expect(state.get("status") == "passed", f"state.status={state.get('status')!r}, want 'passed'"), + expect((ws / "run-log.md").exists() and (ws / "run-log.md").stat().st_size > 0, "run log missing or empty"), + ) + + +def scenario_judge_degrade(h: Harness) -> Optional[str]: + # Unparseable judge output must degrade to revise (never pass) and the + # run must end failed at the revision cap. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_script="bad_judge") + result = h.run(work, spec) + state = h.state(work) + return check_all( + expect(result.returncode != 0, "expected non-zero exit for unparseable judge"), + expect(state.get("status") not in {"passed", "running", None}, f"state.status={state.get('status')!r} must be terminal and not passed"), + ) + + +def scenario_consent_fail_closed(h: Harness) -> Optional[str]: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_local=False) + refused = h.run(work, spec, stdin="") + judge_called = (work / "judge-state.json").exists() + state_refused = h.state(work) + first = check_all( + expect(refused.returncode != 0, "closed stdin must not allow the send"), + expect(not judge_called, "judge was invoked without consent"), + expect(state_refused.get("status") != "passed", "run must not pass without consent"), + ) + if first: + return first + # Fresh workspace, explicit consent: the run must proceed and record it. + import shutil as _shutil + _shutil.rmtree(work / "loop-workspace", ignore_errors=True) + granted = h.run(work, spec, stdin="yes\n") + state_granted = h.state(work) + return check_all( + expect(granted.returncode == 0, f"consented run failed: {granted.stderr}"), + expect((work / "judge-state.json").exists(), "judge never invoked after consent"), + expect(bool(state_granted.get("consent")), "granted consent not recorded in state"), + ) + + +def scenario_prompt_redaction(h: Harness) -> Optional[str]: + # Flagged-file content seeded into an artifact must not reach the judge. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/**"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress) + ws = work / "loop-workspace" + ws.mkdir() + (ws / "plan.md").write_text(f"Plan leaked {SECRET} before redaction.\nOwner: host\nNo TBD\n", encoding="utf-8") + result = h.run(work, spec) + capture = work / "judge-capture.txt" + captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(captured, "judge capture missing - judge never called"), + expect(SECRET not in captured, "flagged content reached the judge"), + ) + + +def scenario_host_prompt_scrub(h: Harness) -> Optional[str]: + # Flagged-file content seeded into an artifact must not reach the host + # either - the host is a send like any other (contract section 7). + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/**"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress) + ws = work / "loop-workspace" + ws.mkdir() + (ws / "plan.md").write_text(f"Plan leaked {SECRET} before redaction.\nOwner: host\nNo TBD\n", encoding="utf-8") + result = h.run(work, spec) + capture = work / "host-capture.txt" + captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(captured, "host capture missing - host never called"), + expect(SECRET not in captured, "flagged content reached the host"), + ) + + +def scenario_context_non_send(h: Harness) -> Optional[str]: + # A context source naming a flagged file must not inline its content. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/secret.txt"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress, extra_context=" - file: ./inputs/secret.txt") + result = h.run(work, spec) + capture = work / "judge-capture.txt" + captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(SECRET not in captured, "flagged context file content reached the judge"), + ) + + +def scenario_cmd_output_scrub(h: Harness) -> Optional[str]: + # A cmd context source that prints a flagged file must be scrubbed. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + python = Path(h.python).as_posix() + cmd_line = ( + f' - cmd: ["{python}", "-c", ' + '"import pathlib; print(pathlib.Path(\'inputs/secret.txt\').read_text())"]' + ) + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/secret.txt"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress, extra_context=cmd_line) + result = h.run(work, spec) + capture = work / "judge-capture.txt" + captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(SECRET not in captured, "cmd output leaked flagged content to the judge"), + ) + + +def scenario_workspace_escape(h: Harness) -> Optional[str]: + with tempfile.TemporaryDirectory() as tmp: + outer = Path(tmp) + work = outer / "loop" + work.mkdir() + spec = h.scaffold(work) + resolved = json.loads(spec.read_text(encoding="utf-8")) + resolved["workspace"]["dir"] = "../escaped-workspace" + spec.write_text(json.dumps(resolved), encoding="utf-8") + result = h.run(work, spec) + return check_all( + expect(result.returncode != 0, "runner accepted a workspace outside the loop directory"), + expect(not (outer / "escaped-workspace").exists(), "runner created a directory outside the loop directory"), + ) + + +def scenario_revision_cap(h: Harness) -> Optional[str]: + # A judge that keeps revising must exhaust max_revisions and fail the + # run with a terminal state, not loop forever or pass. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_revise_count=99, max_revisions=1) + result = h.run(work, spec) + state = h.state(work) + return check_all( + expect(result.returncode != 0, "expected non-zero exit at revision cap"), + expect(state.get("status") not in {"passed", "running", None}, f"state.status={state.get('status')!r} must be terminal and not passed"), + ) + + +SCENARIOS = [ + Scenario("happy-path", "gates pass after one revision; artifacts, state, log exist", scenario_happy_path), + Scenario("judge-degrade", "unparseable judge output degrades to revise and fails at cap", scenario_judge_degrade), + Scenario("consent-fail-closed", "non-local member requires recorded consent before first send", scenario_consent_fail_closed), + Scenario("prompt-redaction", "flagged-file content in an artifact never reaches a member", scenario_prompt_redaction), + Scenario("host-prompt-scrub", "flagged-file content in an artifact never reaches the host either", scenario_host_prompt_scrub), + Scenario("context-non-send", "flagged context files are not inlined into prompts", scenario_context_non_send), + Scenario("cmd-output-scrub", "cmd context-source output is scrubbed of flagged content", scenario_cmd_output_scrub), + Scenario("workspace-escape", "workspace.dir outside the loop directory is refused", scenario_workspace_escape), + Scenario("revision-cap", "max_revisions bounds the gate and ends the run failed", scenario_revision_cap), +] + + +def main(argv: "Optional[list]" = None) -> int: + parser = argparse.ArgumentParser(description="Looper runner conformance harness (contract v1)") + parser.add_argument("runner", type=Path, help="Path to the runner under test") + parser.add_argument("--only", help="Run a single scenario by name") + parser.add_argument("--python", default=sys.executable, help="Interpreter for .py runners and fixtures") + args = parser.parse_args(argv) + + runner = args.runner.resolve() + if not runner.exists(): + print(f"conformance: error: runner not found: {runner}", file=sys.stderr) + return 2 + + harness = Harness(runner, args.python) + selected = [s for s in SCENARIOS if args.only in (None, s.name)] + if not selected: + print(f"conformance: error: unknown scenario {args.only!r}", file=sys.stderr) + return 2 + + failures = 0 + for scenario in selected: + try: + problem = scenario.fn(harness) + except Exception as exc: # a crashed scenario is a failure, not a crash + problem = f"scenario raised {type(exc).__name__}: {exc}" + if problem: + failures += 1 + print(f"FAIL {scenario.name}: {problem}") + else: + print(f"pass {scenario.name}") + + total = len(selected) + print(f"conformance: {total - failures}/{total} scenarios passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_looper.py b/tests/test_looper.py index 7a5aa45..fa0dd30 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -1277,5 +1277,19 @@ def test_citations_reject_path_outside_sources_dir(self) -> None: self.assertEqual(inside.returncode, 0, inside.stderr) +class ConformanceTests(unittest.TestCase): + def test_reference_runner_passes_conformance_suite(self) -> None: + result = run_cmd( + [ + sys.executable, + str(ROOT / "conformance" / "check_runner.py"), + str(RUNNER_TEMPLATE), + ], + ROOT, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("FAIL", result.stdout) + + if __name__ == "__main__": unittest.main() From 4f01f6b503ec257d91ed9174adf9b09cba9113b8 Mon Sep 17 00:00:00 2001 From: Kevin Simback <16635828+ksimback@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:10:31 -0700 Subject: [PATCH 2/2] Fix vacuous conformance scenarios and contract mismatches from review The high-effort review proved four of five redaction scenarios could false-PASS (a runner with all scrub layers disabled passed them) and found normative text contradicting the reference runner. All 10 findings addressed: Harness (now 10 scenarios; sabotage-tested): - The fake host gains a "leak file" mode: flagged content enters plan.md through the normal draft flow instead of a pre-seeded artifact no contract clause requires runners to reuse. - All redaction scenarios assert on the host capture as well as the judge capture (flagged context/cmd content lands in the HOST prompt first - asserting only on the judge was vacuous). - New default-redactions scenario: the .env default glob set must apply with an empty privacy.egress. - revision-cap uses a stall-proof judge (unique blocking issue per round) so a no-progress detector cannot mask a missing cap, and asserts the exact judge call count and that no delivery is drafted. - judge-degrade asserts review artifacts from real revision rounds (degrade-to-revise, not crash-on-bad-output). - Verified against a sabotaged runner (redaction disabled): the five redaction scenarios all FAIL, everything else still passes. Contract: - Default redaction globs enumerated normatively (.env, .env.*, secrets/**, **/*.key) - they are absent from the resolved spec and existed only in code. - Invocation conventions written down (spec path as sole positional arg, loop dir as cwd, subprocess cwd = loop dir). - Context-path escape: MUST NOT read; abort or blocked-marker both conformant (matches the reference, which continues with a marker). - Exit codes: 0=passed is the only MUST; the 1/2 split is SHOULD, with the state file as the normative failure record (the reference maps wall-clock aborts to 2). Runner: - Any crash - not just RunnerError - now leaves state.json terminal instead of a phantom "running" (contract section 5 was right and the runner was wrong; non-UTF-8 context files reproduced it). 49 tests total; conformance 10/10 on the reference runner. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 16 ++-- README.md | 2 +- RUNNER-CONTRACT.md | 60 ++++++++++----- conformance/check_runner.py | 145 +++++++++++++++++++++++++----------- templates/run-loop.py | 5 +- tests/test_looper.py | 24 ++++++ 6 files changed, 182 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed4ad9..44b5c50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,17 @@ separately via `version:` in `loop.yaml` (currently `1`). model invocation, gate/verdict semantics, caps and termination, fail-closed consent, two-layer redaction with surfacing, state/log obligations, exit codes. -- `conformance/check_runner.py` — nine-scenario conformance harness any +- `conformance/check_runner.py` — ten-scenario conformance harness any runner can be tested against (`python conformance/check_runner.py - path/to/runner`): happy path, judge-degrade, consent fail-closed, prompt - redaction, host-prompt scrub, context non-send, cmd-output scrub, - workspace escape refusal, revision cap. Self-contained deterministic - fixtures — no model CLIs needed. The reference `templates/run-loop.py` - is held to the suite in CI. + path/to/runner`): happy path, judge-degrade (verifying real revision + rounds), consent fail-closed, prompt redaction, host-prompt scrub, + default redaction globs, context non-send, cmd-output scrub, workspace + escape refusal, revision cap (stall-proof judge so a no-progress detector + cannot mask a missing cap). Self-contained deterministic fixtures — no + model CLIs needed. The reference `templates/run-loop.py` is held to the + suite in CI. +- Runner: any crash — not just a `RunnerError` — now leaves `state.json` in + a terminal state instead of a phantom `running` (contract section 5). ### Fixed — redaction covers every send (runner) - **Host prompts are now scrubbed.** The host was the one recipient whose diff --git a/README.md b/README.md index d5b21cb..16eb03a 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ Prove a runner honors it with the conformance suite: python conformance/check_runner.py path/to/your-runner ``` -Nine scenarios, deterministic fixtures, no real model CLIs needed. The +Ten scenarios, deterministic fixtures, no real model CLIs needed. The reference runner is held to the same suite in CI. For local development, this repository root is the skill root. Edit and test it diff --git a/RUNNER-CONTRACT.md b/RUNNER-CONTRACT.md index 6388bd4..48f88b8 100644 --- a/RUNNER-CONTRACT.md +++ b/RUNNER-CONTRACT.md @@ -11,21 +11,29 @@ Looper loops. versioned with the spec format: this file describes runners for spec `version: 1` and the resolved-spec schema `schemas/loop.resolved.v1.schema.json`. -## 1. Inputs +## 1. Inputs and invocation - A runner MUST take a single `loop.resolved.json` as its input and MUST NOT read or re-interpret `loop.yaml`. Compilation is the compiler's job; the resolved spec is the whole interface. +- Invocation convention (what the conformance suite exercises): the resolved + spec path is the runner's sole positional argument, and the runner operates + with the *loop directory* (the spec file's parent) as its working + directory. A runner MAY offer additional interfaces, but MUST support this + one to claim conformance. - A runner MUST reject a spec whose required fields are missing with a clear error rather than guessing defaults for: `goal.statement`, `gates.*`, `loop_control.max_iterations`, `workspace.dir`, `host.invoke`. ## 2. Paths and workspace -- `workspace.dir` and every context-source `file` path MUST resolve inside - the directory containing the spec (the *loop directory*). A path that - escapes it (absolute, `..`, symlink tricks) MUST abort the run before any - filesystem writes. +- `workspace.dir` MUST resolve inside the directory containing the spec (the + *loop directory*). A workspace path that escapes it (absolute, `..`, + symlink tricks) MUST abort the run before any filesystem writes. +- A context-source `file` path that escapes the loop directory MUST NOT be + read. The runner MAY abort, or MAY continue with a blocked-path marker in + place of the content (the reference runner does the latter); what it MUST + NOT do is read or send the file. - Artifacts live in the workspace: `plan.md`, `delivery-.md`, `review--.md`, plus the configured `observability.state_file` and `observability.run_log`. @@ -38,6 +46,10 @@ versioned with the spec format: this file describes runners for spec - Programmatic checks MUST run their `check` argv with `timeout_sec` and evaluate `expect` as: `exit_zero` (return code 0), `exit_nonzero` (non-zero), `stdout_contains` (the `contains` string appears in stdout). +- Programmatic-check and `cmd` context-source subprocesses MUST run with the + loop directory as their working directory — specs are written with + loop-directory-relative paths (`loop-workspace/delivery-1.md`, + `inputs/...`). - Model and check subprocess output MUST be decoded as UTF-8 (with replacement on errors), not the platform locale codepage. @@ -100,11 +112,14 @@ versioned with the spec format: this file describes runners for spec ## 7. Redaction (two layers) -- **Layer 1 — path-based non-send.** Files matching the redaction globs (the - default set plus every `privacy.egress[].redact` pattern; matching must - cover nested paths, e.g. bare `.env` also matches `config/.env`) MUST NOT - be read into any prompt. A context source naming a flagged file yields a - `[redacted]` marker instead of content. +- The **default redaction globs** are normative and MUST always apply, even + when `privacy.egress` is empty or absent (the resolved spec does not carry + them): `.env`, `.env.*`, `secrets/**`, `**/*.key`. Every + `privacy.egress[].redact` pattern extends this set. +- **Layer 1 — path-based non-send.** Files matching the redaction globs + (matching must cover nested paths, e.g. bare `.env` also matches + `config/.env`) MUST NOT be read into any prompt. A context source naming a + flagged file yields a `[redacted]` marker instead of content. - **Layer 2 — content scrub with surfacing.** Prompts to **any** model — the host as much as council members — and the output of `cmd` context sources MUST be scrubbed against the content of flagged files @@ -129,10 +144,15 @@ versioned with the spec format: this file describes runners for spec ## 9. Exit codes -- `0` — the loop passed all gates. -- `1` — the loop ran and failed (caps, gate exhaustion, no-progress). -- `2` — the runner itself refused or errored (bad spec, bad paths, consent - refused, interactive input unavailable). +- `0` MUST mean exactly one thing: the loop passed all gates. Any other + outcome MUST exit non-zero. +- The split among non-zero codes is a SHOULD: `1` for a loop that ran and + failed (gate exhaustion, iteration cap, no-progress), `2` for a runner + refusal or error (bad spec, bad paths, consent refused, interactive input + unavailable). The reference runner maps a wall-clock budget abort to `2` + (it is raised as a runner error); tooling that needs to distinguish + failure classes should read the state file's `status`/`failure`, which is + the normative record, rather than the 1-vs-2 split. ## 10. Conformance @@ -144,8 +164,10 @@ python conformance/check_runner.py path/to/your-runner The harness scaffolds fixture loops (deterministic fake host/judge scripts — no real model CLIs needed), invokes your runner in each, and asserts the -observable obligations above: happy path, judge-degrade, consent fail-closed, -prompt redaction, host-prompt scrub, context non-send, cmd-output scrub, -workspace escape refusal, and revision-cap failure. Every scenario must pass -for a conformance claim. The reference runner is checked in CI against this -same suite. +observable obligations above: happy path, judge-degrade (with real revision +rounds), consent fail-closed, prompt redaction, host-prompt scrub, default +redaction globs, context non-send, cmd-output scrub, workspace escape +refusal, and revision-cap failure (with a stall-proof judge, so a no-progress +detector cannot mask a missing cap). Every scenario must pass for a +conformance claim. The reference runner is checked in CI against this same +suite. diff --git a/conformance/check_runner.py b/conformance/check_runner.py index b405e5c..6cf81d0 100644 --- a/conformance/check_runner.py +++ b/conformance/check_runner.py @@ -30,13 +30,16 @@ ROOT = Path(__file__).resolve().parents[1] COMPILER = ROOT / "scripts" / "looper.py" +# Fake host: argv[1] captures every prompt received; optional argv[2] names a +# "leak file" whose content the host copies into the plan it drafts - the +# legitimate path by which flagged content enters an artifact. FAKE_HOST = ''' import pathlib, sys prompt = sys.stdin.read() -if len(sys.argv) > 1: - capture = pathlib.Path(sys.argv[1]) - previous = capture.read_text(encoding="utf-8") if capture.exists() else "" - capture.write_text(previous + "\\n---CALL---\\n" + prompt, encoding="utf-8") +capture = pathlib.Path(sys.argv[1]) +previous = capture.read_text(encoding="utf-8") if capture.exists() else "" +capture.write_text(previous + "\\n---CALL---\\n" + prompt, encoding="utf-8") +leak = pathlib.Path(sys.argv[2]) if len(sys.argv) > 2 else None if "Revise the artifact" in prompt: print("Revised artifact") print() @@ -46,6 +49,8 @@ print("# Plan") print() print("Owner: host") + if leak is not None and leak.exists(): + print(leak.read_text(encoding="utf-8")) print("No TBD") else: print("# Delivery") @@ -54,7 +59,9 @@ print("No TBD") ''' -# Verdict-emitting judge: revises `revise_count` times, then passes. Records +# Verdict-emitting judge: revises `revise_count` times, then passes; a +# negative revise_count means revise forever with a UNIQUE blocking issue per +# call (so no-progress detectors cannot mask a missing revision cap). Records # every prompt it receives so scenarios can assert what crossed the boundary. FAKE_JUDGE = ''' import json, pathlib, sys @@ -71,11 +78,12 @@ count = int(state.get("count", 0)) state["count"] = count + 1 state_path.write_text(json.dumps(state), encoding="utf-8") -verdict = ( - {"verdict": "revise", "blocking_issues": ["Add an explicit owner."], "confidence": 0.9, "notes": ""} - if count < revise_count - else {"verdict": "pass", "blocking_issues": [], "confidence": 0.9, "notes": ""} -) +if revise_count < 0: + verdict = {"verdict": "revise", "blocking_issues": [f"Unique issue {count}."], "confidence": 0.9, "notes": ""} +elif count < revise_count: + verdict = {"verdict": "revise", "blocking_issues": ["Add an explicit owner."], "confidence": 0.9, "notes": ""} +else: + verdict = {"verdict": "pass", "blocking_issues": [], "confidence": 0.9, "notes": ""} print("```json") print(json.dumps(verdict)) print("```") @@ -126,6 +134,7 @@ def scaffold( egress_yaml: str = " egress: []", extra_context: str = "", max_revisions: int = 2, + host_leak: str = "", ) -> Path: """Write fixture scripts + loop.yaml into `work`, compile, return spec path.""" fixtures = work / "fixtures" @@ -140,6 +149,10 @@ def scaffold( python = Path(self.python).as_posix() judge_state = (work / "judge-state.json").as_posix() capture = (work / "judge-capture.txt").as_posix() + host_invoke = [python, (fixtures / "fake_host.py").as_posix(), (work / "host-capture.txt").as_posix()] + if host_leak: + host_invoke.append(host_leak) + host_invoke_tail = ", ".join(f'"{item}"' for item in host_invoke) if judge_script == "fake_judge": judge_invoke = ( f'["{python}", "{(fixtures / "fake_judge.py").as_posix()}", ' @@ -175,7 +188,7 @@ def scaffold( host: cli: fake-host model: fixture - invoke: ["{python}", "{(fixtures / 'fake_host.py').as_posix()}", "{(work / 'host-capture.txt').as_posix()}"] + invoke: [{host_invoke_tail}] timeout_sec: 30 council: @@ -279,16 +292,20 @@ def scenario_happy_path(h: Harness) -> Optional[str]: def scenario_judge_degrade(h: Harness) -> Optional[str]: - # Unparseable judge output must degrade to revise (never pass) and the - # run must end failed at the revision cap. + # Unparseable judge output must degrade to revise (never pass, never + # crash): revision rounds must actually happen (review artifacts per the + # contract's workspace layout) before the run ends failed at the cap. with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) spec = h.scaffold(work, judge_script="bad_judge") result = h.run(work, spec) state = h.state(work) + ws = work / "loop-workspace" return check_all( expect(result.returncode != 0, "expected non-zero exit for unparseable judge"), expect(state.get("status") not in {"passed", "running", None}, f"state.status={state.get('status')!r} must be terminal and not passed"), + expect((ws / "review-plan_gate-1.md").exists(), "no review artifact - runner crashed instead of degrading to revise"), + expect((ws / "review-plan_gate-2.md").exists(), "no second review round - runner did not revise after degraded verdict"), ) @@ -318,8 +335,18 @@ def scenario_consent_fail_closed(h: Harness) -> Optional[str]: ) +def read_captures(work: Path) -> tuple[str, str]: + judge = work / "judge-capture.txt" + host = work / "host-capture.txt" + return ( + judge.read_text(encoding="utf-8") if judge.exists() else "", + host.read_text(encoding="utf-8") if host.exists() else "", + ) + + def scenario_prompt_redaction(h: Harness) -> Optional[str]: - # Flagged-file content seeded into an artifact must not reach the judge. + # The host copies a flagged file's content into the plan it drafts (the + # legitimate leak path); the artifact send to the judge must be scrubbed. with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) (work / "inputs").mkdir() @@ -331,23 +358,26 @@ def scenario_prompt_redaction(h: Harness) -> Optional[str]: ' redact: ["inputs/**"]\n' " consent: required\n" ) - spec = h.scaffold(work, egress_yaml=egress) - ws = work / "loop-workspace" - ws.mkdir() - (ws / "plan.md").write_text(f"Plan leaked {SECRET} before redaction.\nOwner: host\nNo TBD\n", encoding="utf-8") + # judge_revise_count=0: no revision round, so the leaked line stays in + # plan.md for the whole run (the fake host's revision output would + # otherwise drop it). + spec = h.scaffold(work, egress_yaml=egress, host_leak="inputs/secret.txt", judge_revise_count=0) result = h.run(work, spec) - capture = work / "judge-capture.txt" - captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + judge_cap, _ = read_captures(work) + plan = work / "loop-workspace" / "plan.md" + plan_text = plan.read_text(encoding="utf-8") if plan.exists() else "" return check_all( expect(result.returncode == 0, f"run failed: {result.stderr}"), - expect(captured, "judge capture missing - judge never called"), - expect(SECRET not in captured, "flagged content reached the judge"), + expect(SECRET in plan_text, "fixture broken: leak never entered the artifact"), + expect(judge_cap, "judge capture missing - judge never called"), + expect(SECRET not in judge_cap, "flagged content reached the judge"), ) def scenario_host_prompt_scrub(h: Harness) -> Optional[str]: - # Flagged-file content seeded into an artifact must not reach the host - # either - the host is a send like any other (contract section 7). + # The host EMITS the secret into plan.md, but must never RECEIVE it back: + # the delivery/revise prompts embedding that artifact must be scrubbed + # (contract section 7 - the host is a send like any other). with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) (work / "inputs").mkdir() @@ -359,22 +389,22 @@ def scenario_host_prompt_scrub(h: Harness) -> Optional[str]: ' redact: ["inputs/**"]\n' " consent: required\n" ) - spec = h.scaffold(work, egress_yaml=egress) - ws = work / "loop-workspace" - ws.mkdir() - (ws / "plan.md").write_text(f"Plan leaked {SECRET} before redaction.\nOwner: host\nNo TBD\n", encoding="utf-8") + spec = h.scaffold(work, egress_yaml=egress, host_leak="inputs/secret.txt", judge_revise_count=0) result = h.run(work, spec) - capture = work / "host-capture.txt" - captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + _, host_cap = read_captures(work) + plan = work / "loop-workspace" / "plan.md" + plan_text = plan.read_text(encoding="utf-8") if plan.exists() else "" return check_all( expect(result.returncode == 0, f"run failed: {result.stderr}"), - expect(captured, "host capture missing - host never called"), - expect(SECRET not in captured, "flagged content reached the host"), + expect(SECRET in plan_text, "fixture broken: leak never entered the artifact"), + expect(host_cap.count("---CALL---") >= 2, "host was not called for a delivery after the plan"), + expect(SECRET not in host_cap, "flagged content was sent back to the host"), ) def scenario_context_non_send(h: Harness) -> Optional[str]: - # A context source naming a flagged file must not inline its content. + # A context source naming a flagged file must not inline its content into + # any prompt - the host's plan prompt is where context lands first. with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) (work / "inputs").mkdir() @@ -388,16 +418,18 @@ def scenario_context_non_send(h: Harness) -> Optional[str]: ) spec = h.scaffold(work, egress_yaml=egress, extra_context=" - file: ./inputs/secret.txt") result = h.run(work, spec) - capture = work / "judge-capture.txt" - captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + judge_cap, host_cap = read_captures(work) return check_all( expect(result.returncode == 0, f"run failed: {result.stderr}"), - expect(SECRET not in captured, "flagged context file content reached the judge"), + expect(host_cap, "host capture missing - host never called"), + expect(SECRET not in host_cap, "flagged context file content reached the host"), + expect(SECRET not in judge_cap, "flagged context file content reached the judge"), ) def scenario_cmd_output_scrub(h: Harness) -> Optional[str]: - # A cmd context source that prints a flagged file must be scrubbed. + # A cmd context source that prints a flagged file must be scrubbed before + # its output reaches any prompt (host first, judge downstream). with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) (work / "inputs").mkdir() @@ -416,11 +448,29 @@ def scenario_cmd_output_scrub(h: Harness) -> Optional[str]: ) spec = h.scaffold(work, egress_yaml=egress, extra_context=cmd_line) result = h.run(work, spec) - capture = work / "judge-capture.txt" - captured = capture.read_text(encoding="utf-8") if capture.exists() else "" + judge_cap, host_cap = read_captures(work) + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(host_cap, "host capture missing - host never called"), + expect(SECRET not in host_cap, "cmd output leaked flagged content to the host"), + expect(SECRET not in judge_cap, "cmd output leaked flagged content to the judge"), + ) + + +def scenario_default_redactions(h: Harness) -> Optional[str]: + # The default redaction globs (.env etc.) apply even when privacy.egress + # is empty - a runner that only honors explicit redact lists leaks here. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / ".env").write_text(f"API_KEY={SECRET}\n", encoding="utf-8") + spec = h.scaffold(work, host_leak=".env", judge_revise_count=0) + result = h.run(work, spec) + judge_cap, host_cap = read_captures(work) return check_all( expect(result.returncode == 0, f"run failed: {result.stderr}"), - expect(SECRET not in captured, "cmd output leaked flagged content to the judge"), + expect(judge_cap, "judge capture missing - judge never called"), + expect(SECRET not in judge_cap, "default-glob (.env) content reached the judge"), + expect(SECRET not in host_cap, "default-glob (.env) content was sent back to the host"), ) @@ -441,16 +491,24 @@ def scenario_workspace_escape(h: Harness) -> Optional[str]: def scenario_revision_cap(h: Harness) -> Optional[str]: - # A judge that keeps revising must exhaust max_revisions and fail the - # run with a terminal state, not loop forever or pass. + # A judge that keeps revising - with a UNIQUE blocking issue every round, + # so a no-progress detector cannot mask a missing cap - must be stopped + # by max_revisions exactly: initial round + max_revisions rounds, then a + # terminal failure with no delivery ever drafted. with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) - spec = h.scaffold(work, judge_revise_count=99, max_revisions=1) + spec = h.scaffold(work, judge_revise_count=-1, max_revisions=1) result = h.run(work, spec) state = h.state(work) + judge_state = work / "judge-state.json" + calls = 0 + if judge_state.exists(): + calls = int(json.loads(judge_state.read_text(encoding="utf-8")).get("count", 0)) return check_all( expect(result.returncode != 0, "expected non-zero exit at revision cap"), expect(state.get("status") not in {"passed", "running", None}, f"state.status={state.get('status')!r} must be terminal and not passed"), + expect(calls == 2, f"judge called {calls} times; max_revisions=1 means exactly 2 rounds (initial + one revision)"), + expect(not (work / "loop-workspace" / "delivery-1.md").exists(), "delivery drafted despite the plan gate never passing"), ) @@ -460,6 +518,7 @@ def scenario_revision_cap(h: Harness) -> Optional[str]: Scenario("consent-fail-closed", "non-local member requires recorded consent before first send", scenario_consent_fail_closed), Scenario("prompt-redaction", "flagged-file content in an artifact never reaches a member", scenario_prompt_redaction), Scenario("host-prompt-scrub", "flagged-file content in an artifact never reaches the host either", scenario_host_prompt_scrub), + Scenario("default-redactions", "the default redaction globs apply even with no egress entries", scenario_default_redactions), Scenario("context-non-send", "flagged context files are not inlined into prompts", scenario_context_non_send), Scenario("cmd-output-scrub", "cmd context-source output is scrubbed of flagged content", scenario_cmd_output_scrub), Scenario("workspace-escape", "workspace.dir outside the loop directory is refused", scenario_workspace_escape), diff --git a/templates/run-loop.py b/templates/run-loop.py index 12d4a04..c31baff 100644 --- a/templates/run-loop.py +++ b/templates/run-loop.py @@ -739,7 +739,10 @@ def human_checkpoint(self, name: str) -> None: def run(self) -> int: try: return self._run() - except RunnerError as exc: + except Exception as exc: + # Any crash - not just RunnerError - must leave a terminal state: + # a state file claiming "running" after the process died is a + # phantom in-flight run for resume logic and audits. if self.state.get("status") not in {"failed", "blocked"}: self.save_state(status="failed", failure=str(exc)) self.append_log("run_error", error=str(exc)) diff --git a/tests/test_looper.py b/tests/test_looper.py index fa0dd30..e56c1aa 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -1277,6 +1277,30 @@ def test_citations_reject_path_outside_sources_dir(self) -> None: self.assertEqual(inside.returncode, 0, inside.stderr) +class CrashStateTests(unittest.TestCase): + def test_unexpected_crash_leaves_terminal_state_not_running(self) -> None: + # Contract section 5: a crash must not leave state claiming + # "running". A non-UTF-8 context file used to do exactly that. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_bytes( + "caf\xe9 notes\n".encode("latin-1") + ) + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertNotEqual(result.returncode, 0) + state = json.loads((work / "loop-workspace" / "state.json").read_text(encoding="utf-8")) + self.assertNotEqual(state["status"], "running", state) + + class ConformanceTests(unittest.TestCase): def test_reference_runner_passes_conformance_suite(self) -> None: result = run_cmd(