diff --git a/CHANGELOG.md b/CHANGELOG.md index 8248219..44b5c50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ separately via `version:` in `loop.yaml` (currently `1`). ## Unreleased +### Added — runner contract v1 + conformance suite +- `RUNNER-CONTRACT.md` — the normative contract for third-party runners + executing `loop.resolved.json` (spec version 1): inputs, path safety, + model invocation, gate/verdict semantics, caps and termination, + fail-closed consent, two-layer redaction with surfacing, state/log + obligations, exit codes. +- `conformance/check_runner.py` — ten-scenario conformance harness any + runner can be tested against (`python conformance/check_runner.py + path/to/runner`): happy path, judge-degrade (verifying real revision + rounds), consent fail-closed, prompt redaction, host-prompt scrub, + default redaction globs, context non-send, cmd-output scrub, workspace + escape refusal, revision cap (stall-proof judge so a no-progress detector + cannot mask a missing cap). Self-contained deterministic fixtures — no + model CLIs needed. The reference `templates/run-loop.py` is held to the + suite in CI. +- Runner: any crash — not just a `RunnerError` — now leaves `state.json` in + a terminal state instead of a phantom `running` (contract section 5). + ### Fixed — redaction covers every send (runner) - **Host prompts are now scrubbed.** The host was the one recipient whose prompts never passed through the content scrub: flagged-file content that diff --git a/README.md b/README.md index 648b332..16eb03a 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,22 @@ another terminal, or outside the LLM session: python3 ./looper-output/run-loop.py ``` +### Building your own runner + +`loop.resolved.json` is an interchange format, not just this repo's internal +detail — a Looper spec can be executed by runners other than the bundled +`run-loop.py` (other languages, other host platforms, other agent stacks). +[`RUNNER-CONTRACT.md`](RUNNER-CONTRACT.md) is the normative contract: what a +runner MUST do about gates, caps, consent, redaction, state, and exit codes. +Prove a runner honors it with the conformance suite: + +```bash +python conformance/check_runner.py path/to/your-runner +``` + +Ten scenarios, deterministic fixtures, no real model CLIs needed. The +reference runner is held to the same suite in CI. + For local development, this repository root is the skill root. Edit and test it here, then install or update the global skill by cloning or copying the repo to `$HOME/.claude/skills/looper` and copying `commands/looper.md` to diff --git a/RUNNER-CONTRACT.md b/RUNNER-CONTRACT.md new file mode 100644 index 0000000..48f88b8 --- /dev/null +++ b/RUNNER-CONTRACT.md @@ -0,0 +1,173 @@ +# Looper Runner Contract (v1) + +This document is the normative contract for **runners**: programs that execute +a compiled Looper spec (`loop.resolved.json`, spec `version: 1`). The bundled +`templates/run-loop.py` is the reference implementation; third-party runners +(other languages, other host platforms) that satisfy every MUST below — and +pass the conformance suite in `conformance/` — may honestly claim to run +Looper loops. + +"MUST" / "MUST NOT" / "SHOULD" are used as in RFC 2119. The contract is +versioned with the spec format: this file describes runners for spec +`version: 1` and the resolved-spec schema `schemas/loop.resolved.v1.schema.json`. + +## 1. Inputs and invocation + +- A runner MUST take a single `loop.resolved.json` as its input and MUST NOT + read or re-interpret `loop.yaml`. Compilation is the compiler's job; the + resolved spec is the whole interface. +- Invocation convention (what the conformance suite exercises): the resolved + spec path is the runner's sole positional argument, and the runner operates + with the *loop directory* (the spec file's parent) as its working + directory. A runner MAY offer additional interfaces, but MUST support this + one to claim conformance. +- A runner MUST reject a spec whose required fields are missing with a clear + error rather than guessing defaults for: `goal.statement`, `gates.*`, + `loop_control.max_iterations`, `workspace.dir`, `host.invoke`. + +## 2. Paths and workspace + +- `workspace.dir` MUST resolve inside the directory containing the spec (the + *loop directory*). A workspace path that escapes it (absolute, `..`, + symlink tricks) MUST abort the run before any filesystem writes. +- A context-source `file` path that escapes the loop directory MUST NOT be + read. The runner MAY abort, or MAY continue with a blocked-path marker in + place of the content (the reference runner does the latter); what it MUST + NOT do is read or send the file. +- Artifacts live in the workspace: `plan.md`, `delivery-.md`, + `review--.md`, plus the configured `observability.state_file` and + `observability.run_log`. + +## 3. Model invocations + +- Every model call MUST use the member's `invoke` argv array verbatim — no + shell string interpolation — with the prompt on stdin, and MUST apply the + member's `timeout_sec`. +- Programmatic checks MUST run their `check` argv with `timeout_sec` and + evaluate `expect` as: `exit_zero` (return code 0), `exit_nonzero` + (non-zero), `stdout_contains` (the `contains` string appears in stdout). +- Programmatic-check and `cmd` context-source subprocesses MUST run with the + loop directory as their working directory — specs are written with + loop-directory-relative paths (`loop-workspace/delivery-1.md`, + `inputs/...`). +- Model and check subprocess output MUST be decoded as UTF-8 (with + replacement on errors), not the platform locale codepage. + +## 4. Gates and verdicts + +- The plan gate MUST pass on `plan.md` before any delivery is produced; the + delivery gate MUST run against each `delivery-.md`. +- Criterion types: + - `programmatic` and `human` criteria are evaluated on every gate round. + - `judge` criteria are evaluated **only** by the gate's `verdict_source` + when `verdict_policy: revise_until_clean` names a judge member. A runner + MUST NOT silently skip judge criteria in other configurations — if a + gate's configuration makes a judge criterion unreachable, the runner + SHOULD surface that (the compiler's `lint` flags it statically). +- `revise_until_clean`: the gate passes only when no programmatic/human + failures remain AND the verdict source returns `pass`. A `human` verdict + source is an interactive approval. +- `fixed_passes`: the gate runs reviewer passes until `max_revisions` + review rounds have completed with no programmatic/human failures. The + synthetic "one more reviewer pass" marker MUST NOT feed the no-progress + detector or be reported as a real failure. +- Judge output MUST be parsed as a JSON object with `verdict` of `pass` or + `revise`. Unparseable or malformed output MUST degrade to `revise` with a + recorded warning — never to `pass`. +- On a failed round the runner MUST write the review artifact, request a + host revision, and stop the gate at `max_revisions`, marking the run + failed. + +## 5. Caps and termination + +- `loop_control.max_iterations` MUST bound delivery iterations; exhausting it + fails the run. +- The wall-clock budget (`loop_control.budget.wall_clock_min`) MUST be + enforced at least at step boundaries, and elapsed time MUST accumulate + across resumed runs (persisted in state). +- Token / USD budgets are advisory unless the runner has real accounting; a + runner MUST NOT claim to enforce them when it cannot measure them. +- No-progress: the runner MUST derive a signature from a gate round's + failures; when the same gate yields the same signature + `max_stalled_iterations` times consecutively, the configured action fires: + `stop` fails the **entire run** (not just the current artifact); + `human_checkpoint` asks for an interactive override. +- A run MUST end in exactly one terminal state: `passed`, `failed`, or + `blocked`. A crash or model failure MUST NOT leave state claiming + `running`. + +## 6. Consent (fail closed) + +- Before the first send to any council member not flagged `local: true`, the + runner MUST obtain interactive consent — unless every `privacy.egress` + entry targeting that member declares `consent: granted`. +- No matching egress entry means consent is REQUIRED, not waived. +- Granted consent MUST be recorded in state (timestamp, what is sent, active + redactions) so later runs and audits can see it. Refused consent MUST + block the run (`blocked`, non-zero exit) without invoking the member. +- The prompt content MUST be scrubbed (section 7) before consent is + requested, and the consent prompt SHOULD display any leak warnings + already recorded for that member — the consent decision is made with the + leak signal visible, not before it exists. + +## 7. Redaction (two layers) + +- The **default redaction globs** are normative and MUST always apply, even + when `privacy.egress` is empty or absent (the resolved spec does not carry + them): `.env`, `.env.*`, `secrets/**`, `**/*.key`. Every + `privacy.egress[].redact` pattern extends this set. +- **Layer 1 — path-based non-send.** Files matching the redaction globs + (matching must cover nested paths, e.g. bare `.env` also matches + `config/.env`) MUST NOT be read into any prompt. A context source naming a + flagged file yields a `[redacted]` marker instead of content. +- **Layer 2 — content scrub with surfacing.** Prompts to **any** model — + the host as much as council members — and the output of `cmd` context + sources MUST be scrubbed against the content of flagged files + (full-content and per-line). When a scrub catches anything, the runner + MUST log a redaction event and record a state warning naming **every** + source file whose content matched and the destination. Scrubbing is + best-effort and errs toward over-redaction; surfacing it is what keeps + the contract honest. +- A flagged file the runner cannot content-scrub (too large, undecodable) + MUST be surfaced as a blind spot (log event + state warning) rather than + silently skipped — the operator must know the scrub cannot see it. + +## 8. State and run log + +- The state file MUST track at minimum: `status`, `iteration`, `consent`, + `warnings`, accumulated wall-clock, and timestamps; it is the resumable + snapshot. +- The run log MUST be append-only and record at minimum: run start, context + reads (with redaction status), host calls, programmatic check results, + judge verdicts, gate transitions, revisions, no-progress detections, and + every stop decision with its reason. + +## 9. Exit codes + +- `0` MUST mean exactly one thing: the loop passed all gates. Any other + outcome MUST exit non-zero. +- The split among non-zero codes is a SHOULD: `1` for a loop that ran and + failed (gate exhaustion, iteration cap, no-progress), `2` for a runner + refusal or error (bad spec, bad paths, consent refused, interactive input + unavailable). The reference runner maps a wall-clock budget abort to `2` + (it is raised as a runner error); tooling that needs to distinguish + failure classes should read the state file's `status`/`failure`, which is + the normative record, rather than the 1-vs-2 split. + +## 10. Conformance + +Run the suite against your runner: + +```bash +python conformance/check_runner.py path/to/your-runner +``` + +The harness scaffolds fixture loops (deterministic fake host/judge scripts — +no real model CLIs needed), invokes your runner in each, and asserts the +observable obligations above: happy path, judge-degrade (with real revision +rounds), consent fail-closed, prompt redaction, host-prompt scrub, default +redaction globs, context non-send, cmd-output scrub, workspace escape +refusal, and revision-cap failure (with a stall-proof judge, so a no-progress +detector cannot mask a missing cap). Every scenario must pass for a +conformance claim. The reference runner is checked in CI against this same +suite. diff --git a/conformance/check_runner.py b/conformance/check_runner.py new file mode 100644 index 0000000..6cf81d0 --- /dev/null +++ b/conformance/check_runner.py @@ -0,0 +1,565 @@ +#!/usr/bin/env python3 +"""Looper runner conformance harness (contract v1). + +Checks that a runner honors the observable obligations in RUNNER-CONTRACT.md: +gates, caps, consent, redaction, state, and exit codes. Scenarios scaffold a +fixture loop in a temp directory (deterministic fake host/judge scripts - no +real model CLIs), compile it with the repo compiler, invoke the candidate +runner, and assert on what the contract makes observable: exit codes, state +file contents, artifacts, and what reached the fake judge's stdin. + +Usage: + python conformance/check_runner.py path/to/runner[.py] [--only NAME] + +A .py runner is invoked with this same Python interpreter; anything else is +executed directly. The resolved spec path is passed as the single argument, +with the loop directory as the working directory. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +from typing import Any, Callable, Optional + +ROOT = Path(__file__).resolve().parents[1] +COMPILER = ROOT / "scripts" / "looper.py" + +# Fake host: argv[1] captures every prompt received; optional argv[2] names a +# "leak file" whose content the host copies into the plan it drafts - the +# legitimate path by which flagged content enters an artifact. +FAKE_HOST = ''' +import pathlib, sys +prompt = sys.stdin.read() +capture = pathlib.Path(sys.argv[1]) +previous = capture.read_text(encoding="utf-8") if capture.exists() else "" +capture.write_text(previous + "\\n---CALL---\\n" + prompt, encoding="utf-8") +leak = pathlib.Path(sys.argv[2]) if len(sys.argv) > 2 else None +if "Revise the artifact" in prompt: + print("Revised artifact") + print() + print("Owner: host") + print("No TBD") +elif "Draft plan.md" in prompt: + print("# Plan") + print() + print("Owner: host") + if leak is not None and leak.exists(): + print(leak.read_text(encoding="utf-8")) + print("No TBD") +else: + print("# Delivery") + print() + print("Owner: host") + print("No TBD") +''' + +# Verdict-emitting judge: revises `revise_count` times, then passes; a +# negative revise_count means revise forever with a UNIQUE blocking issue per +# call (so no-progress detectors cannot mask a missing revision cap). Records +# every prompt it receives so scenarios can assert what crossed the boundary. +FAKE_JUDGE = ''' +import json, pathlib, sys +state_path = pathlib.Path(sys.argv[1]) +revise_count = int(sys.argv[2]) +capture = pathlib.Path(sys.argv[3]) if len(sys.argv) > 3 else None +prompt = sys.stdin.read() +if capture is not None: + previous = capture.read_text(encoding="utf-8") if capture.exists() else "" + capture.write_text(previous + "\\n---CALL---\\n" + prompt, encoding="utf-8") +state = {"count": 0} +if state_path.exists(): + state = json.loads(state_path.read_text(encoding="utf-8")) +count = int(state.get("count", 0)) +state["count"] = count + 1 +state_path.write_text(json.dumps(state), encoding="utf-8") +if revise_count < 0: + verdict = {"verdict": "revise", "blocking_issues": [f"Unique issue {count}."], "confidence": 0.9, "notes": ""} +elif count < revise_count: + verdict = {"verdict": "revise", "blocking_issues": ["Add an explicit owner."], "confidence": 0.9, "notes": ""} +else: + verdict = {"verdict": "pass", "blocking_issues": [], "confidence": 0.9, "notes": ""} +print("```json") +print(json.dumps(verdict)) +print("```") +''' + +BAD_JUDGE = ''' +print("Looks pretty good, but this is not JSON.") +''' + +CHECK_CONTAINS = ''' +import pathlib, sys +path = pathlib.Path(sys.argv[1]) +needle = sys.argv[2] +if path.exists() and needle in path.read_text(encoding="utf-8"): + raise SystemExit(0) +print(f"{needle!r} not found in {path}", file=sys.stderr) +raise SystemExit(1) +''' + +SECRET = "CONFORMANCE-SECRET-VALUE-93af" + + +class Scenario: + def __init__(self, name: str, doc: str, fn: Callable[["Harness"], Optional[str]]): + self.name = name + self.doc = doc + self.fn = fn + + +class Harness: + def __init__(self, runner: Path, python: str): + self.runner = runner + self.python = python + + def runner_argv(self, spec: Path) -> list: + if self.runner.suffix == ".py": + return [self.python, str(self.runner), str(spec)] + return [str(self.runner), str(spec)] + + def scaffold( + self, + work: Path, + *, + judge_role: str = "judge", + judge_local: bool = True, + judge_script: str = "fake_judge", + judge_revise_count: int = 1, + egress_yaml: str = " egress: []", + extra_context: str = "", + max_revisions: int = 2, + host_leak: str = "", + ) -> Path: + """Write fixture scripts + loop.yaml into `work`, compile, return spec path.""" + fixtures = work / "fixtures" + fixtures.mkdir(parents=True, exist_ok=True) + (fixtures / "fake_host.py").write_text(textwrap.dedent(FAKE_HOST), encoding="utf-8") + (fixtures / "fake_judge.py").write_text(textwrap.dedent(FAKE_JUDGE), encoding="utf-8") + (fixtures / "bad_judge.py").write_text(textwrap.dedent(BAD_JUDGE), encoding="utf-8") + (fixtures / "check_contains.py").write_text(textwrap.dedent(CHECK_CONTAINS), encoding="utf-8") + (work / "inputs").mkdir(exist_ok=True) + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + + python = Path(self.python).as_posix() + judge_state = (work / "judge-state.json").as_posix() + capture = (work / "judge-capture.txt").as_posix() + host_invoke = [python, (fixtures / "fake_host.py").as_posix(), (work / "host-capture.txt").as_posix()] + if host_leak: + host_invoke.append(host_leak) + host_invoke_tail = ", ".join(f'"{item}"' for item in host_invoke) + if judge_script == "fake_judge": + judge_invoke = ( + f'["{python}", "{(fixtures / "fake_judge.py").as_posix()}", ' + f'"{judge_state}", "{judge_revise_count}", "{capture}"]' + ) + else: + judge_invoke = f'["{python}", "{(fixtures / "bad_judge.py").as_posix()}"]' + + loop_yaml = textwrap.dedent( + f""" + version: 1 + meta: + name: conformance-loop + description: Conformance fixture loop + author: conformance + created: 2026-07-07 + + goal: + statement: Produce a checked delivery. + context_sources: + - file: ./inputs/process-notes.md + __EXTRA_CONTEXT__ + definition_of_done: The delivery includes an owner and no TBD. + verification: + - id: has-owner + type: programmatic + check: ["{python}", "{(fixtures / 'check_contains.py').as_posix()}", "loop-workspace/delivery-1.md", "Owner:"] + expect: exit_zero + - id: covers-goal + type: judge + rubric: The artifact includes an owner and no unresolved TBD. + + host: + cli: fake-host + model: fixture + invoke: [{host_invoke_tail}] + timeout_sec: 30 + + council: + - id: reviewer-1 + role: {judge_role} + cli: fake-judge + model: fixture + invoke: {judge_invoke} + timeout_sec: 30 + scope: [plan, delivery] + local: {str(judge_local).lower()} + + gates: + plan_gate: + when: after_plan + members: [reviewer-1] + verdict_policy: revise_until_clean + verdict_source: reviewer-1 + criteria: [covers-goal] + max_revisions: {max_revisions} + delivery_gate: + when: after_each_delivery + members: [reviewer-1] + verdict_policy: revise_until_clean + verdict_source: reviewer-1 + criteria: [has-owner, covers-goal] + max_revisions: {max_revisions} + + loop_control: + max_iterations: 2 + budget: + wall_clock_min: 5 + human_checkpoints: [] + stop_conditions: + - all deliveries pass their gate clean + - max_iterations reached + + privacy: + __EGRESS__ + + workspace: + dir: ./loop-workspace + layout: [plan.md, "delivery-{{n}}.md", "review-{{n}}.md", state.json] + """ + ).strip() + "\n" + # Sentinels are replaced after dedent so injected blocks supply their + # own indentation relative to the dedented document. + loop_yaml = loop_yaml.replace( + "\n__EXTRA_CONTEXT__", ("\n" + extra_context.rstrip()) if extra_context else "" + ) + loop_yaml = loop_yaml.replace("__EGRESS__", egress_yaml.strip("\n")) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = subprocess.run( + [self.python, str(COMPILER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + cwd=str(work), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False, + ) + if compiled.returncode != 0: + raise RuntimeError(f"fixture failed to compile: {compiled.stderr}") + return work / "loop.resolved.json" + + def run(self, work: Path, spec: Path, stdin: str = "") -> "subprocess.CompletedProcess[str]": + return subprocess.run( + self.runner_argv(spec), cwd=str(work), input=stdin, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + encoding="utf-8", errors="replace", timeout=300, check=False, + ) + + def state(self, work: Path) -> dict: + path = work / "loop-workspace" / "state.json" + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def expect(condition: Any, message: str) -> Optional[str]: + return None if condition else message + + +def check_all(*failures: Optional[str]) -> Optional[str]: + real = [item for item in failures if item] + return "; ".join(real) if real else None + + +# --- Scenarios ------------------------------------------------------------- + +def scenario_happy_path(h: Harness) -> Optional[str]: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_revise_count=1) + result = h.run(work, spec) + ws = work / "loop-workspace" + state = h.state(work) + return check_all( + expect(result.returncode == 0, f"expected exit 0, got {result.returncode}: {result.stderr}"), + expect((ws / "plan.md").exists(), "plan.md missing"), + expect((ws / "delivery-1.md").exists(), "delivery-1.md missing"), + expect(state.get("status") == "passed", f"state.status={state.get('status')!r}, want 'passed'"), + expect((ws / "run-log.md").exists() and (ws / "run-log.md").stat().st_size > 0, "run log missing or empty"), + ) + + +def scenario_judge_degrade(h: Harness) -> Optional[str]: + # Unparseable judge output must degrade to revise (never pass, never + # crash): revision rounds must actually happen (review artifacts per the + # contract's workspace layout) before the run ends failed at the cap. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_script="bad_judge") + result = h.run(work, spec) + state = h.state(work) + ws = work / "loop-workspace" + return check_all( + expect(result.returncode != 0, "expected non-zero exit for unparseable judge"), + expect(state.get("status") not in {"passed", "running", None}, f"state.status={state.get('status')!r} must be terminal and not passed"), + expect((ws / "review-plan_gate-1.md").exists(), "no review artifact - runner crashed instead of degrading to revise"), + expect((ws / "review-plan_gate-2.md").exists(), "no second review round - runner did not revise after degraded verdict"), + ) + + +def scenario_consent_fail_closed(h: Harness) -> Optional[str]: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_local=False) + refused = h.run(work, spec, stdin="") + judge_called = (work / "judge-state.json").exists() + state_refused = h.state(work) + first = check_all( + expect(refused.returncode != 0, "closed stdin must not allow the send"), + expect(not judge_called, "judge was invoked without consent"), + expect(state_refused.get("status") != "passed", "run must not pass without consent"), + ) + if first: + return first + # Fresh workspace, explicit consent: the run must proceed and record it. + import shutil as _shutil + _shutil.rmtree(work / "loop-workspace", ignore_errors=True) + granted = h.run(work, spec, stdin="yes\n") + state_granted = h.state(work) + return check_all( + expect(granted.returncode == 0, f"consented run failed: {granted.stderr}"), + expect((work / "judge-state.json").exists(), "judge never invoked after consent"), + expect(bool(state_granted.get("consent")), "granted consent not recorded in state"), + ) + + +def read_captures(work: Path) -> tuple[str, str]: + judge = work / "judge-capture.txt" + host = work / "host-capture.txt" + return ( + judge.read_text(encoding="utf-8") if judge.exists() else "", + host.read_text(encoding="utf-8") if host.exists() else "", + ) + + +def scenario_prompt_redaction(h: Harness) -> Optional[str]: + # The host copies a flagged file's content into the plan it drafts (the + # legitimate leak path); the artifact send to the judge must be scrubbed. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/**"]\n' + " consent: required\n" + ) + # judge_revise_count=0: no revision round, so the leaked line stays in + # plan.md for the whole run (the fake host's revision output would + # otherwise drop it). + spec = h.scaffold(work, egress_yaml=egress, host_leak="inputs/secret.txt", judge_revise_count=0) + result = h.run(work, spec) + judge_cap, _ = read_captures(work) + plan = work / "loop-workspace" / "plan.md" + plan_text = plan.read_text(encoding="utf-8") if plan.exists() else "" + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(SECRET in plan_text, "fixture broken: leak never entered the artifact"), + expect(judge_cap, "judge capture missing - judge never called"), + expect(SECRET not in judge_cap, "flagged content reached the judge"), + ) + + +def scenario_host_prompt_scrub(h: Harness) -> Optional[str]: + # The host EMITS the secret into plan.md, but must never RECEIVE it back: + # the delivery/revise prompts embedding that artifact must be scrubbed + # (contract section 7 - the host is a send like any other). + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/**"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress, host_leak="inputs/secret.txt", judge_revise_count=0) + result = h.run(work, spec) + _, host_cap = read_captures(work) + plan = work / "loop-workspace" / "plan.md" + plan_text = plan.read_text(encoding="utf-8") if plan.exists() else "" + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(SECRET in plan_text, "fixture broken: leak never entered the artifact"), + expect(host_cap.count("---CALL---") >= 2, "host was not called for a delivery after the plan"), + expect(SECRET not in host_cap, "flagged content was sent back to the host"), + ) + + +def scenario_context_non_send(h: Harness) -> Optional[str]: + # A context source naming a flagged file must not inline its content into + # any prompt - the host's plan prompt is where context lands first. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/secret.txt"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress, extra_context=" - file: ./inputs/secret.txt") + result = h.run(work, spec) + judge_cap, host_cap = read_captures(work) + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(host_cap, "host capture missing - host never called"), + expect(SECRET not in host_cap, "flagged context file content reached the host"), + expect(SECRET not in judge_cap, "flagged context file content reached the judge"), + ) + + +def scenario_cmd_output_scrub(h: Harness) -> Optional[str]: + # A cmd context source that prints a flagged file must be scrubbed before + # its output reaches any prompt (host first, judge downstream). + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "secret.txt").write_text(SECRET + "\n", encoding="utf-8") + python = Path(h.python).as_posix() + cmd_line = ( + f' - cmd: ["{python}", "-c", ' + '"import pathlib; print(pathlib.Path(\'inputs/secret.txt\').read_text())"]' + ) + egress = ( + " egress:\n" + " - to: reviewer-1\n" + " sends: [plan, deliveries]\n" + ' redact: ["inputs/secret.txt"]\n' + " consent: required\n" + ) + spec = h.scaffold(work, egress_yaml=egress, extra_context=cmd_line) + result = h.run(work, spec) + judge_cap, host_cap = read_captures(work) + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(host_cap, "host capture missing - host never called"), + expect(SECRET not in host_cap, "cmd output leaked flagged content to the host"), + expect(SECRET not in judge_cap, "cmd output leaked flagged content to the judge"), + ) + + +def scenario_default_redactions(h: Harness) -> Optional[str]: + # The default redaction globs (.env etc.) apply even when privacy.egress + # is empty - a runner that only honors explicit redact lists leaks here. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / ".env").write_text(f"API_KEY={SECRET}\n", encoding="utf-8") + spec = h.scaffold(work, host_leak=".env", judge_revise_count=0) + result = h.run(work, spec) + judge_cap, host_cap = read_captures(work) + return check_all( + expect(result.returncode == 0, f"run failed: {result.stderr}"), + expect(judge_cap, "judge capture missing - judge never called"), + expect(SECRET not in judge_cap, "default-glob (.env) content reached the judge"), + expect(SECRET not in host_cap, "default-glob (.env) content was sent back to the host"), + ) + + +def scenario_workspace_escape(h: Harness) -> Optional[str]: + with tempfile.TemporaryDirectory() as tmp: + outer = Path(tmp) + work = outer / "loop" + work.mkdir() + spec = h.scaffold(work) + resolved = json.loads(spec.read_text(encoding="utf-8")) + resolved["workspace"]["dir"] = "../escaped-workspace" + spec.write_text(json.dumps(resolved), encoding="utf-8") + result = h.run(work, spec) + return check_all( + expect(result.returncode != 0, "runner accepted a workspace outside the loop directory"), + expect(not (outer / "escaped-workspace").exists(), "runner created a directory outside the loop directory"), + ) + + +def scenario_revision_cap(h: Harness) -> Optional[str]: + # A judge that keeps revising - with a UNIQUE blocking issue every round, + # so a no-progress detector cannot mask a missing cap - must be stopped + # by max_revisions exactly: initial round + max_revisions rounds, then a + # terminal failure with no delivery ever drafted. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + spec = h.scaffold(work, judge_revise_count=-1, max_revisions=1) + result = h.run(work, spec) + state = h.state(work) + judge_state = work / "judge-state.json" + calls = 0 + if judge_state.exists(): + calls = int(json.loads(judge_state.read_text(encoding="utf-8")).get("count", 0)) + return check_all( + expect(result.returncode != 0, "expected non-zero exit at revision cap"), + expect(state.get("status") not in {"passed", "running", None}, f"state.status={state.get('status')!r} must be terminal and not passed"), + expect(calls == 2, f"judge called {calls} times; max_revisions=1 means exactly 2 rounds (initial + one revision)"), + expect(not (work / "loop-workspace" / "delivery-1.md").exists(), "delivery drafted despite the plan gate never passing"), + ) + + +SCENARIOS = [ + Scenario("happy-path", "gates pass after one revision; artifacts, state, log exist", scenario_happy_path), + Scenario("judge-degrade", "unparseable judge output degrades to revise and fails at cap", scenario_judge_degrade), + Scenario("consent-fail-closed", "non-local member requires recorded consent before first send", scenario_consent_fail_closed), + Scenario("prompt-redaction", "flagged-file content in an artifact never reaches a member", scenario_prompt_redaction), + Scenario("host-prompt-scrub", "flagged-file content in an artifact never reaches the host either", scenario_host_prompt_scrub), + Scenario("default-redactions", "the default redaction globs apply even with no egress entries", scenario_default_redactions), + Scenario("context-non-send", "flagged context files are not inlined into prompts", scenario_context_non_send), + Scenario("cmd-output-scrub", "cmd context-source output is scrubbed of flagged content", scenario_cmd_output_scrub), + Scenario("workspace-escape", "workspace.dir outside the loop directory is refused", scenario_workspace_escape), + Scenario("revision-cap", "max_revisions bounds the gate and ends the run failed", scenario_revision_cap), +] + + +def main(argv: "Optional[list]" = None) -> int: + parser = argparse.ArgumentParser(description="Looper runner conformance harness (contract v1)") + parser.add_argument("runner", type=Path, help="Path to the runner under test") + parser.add_argument("--only", help="Run a single scenario by name") + parser.add_argument("--python", default=sys.executable, help="Interpreter for .py runners and fixtures") + args = parser.parse_args(argv) + + runner = args.runner.resolve() + if not runner.exists(): + print(f"conformance: error: runner not found: {runner}", file=sys.stderr) + return 2 + + harness = Harness(runner, args.python) + selected = [s for s in SCENARIOS if args.only in (None, s.name)] + if not selected: + print(f"conformance: error: unknown scenario {args.only!r}", file=sys.stderr) + return 2 + + failures = 0 + for scenario in selected: + try: + problem = scenario.fn(harness) + except Exception as exc: # a crashed scenario is a failure, not a crash + problem = f"scenario raised {type(exc).__name__}: {exc}" + if problem: + failures += 1 + print(f"FAIL {scenario.name}: {problem}") + else: + print(f"pass {scenario.name}") + + total = len(selected) + print(f"conformance: {total - failures}/{total} scenarios passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/templates/run-loop.py b/templates/run-loop.py index 12d4a04..c31baff 100644 --- a/templates/run-loop.py +++ b/templates/run-loop.py @@ -739,7 +739,10 @@ def human_checkpoint(self, name: str) -> None: def run(self) -> int: try: return self._run() - except RunnerError as exc: + except Exception as exc: + # Any crash - not just RunnerError - must leave a terminal state: + # a state file claiming "running" after the process died is a + # phantom in-flight run for resume logic and audits. if self.state.get("status") not in {"failed", "blocked"}: self.save_state(status="failed", failure=str(exc)) self.append_log("run_error", error=str(exc)) diff --git a/tests/test_looper.py b/tests/test_looper.py index 7a5aa45..e56c1aa 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -1277,5 +1277,43 @@ def test_citations_reject_path_outside_sources_dir(self) -> None: self.assertEqual(inside.returncode, 0, inside.stderr) +class CrashStateTests(unittest.TestCase): + def test_unexpected_crash_leaves_terminal_state_not_running(self) -> None: + # Contract section 5: a crash must not leave state claiming + # "running". A non-UTF-8 context file used to do exactly that. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_bytes( + "caf\xe9 notes\n".encode("latin-1") + ) + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertNotEqual(result.returncode, 0) + state = json.loads((work / "loop-workspace" / "state.json").read_text(encoding="utf-8")) + self.assertNotEqual(state["status"], "running", state) + + +class ConformanceTests(unittest.TestCase): + def test_reference_runner_passes_conformance_suite(self) -> None: + result = run_cmd( + [ + sys.executable, + str(ROOT / "conformance" / "check_runner.py"), + str(RUNNER_TEMPLATE), + ], + ROOT, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotIn("FAIL", result.stdout) + + if __name__ == "__main__": unittest.main()